auto: docs sync 2026-07-12T00:00:23+00:00

Files changed: ", c.get(k))\nPY\n\\\"\n\"" engine/config.yaml engine/lib/lint.py engine/lib/vocab_gen.py engine/lint-report.md engine/sweep.sh vault/.obsidian/workspace.json vault/archive/projects/meshai-native-fire-severity-audit-cc-handoff.md vault/archive/projects/vaultwarden-plan.md vault/docs/matrix/matrix_host.md vault/docs/matrix/synapse.md vault/docs/services/services.md vault/docs/software/authentik.md vault/docs/software/caddy.md vault/docs/software/dns.md vault/docs/software/recon.md vault/docs/software/searxng.md vault/glossary.md vault/notes/echo6-landing-page-data-export.md vault/projects/matrix-synapse-deployment.md vault/projects/meshai.md vault/projects/meshtastic-headscale-runbook.md vault/projects/mmud-project.md vault/runbooks/add-peertube-channel.md vault/runbooks/authentik-access-groups.md vault/runbooks/authentik-create-invitation.md vault/runbooks/authentik-oidc-application.md vault/runbooks/authentik-upgrade.md vault/runbooks/expose-service-contabo.md vault/runbooks/lxc-service-migration.md vault/runbooks/mailcow-create-mailbox.md vault/runbooks/meshtastic-sidecar-node.md vault/runbooks/meshtasticd-sim-nodes-runbook.md vault/runbooks/proxmox-create-ubuntu-vm.md vault/runbooks/recon-operations.md vault/runbooks/recon-service-integration.md vault/runbooks/syncthing-add-node.md
This commit is contained in:
echo6-autocommit 2026-07-12 00:00:23 +00:00
commit cce29c595d
37 changed files with 336 additions and 262 deletions

1
, c.get(k)) PY \" " Normal file
View file

@ -0,0 +1 @@
bash: -c: line 1: unexpected EOF while looking for matching `"'

View file

@ -32,10 +32,10 @@ topic_categories:
# Read-only inventory sources for the generated entity lexicon (tier 2)
inventory:
proxmox_nodes: [data, utility, cloud, media, toc] # pct list / qm list
proxmox_vps: [contabo, edge2] # pct list
docker_hosts: [cortex, utility, media, contabo] # docker ps
headscale_host: contabo # headscale nodes list
proxmox_nodes: [data, utility, cloud, media, toc] # pct list / qm list (root)
proxmox_vps: [edge1, edge2] # edge1=root; edge2=admin+sudo (hardened, no root login)
docker_hosts: [cortex, utility, media] # host-level docker ps; edge1/edge2 services are LXC-nested (not visible at host level)
headscale_host: edge2 # Headscale runs in edge2 CT 107 (dockerized)
ssh_user: zvx
# Frontmatter property schema enforced by lint

View file

@ -290,7 +290,7 @@ def _normalize_name(name: str) -> str:
def build_note_index(vault_dir: Path) -> dict[str, Path]:
index: dict[str, Path] = {}
for p in vault_dir.rglob("*.md"):
if "archive" in p.parts:
if "archive" in p.parts or ".trash" in p.parts:
continue
key = _normalize_name(p.stem)
index[key] = p
@ -328,7 +328,7 @@ def build_backlink_counts(
) -> dict[str, int]:
counts: dict[str, int] = {k: 0 for k in note_index}
for p in vault_dir.rglob("*.md"):
if "archive" in p.parts:
if "archive" in p.parts or ".trash" in p.parts:
continue
try:
_, body = parse_frontmatter(p)
@ -386,7 +386,7 @@ def build_earned_a_doc_candidates(
doc_mentions: dict[str, set] = defaultdict(set)
for p in vault_dir.rglob("*.md"):
if "archive" in p.parts:
if "archive" in p.parts or ".trash" in p.parts:
continue
try:
text = p.read_text(encoding="utf-8", errors="replace")
@ -448,7 +448,7 @@ def build_tag_coverage(
tag_to_paths: dict[str, list[str]] = defaultdict(list)
for p in vault_dir.rglob("*.md"):
if "archive" in p.parts:
if "archive" in p.parts or ".trash" in p.parts:
continue
try:
fm, _ = parse_frontmatter(p)
@ -695,7 +695,7 @@ def run_lint(vault_dir: Path, engine_dir: Path) -> tuple[list[LintResult], dict]
t0 = time.monotonic()
for path in all_paths:
if "archive" in path.parts:
if "archive" in path.parts or ".trash" in path.parts:
continue
findings: list[dict] = []
try:

View file

@ -34,9 +34,9 @@ def load_config(config_path: str) -> dict:
],
"inventory": {
"proxmox_nodes": ["data", "utility", "cloud", "media", "toc"],
"proxmox_vps": ["contabo", "edge2"],
"docker_hosts": ["cortex", "utility", "media", "contabo"],
"headscale_host": "contabo",
"proxmox_vps": ["edge1", "edge2"],
"docker_hosts": ["cortex", "utility", "media"],
"headscale_host": "edge2",
"ssh_user": "zvx",
},
}
@ -114,7 +114,8 @@ HOST_IPS = {
"mt-burleybutte": ("192.168.1.185", None),
"pi-nas": ("192.168.1.245", "100.64.0.21"),
"matt-desktop": ("192.168.1.111", "100.64.0.10"),
"contabo": ("5.189.158.149", "100.64.0.1"),
"contabo": ("5.189.158.149", "100.64.0.40"),
"edge1": ("5.189.158.149", "100.64.0.40"),
"edge2": ("184.174.35.153", "100.64.0.26"),
}
@ -426,7 +427,7 @@ def parse_environment_md(path: Path) -> list[dict]:
aliases.append(node_id_m.group(1))
# Infer type
if 'Contabo' in name or 'VPS' in name.upper():
if 'Contabo' in name or 'edge1' in name.lower() or 'edge2' in name.lower() or 'VPS' in name.upper():
etype = 'vps'
elif re.search(r'pi|Pi|raspberry', purpose, re.I) or name.startswith('pi-') or name.startswith('mt-') or name == 'aida-nebra':
etype = 'pi'
@ -549,7 +550,7 @@ def parse_services_md(path: Path, known_entities: list[dict]) -> list[dict]:
host_map = {
'utility': 'utility', 'cloud': 'cloud', 'media': 'media',
'data': 'data', 'toc': 'toc', 'cortex': 'cortex',
'contabo': 'contabo', 'edge2': 'edge2',
'contabo': 'contabo', 'edge1': 'edge1', 'edge2': 'edge2',
'aida-nebra': 'aida-nebra', 'pi-nas': 'pi-nas',
}
runs_on = host_map.get(first_word, first_word)
@ -813,12 +814,21 @@ def _proxmox_ip_for_name(name: str) -> str | None:
"cloud": "192.168.1.242",
"media": "192.168.1.243",
"toc": "192.168.1.244",
"contabo": "5.189.158.149",
"edge1": "5.189.158.149",
"edge2": "184.174.35.153",
}
return ip_map.get(name)
def _host_access(name: str) -> tuple[str, str]:
"""Return (ssh_user, sudo_prefix) for a host.
edge2 is hardened: root login is disabled; use admin + passwordless sudo.
All other Proxmox hosts (home nodes, edge1) use root."""
if name == "edge2":
return ("admin", "sudo ")
return ("root", "")
def query_proxmox_node(host_name: str) -> list[dict]:
"""Try pct list + qm list on a Proxmox host; return entity dicts."""
ip = _proxmox_ip_for_name(host_name)
@ -826,9 +836,9 @@ def query_proxmox_node(host_name: str) -> list[dict]:
return []
entities = []
# Try root SSH (Proxmox hosts use root)
out_pct = _ssh_run(ip, "root", "pct list 2>/dev/null")
out_qm = _ssh_run(ip, "root", "qm list 2>/dev/null")
user, sudo = _host_access(host_name)
out_pct = _ssh_run(ip, user, f"{sudo}pct list 2>/dev/null")
out_qm = _ssh_run(ip, user, f"{sudo}qm list 2>/dev/null")
if out_pct:
for line in out_pct.splitlines():
@ -861,7 +871,6 @@ def query_docker_host(host_name: str, user: str) -> list[dict]:
"cortex": "192.168.1.150",
"utility": "192.168.1.241",
"media": "192.168.1.160",
"contabo": "5.189.158.149",
}
ip = ip_map.get(host_name)
if not ip:
@ -889,9 +898,14 @@ def query_docker_host(host_name: str, user: str) -> list[dict]:
def query_headscale(headscale_host: str) -> list[dict]:
"""Try headscale nodes list on contabo; return node entities."""
"""Try headscale nodes list; Headscale runs dockerized inside edge2 CT 107."""
ip = _proxmox_ip_for_name(headscale_host) or headscale_host
out = _ssh_run(ip, "root", "headscale nodes list 2>/dev/null")
user, sudo = _host_access(headscale_host)
if headscale_host == "edge2":
cmd = f"{sudo}pct exec 107 -- docker exec headscale headscale nodes list 2>/dev/null"
else:
cmd = "headscale nodes list 2>/dev/null"
out = _ssh_run(ip, user, cmd)
if not out:
return []
@ -952,10 +966,10 @@ def run_live_inventory(config: dict) -> tuple[list[dict], list[str]]:
# Headscale
try:
ents = query_headscale(inv.get("headscale_host", "contabo"))
ents = query_headscale(inv.get("headscale_host", "edge2"))
if ents:
live_entities.extend(ents)
succeeded.append("headscale:contabo")
succeeded.append(f"headscale:{inv.get('headscale_host', 'edge2')}")
except Exception:
pass

View file

@ -1,17 +1,17 @@
# Vault Lint Report
Generated: 2026-07-11T12:00:06Z | Docs scanned: 105 | Elapsed: 0.0s
Generated: 2026-07-11T22:05:39Z | Docs scanned: 105 | Elapsed: 0.0s
## Summary
| Severity | Count |
|----------|-------|
| ERROR (dead links) | 0 |
| WARN (schema) | 2 |
| INFO (orphans) | 40 |
| WARN (schema) | 0 |
| INFO (orphans) | 39 |
### WARN breakdown
- Missing frontmatter block: 2
- Missing frontmatter block: 0
- Invalid/missing frontmatter fields: 0
- Unknown tags: 0
@ -19,14 +19,8 @@ Generated: 2026-07-11T12:00:06Z | Docs scanned: 105 | Elapsed: 0.0s
_None. All wikilinks resolve._
## WARN — Schema & Tag Violations
- `.trash/2026-06-19.md` — missing frontmatter block entirely
- `projects/meshai-native-fire-severity-audit-cc-handoff.md` — missing frontmatter block entirely
## INFO — Orphan Notes (no incoming links, capped at 40)
- no incoming links: .trash/2026-06-19.md
- no incoming links: runbooks/add-peertube-channel.md
- no incoming links: runbooks/authentik-access-groups.md
- no incoming links: runbooks/authentik-create-invitation.md
@ -34,9 +28,8 @@ _None. All wikilinks resolve._
- no incoming links: runbooks/authentik-upgrade.md
- no incoming links: docs/navi/cc-rules.md
- no incoming links: CLAUDE-baseline.md
- no incoming links: runbooks/ct-runbook.md
- no incoming links: notes/echo6-landing-page-data-export.md
- no incoming links: runbooks/expose-service-contabo.md
- no incoming links: runbooks/expose-service-edge2.md
- no incoming links: runbooks/expose-service-home.md
- no incoming links: runbooks/fleet-magicdns-resolved-migration.md
- no incoming links: projects/fleet-platform-baseline.md
@ -51,9 +44,7 @@ _None. All wikilinks resolve._
- no incoming links: runbooks/idahomesh-vpn-device-setup.md
- no incoming links: runbooks/lxc-service-migration.md
- no incoming links: runbooks/mailcow-create-mailbox.md
- no incoming links: docs/matrix/matrix_host.md
- no incoming links: projects/meshai-config-hot-apply.md
- no incoming links: projects/meshai-native-fire-severity-audit-cc-handoff.md
- no incoming links: runbooks/meshai-prod-compose-override.md
- no incoming links: runbooks/meshmonitor-password-reset.md
- no incoming links: runbooks/meshtasticd-sim-nodes-runbook.md
@ -66,18 +57,19 @@ _None. All wikilinks resolve._
- no incoming links: runbooks/pymc-repeater-kiss-tnc-reenumeration.md
- no incoming links: runbooks/recon-operations.md
- no incoming links: runbooks/recon-service-integration.md
- no incoming links: session-resume/SESSION-HANDOFF-meshai-test.md
- no incoming links: docs/matrix/synapse_retention_discovery.md
- no incoming links: runbooks/syncthing-add-node.md
## Gaps & suggestions
### Docs with no tags
- `.trash/2026-06-19.md`
- `projects/meshai-native-fire-severity-audit-cc-handoff.md`
_None — all docs have at least one tag._
### True orphans (no inbound link, no shared tag)
- `.trash/2026-06-19.md`
- `projects/meshai-native-fire-severity-audit-cc-handoff.md`
_None._
### Dead wikilinks
@ -95,17 +87,17 @@ Matt decides whether to create a real doc — when he does, future sweeps will l
| Term | Docs mentioning it |
|------|--------------------|
| `tailscale` | 39 |
| `docker` | 36 |
| `docker` | 34 |
| `proxmox` | 31 |
| `headscale` | 25 |
| `headscale` | 24 |
| `meshtastic` | 19 |
| `peertube` | 19 |
| `mailcow` | 17 |
| `element` | 14 |
| `forgejo` | 13 |
| `immich` | 13 |
| `nextcloud` | 13 |
| `vaultwarden` | 13 |
| `element` | 13 |
| `livesync` | 12 |
| `vaultwarden` | 12 |
| `aida-nebra` | 11 |
| `jellyfin` | 11 |

View file

@ -58,8 +58,13 @@ GPU_DEFERRED=0
DEFER_REASON=""
# Guard 1 — competing GPU processes (share the A4000)
if pgrep -fa 'peertube-runner|whisper|ffmpeg' >/dev/null 2>&1; then
COMPETING="$(pgrep -fa 'peertube-runner|whisper|ffmpeg' | head -1)"
# Note: peertube-runner is intentionally excluded — it's a persistent server
# daemon that's always running, so its mere presence doesn't mean the GPU is
# busy (only an active transcode job does, and that spawns ffmpeg, which is
# still caught here; actual peertube contention is caught by the util/VRAM
# guards below).
if pgrep -fa 'whisper|ffmpeg' >/dev/null 2>&1; then
COMPETING="$(pgrep -fa 'whisper|ffmpeg' | head -1)"
DEFER_REASON="competing GPU process: ${COMPETING}"
GPU_DEFERRED=1
fi

View file

@ -199,20 +199,23 @@
},
"active": "17bd4a6166f789d0",
"lastOpenFiles": [
"runbooks/lxc-service-migration.md.tmp.2217597.ef0b7b5fefc5",
"runbooks/lxc-service-migration.md.tmp.2217597.ff9206e1ecc9",
"runbooks/authentik-create-invitation.md.tmp.2217597.48bf7c4dd5b1",
"runbooks/authentik-create-invitation.md.tmp.2217597.e42e1524f89f",
"runbooks/authentik-create-invitation.md.tmp.2217597.28c10044e078",
"runbooks/authentik-oidc-application.md.tmp.2217597.1beb148d6685",
"runbooks/authentik-oidc-application.md.tmp.2217597.8c872b0d8558",
"runbooks/authentik-oidc-application.md.tmp.2217597.f9c4cbed7ec4",
"runbooks/authentik-oidc-application.md.tmp.2217597.6838e5562889",
"glossary.md.tmp.2217597.98abda89426b",
"glossary.md.tmp.2217597.b4ff5128b973",
"archive/projects/vaultwarden-plan.md",
"archive/projects/meshai-native-fire-severity-audit-cc-handoff.md",
"projects/meshai-native-fire-severity-audit-cc-handoff.md",
"projects/meshai-config-hot-apply.md",
"projects/meshai-config-hot-apply.md.tmp.5281.28f86510539f",
"projects/meshai-region-routing-plan.md.tmp.5281.292f05969651",
"projects/meshai-region-routing-plan.md.tmp.5281.9915ac2ff82e",
"projects/meshai-region-routing-plan.md.tmp.5281.d93b011e3329",
"projects/meshai-region-routing-plan.md.tmp.5281.ce38ed145504",
"projects/meshai-region-routing-plan.md.tmp.5281.c1e4202de220",
"projects/meshai-region-routing-plan.md.tmp.5281.4ae73bc6fce2",
"projects/meshai-region-routing-plan.md",
"projects/meshai-region-routing-plan.md.tmp.5281.a1cf56fc940f",
"runbooks/meshai-prod-compose-override.md",
"runbooks/meshai-prod-compose-override.md.tmp.5281.6ff00a45f3af",
"docs/hardware/environment.md.tmp.246153.593a4dd35d71",
"projects/meshai.md",
"projects/meshcore-transport.md",
"runbooks/pymc-repeater-kiss-tnc-reenumeration.md",
@ -233,9 +236,6 @@
"docs/software/navi.md",
"archive/projects/mmud/mmud-phase6-prompt.md",
"archive/projects/last-ember-project.md",
"projects/mmud-project.md",
"concepts/lxc-container.md",
"concepts/osint.md",
"assets/echo6yellow_logo_422x422_square.png",
"assets/echo6yellow_logo_422x81.png",
"assets/echo6_logo.png",

View file

@ -1,3 +1,13 @@
---
title: "MeshAI Native Fire-Severity Audit — CC Handoff (archived)"
type: project
tags:
- mesh
status: archived
updated: 2026-07-11
---
> **ARCHIVED / SUPERSEDED — fix deployed.** This was a one-time Claude Code handoff for the native fire-severity regression (audit 2026-07-08). The fix has been deployed; this runbook is retained for history only. Its host/access details are STALE — meshai runs as utility CT 108 (`pct exec 108` from root@utility; zvx has no sudo), NOT at 192.168.1.144 with sudo. Do not execute.
# MeshAI Audit Handoff — Native Fire Severity Regression
**For:** Claude Code (CC), operating from cortex against the MeshAI LXC (192.168.1.144)

View file

@ -10,8 +10,11 @@ related:
- [[edge2-access-reference]]
- [[caddy]]
- [[ip-allocation]]
updated: 2026-06-18
status: archived
updated: 2026-07-11
---
> **ARCHIVED / COMPLETED 2026-06-16.** Vaultwarden was migrated to edge2 CT 102 (100.64.0.33) on 2026-06-16 — see [[services]]. This plan is retained for history only; its future-tense provisioning steps and `root@100.64.0.1` (dead Contabo) access are STALE. Do not re-execute.
# Vaultwarden → edge2 LXC — Migration Pilot (+ reusable LXC-migration runbook) — v2
## Changes from v1 (what was corrected and why)

View file

@ -1,5 +1,5 @@
---
title: Matrix Host Reference — Contabo VPS
title: Matrix Host Reference — edge2 CT 106 (formerly Contabo VPS)
type: reference
tags:
- matrix
@ -10,14 +10,17 @@ related:
- [[expose-service-contabo]]
- [[ct-runbook]]
- [[lxc-service-migration]]
updated: 2026-06-18
updated: 2026-07-11
---
# Matrix Host Reference — Contabo VPS
# Matrix Host Reference — edge2 CT 106
> Migrated off Contabo → edge2 CT 106 on 2026-06-18.
# Generated: 2026-04-09 (Phase 0)
## SSH Access
- Host: 100.64.0.1 (Tailscale) / 5.189.158.149 (public)
- Auth: SSH key as root — CONFIRMED WORKING
- Host: 100.64.0.37 (Tailscale, edge2 CT 106) — was 100.64.0.1 (Contabo) before the 2026-06-18 migration
- Auth: SSH key as root — CONFIRMED WORKING on Contabo; re-verify on edge2 CT 106
- sudo: root user, no sudo needed
## System Info

View file

@ -10,7 +10,7 @@ related:
- [[matrix_host]]
- [[synapse_retention_discovery]]
- [[caddy]]
updated: 2026-06-18
updated: 2026-07-11
---
# Synapse Deployment Reference
# Generated: 2026-04-09 (Phase 1)
@ -60,7 +60,7 @@ updated: 2026-06-18
- Auth: password from .env file (POSTGRES_PASSWORD)
- New DB/user can be created without collision — synapse user has Superuser/Create role privileges
## Reverse Proxy (Caddy on Contabo)
## Reverse Proxy (Caddy on edge2)
- matrix.echo6.co routes:
- /_matrix/client/*/login|logout|refresh|auth_metadata → MAS (127.0.0.1:8085)
- /_matrix/* → Synapse (127.0.0.1:8008)

View file

@ -10,7 +10,7 @@ related:
- [[glossary]]
- [[meshtastic-headscale-runbook]]
- [[lxc-service-migration]]
updated: 2026-06-19
updated: 2026-07-11
---
# Current Services Inventory
@ -28,7 +28,7 @@ updated: 2026-06-19
| meshtasticd | mt-burleybutte | 192.168.1.185:4403 | Internal | Software Meshtastic node (Nebra 2W hat) |
| IdahoMesh Headscale | utility (CT 106) | 192.168.1.106:8080 | https://vpn.idahomesh.com | Meshtastic mesh VPN coordination |
| mesh-bridge | utility (CT 107) | 192.168.1.107 | Internal | Dual-tailscaled bridge (echo6 ↔ idahomesh) |
| MeshAI | utility (CT 108) | 192.168.1.144:4403 / :8080 | Internal | LLM-powered Meshtastic assistant (Docker, work-meshai local build, Gemini Flash, Google grounding) |
| MeshAI | utility (CT 108) | 192.168.1.144:4403 / :8080 | Internal | LLM-powered Meshtastic assistant (Docker, work-meshai local build, gemini-3.1-flash-lite, Google grounding) |
| [[argus]] | utility (CT 103) | 192.168.1.103:8080 | Internal | Python app on :8080 — OSINT intelligence gathering platform |
| [[central]] | utility (CT 104) | 192.168.1.104:8000 / 100.64.0.12 | central.echo6.mesh (mesh) | Data-hub spine — ~25 adapters → NATS/JetStream → TimescaleDB; serves traffic tiles to navi — see [[central]] |
| NATS/JetStream (central) | utility (CT 104) | 192.168.1.104:4222 / :8222 | Internal | Central backend message bus (NATS :4222 client, :8222 monitoring) |
@ -160,7 +160,7 @@ updated: 2026-06-19
- MeshAI — LLM-powered Meshtastic mesh assistant (Docker)
- Bot name: AIDA, node ID !27780c47, channel 8 whitelist
- Image: work-meshai (local build, not ghcr.io/zvx-echo6/meshai:latest)
- Backend: Gemini 2.5 Flash with Google Search grounding
- Backend: gemini-3.1-flash-lite with Google Search grounding
- Connects to meshtasticd **on aida-nebra** (192.168.1.253:4403) — the AIDA-N2 node !27780c47
- Exposes port 8080 (web UI)
- Config TUI on port 7682 (`meshai --config`)

View file

@ -10,13 +10,15 @@ related:
- [[caddy]]
- [[echo6-landing-page-data-export]]
- [[authentik-access-groups]]
updated: 2026-06-18
updated: 2026-07-11
---
# Authentik SSO Configuration
> Migrated off Contabo → edge2 on 2026-06-18.
## Location
- **Server:** Contabo (5.189.158.149 / 100.64.0.1)
- **Server:** edge2 CT 105 (100.64.0.36)
- **URL:** https://auth.echo6.co
- **Internal Port:** 9000
@ -108,7 +110,7 @@ Users must be in at least one bound group to access the application.
## Create New API Token
```bash
ssh root@100.64.0.1 'docker exec -i authentik-server ak shell' <<'PYEOF'
ssh edge2 'sudo pct exec 105 -- docker exec -i authentik-server ak shell' <<'PYEOF'
from authentik.core.models import Token, TokenIntents, User
user = User.objects.get(username="akadmin")
Token.objects.filter(identifier="token-name").delete()
@ -331,11 +333,12 @@ Echo6 cyberpunk branding applied to Authentik 2025.12.4 via System → Brands.
The custom CSS is stored in the Brand model's `branding_custom_css` field. To update:
```bash
# Copy CSS to Contabo
scp /path/to/echo6-authentik.css root@100.64.0.1:/opt/authentik/branding/custom.css
# Copy CSS to edge2 CT 105 (via edge2 host, then pct push into the CT)
scp /path/to/echo6-authentik.css edge2:/tmp/echo6-authentik.css
ssh edge2 'sudo pct push 105 /tmp/echo6-authentik.css /opt/authentik/branding/custom.css'
# Load into Brand model via ak shell
ssh root@100.64.0.1 'docker exec -i authentik-server ak shell' <<'PYEOF'
ssh edge2 'sudo pct exec 105 -- docker exec -i authentik-server ak shell' <<'PYEOF'
from authentik.brands.models import Brand
b = Brand.objects.get(domain="auth.echo6.co")
b.branding_custom_css = open("/media/custom/custom.css").read()
@ -343,7 +346,7 @@ b.save()
PYEOF
# Restart to apply
ssh root@100.64.0.1 'cd /opt/authentik && docker compose restart server worker'
ssh edge2 'sudo pct exec 105 -- bash -c "cd /opt/authentik && docker compose restart server worker"'
```
### SSO Launch URL Pattern

View file

@ -10,13 +10,15 @@ related:
- [[headscale-onboard-node]]
- [[expose-service-home]]
- [[lxc-service-migration]]
updated: 2026-06-18
updated: 2026-07-11
---
# Caddy & DNS Reference
## Contabo Caddy
> Migrated off Contabo → edge2 on 2026-06-19.
**Config:** `/etc/caddy/Caddyfile` on Contabo (ssh root@100.64.0.1)
## edge2 Caddy (front door)
**Config:** `/etc/caddy/Caddyfile` on edge2 (`ssh edge2`; `root@<IP>` is stale)
**Global options:** `email admin@echo6.co`, `admin off` (no live reload — must `systemctl restart caddy`)
@ -24,25 +26,25 @@ updated: 2026-06-18
| Domain | Backend | Service |
|--------|---------|---------|
| auth.echo6.co | 127.0.0.1:9000 | [[authentik]] SSO |
| forge.echo6.co | 127.0.0.1:3001 | Forgejo Git |
| mail.echo6.co | https://127.0.0.1:8453 | Mailcow (tls_insecure_skip_verify, r/w timeout 3600s) |
| vpn.echo6.co | 127.0.0.1:8084 | Headscale |
| vpn.echo6.co/admin* | 127.0.0.1:3100 | Headplane |
| autodiscover.echo6.co | https://127.0.0.1:8443 | Mailcow autodiscover |
| autoconfig.echo6.co | https://127.0.0.1:8443 | Mailcow autoconfig |
| vault.echo6.co | 127.0.0.1:8086 | vaultwarden |
| proxmox.echo6.co | https://100.64.0.6:8006 (via Tailscale) | Proxmox VE (data node) |
| wt.echo6.co | 127.0.0.1:8099 ([[authentik]] forward auth) | WATCHTOWER ops dashboard |
| matrix.echo6.co | 127.0.0.1:8008 + 127.0.0.1:8085 | Matrix [[synapse]] + MAS (login/logout/refresh/auth_metadata → MAS:8085, _matrix/* → [[synapse]]:8008, default → MAS:8085) |
| element.echo6.co | 127.0.0.1:8088 | Element Web client |
| notes.echo6.co | 127.0.0.1:5984 + 127.0.0.1:5985 | LiveSync (CouchDB + provisioner, forward auth on /_provision*, CORS for Obsidian) |
| tak.echo6.co | https://100.64.0.1:8446 + 100.64.0.1:8990 | TAK Server admin (8446, [[authentik]] forward auth) + SIGIL console (/sigil, 8990) |
| auth.echo6.co | 100.64.0.36:9000 (Tailscale, CT 105) | [[authentik]] SSO |
| forge.echo6.co | 100.64.0.34:3001 (Tailscale, CT 103) | Forgejo Git |
| mail.echo6.co | — | Mailcow — moved to **edge1** (separate mail-only host, 10.10.10.2:8453 via edge1's own Caddy); not on edge2 |
| vpn.echo6.co | 100.64.0.38:8084 (Tailscale, CT 107) | Headscale |
| vpn.echo6.co/admin* | 100.64.0.38:3100 (Tailscale, CT 107) | Headplane |
| autodiscover.echo6.co | — | Mailcow autodiscover — moved to **edge1** (10.10.10.2:8453); not on edge2 |
| autoconfig.echo6.co | — | Mailcow autoconfig — moved to **edge1** (10.10.10.2:8453); not on edge2 |
| vault.echo6.co | 100.64.0.33:8086 (Tailscale, CT 102) | vaultwarden |
| proxmox.echo6.co | https://100.64.0.6:8006 (via Tailscale) | Proxmox VE (data node) — unchanged |
| wt.echo6.co | — | ~~WATCHTOWER ops dashboard~~**decommissioned 2026-06-16** (was 100.64.0.1, now dead) |
| matrix.echo6.co | 100.64.0.37:8008 + 100.64.0.37:8085 (Tailscale, CT 106) | Matrix [[synapse]] + MAS (login/logout/refresh/auth_metadata → MAS:8085, _matrix/* → [[synapse]]:8008, default → MAS:8085) |
| element.echo6.co | 100.64.0.37:8088 (Tailscale, CT 106) | Element Web client |
| notes.echo6.co | 100.64.0.35:5984 + 100.64.0.35:5985 (Tailscale, CT 104) | LiveSync (CouchDB + provisioner, forward auth on /_provision*, CORS for Obsidian) |
| tak.echo6.co | — | ~~TAK Server admin + SIGIL console~~**decommissioned 2026-06-16** (was 100.64.0.1, now dead) |
### Commands
```bash
ssh root@100.64.0.1
ssh edge2
caddy validate --config /etc/caddy/Caddyfile
systemctl restart caddy # admin off, so reload won't work
journalctl -u caddy -f
@ -88,10 +90,12 @@ ssh root@192.168.1.241 'pct exec 101 -- journalctl -u caddy -f'
---
## dnsmasq (Tailscale Split DNS)
## dnsmasq (Tailscale Split DNS) — HISTORICAL / OBSOLETE
**Config:** `/etc/dnsmasq.d/tailscale-dns.conf` on Contabo
**Listens on:** 100.64.0.1:53
> **Not in use.** Tailnet split-DNS for echo6.co was retired: echo6.co now resolves via public GoDaddy DNS, not internal dnsmasq (see [[services]]). This section documents the OLD setup that ran on the original Contabo VPS (100.64.0.1) before its 2026-06-19 rebuild into edge1 (mail-only). That host and its 100.64.0.1 tailnet identity are dead — do not repoint these records to edge1 or edge2; kept below for historical reference only.
**Config:** `/etc/dnsmasq.d/tailscale-dns.conf` on Contabo (dead host, pre-2026-06-19)
**Listens on:** 100.64.0.1:53 (dead)
### Current Records
@ -121,6 +125,7 @@ ssh root@192.168.1.241 'pct exec 101 -- journalctl -u caddy -f'
### Commands
```bash
# HISTORICAL — host is dead, commands no longer work
ssh root@100.64.0.1
nano /etc/dnsmasq.d/tailscale-dns.conf
systemctl restart dnsmasq
@ -131,21 +136,31 @@ dig +short forge.echo6.co @100.64.0.1 # Test
## GoDaddy DNS Records (echo6.co)
### Contabo Services → 5.189.158.149
### edge2 Services → 184.174.35.153
| Subdomain | Service |
|-----------|---------|
| auth | Authentik SSO |
| forge | Forgejo Git |
| mail | Mailcow Email |
| vpn | Headscale VPN |
| vault | Vaultwarden |
| wt | WATCHTOWER ops dashboard |
| matrix | Matrix Synapse |
| element | Element Web |
| notes | LiveSync (CouchDB + provisioner) |
| proxmox | Proxmox VE (via Tailscale to data node) |
| tak | TAK Server + SIGIL |
### edge1 Services (mail-only host) → 5.189.158.149
| Subdomain | Service |
|-----------|---------|
| mail | Mailcow Email |
### Decommissioned records (removed from GoDaddy, 2026-06-16)
| Subdomain | Service |
|-----------|---------|
| wt | ~~WATCHTOWER ops dashboard~~ |
| tak | ~~TAK Server + SIGIL~~ |
### Home Services → 199.6.36.163
@ -181,7 +196,7 @@ dig +short forge.echo6.co @100.64.0.1 # Test
## Headscale Config
**Location:** `/opt/headscale/` on Contabo
**Location:** `/opt/headscale/` on edge2 (CT 107, managed via `headscale-stack.service` systemd reboot guard)
**Data:** Named Docker volume `headscale_headscale-data`
**Config:** `/opt/headscale/config.yaml`
@ -197,32 +212,37 @@ oidc:
client_id: "headscale"
```
**Split [[dns]]:** Configured via dnsmasq on Contabo.
**Split [[dns]]:** Previously configured via dnsmasq on Contabo (pre-2026-06-19). Not verified on edge2/edge1 in this pass — flagged stale/unresolved; see `## dnsmasq (Tailscale Split DNS)` section below (also unverified).
**Headplane:** Deployed at `vpn.echo6.co/admin` - OIDC via Authentik. First login gets Owner.
---
## Port Map (Contabo)
## Port Map (edge2)
| Service | Container Port | Host Binding (Tailscale) | Public Domain |
|---------|---------------|--------------|---------------|
| Authentik | 9000 | 100.64.0.36:9000 (CT 105) | auth.echo6.co |
| Forgejo | 3000 | 100.64.0.34:3001 (CT 103) | forge.echo6.co |
| Forgejo SSH | 22 | edge2 iptables DNAT :2222 → 100.64.0.34:2222 (CT 103) | Direct (not proxied) |
| Headscale | 8080 | 100.64.0.38:8084 (CT 107) | vpn.echo6.co |
| Headplane | 3000 | 100.64.0.38:3100 (CT 107) | vpn.echo6.co/admin |
| Vaultwarden | 80 | 100.64.0.33:8086 (CT 102) | vault.echo6.co |
| Vaultwarden WS | 3012 | 100.64.0.33:3012 (CT 102) | vault.echo6.co/notifications/hub |
| ~~WATCHTOWER~~ | 8084 | — | ~~wt.echo6.co~~ — decommissioned 2026-06-16 |
| Matrix Synapse | 8008 | 100.64.0.37:8008 (CT 106) | matrix.echo6.co (/_matrix/*, /_synapse/*) |
| Matrix MAS | 8080 | 100.64.0.37:8085 (CT 106) | matrix.echo6.co (login/logout/refresh/auth_metadata, default) |
| Element Web | 80 | 100.64.0.37:8088 (CT 106) | element.echo6.co |
| LiveSync CouchDB | 5984 | 100.64.0.35:5984 (CT 104) | notes.echo6.co |
| LiveSync Provisioner | 8080 | 100.64.0.35:5985 (CT 104) | notes.echo6.co/_provision/* |
| ~~TAK Server Admin~~ | 8446 | — | ~~tak.echo6.co~~ — decommissioned 2026-06-16 |
| ~~SIGIL Console~~ | 8990 | — | ~~tak.echo6.co/sigil~~ — decommissioned 2026-06-16 |
## Port Map (edge1 — mail-only, separate host)
| Service | Container Port | Host Binding | Public Domain |
|---------|---------------|--------------|---------------|
| Authentik | 9000 | 127.0.0.1:9000 | auth.echo6.co |
| Forgejo | 3000 | 127.0.0.1:3001 | forge.echo6.co |
| Forgejo SSH | 22 | 0.0.0.0:2222 | Direct (not proxied) |
| Headscale | 8080 | 127.0.0.1:8084 | vpn.echo6.co |
| Headplane | 3000 | 127.0.0.1:3100 | vpn.echo6.co/admin |
| Mailcow | 8443 | 127.0.0.1:8443 | mail.echo6.co |
| Vaultwarden | 80 | 127.0.0.1:8086 | vault.echo6.co |
| Vaultwarden WS | 3012 | 127.0.0.1:3012 | vault.echo6.co/notifications/hub |
| WATCHTOWER | 8084 | host network :8099 | wt.echo6.co |
| Matrix Synapse | 8008 | 127.0.0.1:8008 | matrix.echo6.co (/_matrix/*, /_synapse/*) |
| Matrix MAS | 8080 | 127.0.0.1:8085 | matrix.echo6.co (login/logout/refresh/auth_metadata, default) |
| Element Web | 80 | 127.0.0.1:8088 | element.echo6.co |
| LiveSync CouchDB | 5984 | 127.0.0.1:5984 | notes.echo6.co |
| LiveSync Provisioner | 8080 | 127.0.0.1:5985 | notes.echo6.co/_provision/* |
| TAK Server Admin | 8446 | https://100.64.0.1:8446 (Tailscale) | tak.echo6.co |
| SIGIL Console | 8990 | 100.64.0.1:8990 | tak.echo6.co/sigil |
| Mailcow | 8453 | 10.10.10.2:8453 (CT 101, via edge1 host Caddy + DNAT) | mail.echo6.co, autodiscover.echo6.co, autoconfig.echo6.co |
---
*Last updated: 2026-04-13 — Audit sync: added MAS routing on matrix.echo6.co, lidarr/navidrome/vpn.idahomesh.com to utility Caddy, proxmox/tak to GoDaddy, removed ghost docs.echo6.co entries, added dnsmasq lidarr/navidrome*
*Last updated: 2026-07-11 — Flip off Contabo completed: "Contabo Caddy" section → "edge2 Caddy" (front door for auth/forge/vpn/vault/matrix/element/notes/proxmox, CTs verified against [[ip-allocation]]/[[services]]); Mailcow + autodiscover/autoconfig moved to edge1 (separate mail-only host, not on edge2); WATCHTOWER + TAK/SIGIL marked decommissioned (dead 100.64.0.1 backends removed); Headscale config location + Port Map updated to edge2; `ssh root@100.64.0.1``ssh edge2`. dnsmasq split-DNS section marked HISTORICAL/OBSOLETE (echo6.co split-DNS retired, ran on the dead pre-2026-06-19 Contabo host, not repointed to edge1/edge2 per [[services]]); GoDaddy DNS Records section corrected — edge2 services (auth/forge/vpn/vault/matrix/element/notes/proxmox) → 184.174.35.153, mail → edge1 5.189.158.149 (unchanged public IP), wt/tak marked as removed records. Prior: 2026-04-13 — Audit sync: added MAS routing on matrix.echo6.co, lidarr/navidrome/vpn.idahomesh.com to utility Caddy, proxmox/tak to GoDaddy, removed ghost docs.echo6.co entries, added dnsmasq lidarr/navidrome*

View file

@ -10,7 +10,7 @@ related:
- [[expose-service-contabo]]
- [[headscale-onboard-node]]
- [[authentik-oidc-application]]
updated: 2026-06-18
updated: 2026-07-11
---
# GoDaddy DNS Management
@ -29,7 +29,8 @@ Stored in `/home/zvx/projects/.ref/credentials` as:
| Purpose | IP |
|---------|-----|
| External (home [[services]]) | `199.6.36.163` |
| Contabo Server | `5.189.158.149` |
| edge1 (mail/autodiscover/autoconfig only — mail-only host, rebuilt 2026-06-19) | `5.189.158.149` |
| edge2 (front door: auth/forge/vpn/vault/matrix/element/notes/proxmox) | `184.174.35.153` |
## Managed Domains
@ -67,9 +68,14 @@ godaddy-dns.py setup-mail
godaddy-dns.py add-a echo6.co newservice 199.6.36.163
```
### Point subdomain to Contabo
### Point subdomain to edge2 (front door — auth, forge, vpn, vault, matrix, element, notes, proxmox)
```bash
godaddy-dns.py add-a echo6.co auth 5.189.158.149
godaddy-dns.py add-a echo6.co auth 184.174.35.153
```
### Point subdomain to edge1 (mail, autodiscover, autoconfig only)
```bash
godaddy-dns.py add-a echo6.co mail 5.189.158.149
```
### Create CNAME alias

View file

@ -10,7 +10,7 @@ related:
- [[services]]
- [[usenet]]
- [[caddy]]
updated: 2026-06-18
updated: 2026-07-11
---
# RECON — Knowledge Extraction Pipeline
@ -40,8 +40,8 @@ RECON extracts knowledge from PDFs and web content into a searchable vector data
| Vector DB | Qdrant | cortex:6333 (Docker) |
| Embeddings | TEI (bge-m3, 1024-dim) | cortex:8090 (Docker) |
| Sparse embeddings | recon-sparse (bge-m3 SPLADE) | cortex:8091 (systemd) |
| Enrichment | Gemini `gemini-2.5-flash-lite` (enforced) | Google API (4 keys) |
| Vision Ocr | Gemini 2.5 Flash Lite | Google API (shared keys) |
| Enrichment | Gemini `gemini-3.1-flash-lite` (enforced) | Google API (4 keys) |
| Vision Ocr | gemini-3.1-flash-lite | Google API (shared keys) |
| Text extraction | PyPDF2, poppler-utils, Tesseract | Local |
| PDF source | NFS | pi-nas:/export/library → /mnt/library |
| File server | nginx | localhost:8888 → files.echo6.co |
@ -62,7 +62,7 @@ Per page, in order. Each method only runs if the previous returned <50 chars:
1. **PyPDF2** — fast, free, works on text-based PDFs
2. **pdftotext** (poppler) — handles some PDFs PyPDF2 misses
3. **Tesseract OCR** — renders page to image, runs local OCR
4. **Gemini Vision** — renders page to PNG, sends to Gemini 2.5 Flash Lite vision API
4. **Gemini Vision** — renders page to PNG, sends to gemini-3.1-flash-lite vision API
Method tracking saved in `data/text/{hash}/meta.json` as `ocr_methods` dict.
@ -88,7 +88,7 @@ Key sections:
- `processing.extract_workers` (4), `enrich_workers` (16), `embed_workers` (4)
- `processing.extract_timeout` (1800s), `page_timeout` (30s)
- `processing.enrich_max_retries` (5), `enrich_base_delay` (5.0)
- `gemini.model` (gemini-2.5-flash-lite), `gemini.response_mime_type` (application/json)
- `gemini.model` (gemini-3.1-flash-lite), `gemini.response_mime_type` (application/json)
- `service.scan_interval` (3600), `stage_poll_interval` (30)
**API keys:** `/opt/recon/.env` — GEMINI_KEY_1 through GEMINI_KEY_4
@ -112,10 +112,10 @@ Key sections:
## Backups
- **Destination:** `root@100.64.0.1:/opt/backups/recon/`
- **Destination:** `root@100.64.0.40:/opt/backups/recon/` (edge1 — retains the storage/public IP post-rebuild)
- **Full sync:** every 6 hours (concepts, text, DB, config)
- **DB snapshot:** every 2 hours
- **Recovery:** restore from Contabo`recon rebuild` (reconstructs Qdrant from concept JSONs)
- **Recovery:** restore from edge1`recon rebuild` (reconstructs Qdrant from concept JSONs)
- **Critical data:** `data/concepts/` — Gemini extraction work, costs money to regenerate
## Key Files
@ -134,7 +134,7 @@ Key sections:
│ ├── status.py # SQLite DB (WAL, thread-safe)
│ └── utils.py # Config, hashing, logging
├── scripts/
│ ├── backup.sh # Backup to Contabo
│ ├── backup.sh # Backup to edge1
│ ├── validate.py # Pipeline consistency checker
│ └── rebuild_qdrant.py # Nuclear Qdrant rebuild
└── data/
@ -145,4 +145,4 @@ Key sections:
---
*Last updated: 2026-06-18 — Updated: repo/branch, recon-sparse :8091, recon_knowledge_hybrid collection, Entrypoints; PROJECT-BIBLE.md dated 2026-02-16 (predates current deployment) — verified against live 2026-06-18*
*Last updated: 2026-07-11 — Gemini model refs updated to gemini-3.1-flash-lite (retired gemini-2.5-flash-lite); backup destination corrected Contabo (100.64.0.1, dead) → edge1 (100.64.0.40). Prior: 2026-06-18 — Updated: repo/branch, recon-sparse :8091, recon_knowledge_hybrid collection, Entrypoints; PROJECT-BIBLE.md dated 2026-02-16 (predates current deployment) — verified against live 2026-06-18*

View file

@ -10,7 +10,7 @@ related:
- [[ip-allocation]]
- [[headscale-onboard-node]]
- [[services]]
updated: 2026-06-18
updated: 2026-07-11
---
# SearXNG — Echo6 Search Homepage
@ -132,8 +132,9 @@ curl -s http://192.168.1.102:8080 | head -30
- `echo6.co``100.64.0.15:8080` + Matrix `.well-known` handlers
- `search.echo6.co` → 301 redirect to `https://echo6.co`
**dnsmasq (Contabo):**
**dnsmasq (historical — this ran on Contabo, decommissioned 2026-06-19):**
- `echo6.co``100.64.0.8` (utility Caddy)
- Per [[services]]: tailnet split-DNS is NOT used for `echo6.co` — it resolves via public GoDaddy DNS. This dnsmasq entry describes the pre-migration setup and should not be assumed current.
**GoDaddy [[dns]]:**
- `@` (echo6.co) → `199.6.36.163` (home)

View file

@ -10,7 +10,7 @@ related:
- [[caddy]]
- [[authentik]]
- [[usenet]]
updated: 2026-06-18
updated: 2026-07-11
---
# Glossary & Vocabulary
@ -38,7 +38,7 @@ mesh · matrix · [[recon]] · media · auth · [[dns]] · vpn · storage · pro
- **[[authentik]]** — aliases: 100.64.0.36
- **bluefin** — aliases: 100.64.0.30
- **cloud** (Cloud) — aliases: 192.168.1.242, 100.64.0.4
- **contabo** — aliases: 100.64.0.1
- **contabo** (edge1, mail-only — rebuilt in-place 2026-06-19, tailnet identity re-registered as `contabo`) — aliases: 100.64.0.40, 5.189.158.149
- **data** (Data) — aliases: 192.168.1.240, 100.64.0.6
- **edge2** — aliases: 184.174.35.153, 100.64.0.26
- **forgejo** — aliases: 100.64.0.34
@ -56,7 +56,6 @@ mesh · matrix · [[recon]] · media · auth · [[dns]] · vpn · storage · pro
- **arr** — aliases: 192.168.1.160, 100.64.0.18, VM 105 — on: media
- **cortex** — aliases: 192.168.1.150, 100.64.0.14, VM 150 — on: toc
- **recon** — aliases: 100.64.0.24
- **recon-vm** — aliases: 192.168.1.130, 100.64.0.24, VM 1130 — on: data
### LXC Containers
@ -67,7 +66,6 @@ mesh · matrix · [[recon]] · media · auth · [[dns]] · vpn · storage · pro
- **livesync** — aliases: 10.10.10.22, CT 104, 100.64.0.35 — on: edge2
- **meshtastic-hs** — aliases: 192.168.1.106, CT 106 — on: utility
- **pdm** — aliases: 10.10.10.10, CT 100, 100.64.0.28 — on: edge2
- **running** — aliases: CT 100 — on: utility _(live)_
- **[[searxng]]** — aliases: 192.168.1.102, CT 102, 100.64.0.15 — on: utility
- **vaultwarden** — aliases: 10.10.10.20, CT 102, 100.64.0.33 — on: edge2
- **wordpress** — aliases: 10.10.10.11, CT 101, 100.64.0.31 — on: edge2
@ -91,42 +89,42 @@ mesh · matrix · [[recon]] · media · auth · [[dns]] · vpn · storage · pro
### Services
- **echo6-agent** — aliases: echo6-agent — on: 2026-06-16
- **echo6-contabo-agent** (Echo6 Contabo Agent) — aliases: Echo6 Contabo Agent — on: contabo
- **echo6-contabo-agent** (Echo6 Contabo Agent) — aliases: Echo6 Contabo Agent — on: contabo _(decommissioned/historical — Contabo-local agent; host rebuilt as edge1, mail-only, 2026-06-19)_
- **echo6-cortex-agent** (Echo6 Cortex Agent) — aliases: Echo6 Cortex Agent — on: cortex
- **echo6-search-searxng** (Echo6 Search (SearXNG)) — aliases: Echo6 Search (SearXNG) — on: utility
- **element-web** (Element Web) — aliases: Element Web, element — on: contabo
- **element-web** (Element Web) — aliases: Element Web, element — on: edge2
- **files** (Files) — aliases: Files, files — on: data
- **forge-forgejo** (Forge (Forgejo)) — aliases: Forge (Forgejo), forge — on: edge2
- **headplane** (Headplane) — aliases: Headplane, vpn — on: contabo
- **headscale** (Headscale) — aliases: Headscale, vpn — on: contabo
- **headplane** (Headplane) — aliases: Headplane, vpn — on: edge2
- **headscale** (Headscale) — aliases: Headscale, vpn — on: edge2
- **idahomesh-headscale** (IdahoMesh Headscale) — aliases: IdahoMesh Headscale — on: utility
- **jellyfin** (Jellyfin) — aliases: Jellyfin, jellyfin — on: media
- **jellyseer** (Jellyseer) — aliases: Jellyseer, requests — on: media
- **lidarr** — aliases: lidarr — on: media _(live)_
- **mailcow** (Mailcow) — aliases: Mailcow, mail — on: contabo
- **mailcowdockerized-acme-mailcow-1** — aliases: mailcowdockerized-acme-mailcow-1 — on: contabo _(live)_
- **mailcowdockerized-clamd-mailcow-1** — aliases: mailcowdockerized-clamd-mailcow-1 — on: contabo _(live)_
- **mailcowdockerized-dockerapi-mailcow-1** — aliases: mailcowdockerized-dockerapi-mailcow-1 — on: contabo _(live)_
- **mailcowdockerized-dovecot-mailcow-1** — aliases: mailcowdockerized-dovecot-mailcow-1 — on: contabo _(live)_
- **mailcowdockerized-memcached-mailcow-1** — aliases: mailcowdockerized-memcached-mailcow-1 — on: contabo _(live)_
- **mailcowdockerized-mysql-mailcow-1** — aliases: mailcowdockerized-mysql-mailcow-1 — on: contabo _(live)_
- **mailcowdockerized-netfilter-mailcow-1** — aliases: mailcowdockerized-netfilter-mailcow-1 — on: contabo _(live)_
- **mailcowdockerized-nginx-mailcow-1** — aliases: mailcowdockerized-nginx-mailcow-1 — on: contabo _(live)_
- **mailcowdockerized-ofelia-mailcow-1** — aliases: mailcowdockerized-ofelia-mailcow-1 — on: contabo _(live)_
- **mailcowdockerized-olefy-mailcow-1** — aliases: mailcowdockerized-olefy-mailcow-1 — on: contabo _(live)_
- **mailcowdockerized-php-fpm-mailcow-1** — aliases: mailcowdockerized-php-fpm-mailcow-1 — on: contabo _(live)_
- **mailcowdockerized-postfix-mailcow-1** — aliases: mailcowdockerized-postfix-mailcow-1 — on: contabo _(live)_
- **mailcowdockerized-postfix-tlspol-mailcow-1** — aliases: mailcowdockerized-postfix-tlspol-mailcow-1 — on: contabo _(live)_
- **mailcowdockerized-redis-mailcow-1** — aliases: mailcowdockerized-redis-mailcow-1 — on: contabo _(live)_
- **mailcowdockerized-rspamd-mailcow-1** — aliases: mailcowdockerized-rspamd-mailcow-1 — on: contabo _(live)_
- **mailcowdockerized-sogo-mailcow-1** — aliases: mailcowdockerized-sogo-mailcow-1 — on: contabo _(live)_
- **mailcowdockerized-unbound-mailcow-1** — aliases: mailcowdockerized-unbound-mailcow-1 — on: contabo _(live)_
- **mailcowdockerized-watchdog-mailcow-1** — aliases: mailcowdockerized-watchdog-mailcow-1 — on: contabo _(live)_
- **matrix-element** — aliases: matrix-element — on: contabo _(live)_
- **matrix-mas** (Matrix MAS) — aliases: Matrix MAS — on: contabo
- **mailcow** (Mailcow) — aliases: Mailcow, mail — on: edge1
- **mailcowdockerized-acme-mailcow-1** — aliases: mailcowdockerized-acme-mailcow-1 — on: edge1 _(live)_
- **mailcowdockerized-clamd-mailcow-1** — aliases: mailcowdockerized-clamd-mailcow-1 — on: edge1 _(live)_
- **mailcowdockerized-dockerapi-mailcow-1** — aliases: mailcowdockerized-dockerapi-mailcow-1 — on: edge1 _(live)_
- **mailcowdockerized-dovecot-mailcow-1** — aliases: mailcowdockerized-dovecot-mailcow-1 — on: edge1 _(live)_
- **mailcowdockerized-memcached-mailcow-1** — aliases: mailcowdockerized-memcached-mailcow-1 — on: edge1 _(live)_
- **mailcowdockerized-mysql-mailcow-1** — aliases: mailcowdockerized-mysql-mailcow-1 — on: edge1 _(live)_
- **mailcowdockerized-netfilter-mailcow-1** — aliases: mailcowdockerized-netfilter-mailcow-1 — on: edge1 _(live)_
- **mailcowdockerized-nginx-mailcow-1** — aliases: mailcowdockerized-nginx-mailcow-1 — on: edge1 _(live)_
- **mailcowdockerized-ofelia-mailcow-1** — aliases: mailcowdockerized-ofelia-mailcow-1 — on: edge1 _(live)_
- **mailcowdockerized-olefy-mailcow-1** — aliases: mailcowdockerized-olefy-mailcow-1 — on: edge1 _(live)_
- **mailcowdockerized-php-fpm-mailcow-1** — aliases: mailcowdockerized-php-fpm-mailcow-1 — on: edge1 _(live)_
- **mailcowdockerized-postfix-mailcow-1** — aliases: mailcowdockerized-postfix-mailcow-1 — on: edge1 _(live)_
- **mailcowdockerized-postfix-tlspol-mailcow-1** — aliases: mailcowdockerized-postfix-tlspol-mailcow-1 — on: edge1 _(live)_
- **mailcowdockerized-redis-mailcow-1** — aliases: mailcowdockerized-redis-mailcow-1 — on: edge1 _(live)_
- **mailcowdockerized-rspamd-mailcow-1** — aliases: mailcowdockerized-rspamd-mailcow-1 — on: edge1 _(live)_
- **mailcowdockerized-sogo-mailcow-1** — aliases: mailcowdockerized-sogo-mailcow-1 — on: edge1 _(live)_
- **mailcowdockerized-unbound-mailcow-1** — aliases: mailcowdockerized-unbound-mailcow-1 — on: edge1 _(live)_
- **mailcowdockerized-watchdog-mailcow-1** — aliases: mailcowdockerized-watchdog-mailcow-1 — on: edge1 _(live)_
- **matrix-element** — aliases: matrix-element — on: edge2 _(live)_
- **matrix-mas** (Matrix MAS) — aliases: Matrix MAS — on: edge2
- **matrix-postgres** — aliases: matrix-postgres — on: contabo _(live)_
- **matrix-synapse** (Matrix [[synapse]]) — aliases: Matrix Synapse, matrix — on: contabo
- **[[mautrix_signal]]** — aliases: mautrix-signal — on: contabo
- **matrix-synapse** (Matrix [[synapse]]) — aliases: Matrix Synapse, matrix — on: edge2
- **[[mautrix_signal]]** — aliases: mautrix-signal — on: edge2
- **meshtastic-cli** (Meshtastic CLI) — aliases: Meshtastic CLI — on: mt-isr
- **meshtasticd** — aliases: meshtasticd — on: mt-burleybutte
- **meshtasticd-aida-n2** (meshtasticd (AIDA-N2)) — aliases: meshtasticd (AIDA-N2) — on: aida-nebra
@ -147,10 +145,10 @@ mesh · matrix · [[recon]] · media · auth · [[dns]] · vpn · storage · pro
- **samba** (Samba) — aliases: Samba — on: data
- **sigil** (SIGIL) — aliases: SIGIL — on: 2026-06-16
- **sonarr** (Sonarr) — aliases: Sonarr — on: media
- **syncthing** (Syncthing) — aliases: Syncthing — on: contabo
- **syncthing** (Syncthing) — aliases: Syncthing — on: contabo _(decommissioned 2026-06-19 with edge1 rebuild)_
- **tak-server** (TAK Server) — aliases: TAK Server — on: 2026-06-16
- **tei** (TEI) — aliases: TEI — on: cortex
- **termix** (Termix) — aliases: Termix — on: contabo
- **termix** (Termix) — aliases: Termix — on: contabo _(decommissioned — wiped with edge1 rebuild 2026-06-19, not migrated to edge2)_
- **utility-caddy** (Utility Caddy) — aliases: Utility Caddy — on: utility
- **watchtower** (WATCHTOWER) — aliases: WATCHTOWER — on: 2026-06-16

View file

@ -10,7 +10,7 @@ related:
- [[ip-allocation]]
- [[caddy]]
- [[CLAUDE-baseline]]
updated: 2026-06-18
updated: 2026-07-11
---
# Echo6 Landing Page — Data Export
## Echo6 Platform Reference — Infrastructure, Services & Brand Identity
@ -317,7 +317,7 @@ Core content, classification (domain/subdomain/skill level/scenario), provenance
| Stage | Workers | Bottleneck | Description |
|-------|---------|------------|-------------|
| Extract | 4 | CPU-bound | PyPDF2 → pdftotext → Tesseract → Gemini Vision (4-method fallback chain) |
| Enrich | 16 | I/O-bound (Gemini API) | 10-page windows → Gemini 2.5 Flash Lite → structured JSON concepts |
| Enrich | 16 | I/O-bound (Gemini API) | 10-page windows → gemini-3.1-flash-lite → structured JSON concepts |
| Embed | batch | I/O-bound (TEI) | bge-m3 1024-dim → Qdrant insert, 128/batch |
| Scanner | 1 | Hourly cron | Auto-discovers new PDFs from NFS mount |
@ -447,7 +447,7 @@ files.echo6.co → Document/PDF download server
| Virtualization | Proxmox (5 nodes) |
| Networking | Tailscale/Headscale, Caddy, nginx, dnsmasq |
| GPU compute | NVIDIA RTX A4000 (CUDA, NVENC, Tensor) |
| AI/ML | Gemini 2.5 Flash Lite, Ollama, TEI (bge-m3), JOSIEFIED Qwen3 8B |
| AI/ML | gemini-3.1-flash-lite, Ollama, TEI (bge-m3), JOSIEFIED Qwen3 8B |
| Vector DB | Qdrant (HNSW index, cosine similarity) |
| Databases | SQLite (RECON), PostgreSQL (PeerTube) |
| Video | PeerTube v8, yt-dlp, ffmpeg/NVENC, Whisper |

View file

@ -10,12 +10,12 @@ related:
- [[mautrix_signal]]
- [[caddy]]
- [[lxc-service-migration]]
updated: 2026-06-18
updated: 2026-07-11
---
# Matrix Synapse Deployment
**Status:** Deployed 2026-02-15, migrated to Contabo 2026-02-15
**Target:** Contabo VPS (5.189.158.149 / 100.64.0.1)
**Status:** Deployed 2026-02-15, migrated to Contabo 2026-02-15. Migrated to edge2 CT 106 2026-06-19.
**Target (historical, at time of deployment):** Contabo VPS (5.189.158.149 / 100.64.0.1)
**URLs:** https://matrix.echo6.co ([[synapse]]), https://element.echo6.co (Element Web)
**Server Name:** echo6.co (federated identity: @user:echo6.co)
@ -25,9 +25,11 @@ updated: 2026-06-18
| Component | Detail |
|-----------|--------|
| Host | Contabo VPS (5.189.158.149 / 100.64.0.1) |
| Host (historical, at deployment time) | Contabo VPS (5.189.158.149 / 100.64.0.1) |
| Host (current) | edge2 CT 106 (100.64.0.37) — migrated 2026-06-18 |
| Docker [[services]] | Synapse (127.0.0.1:8008), Element Web (127.0.0.1:8088), PostgreSQL 16 |
| Reverse proxy | Contabo [[caddy]] (auto ACME certs) |
| Reverse proxy (historical) | Contabo [[caddy]] (auto ACME certs) |
| Reverse proxy (current) | edge2 host Caddy |
| SSO | [[authentik]] OIDC → communication-users group |
| Federation | Well-known delegation on echo6.co base domain (served by utility Caddy) |
| Compose path | `/opt/matrix/docker-compose.yml` |
@ -297,7 +299,7 @@ If `echo6.co` doesn't have a cert yet, issue one via acme.sh following the same
### dnsmasq split DNS
Add to `/etc/dnsmasq.d/tailscale-dns.conf` on Contabo:
**Historical (Contabo, pre-2026-06-19 migration).** Was added to `/etc/dnsmasq.d/tailscale-dns.conf` on Contabo:
```
address=/matrix.echo6.co/100.64.0.8
@ -306,12 +308,14 @@ address=/element.echo6.co/100.64.0.8
Both point to the Utility Caddy Tailscale IP (100.64.0.8), which proxies to CT 108.
Restart dnsmasq:
Restart dnsmasq (historical — ran on Contabo):
```bash
ssh root@100.64.0.1 "systemctl restart dnsmasq"
```
**Current (post-migration):** Matrix and Element now run on edge2 CT 106 (100.64.0.37), fronted by edge2's own host Caddy — see [[services]] and [[matrix_host]]. dnsmasq split-DNS on the old Contabo host is no longer part of this path.
---
## Step 8: Configure Authentik SSO

View file

@ -11,7 +11,7 @@ related:
- [[services]]
- [[meshcore-transport]]
- [[central]]
updated: 2026-07-02
updated: 2026-07-11
---
# meshai
@ -25,7 +25,7 @@ meshai (bot name **AIDA**) attaches to the mesh as a physical node and does two
- **Outbound alerts.** It consumes real-world situational-awareness feeds (weather, wildfire, traffic/511, space weather, earthquakes, water, satellite passes, …), normalizes them into events, and broadcasts short, chunked alerts to the mesh.
- **Interactive assistant.** Mesh users message it directly and get data-driven LLM answers over LoRa — mesh-health questions ("how's the mesh?"), knowledge/weather queries, and command handlers (`!health`, `!region`, `!neighbors`, `!fires`, `!quakes`, `!space`, `!water`, subscriptions, etc.).
The LLM backend is Gemini 2.5 Flash with Google Search grounding (multi-backend capable — Gemini / OpenAI / Anthropic / local via LiteLLM). Conversation memory is a rolling window persisted to SQLite.
The LLM backend is gemini-3.1-flash-lite with Google Search grounding (multi-backend capable — Gemini / OpenAI / Anthropic / local via LiteLLM). Conversation memory is a rolling window persisted to SQLite.
## Where it runs

View file

@ -10,7 +10,7 @@ related:
- [[meshtastic-sidecar-node]]
- [[headscale-onboard-node]]
- [[caddy]]
updated: 2026-06-21
updated: 2026-07-11
---
# IdahoMesh Tailnet Runbook
@ -369,9 +369,9 @@ tailscale --socket=/var/run/tailscale-meshtastic/tailscaled.sock up \
After joining, approve the advertised route on Echo6 Headscale only:
```bash
# On Echo6 Headscale (Contabo) — enable the 100.100.0.0/16 route
docker exec headscale-vanilla headscale routes list
docker exec headscale-vanilla headscale routes enable -r <route-id>
# On Echo6 Headscale (edge2 CT 107) — enable the 100.100.0.0/16 route
ssh edge2 "sudo pct exec 107 -- docker exec headscale headscale routes list"
ssh edge2 "sudo pct exec 107 -- docker exec headscale headscale routes enable -r <route-id>"
# NO route approval needed on IdahoMesh Headscale — nothing is advertised
```

View file

@ -10,7 +10,7 @@ related:
- [[ip-allocation]]
- [[services]]
- [[meshtastic-headscale-runbook]]
updated: 2026-06-18
updated: 2026-07-11
---
# MMUD — Mesh Multi-User Dungeon
@ -18,7 +18,7 @@ Text-based multiplayer dungeon crawler for Meshtastic LoRa mesh networks. BBS do
## Status
**Phase:** Deployed and running — all 6 phases implemented, NPC conversation system live with Gemini 2.5 Flash.
**Phase:** Deployed and running — all 6 phases implemented, NPC conversation system live with gemini-3.1-flash-lite.
## Deployment
@ -31,7 +31,7 @@ Text-based multiplayer dungeon crawler for Meshtastic LoRa mesh networks. BBS do
- MRN (CT 114) — Maren healer NPC
- TRVL (CT 115) — Torval merchant NPC
- WSPR (CT 116) — Whisper sage NPC
- **LLM Backend:** Gemini 2.5 Flash via Google genai SDK (configured in DB `llm_config` table)
- **LLM Backend:** gemini-3.1-flash-lite via Google genai SDK (configured in DB `llm_config` table)
- **Compose:** `/opt/mmud/docker-compose.yml` on CT 109
- **Admin:** https://mmud.echo6.co/admin (session auth, password in docker env)
@ -75,9 +75,9 @@ NPCs use runtime LLM calls (the one exception to the "no runtime LLM" rule). Key
## LLM Configuration
- Model configured via `llm_config` table in SQLite DB (not env vars)
- Currently: Gemini 2.5 Flash (`gemini-2.5-flash`)
- Currently: gemini-3.1-flash-lite
- API key stored in DB, manageable via admin panel at /admin/llm
- **No `max_output_tokens` restrictions**Gemini 2.5 Flash thinking tokens consume the budget, causing truncation. All backends have token limits removed.
- **No `max_output_tokens` restrictions**gemini-3.1-flash-lite thinking tokens consume the budget, causing truncation. All backends have token limits removed.
- Supports: Google (Gemini), Anthropic (Claude), OpenAI-compatible backends
## Development Phases (All Complete)
@ -91,7 +91,7 @@ NPCs use runtime LLM calls (the one exception to the "no runtime LLM" rule). Key
## Gotchas
- Gemini 2.5 Flash thinking tokens count against `max_output_tokens` — never set token limits
- gemini-3.1-flash-lite thinking tokens count against `max_output_tokens` — never set token limits
- NPC greeting path uses `complete()`, conversation path uses `chat()` — different code paths
- `npc_memory` table stores `turn_count` for interaction depth tracking
- Death log table (`death_log`) tracks monster kills for Maren's memory feature

View file

@ -10,7 +10,7 @@ related:
- [[recon-service-integration]]
- [[proxmox-onboard-node]]
- [[ct-runbook]]
updated: 2026-06-18
updated: 2026-07-11
---
# Add PeerTube Channel
@ -157,7 +157,7 @@ curl -s http://192.168.1.130:8420/api/peertube/channels \
If `tee` race condition empties the file:
1. Check Contabo backup: `ssh root@100.64.0.1 ls -la /opt/backups/recon/`
1. Check edge1 backup (was Contabo pre-2026-06-19 rebuild): `ssh root@100.64.0.40 ls -la /opt/backups/recon/`
2. Or rebuild from PeerTube DB:
```bash
ssh zvx@192.168.1.170 "sudo -u peertube psql peertube_prod -t -A -c \
@ -178,7 +178,7 @@ If `tee` race condition empties the file:
| Issue | Cause | Fix |
|-------|-------|-----|
| yt-dlp "Join this channel" error | Members-only first video | API auto-retries with `/videos` tab. CLI: add `--ignore-errors --playlist-items 1:5` and use `/videos` URL |
| channel-map.json empty (0 bytes) | tee race condition | Always write to temp file first, then tee. Restore from backup or Contabo |
| channel-map.json empty (0 bytes) | tee race condition | Always write to temp file first, then tee. Restore from backup or edge1 |
| sudo: password required | Sudoers not set up | Create `/etc/sudoers.d/recon-mgmt` via `pct exec 110` from root@192.168.1.243 |
| PeerTube "actor name already exists" | Channel exists in PeerTube but not in JSON | Add entry to JSON manually with correct `peertube_channel_id` |

View file

@ -10,13 +10,13 @@ related:
- [[authentik-create-invitation]]
- [[deploy-livesync]]
- [[proxmox-onboard-node]]
updated: 2026-06-18
updated: 2026-07-11
---
# Authentik Access Groups
Manage group-based application access via the [[authentik]] API. No web UI interaction required.
**[[authentik]] instance:** https://auth.echo6.co (Contabo, 100.64.0.1)
**[[authentik]] instance:** https://auth.echo6.co (edge2 CT 105, 100.64.0.36)
**Key behavior:** Users in `authentik Admins` (is_superuser=true) bypass ALL policy checks automatically. Group bindings only restrict non-superuser access.

View file

@ -10,7 +10,7 @@ related:
- [[authentik-oidc-application]]
- [[authentik-upgrade]]
- [[mailcow-create-mailbox]]
updated: 2026-06-18
updated: 2026-07-11
---
# Authentik: Create Invitation
@ -156,13 +156,12 @@ See the [Access Groups runbook](authentik-access-groups.md) for detailed group m
1. **Check custom attributes** — the `email` field must be present and correctly formatted
2. **Check SMTP** — verify Authentik can send email:
```bash
ssh root@100.64.0.1
docker exec authentik-server ak test_email matt@echo6.co
ssh edge2 'sudo pct exec 105 -- docker exec authentik-server ak test_email matt@echo6.co'
```
3. **Check Mailcow authsource** — if SMTP auth fails, the no-reply@echo6.co mailbox may have reverted to `generic-oidc`. See [Mailcow Create Mailbox runbook](mailcow-create-mailbox.md), Step 2
4. **Check Authentik logs**:
```bash
docker compose -f /opt/authentik/docker-compose.yml logs server --since 5m 2>&1 | grep -i email
ssh edge2 'sudo pct exec 105 -- docker compose -f /opt/authentik/docker-compose.yml logs server --since 5m 2>&1 | grep -i email'
```
### "Invalid invite/invite not found" when clicking link

View file

@ -10,15 +10,15 @@ related:
- [[authentik-upgrade]]
- [[mailcow-create-mailbox]]
- [[expose-service-home]]
updated: 2026-06-18
updated: 2026-07-11
---
# Add Authentik OIDC to an Application
Fully automated via [[authentik]] API. No web UI interaction required.
**Prerequisite:** [[dns]] must already exist for the service (run expose-service-contabo.md or expose-service-home.md first).
**Prerequisite:** [[dns]] must already exist for the service (run expose-service-edge2.md or expose-service-home.md first).
**Authentik instance:** https://auth.echo6.co (Contabo, 100.64.0.1)
**Authentik instance:** https://auth.echo6.co (edge2 CT 105, 100.64.0.36)
---
@ -54,7 +54,7 @@ These conflict with Authentik's internal OAuth2 endpoints and **cannot be used**
Create an API token from the Authentik admin account. This only needs to happen once — reuse the token across all OIDC setups.
```bash
ssh root@100.64.0.1 "docker exec authentik-server \
ssh edge2 "sudo pct exec 105 -- docker exec authentik-server \
ak create_token --user akadmin --identifier oidc-automation --expiring 2>/dev/null \
|| echo 'Token may already exist — check credentials file'"
```
@ -77,7 +77,7 @@ The API requires UUIDs for flows, scope mappings, and signing keys. These are st
### Authorization flow
```bash
ssh root@100.64.0.1 "curl -s \
ssh edge2 "sudo pct exec 105 -- curl -s \
-H 'Authorization: Bearer $AK_TOKEN' \
'$AK_API/flows/instances/?slug=default-provider-authorization-implicit-consent' \
| jq -r '.results[0].pk'"
@ -89,7 +89,7 @@ Store as `AUTH_FLOW_PK`.
```bash
# Get all scope mapping UUIDs at once
ssh root@100.64.0.1 "curl -s \
ssh edge2 "sudo pct exec 105 -- curl -s \
-H 'Authorization: Bearer $AK_TOKEN' \
'$AK_API/propertymappings/provider/scope/?ordering=scope_name' \
| jq -r '.results[] | select(.scope_name == \"openid\" or .scope_name == \"email\" or .scope_name == \"profile\" or .scope_name == \"offline_access\") | \"\(.scope_name): \(.pk)\"'"
@ -100,7 +100,7 @@ Store each UUID: `SCOPE_OPENID_PK`, `SCOPE_EMAIL_PK`, `SCOPE_PROFILE_PK`, `SCOPE
### Signing key
```bash
ssh root@100.64.0.1 "curl -s \
ssh edge2 "sudo pct exec 105 -- curl -s \
-H 'Authorization: Bearer $AK_TOKEN' \
'$AK_API/crypto/certificatekeypairs/?name=authentik+Self-signed+Certificate&has_key=true' \
| jq -r '.results[0].pk'"
@ -131,7 +131,7 @@ fi
Create the provider:
```bash
PROVIDER_RESPONSE=$(ssh root@100.64.0.1 "curl -s \
PROVIDER_RESPONSE=$(ssh edge2 "sudo pct exec 105 -- curl -s \
-X POST '$AK_API/providers/oauth2/' \
-H 'Authorization: Bearer $AK_TOKEN' \
-H 'Content-Type: application/json' \
@ -172,7 +172,7 @@ echo "Client Secret: $CLIENT_SECRET"
## Step 4: Create the Application
```bash
ssh root@100.64.0.1 "curl -s \
ssh edge2 "sudo pct exec 105 -- curl -s \
-X POST '$AK_API/core/applications/' \
-H 'Authorization: Bearer $AK_TOKEN' \
-H 'Content-Type: application/json' \
@ -296,7 +296,7 @@ Check in order:
Debug via API:
```bash
ssh root@100.64.0.1 "curl -s \
ssh edge2 "sudo pct exec 105 -- curl -s \
-H 'Authorization: Bearer $AK_TOKEN' \
'$AK_API/providers/oauth2/?search=$SERVICE_NAME' \
| jq '.results[0] | {name, client_id, signing_key, access_token_validity, refresh_token_validity, property_mappings}'"
@ -305,7 +305,7 @@ ssh root@100.64.0.1 "curl -s \
Or via ak shell:
```bash
ssh root@100.64.0.1 "docker exec authentik-server ak shell -c \"
ssh edge2 "sudo pct exec 105 -- docker exec authentik-server ak shell -c \"
from authentik.providers.oauth2.models import OAuth2Provider
p = OAuth2Provider.objects.get(name='$SERVICE_NAME')
print(f'Access Token: {p.access_token_validity}')
@ -330,7 +330,7 @@ The redirect URI in the app config must **exactly** match what's in Authentik
User isn't authorized for the application. By default all authenticated users have access. If you've added group restrictions via policy bindings, verify the user is in the correct group:
```bash
ssh root@100.64.0.1 "curl -s \
ssh edge2 "sudo pct exec 105 -- curl -s \
-H 'Authorization: Bearer $AK_TOKEN' \
'$AK_API/core/applications/$SERVICE_SLUG/' \
| jq '{name, slug, policy_engine_mode}'"
@ -344,12 +344,12 @@ Missing `offline_access` scope. Without refresh tokens, sessions only last as lo
```bash
# Delete application first (it references the provider)
ssh root@100.64.0.1 "curl -s -X DELETE \
ssh edge2 "sudo pct exec 105 -- curl -s -X DELETE \
-H 'Authorization: Bearer $AK_TOKEN' \
'$AK_API/core/applications/$SERVICE_SLUG/'"
# Then delete provider
ssh root@100.64.0.1 "curl -s -X DELETE \
ssh edge2 "sudo pct exec 105 -- curl -s -X DELETE \
-H 'Authorization: Bearer $AK_TOKEN' \
'$AK_API/providers/oauth2/$PROVIDER_PK/'"
```

View file

@ -10,11 +10,13 @@ related:
- [[authentik]]
- [[authentik-create-invitation]]
- [[ct-runbook]]
updated: 2026-06-18
updated: 2026-07-11
---
# Authentik: Major Version Upgrade
Upgrade [[authentik]] between major versions on Contabo. Covers backup, upgrade, verification, and rollback.
> Migrated off Contabo → edge2 CT 105 on 2026-06-18.
Upgrade [[authentik]] between major versions on edge2 CT 105. Covers backup, upgrade, verification, and rollback.
---
@ -26,7 +28,7 @@ Any time Authentik is upgraded across major versions (e.g., 2024.12 → 2025.6
## Prerequisites
- SSH access to Contabo (`ssh root@100.64.0.1`)
- SSH access to edge2 CT 105 via edge2 host: `ssh edge2 'sudo pct exec 105 -- bash'` (interactive shell inside the CT; root SSH direct to edge2/the CT is refused — see [[edge2-access-reference]])
- Authentik compose directory: `/opt/authentik/`
- Current version: check with `docker exec authentik-server ak --version`
@ -68,14 +70,14 @@ Look for:
## Step 2: Backup
### 2a. Snapshot Contabo (if Proxmox-managed)
### 2a. Snapshot edge2 CT 105 (Proxmox-managed LXC)
If Contabo were a Proxmox VM, take a snapshot. Since it's a bare-metal VPS, skip this and rely on the file-level backups below.
CT 105 is an LXC on edge2 — take a Proxmox snapshot before upgrading: `ssh edge2 "sudo pct snapshot 105 pre-upgrade-$(date +%Y%m%d)"`. Also rely on the file-level backups below.
### 2b. PostgreSQL Dump
```bash
ssh root@100.64.0.1
ssh edge2 'sudo pct exec 105 -- bash' # interactive shell inside CT 105; run the rest of this section inside it
cd /opt/authentik
TIMESTAMP=$(date +%Y%m%d_%H%M%S)

View file

@ -10,10 +10,12 @@ related:
- [[lxc-service-migration]]
- [[headscale-onboard-node]]
- [[caddy]]
updated: 2026-06-18
updated: 2026-07-11
---
# Expose Service on Contabo
> SUPERSEDED — Contabo was decommissioned 2026-06-19. Use [[expose-service-edge2]] (services) or [[expose-service-contabo]]→edge1 for mail. This doc is kept for history only.
## Prerequisites
- Service running in Docker on Contabo
- Port bound to `127.0.0.1` only (never `0.0.0.0`)

View file

@ -10,10 +10,13 @@ related:
- [[headscale-onboard-node]]
- [[caddy]]
- [[expose-service-contabo]]
updated: 2026-06-19
- [[edge2-access-reference]]
updated: 2026-07-11
---
# LXC Service Migration — Contabo → edge2
> **Note on the source host references below:** Contabo (`100.64.0.1`) was the migration **source** host during the 2026-06 service evacuation and was decommissioned/rebuilt as **edge1** (mail-only) on 2026-06-19 — it no longer exists at that tailnet address. The `ssh root@100.64.0.1` commands throughout this runbook are illustrative of "the source host you are migrating from"; for any future migration, substitute the actual current source host and its real access pattern. edge2 targets always use `ssh edge2` + `sudo pct exec` — never `ssh root@<edge2-IP>` (root SSH is refused on edge2). See [[edge2-access-reference]]. The migration **pattern** itself (phases, gates, rollback structure) remains valid regardless of which host is the source.
> Proven pilots: **Vaultwarden → edge2 CT 102** (SQLite, 2026-06-16), **Forgejo → edge2 CT 103** (PostgreSQL + non-Caddy SSH port, 2026-06-16), **LiveSync (CouchDB) → edge2 CT 104** (cold named-volume tar + bind-mounted config, 2026-06-16), **[[authentik]] (PostgreSQL keystone) → edge2 CT 105** (SECRET_KEY-must-travel, multi-block [[caddy]] cutover across 2 site blocks, reboot tailscale-before-docker race, 2026-06-18), **Matrix stack → edge2 CT 106** (multi-DB Postgres + stateful Signal bridge, 5 containers, 2026-06-18), and **Headscale → edge2 CT 107** (tailnet control plane, noise_private.key must travel, 2026-06-19). This runbook generalizes these patterns into a reusable template for evacuating any Contabo-Caddy-fronted service to an edge2 LXC.
---

View file

@ -10,11 +10,13 @@ related:
- [[caddy]]
- [[authentik-create-invitation]]
- [[proxmox-onboard-node]]
updated: 2026-06-18
updated: 2026-07-11
---
# Mailcow: Create Mailbox
Create a new mailbox in Mailcow on the Contabo VPS. Covers both interactive (UI) and API-driven creation, with the critical authsource fix for service accounts.
> Migrated off Contabo → edge1 (mail-only) on 2026-06-19. Mailcow now lives in edge1 CT 101 (internal 10.10.10.2); public IP 5.189.158.149 is unchanged, but the tailscale address is now 100.64.0.40 (not 100.64.0.1).
Create a new mailbox in Mailcow on edge1 (Mailcow CT 101). Covers both interactive (UI) and API-driven creation, with the critical authsource fix for service accounts.
---
@ -26,7 +28,7 @@ Any time a new mailbox is created in Mailcow, but **especially** for service/sys
## Prerequisites
- SSH access to Contabo (`ssh root@100.64.0.1`)
- SSH access to edge1 (`ssh edge1` or `ssh root@100.64.0.40`)
- Mailcow API key (stored in Mailcow admin UI under System → Configuration → API)
- Mailcow DB password: source from `/opt/mailcow-dockerized/.env` (`DBPASS`)
@ -51,7 +53,7 @@ MAILCOW_API_KEY=<api-key> # From Mailcow admin UI
### Option A: Via Mailcow API
```bash
ssh root@100.64.0.1
ssh edge1
curl -sk -X POST "https://127.0.0.1:8443/api/v1/add/mailbox" \
-H "X-API-Key: ${MAILCOW_API_KEY}" \
@ -128,7 +130,7 @@ The failure message gives no indication that OIDC is the cause. The password is
Change the authsource from `generic-oidc` to `mailcow` in the database:
```bash
ssh root@100.64.0.1
ssh edge1
# Source the DB password
DBPASS=$(grep ^DBPASS /opt/mailcow-dockerized/.env | cut -d= -f2)
@ -161,7 +163,7 @@ Rule of thumb: if the account will ever authenticate with a username + password
Wait a few seconds after the authsource fix, then test:
```bash
# From the Contabo host
# From edge1
python3 -c "
import smtplib
s = smtplib.SMTP('mail.echo6.co', 587, timeout=10)
@ -236,7 +238,7 @@ Too many failed SMTP login attempts can trigger Mailcow's brute-force protection
docker logs mailcowdockerized-netfilter-mailcow-1 --since 10m 2>&1 | grep -i ban
```
If the Contabo IP (5.189.158.149) is banned, restart the netfilter container:
If the edge1 public IP (5.189.158.149) is banned, restart the netfilter container:
```bash
cd /opt/mailcow-dockerized && docker compose restart netfilter-mailcow

View file

@ -10,7 +10,7 @@ related:
- [[idahomesh-bridge-setup]]
- [[headscale-onboard-node]]
- [[advbbs-project]]
updated: 2026-06-18
updated: 2026-07-11
---
# Meshtastic Sidecar Node — Modular Deployment Runbook
@ -127,7 +127,7 @@ ping -c 3 192.168.1.1 # Gateway reachable
| Tailnet | Headscale URL | Prefix | Key generation |
|---------|---------------|--------|----------------|
| Echo6 | `https://vpn.echo6.co` | 100.64.0.0/10 | On Contabo |
| Echo6 | `https://vpn.echo6.co` | 100.64.0.0/10 | On edge2 CT 107 |
| IdahoMesh | `https://vpn.idahomesh.com` | 100.100.0.0/16 | On CT 106 |
### Install Tailscale
@ -138,10 +138,10 @@ curl -fsSL https://tailscale.com/install.sh | sh
### Generate preauthkey
**Echo6** (from cortex or any machine with Tailscale access to Contabo):
**Echo6** (from cortex or any machine with Tailscale access to edge2):
```bash
ssh root@100.64.0.1 'docker exec headscale headscale preauthkeys create --user 1 --reusable --expiration 1h'
ssh edge2 "sudo pct exec 107 -- docker exec headscale headscale preauthkeys create --user 1 --reusable --expiration 1h"
```
**IdahoMesh** (from utility Proxmox host):
@ -179,7 +179,7 @@ sudo systemctl daemon-reload
```bash
tailscale status # Should show connected
tailscale ip -4 # Should show 100.64.x.x or 100.100.x.x
ping -c 3 100.64.0.1 # Echo6: ping Contabo
ping -c 3 100.64.0.26 # Echo6: ping edge2 (Headscale control now on edge2 CT 107, 100.64.0.38)
ping -c 3 100.100.0.1 # IdahoMesh: ping Headscale
```

View file

@ -10,7 +10,7 @@ related:
- [[headscale-onboard-node]]
- [[proxmox-onboard-node]]
- [[ip-allocation]]
updated: 2026-06-18
updated: 2026-07-11
---
# Meshtasticd SIM Node Runbook — LXC Deployment
@ -241,10 +241,10 @@ pct exec <CTID> -- bash -c "
"
```
Generate a preauth key on Contabo (user ID 1 = echo6):
Generate a preauth key on edge2 CT 107 (user ID 1 = echo6):
```bash
ssh root@100.64.0.1 'docker exec headscale headscale preauthkeys create --user 1 --reusable --expiration 1h'
ssh edge2 "sudo pct exec 107 -- docker exec headscale headscale preauthkeys create --user 1 --reusable --expiration 1h"
```
Register the node:

View file

@ -10,7 +10,7 @@ related:
- [[proxmox-onboard-node]]
- [[headscale-onboard-node]]
- [[pi-nas-omv-runbook]]
updated: 2026-06-18
updated: 2026-07-11
---
# Proxmox — Create Ubuntu VM (Cloud-Init)
@ -19,7 +19,7 @@ Automated VM creation using Ubuntu cloud images. No interactive installer needed
## Prerequisites
- SSH access to the target Proxmox host (directly or via jump box)
- Headscale running on Contabo with a valid preauth key
- Headscale running on edge2 CT 107 with a valid preauth key
- Target Proxmox host has sufficient resources (check with `pvesm status`, `free -h`, `nproc`)
## Variables — Prompt the User
@ -253,10 +253,10 @@ ssh zvx@$VM_IP 'curl -fsSL https://deb.nodesource.com/setup_22.x | sudo bash - &
## Step 12 — Tailscale Registration
Generate a preauth key on Contabo first:
Generate a preauth key on edge2 CT 107 first:
```bash
docker exec headscale headscale preauthkeys create --user echo6 --reusable --expiration 72h
ssh edge2 "sudo pct exec 107 -- docker exec headscale headscale preauthkeys create --user echo6 --reusable --expiration 72h"
```
Then register the VM:
@ -268,7 +268,7 @@ ssh zvx@$VM_IP "sudo tailscale up --login-server https://vpn.echo6.co --auth-key
# Verify
ssh zvx@$VM_IP 'tailscale status'
docker exec headscale headscale nodes list
ssh edge2 "sudo pct exec 107 -- docker exec headscale headscale nodes list"
```
## Step 13 — Final Verification
@ -287,7 +287,7 @@ ssh zvx@$VM_IP "
echo '=== Disk ===' && df -h /
"
docker exec headscale headscale nodes list
ssh edge2 "sudo pct exec 107 -- docker exec headscale headscale nodes list"
```
## Post-Creation

View file

@ -10,7 +10,7 @@ related:
- [[caddy]]
- [[recon-service-integration]]
- [[services]]
updated: 2026-06-18
updated: 2026-07-11
---
# RECON Operations Runbook
@ -87,11 +87,11 @@ python3 recon.py queue
| Ollama | cortex | 11434 | Chat model for Aurora RAG |
| NFS | pi-nas | — | /mnt/library (PDF source) |
| Gemini API | Google | — | Enrichment + vision OCR (4 keys in .env) |
| Contabo VPS | 100.64.0.1 | — | Backup destination |
| edge1 | 100.64.0.40 | — | Backup destination (was Contabo 100.64.0.1 pre-2026-06-19 rebuild) |
## Backups
- **Destination:** `root@100.64.0.1:/opt/backups/recon/`
- **Destination:** `root@100.64.0.40:/opt/backups/recon/` (edge1; was `root@100.64.0.1` on Contabo pre-2026-06-19 rebuild)
- **Full sync (concepts, text, DB, config):** every 6 hours via cron
- **DB snapshot only:** every 2 hours via cron
- **Script:** `/opt/recon/scripts/backup.sh`
@ -99,7 +99,7 @@ python3 recon.py queue
### Verify backups
```bash
ssh root@100.64.0.1 'ls -lh /opt/backups/recon/recon_*.db && du -sh /opt/backups/recon/'
ssh root@100.64.0.40 'ls -lh /opt/backups/recon/recon_*.db && du -sh /opt/backups/recon/'
```
## Troubleshooting
@ -156,15 +156,15 @@ sqlite3 data/recon.db "UPDATE documents SET status='extracted' WHERE status='enr
sqlite3 data/recon.db "UPDATE documents SET status='enriched' WHERE status='embedding';"
```
### Full recovery from Contabo backup
### Full recovery from edge1 backup
```bash
ssh zvx@100.64.0.24
sudo systemctl stop recon
rsync -av root@100.64.0.1:/opt/backups/recon/concepts/ /opt/recon/data/concepts/
rsync -av root@100.64.0.1:/opt/backups/recon/text/ /opt/recon/data/text/
rsync -av root@100.64.0.40:/opt/backups/recon/concepts/ /opt/recon/data/concepts/
rsync -av root@100.64.0.40:/opt/backups/recon/text/ /opt/recon/data/text/
# Pick the latest DB backup
rsync -av root@100.64.0.1:/opt/backups/recon/recon_latest.db /opt/recon/data/recon.db
rsync -av root@100.64.0.40:/opt/backups/recon/recon_latest.db /opt/recon/data/recon.db
cd /opt/recon && source venv/bin/activate
python3 recon.py rebuild # Rebuilds Qdrant from concept JSONs
sudo systemctl start recon

View file

@ -10,7 +10,7 @@ related:
- [[headscale-onboard-node]]
- [[lxc-service-migration]]
- [[caddy]]
updated: 2026-06-18
updated: 2026-07-11
---
# RECON Dashboard Service Integration
@ -22,7 +22,7 @@ Use this when you have a service running on a remote LXC/VM that needs a web man
## Prerequisites
- A running Flask or FastAPI dashboard (e.g., [[recon]] on VM 1130, WATCHTOWER on Contabo)
- A running Flask or FastAPI dashboard (e.g., [[recon]] on VM 1130; WATCHTOWER on Contabo formerly served as an example here — **WATCHTOWER was decommissioned 2026-06-16** and is no longer a live integration target)
- The target service running on a reachable host (LXC, VM, or bare metal)
- SSH access from the dashboard host to the target host
- The dashboard runs as a known user (e.g., `zvx`, `recon`, `watchtower`)
@ -456,10 +456,12 @@ API endpoints:
Dashboard panel: green/red dot + Restart/Stop/Start/Logs buttons + feedback box
```
### WATCHTOWER monitoring remote services (Contabo → multiple hosts)
### WATCHTOWER monitoring remote services (Contabo → multiple hosts) — HISTORICAL, DECOMMISSIONED
> **WATCHTOWER was decommissioned 2026-06-16** (archived to forge.echo6.co/matt/archive-watchtower). The example below reflects the pattern as it existed on the old Contabo VPS and is kept for reference only — it is not a live integration target and should not be used to route new work.
```
DASHBOARD_HOST=5.189.158.149 (Contabo)
DASHBOARD_HOST=5.189.158.149 (Contabo — decommissioned 2026-06-16)
DASHBOARD_USER=root
Services managed:

View file

@ -10,20 +10,22 @@ related:
- [[meshtasticd-sim-nodes-runbook]]
- [[idahomesh-vpn-device-setup]]
- [[headscale-onboard-node]]
updated: 2026-06-18
updated: 2026-07-11
---
# Syncthing: Add a New Node to the Project Sync Cluster
> **Syncthing on Contabo was decommissioned 2026-06-19** with the edge1 rebuild (state removed; Forge is now the durable backup via the `echo6-docs-autocommit` cron). The `contabo` row below and its device ID are historical — do not treat it as a live cluster member. Any new-node onboarding should reassess whether this cluster still has a live counterpart before assuming `contabo` is reachable.
## Overview
Adds a new machine to the Syncthing `projects` folder mesh. All nodes sync bidirectionally — new files merge, nothing is overwritten or deleted.
**Current cluster:**
**Current cluster (as of last update — `contabo` decommissioned 2026-06-19, kept below for history):**
| Node | Device ID (short) | Path | OS |
|------|--------------------|------|----|
| cortex | `6VP7KIB` | `/home/zvx/projects` | Ubuntu 24.04 |
| contabo | `SBYGD4P` | `/home/zvx/projects` | Ubuntu 24.04 |
| ~~contabo~~ (decommissioned 2026-06-19) | `SBYGD4P` | `/home/zvx/projects` | Ubuntu 24.04 |
| bluefin | `5ZTWIXM` | `/var/home/malice/projects` | Fedora Atomic |
| matt-desktop | `GCH6AAG` | `E:\Documents\projects` | Windows |
@ -164,9 +166,11 @@ MY_DEVICE_ID = "<new node device ID>"
PROJECTS_PATH = "<local path to projects folder>" # e.g. /home/zvx/projects
# All cluster nodes — add the new node's ID to this list when updating existing nodes
# NOTE: "contabo" is decommissioned (2026-06-19, edge1 rebuild) and kept here only for history.
# Do not register new nodes against it; remove once the cluster config is next touched for real.
DEVICES = {
"cortex": {"id": "6VP7KIB-ZHBI3AT-XO5FMY2-LFAZYM6-UMAV75U-MZZADW3-ZOBHJXY-GF26DAC", "addr": "tcp://100.64.0.14:22000"},
"contabo": {"id": "SBYGD4P-BUWMWRQ-JJYYG75-YBR4WOO-OH42WH4-IAAO33D-STJZX6O-SZA2SQ4", "addr": "tcp://100.64.0.1:22000"},
# "contabo": {"id": "SBYGD4P-BUWMWRQ-JJYYG75-YBR4WOO-OH42WH4-IAAO33D-STJZX6O-SZA2SQ4", "addr": "tcp://100.64.0.1:22000"}, # DECOMMISSIONED 2026-06-19
"bluefin": {"id": "5ZTWIXM-XNBUEW5-XWJM7PG-FJDMX5H-YMXM3CC-ZVS2PNO-NG2E3KJ-D5HXKQB", "addr": "dynamic"},
"matt-desktop": {"id": "GCH6AAG-IWPH6TR-7GI7THZ-DIVXRRQ-EQMRBNN-IZG7Y2F-HM6BRLX-AC3MIQ6", "addr": "dynamic"},
}