From 5f030499146fa72279b7f2da5010b7e8672f512e Mon Sep 17 00:00:00 2001 From: echo6-autocommit Date: Thu, 18 Jun 2026 00:00:07 +0000 Subject: [PATCH] auto: docs sync 2026-06-18T00:00:07+00:00 Files changed: .obsidian/graph.json .obsidian/workspace.json INDEX.md docs/hardware/environment.md docs/hardware/ip-allocation.md docs/services/services.md runbooks/lxc-service-migration.md session-resume/SESSION-HANDOFF-meshai-test.md --- .obsidian/graph.json | 2 +- .obsidian/workspace.json | 84 +++++++++---------- INDEX.md | 10 ++- docs/hardware/environment.md | 20 +++-- docs/hardware/ip-allocation.md | 4 +- docs/services/services.md | 58 ++++++++----- runbooks/lxc-service-migration.md | 29 ++++++- session-resume/SESSION-HANDOFF-meshai-test.md | 53 ++++++++++++ 8 files changed, 181 insertions(+), 79 deletions(-) create mode 100644 session-resume/SESSION-HANDOFF-meshai-test.md diff --git a/.obsidian/graph.json b/.obsidian/graph.json index e21a18d..3dda752 100644 --- a/.obsidian/graph.json +++ b/.obsidian/graph.json @@ -17,6 +17,6 @@ "repelStrength": 10, "linkStrength": 1, "linkDistance": 250, - "scale": 1, + "scale": 2.25, "close": false } \ No newline at end of file diff --git a/.obsidian/workspace.json b/.obsidian/workspace.json index a3e170e..3999d07 100644 --- a/.obsidian/workspace.json +++ b/.obsidian/workspace.json @@ -11,14 +11,10 @@ "id": "ea4cc678c44e8b67", "type": "leaf", "state": { - "type": "markdown", - "state": { - "file": "rules/proxmox.md", - "mode": "source", - "source": false - }, - "icon": "lucide-file", - "title": "proxmox" + "type": "graph", + "state": {}, + "icon": "lucide-git-fork", + "title": "Graph view" } } ] @@ -189,42 +185,42 @@ }, "active": "ea4cc678c44e8b67", "lastOpenFiles": [ - "runbooks/lxc-service-migration.md.tmp.40509.4536b3c3851b", - "runbooks/lxc-service-migration.md.tmp.40509.bb32c76bdcf4", - "runbooks/lxc-service-migration.md.tmp.40509.052f55354b7d", - "runbooks/lxc-service-migration.md.tmp.40509.d17d83b34e07", - "runbooks/lxc-service-migration.md.tmp.40509.519078fbf870", - "runbooks/lxc-service-migration.md.tmp.40509.10b3a76d106e", - "credentials.tmp.40509.1256ea803f58", - "docs/hardware/ip-allocation.md.tmp.40509.9aa60f8e7a54", - "docs/hardware/ip-allocation.md.tmp.40509.b1093f9672ae", - "docs/hardware/ip-allocation.md.tmp.40509.94cbb0537cfb", - "docs/hardware/environment.md.tmp.40509.b4c1d593d6b6", - "runbooks/lxc-service-migration.md", - "runbooks/edge2-access-reference.md", - "runbooks/expose-service-edge2.md", - "plans/vaultwarden-plan.md", - "plans/vaultwarden-migration.md", - "rules/radio.md", - "rules/tak.md", - "archive/matrix/PHASE6_DECISION.md", - "INDEX.md", - "runbooks/headscale-onboard-node.md", - "docs/services/ots-setup.md", - "docs/hardware/ip-allocation.md", - "CLAUDE-baseline.md", - "notes/ia-download-queue.md", - "notes/echo6-landing-page-data-export.md", - "projects/matrix-synapse-deployment.md", - "projects/meshtastic-headscale-runbook.md", - "projects/mmud-project.md", - "rules/watchtower.md", - "rules/argus.md", - "rules/aurora.md", - "projects/argus.md", - "archive/matrix/PLAN.md", - "archive/matrix/archivist_discovery.md", - "archive/matrix/appservices.md", + "docs/hardware/environment.md", + "nodes/ots-ct.md", + "mocs/mesh.md", + "mocs/matrix.md", + "mocs/media.md", + "hardware/station-g2.md", + "hardware/waveshare-eth-hub.md", + "hardware/nebra-2w-bb.md", + "hardware/nebra-2w-aida.md", + "services/ots.md", + "services/mesh-bridge.md", + "services/idahomesh-headscale.md", + "services/meshai.md", + "services/meshmonitor.md", + "services/meshtasticd-bb.md", + "services/meshtasticd-isr.md", + "services/meshtasticd-aida.md", + "nodes/mt-burleybutte.md", + "nodes/mt-isr.md", + "nodes/aida-nebra.md", + "nodes/mesh-bridge-ct.md", + "nodes/meshtastic-hs-ct.md", + "nodes/meshai-ct.md", + "nodes/meshmonitor-ct.md", + "nodes/utility.md", + "mocs/vault-conventions.md", + "mocs", + "hardware", + "services", + "nodes", + "session-resume", + "runbooks/lxc-service-migration.md.tmp.40509.ede2941a9a27", + "runbooks/lxc-service-migration.md.tmp.40509.f5a73384917a", + "runbooks/lxc-service-migration.md.tmp.40509.0b5e082dca64", + "runbooks/lxc-service-migration.md.tmp.40509.1dd3d6e8eb64", + "docs/hardware/ip-allocation.md.tmp.40509.ca654fd93971", "assets/echo6yellow_logo_422x422_square.png", "assets/echo6yellow_logo_422x81.png", "assets/echo6_logo.png", diff --git a/INDEX.md b/INDEX.md index ee35eed..bfd8858 100644 --- a/INDEX.md +++ b/INDEX.md @@ -1,7 +1,7 @@ --- type: index title: Echo6 Knowledge Base -updated: 2026-06-15 +updated: 2026-06-17 --- # Echo6 Knowledge Base @@ -84,6 +84,14 @@ Living "read-me-first" context for active work: --- +## πŸ”„ Session resume (`session-resume/`) + +Live handoffs for sessions paused on one machine and resumed on another (cross-machine `~/.claude` sync is parked). Tagged `#session-resume`; `status: open` = not yet finished. + +- [[SESSION-HANDOFF-meshai-test]] β€” paused: pulling MeshAI (CT 108) logs to see how it handled the aida-nebra radio drop. Blocked from matt-desktop (no SSH key on utility); resume on cortex. `#open` + +--- + ## πŸ—„οΈ Archive Historical material lives in `archive/` β€” paper trail, not living docs: diff --git a/docs/hardware/environment.md b/docs/hardware/environment.md index 45adad9..426f354 100644 --- a/docs/hardware/environment.md +++ b/docs/hardware/environment.md @@ -4,14 +4,14 @@ Five nodes running Proxmox VE: -| Node | Local IP | Tailscale | Hardware | RAM | Purpose | -|------|----------|-----------|----------|-----|---------| -| data | 192.168.1.240 | 100.64.0.6 | AMD Ryzen 7 PRO 5750GE, 1TB NVMe + 1TB SATA SSD | 32GB DDR4-3200 | Database services | -| utility | 192.168.1.241 | 100.64.0.5 | AMD Ryzen 7 PRO 5750GE, 512GB NVMe | 32GB DDR4-3200 | Utility services, monitoring | -| cloud | 192.168.1.242 | 100.64.0.4 | Intel i7-12700T, 512GB NVMe | 32GB DDR4-3200 | Cloud storage, personal services | -| media | 192.168.1.243 | 100.64.0.3 | Intel i7-14700T, 2x 512GB NVMe | 32GB DDR5-5600 | Media server, *arr stack | -| toc | 192.168.1.244 | 100.64.0.13 | Workstation (i9-10900X) | 64GB DDR4 | GPU compute, AI/ML workloads | - +| Node | Local IP | Tailscale | Hardware | RAM | Purpose | +| ------- | ------------- | ----------- | ----------------------------------------------- | -------------- | -------------------------------- | +| data | 192.168.1.240 | 100.64.0.6 | AMD Ryzen 7 PRO 5750GE, 1TB NVMe + 1TB SATA SSD | 32GB DDR4-3200 | Database services | +| utility | 192.168.1.241 | 100.64.0.5 | AMD Ryzen 7 PRO 5750GE, 512GB NVMe | 32GB DDR4-3200 | Utility services, monitoring | +| cloud | 192.168.1.242 | 100.64.0.4 | Intel i7-12700T, 512GB NVMe | 32GB DDR4-3200 | Cloud storage, personal services | +| media | 192.168.1.243 | 100.64.0.3 | Intel i7-14700T, 2x 512GB NVMe | 32GB DDR5-5600 | Media server, *arr stack | +| toc | 192.168.1.244 | 100.64.0.13 | Workstation (i9-10900X) | 64GB DDR4 | GPU compute, AI/ML workloads | +f ### Node Storage Details | Node | Primary Disk | Secondary Disk | @@ -87,7 +87,7 @@ Five nodes running Proxmox VE: | Contabo Server | 5.189.158.149 | 100.64.0.1 | External VPS: Mail, Authentik, Headscale, Forge, Matrix | | edge2 | 184.174.35.153 | 100.64.0.26 | Contabo Cloud VPS 30 NVMe β€” Proxmox VE 8.4.19 (LXC-only), 8c/24GB/400GB | -*Last updated: 2026-06-17 β€” Added edge2 CT 103 (forgejo, 100.64.0.34, Headscale node 46); previously added CT 102 (vaultwarden), pdm CT 100, wordpress CT 101* +*Last updated: 2026-06-17 β€” Added edge2 CT 104 (livesync, 10.10.10.22, 100.64.0.35, migrated 2026-06-16); previously added CT 103 (forgejo), CT 102 (vaultwarden), pdm CT 100, wordpress CT 101* ## LXC Containers @@ -108,6 +108,7 @@ Five nodes running Proxmox VE: | wordpress | edge2 (CT 101) | 10.10.10.11 | 100.64.0.31 | WordPress for intermountainmesh.com | | vaultwarden | edge2 (CT 102) | 10.10.10.20 | 100.64.0.33 | Vaultwarden password manager (migrated from Contabo 2026-06-16) | | forgejo | edge2 (CT 103) | 10.10.10.21 | 100.64.0.34 | Forgejo git server (migrated from Contabo 2026-06-16) | +| livesync | edge2 (CT 104) | 10.10.10.22 | 100.64.0.35 | LiveSync Obsidian sync (CouchDB + provisioner; migrated from Contabo 2026-06-16) | ## IP Allocation Scheme @@ -157,6 +158,7 @@ Current registered nodes (26 total): | meshai | 100.64.0.32 | LXC | | vaultwarden | 100.64.0.33 | LXC (edge2 CT 102) | | forgejo | 100.64.0.34 | LXC (edge2 CT 103) β€” node id 46 | +| livesync | 100.64.0.35 | LXC (edge2 CT 104) β€” migrated 2026-06-16 | ## IdahoMesh Headscale Node List diff --git a/docs/hardware/ip-allocation.md b/docs/hardware/ip-allocation.md index 8189811..69f0e17 100755 --- a/docs/hardware/ip-allocation.md +++ b/docs/hardware/ip-allocation.md @@ -57,6 +57,7 @@ edge2 (Contabo Cloud VPS 184.174.35.153 / Tailscale 100.64.0.26) uses a separate | 10.10.10.11 | wordpress | CT 101 | 100.64.0.31 | WordPress for intermountainmesh.com | | 10.10.10.20 | vaultwarden | CT 102 | 100.64.0.33 | Vaultwarden password manager (migrated from Contabo 2026-06-16) | | 10.10.10.21 | forgejo | CT 103 | 100.64.0.34 | Forgejo git server (migrated from Contabo 2026-06-16) | +| 10.10.10.22 | livesync | CT 104 | 100.64.0.35 | LiveSync Obsidian sync (CouchDB + provisioner; migrated from Contabo 2026-06-16) | ### VMs (.150-.199) | IP | VM | Host | Purpose | @@ -108,6 +109,7 @@ edge2 (Contabo Cloud VPS 184.174.35.153 / Tailscale 100.64.0.26) uses a separate | 100.64.0.28 | pdm (CT 100 on edge2) | 10.10.10.10 (vmbr0) | | 100.64.0.33 | vaultwarden (CT 102 on edge2) β€” node id 45 | 10.10.10.20 (vmbr0) | | 100.64.0.34 | forgejo (CT 103 on edge2) β€” node id 46 | 10.10.10.21 (vmbr0) | +| 100.64.0.35 | livesync (CT 104 on edge2) β€” hostname `livesync` | 10.10.10.22 (vmbr0) | --- @@ -120,4 +122,4 @@ edge2 (Contabo Cloud VPS 184.174.35.153 / Tailscale 100.64.0.26) uses a separate --- -*Last updated: 2026-06-17 β€” Added edge2 CT 103 (forgejo) at 10.10.10.21 / 100.64.0.34 (Headscale node 46); previously added CT 102 (vaultwarden)* +*Last updated: 2026-06-17 β€” Added edge2 CT 104 (livesync) at 10.10.10.22 / 100.64.0.35; previously added CT 103 (forgejo), CT 102 (vaultwarden)* diff --git a/docs/services/services.md b/docs/services/services.md index 90aa341..310353c 100644 --- a/docs/services/services.md +++ b/docs/services/services.md @@ -32,7 +32,7 @@ | Prowlarr | media (VM 105) | 192.168.1.160:9696 | Internal | Indexer manager (Docker) | | SABnzbd | media (VM 105) | 192.168.1.160:8080 | Internal | Usenet download client (Docker) | | PeerTube | media (CT 110) | 192.168.1.170:9000 | https://stream.echo6.co | Video streaming (native, NFS on pi-nas, SSO) | -| WATCHTOWER | Contabo | 127.0.0.1:8099 | https://wt.echo6.co | Ops dashboard (Docker, Authentik forward auth) | +| WATCHTOWER | **Decommissioned (2026-06-16)** | β€” | ~~wt.echo6.co~~ | Was Docker on Contabo `/opt/watchtower`; stopped & archived to forge.echo6.co/matt/archive-watchtower | | Open WebUI | cortex (VM 150) | 192.168.1.150:8080 | https://ai.echo6.co | AI chat interface (Docker, Ollama backend, SSO) | | Qdrant | cortex (VM 150) | 192.168.1.150:6333 | Internal | Vector database (Docker, RECON knowledge store) | | TEI | cortex (VM 150) | 192.168.1.150:8090 | Internal | Text embeddings (Docker, bge-m3 1024-dim) | @@ -42,9 +42,9 @@ | Matrix Synapse | Contabo | 127.0.0.1:8008 | https://matrix.echo6.co | Matrix homeserver (Docker, SSO) | | Element Web | Contabo | 127.0.0.1:8088 | https://element.echo6.co | Matrix web client (Docker) | | mautrix-signal | Contabo | internal (29328) | DM @signalbot:echo6.co | Signal bridge (Docker, E2BE, MSC4190, double puppeting) | -| LiveSync | Contabo | 127.0.0.1:5984 | https://notes.echo6.co | Obsidian sync (CouchDB + provisioner, Docker, JWT auth) | -| TAK Server | Contabo | 100.64.0.1:8446/8443/8089 | https://tak.echo6.co | TAK Server (Docker, Authentik forward auth on admin portal) | -| SIGIL | Contabo | 100.64.0.1:8990 | https://tak.echo6.co/sigil | TAK management console (Docker, Authentik forward auth) | +| LiveSync | edge2 (CT 104) | 100.64.0.35:5984 (CouchDB) / :5985 (provisioner) | https://notes.echo6.co | Obsidian sync (CouchDB + provisioner, Docker, JWT auth) β€” fronted by Contabo Caddy (reverse_proxy 100.64.0.35:5984 / :5985); **migrated from Contabo 2026-06-16** | +| TAK Server | **Decommissioned (2026-06-16)** | β€” | ~~tak.echo6.co~~ | Was Docker on Contabo `/opt/tak-server-deploy`; stopped & archived to forge.echo6.co/matt/archive-tak-server | +| SIGIL | **Decommissioned (2026-06-16)** | β€” | ~~tak.echo6.co/sigil~~ | Was Docker on Contabo `/opt/sigil`; stopped & archived to forge.echo6.co/matt/archive-tak-server | | OpenTAKServer (OTS) | utility (CT 109) | 192.168.1.109:443 | https://ots.k7zvx.com | Live TAK server (native install, nginx+RabbitMQ+PostgreSQL, Meshtastic MQTT gateway on port 8883) β€” see [[ots-setup]] | | Echo6 Cortex Agent | cortex (VM 150) | N/A (Matrix bot) | #cortex:echo6.co in echo6-ops space | Claude Code bridge β€” @cortex:echo6.co, session continuity, E2EE (systemd) | | Echo6 Contabo Agent | Contabo | N/A (Matrix bot) | #contabo:echo6.co in echo6-ops space | Claude Code bridge β€” @contabo:echo6.co, session continuity, E2EE (systemd) | @@ -266,6 +266,20 @@ - DB: PostgreSQL 16 (forgejo_db container); 9 repos, 1 user verified at migration - Source (Contabo `/opt/forgejo`) STOPPED but intact as cold rollback; rollback = `systemctl disable --now forgejo-ssh-dnat` + restore `/etc/caddy/Caddyfile.bak-preforge` + `systemctl restart caddy` + `docker compose up -d` on Contabo +### edge2 - CT 104 (10.10.10.22 / Tailscale: 100.64.0.35, node `livesync`) +- LiveSync Obsidian sync service (https://notes.echo6.co β€” **migrated from Contabo 2026-06-16**) + - Headscale hostname `livesync`, tailnet IP 100.64.0.35 + - Compose path: `/opt/livesync/docker-compose.yml` + - Containers: `livesync-couchdb` (couchdb:3.4) + `livesync-provisioner` (custom image) + - Named volumes: `couchdb-data`, `provisioner-data` + - Bind mounts: `couchdb/local.ini`, `couchdb/local.d/` (incl. `jwt-keys.ini`) + - CouchDB binds to `100.64.0.35:5984`; provisioner binds to `100.64.0.35:5985` + - Contabo Caddy proxies `notes.echo6.co` β†’ `reverse_proxy 100.64.0.35:5984` (CouchDB) + `reverse_proxy 100.64.0.35:5985` (provisioner); Authentik forward_auth on `/_provision`; Obsidian CORS preserved; Authentik outpost stays `127.0.0.1:9000` + - Auth: per-user JWT (ES512) via provisioner; databases: `cc-db`, `userdb-matt` + - Data: ~16 MB in couchdb-data + - Source on Contabo STOPPED but intact as cold rollback; `/etc/caddy/Caddyfile.bak-prelivesync` exists + - **Resources:** 2 cores / 1024 MB RAM / 512 MB swap / 8 GB rootfs on `local`; unprivileged; onboot; Docker + ### edge2 - CT 102 (10.10.10.20 / Tailscale: 100.64.0.33, node 45 `vaultwarden`) - Vaultwarden password manager (port 8086, https://vault.echo6.co, Docker) - Headscale node id 45, name `vaultwarden`, user `echo6` @@ -282,7 +296,7 @@ - Mailcow (email) - Vaultwarden β€” **migrated to edge2 CT 102 on 2026-06-16** (Caddy now proxies to 100.64.0.33:8086) - Syncthing (syncs with cortex) -- WATCHTOWER (ops dashboard, port 8099, https://wt.echo6.co) +- ~~WATCHTOWER~~ β€” **decommissioned 2026-06-16** (was `/opt/watchtower`; archived to forge.echo6.co/matt/archive-watchtower; `wt.echo6.co` Caddy block + GoDaddy A record removed) - Matrix Synapse homeserver (port 8008, https://matrix.echo6.co, Docker, SSO via Authentik) - Element Web client (port 8088, https://element.echo6.co, Docker) - mautrix-signal bridge (port 29328 internal, Docker, E2BE with MSC4190) @@ -297,17 +311,9 @@ - Encryption: E2BE enabled (allow+default+require), MSC4190, self-signed cross-signing keys - Compose path: `/opt/matrix/docker-compose.yml` - Backup: daily at 3AM, 14-day retention (synapse + mas + mautrix_signal databases) -- LiveSync CouchDB (port 5984, https://notes.echo6.co, Docker, JWT auth) -- LiveSync Provisioner (port 5985, https://notes.echo6.co/_provision/, Docker, Authentik forward auth) -- Compose path: `/opt/livesync/docker-compose.yml` -- Per-user ES512 key pairs, encrypted setup URIs, per-user CouchDB databases -- TAK Server (port 8446 web admin, 8443 mutual TLS API, 8089 TLS for EUDs) - - https://tak.echo6.co (Authentik forward auth on admin portal) - - Compose path: `/opt/tak-server-deploy/docker-compose.yml` - - Certs: `/opt/tak-server-deploy/tak/certs/files/` - - Container names: `tak-server-deploy-tak-1`, `tak-server-deploy-db-1` -- SIGIL console (port 8990, https://tak.echo6.co/sigil, Authentik forward auth) - - Compose path: `/opt/sigil/docker-compose.yml` +- ~~LiveSync~~ β€” **migrated to edge2 CT 104 on 2026-06-16** (source `/opt/livesync` STOPPED, intact as cold rollback; Caddy now proxies `notes.echo6.co` β†’ `100.64.0.35:5984/5985`; `/etc/caddy/Caddyfile.bak-prelivesync` exists) +- ~~TAK Server~~ β€” **decommissioned 2026-06-16** (was `/opt/tak-server-deploy`; archived to forge.echo6.co/matt/archive-tak-server; `tak.echo6.co` Caddy block + GoDaddy A record removed) +- ~~SIGIL~~ β€” **decommissioned 2026-06-16** (was `/opt/sigil`; archived to forge.echo6.co/matt/archive-tak-server) - Matrix Authentication Service (MAS) (port 8085, internal, Docker) - Container: `matrix-mas` on `matrix-net` - Handles login/logout/refresh/auth_metadata for Synapse @@ -320,11 +326,8 @@ - Volume: `termix_termix-data` β†’ `/app/data` - No Caddy block β€” direct access only on port 8083 - Compose: `/opt/termix/` (inferred from Docker volume naming) -- Echo6 Contabo Agent (systemd: echo6-agent.service, matrix-nio bot, @contabo:echo6.co) - - Install path: `/opt/echo6-agent/` - - Claude Code bridge with session continuity + E2EE - - Watches #contabo:echo6.co in echo6-ops space - - CLAUDE_CWD=/root, runs as root +- ~~echo6-agent~~ β€” **decommissioned 2026-06-16** (was systemd `echo6-agent.service` at `/opt/echo6-agent/`; archived to forge.echo6.co/matt/archive-echo6-agent; unit disabled) +- ~~nexus-hub~~ + ~~nexus-agent~~ β€” **decommissioned 2026-06-16** (were systemd units at `/root/nexus-hub` and `/root/nexus-agent`; archived to forge.echo6.co/matt/archive-nexus-hub + archive-nexus-agent; units disabled) - mautrix-signal bridge (mautrix-signal container, port 29328 internal) - Image: dock.mau.dev/mautrix/signal:v0.2603.0 - Config: `/opt/matrix/mautrix-signal/config.yaml` @@ -336,6 +339,19 @@ - Portals auto-create on incoming messages (no autocreate toggle available) - Ref: `/home/zvx/projects/.ref/docs/matrix/mautrix_signal.md` +## Decommissioned Services + +Services stopped, archived, and removed from Caddy/DNS as of their decommission date. On-disk dirs on Contabo retained until edge1 rebuild wipes them. + +| Service | Decommissioned | Archive Repo | Notes | +|---------|---------------|-------------|-------| +| TAK Server | 2026-06-16 | forge.echo6.co/matt/archive-tak-server | Was Docker `/opt/tak-server-deploy` on Contabo; `tak.echo6.co` Caddy block + GoDaddy A record removed | +| SIGIL | 2026-06-16 | forge.echo6.co/matt/archive-tak-server | Was Docker `/opt/sigil` on Contabo; served at `tak.echo6.co/sigil` | +| WATCHTOWER | 2026-06-16 | forge.echo6.co/matt/archive-watchtower | Was Docker `/opt/watchtower` on Contabo; `wt.echo6.co` Caddy block + GoDaddy A record removed | +| echo6-agent | 2026-06-16 | forge.echo6.co/matt/archive-echo6-agent | Was systemd unit at `/opt/echo6-agent/` on Contabo; unit disabled | +| nexus-hub | 2026-06-16 | forge.echo6.co/matt/archive-nexus-hub | Was systemd unit at `/root/nexus-hub` on Contabo; unit disabled | +| nexus-agent | 2026-06-16 | forge.echo6.co/matt/archive-nexus-agent | Was systemd unit at `/root/nexus-agent` on Contabo; unit disabled | + ## Adding New Services When deploying a new service, update this file with: diff --git a/runbooks/lxc-service-migration.md b/runbooks/lxc-service-migration.md index 1351fbc..899d93f 100644 --- a/runbooks/lxc-service-migration.md +++ b/runbooks/lxc-service-migration.md @@ -1,6 +1,6 @@ # LXC Service Migration β€” Contabo β†’ edge2 -> Proven pilots: **Vaultwarden β†’ edge2 CT 102** (SQLite, 2026-06-16) and **Forgejo β†’ edge2 CT 103** (PostgreSQL + non-Caddy SSH port, 2026-06-16). This runbook generalizes both patterns into a reusable template for evacuating any Contabo-Caddy-fronted service to an edge2 LXC. +> Proven pilots: **Vaultwarden β†’ edge2 CT 102** (SQLite, 2026-06-16), **Forgejo β†’ edge2 CT 103** (PostgreSQL + non-Caddy SSH port, 2026-06-16), and **LiveSync (CouchDB) β†’ edge2 CT 104** (cold named-volume tar + bind-mounted config, 2026-06-16). This runbook generalizes these patterns into a reusable template for evacuating any Contabo-Caddy-fronted service to an edge2 LXC. --- @@ -228,12 +228,35 @@ ssh edge2 "sudo pct exec -- bash -c ' SSH host keys MUST travel in the data volume (see G11) so clients see no key-change warning. +**Variant D β€” CouchDB named volumes + bind-mounted config files (proven with LiveSync):** + +CouchDB uses named Docker volumes for data (`couchdb-data`, `provisioner-data`) and bind-mounted config files (`couchdb/local.ini`, `couchdb/local.d/` including `jwt-keys.ini`). Use the same cold named-volume tar as Variant C for each volume, AND copy the compose directory (with all bind-mount source paths) intact: + +```bash +# Stop source entirely (CouchDB does not support online volume export safely) +ssh root@100.64.0.1 'cd /opt/ && docker compose stop' + +# Tar each named volume via throwaway container (same as Variant C) +ssh root@100.64.0.1 'docker run --rm -v couchdb-data:/source -v /tmp:/target alpine tar -czf /target/couchdb-data.tar.gz -C /source .' +ssh root@100.64.0.1 'docker run --rm -v provisioner-data:/source -v /tmp:/target alpine tar -czf /target/provisioner-data.tar.gz -C /source .' + +# Copy the whole compose directory (includes bind-mounted config: local.ini, local.d/*, jwt-keys.ini) +ssh root@100.64.0.1 'tar -czf /tmp/-compose-dir.tar.gz -C /opt ' + +# Transfer + extract in CT +# (transfer tarballs to cortex, push to CT via edge2 pct push, then extract) +``` + +**Critical:** bind-mounted config files (e.g. `jwt-keys.ini` in `couchdb/local.d/`) MUST travel as part of the compose-dir copy alongside the named volumes. If `jwt-keys.ini` is missing, per-user JWT auth breaks on startup. + **Rollback:** destroy CT + `ssh root@100.64.0.1 'cd /opt/ && docker compose up -d'` **Service-specific (example: Vaultwarden):** Also transferred `rsa_key.pem` byte-for-byte (JWT signing key β€” see G9). Did NOT transfer `db.sqlite3-wal`, `db.sqlite3-shm`, `tmp/`. **Service-specific (example: Forgejo):** Used Variant B (PostgreSQL) + Variant C (forgejo-data volume with git repos + SSH host keys). Stopped only the `forgejo` app container; kept `forgejo-db` running for dump. Integrity gate: `SELECT COUNT(*) FROM repository` == 9 source and target. SSH host keys in forgejo-data volume transferred intact β€” no client warning on reconnect. +**Service-specific (example: LiveSync/CouchDB):** Used Variant D. Stopped source entirely; tarred `couchdb-data` + `provisioner-data` named volumes; copied compose dir (incl. `couchdb/local.d/jwt-keys.ini`). Data: ~16 MB. Integrity gate: verified document count in `cc-db` + `userdb-matt` matched source via `curl http://localhost:5984/` against both hosts after cutover. + --- ### Phase 6 β€” Start + health gate `[G/S]` @@ -293,6 +316,8 @@ vault.echo6.co { } ``` +**Multi-token cutover example (LiveSync, 2026-06-16):** LiveSync exposes TWO upstream ports (5984 for CouchDB, 5985 for provisioner) within a single Caddy site block. Both tokens were changed from `127.0.0.1:598x` β†’ `100.64.0.35:598x` in one edit. A third upstream in the same block β€” the Authentik outpost at `127.0.0.1:9000` (used for `forward_auth` on `/_provision`) β€” was left **untouched** because it stays on Contabo. Change only the tokens that move; never touch the Authentik outpost address. + --- #### Phase 7a β€” Non-Caddy public TCP port (iptables DNAT) `[S]` *(Forgejo SSH variant)* @@ -416,4 +441,4 @@ ssh root@100.64.0.1 'systemctl disable --now -ssh-dnat.service && rm /e --- -*Last updated: 2026-06-17 β€” Added Forgejo pilot (PostgreSQL + multi-volume + iptables DNAT); updated Phase 5 Variants B/C, Phase 7a, Gotchas G11-G13, Template Summary* +*Last updated: 2026-06-17 β€” Added LiveSync/CouchDB pilot (Variant D: cold named-volume tar + bind-mounted config, multi-token Caddy cutover); Phase 5 Variant D; Phase 7 multi-token example; previously added Forgejo pilot (Variants B/C, Phase 7a, Gotchas G11-G13)* diff --git a/session-resume/SESSION-HANDOFF-meshai-test.md b/session-resume/SESSION-HANDOFF-meshai-test.md new file mode 100644 index 0000000..e034b60 --- /dev/null +++ b/session-resume/SESSION-HANDOFF-meshai-test.md @@ -0,0 +1,53 @@ +--- +type: session-resume +title: "Session Handoff β€” MeshAI radio-drop resilience test" +status: open +created: 2026-06-17 +origin: matt-desktop (WSL) +resume-on: cortex +tags: [session-resume, handoff, meshai, meshtastic, aida-nebra, diagnostic, resilience-test, open] +--- + +# Session Handoff β€” MeshAI radio-drop resilience test + +**Created:** 2026-06-17, from matt-desktop (WSL). Resume from **cortex**. + +## What we were doing +Matt unplugged/replugged the network cable on **aida-nebra** (AIDA-N2 Meshtastic +node) to reboot it. The link dropped for ~12s then recovered. Matt then realized +this is an accidental **resilience test for MeshAI** β€” he wants to see "how it +dumps": how MeshAI handled losing and regaining its radio TCP connection +(clean reconnect vs. errors/stack traces vs. crash+restart). + +## Key facts +- **aida-nebra**: 192.168.1.253 / TS 100.64.0.9 β€” meshtasticd node !27780c47, + Nebra 2W hat, TCP API port **4403**. Confirmed back online (LAN, :4403, and + Tailscale all UP after the replug). +- **MeshAI**: utility **CT 108**, 192.168.1.144 / TS 100.64.0.32. + Docker image `ghcr.io/zvx-echo6/meshai:latest`. Connects to the radio at + 192.168.1.253:4403. Compose: `/home/zvx/meshai/docker-compose.yml`. +- cortex and its host toc had a brief outage earlier this session; both back UP. + +## Why this stalled (the blocker) +matt-desktop's WSL SSH key is **NOT authorized on utility** (192.168.1.241): +`Permission denied (publickey)` β€” key `SHA256:QL0Tm72T7RYVZfeDhPHJDUYnstRNFx2EF5/k8tKv3AM`. +Could ping everything but not SSH in. Per policy, stopped instead of routing around. +**cortex has working keys** β€” that's why we're resuming there. + +## Next step on cortex (read-only diagnostic β€” do NOT restart/redeploy anything) +```bash +# find the meshai container + current status (did it restart through the blip?) +ssh zvx@192.168.1.241 "pct exec 108 -- docker ps --format '{{.Names}}\t{{.Image}}\t{{.Status}}'" + +# pull logs across the blip window with timestamps +ssh zvx@192.168.1.241 "pct exec 108 -- docker logs --since 30m --timestamps 2>&1 | tail -200" +``` +Report: container uptime/restart count, verbatim disconnectβ†’errorβ†’reconnect lines, +and a one-line verdict (clean recover / crashed+auto-restarted / stuck). + +> NOTE: the blip was ~16:xx MT on 2026-06-17 (a few min before this file's +> timestamp). Logs roll β€” pull soon to still catch the disconnect/reconnect dump. + +## Working model reminder +Matt guides β†’ Opus orchestrates (dispatch Sonnet for the SSH/log pull) β†’ Sonnet +executes the tight prompt above.