diff --git a/.obsidian/workspace.json b/.obsidian/workspace.json index f4671aa..a3e170e 100644 --- a/.obsidian/workspace.json +++ b/.obsidian/workspace.json @@ -189,18 +189,18 @@ }, "active": "ea4cc678c44e8b67", "lastOpenFiles": [ - "INDEX.md.tmp.40509.2a05e7708182", + "runbooks/lxc-service-migration.md.tmp.40509.4536b3c3851b", + "runbooks/lxc-service-migration.md.tmp.40509.bb32c76bdcf4", + "runbooks/lxc-service-migration.md.tmp.40509.052f55354b7d", + "runbooks/lxc-service-migration.md.tmp.40509.d17d83b34e07", + "runbooks/lxc-service-migration.md.tmp.40509.519078fbf870", + "runbooks/lxc-service-migration.md.tmp.40509.10b3a76d106e", + "credentials.tmp.40509.1256ea803f58", + "docs/hardware/ip-allocation.md.tmp.40509.9aa60f8e7a54", + "docs/hardware/ip-allocation.md.tmp.40509.b1093f9672ae", + "docs/hardware/ip-allocation.md.tmp.40509.94cbb0537cfb", + "docs/hardware/environment.md.tmp.40509.b4c1d593d6b6", "runbooks/lxc-service-migration.md", - "runbooks/lxc-service-migration.md.tmp.40509.38316817de79", - "archive/projects/vaultwarden-deployment.md.tmp.40509.88bb66425f5e", - "docs/hardware/ip-allocation.md.tmp.40509.b03329ad68b8", - "docs/hardware/ip-allocation.md.tmp.40509.008cf455ddde", - "docs/hardware/ip-allocation.md.tmp.40509.8b2ec85a60b8", - "docs/hardware/environment.md.tmp.40509.ad3a0bc1d9bf", - "docs/hardware/environment.md.tmp.40509.e90afdbac998", - "docs/hardware/environment.md.tmp.40509.727e35e81611", - "docs/services/services.md.tmp.40509.e97391cd330d", - "docs/services/services.md.tmp.40509.3adb6587d024", "runbooks/edge2-access-reference.md", "runbooks/expose-service-edge2.md", "plans/vaultwarden-plan.md", diff --git a/credentials b/credentials index 5d8a6ec..fc8df6c 100755 --- a/credentials +++ b/credentials @@ -96,11 +96,13 @@ RECON_SMTP_HOST=mail.echo6.co RECON_SMTP_PORT=587 # Forgejo (forge.echo6.co) - Deployed 2026-02-04 -# Location: Contabo VPS (5.189.158.149) +# Migrated to edge2 CT 103 (10.10.10.21 / tailnet 100.64.0.34) on 2026-06-16; SSH via Contabo iptables DNAT (forgejo-ssh-dnat.service). FORGEJO_URL="https://forge.echo6.co" FORGEJO_ADMIN_USER="matt" FORGEJO_ADMIN_PASS="7redditGold" FORGEJO_SSH_PORT="2222" +FORGEJO_DB_NAME=forgejo +FORGEJO_DB_USER=forgejo FORGEJO_DB_PASS="forgejo_db_pass_2026" # Forgejo OAuth2 (Authentik) diff --git a/docs/hardware/environment.md b/docs/hardware/environment.md index 72c2739..45adad9 100644 --- a/docs/hardware/environment.md +++ b/docs/hardware/environment.md @@ -87,7 +87,7 @@ Five nodes running Proxmox VE: | Contabo Server | 5.189.158.149 | 100.64.0.1 | External VPS: Mail, Authentik, Headscale, Forge, Matrix | | edge2 | 184.174.35.153 | 100.64.0.26 | Contabo Cloud VPS 30 NVMe — Proxmox VE 8.4.19 (LXC-only), 8c/24GB/400GB | -*Last updated: 2026-06-17 — Added edge2 CT 102 (vaultwarden), Headscale node 45 (100.64.0.33); previously added edge2 CTs pdm CT 100 / wordpress CT 101* +*Last updated: 2026-06-17 — Added edge2 CT 103 (forgejo, 100.64.0.34, Headscale node 46); previously added CT 102 (vaultwarden), pdm CT 100, wordpress CT 101* ## LXC Containers @@ -107,6 +107,7 @@ Five nodes running Proxmox VE: | pdm | edge2 (CT 100) | 10.10.10.10 | 100.64.0.28 | Proxmox Datacenter Manager | | wordpress | edge2 (CT 101) | 10.10.10.11 | 100.64.0.31 | WordPress for intermountainmesh.com | | vaultwarden | edge2 (CT 102) | 10.10.10.20 | 100.64.0.33 | Vaultwarden password manager (migrated from Contabo 2026-06-16) | +| forgejo | edge2 (CT 103) | 10.10.10.21 | 100.64.0.34 | Forgejo git server (migrated from Contabo 2026-06-16) | ## IP Allocation Scheme @@ -155,6 +156,7 @@ Current registered nodes (26 total): | wordpress | 100.64.0.31 | LXC | | meshai | 100.64.0.32 | LXC | | vaultwarden | 100.64.0.33 | LXC (edge2 CT 102) | +| forgejo | 100.64.0.34 | LXC (edge2 CT 103) — node id 46 | ## IdahoMesh Headscale Node List diff --git a/docs/hardware/ip-allocation.md b/docs/hardware/ip-allocation.md index deddd5d..8189811 100755 --- a/docs/hardware/ip-allocation.md +++ b/docs/hardware/ip-allocation.md @@ -56,6 +56,7 @@ edge2 (Contabo Cloud VPS 184.174.35.153 / Tailscale 100.64.0.26) uses a separate | 10.10.10.10 | pdm | CT 100 | 100.64.0.28 | Proxmox Datacenter Manager | | 10.10.10.11 | wordpress | CT 101 | 100.64.0.31 | WordPress for intermountainmesh.com | | 10.10.10.20 | vaultwarden | CT 102 | 100.64.0.33 | Vaultwarden password manager (migrated from Contabo 2026-06-16) | +| 10.10.10.21 | forgejo | CT 103 | 100.64.0.34 | Forgejo git server (migrated from Contabo 2026-06-16) | ### VMs (.150-.199) | IP | VM | Host | Purpose | @@ -106,6 +107,7 @@ edge2 (Contabo Cloud VPS 184.174.35.153 / Tailscale 100.64.0.26) uses a separate | 100.64.0.26 | edge2 | 184.174.35.153 (external) | | 100.64.0.28 | pdm (CT 100 on edge2) | 10.10.10.10 (vmbr0) | | 100.64.0.33 | vaultwarden (CT 102 on edge2) — node id 45 | 10.10.10.20 (vmbr0) | +| 100.64.0.34 | forgejo (CT 103 on edge2) — node id 46 | 10.10.10.21 (vmbr0) | --- @@ -118,4 +120,4 @@ edge2 (Contabo Cloud VPS 184.174.35.153 / Tailscale 100.64.0.26) uses a separate --- -*Last updated: 2026-06-17 — Added edge2 CT 102 (vaultwarden) at 10.10.10.20 / 100.64.0.33 (Headscale node 45)* +*Last updated: 2026-06-17 — Added edge2 CT 103 (forgejo) at 10.10.10.21 / 100.64.0.34 (Headscale node 46); previously added CT 102 (vaultwarden)* diff --git a/docs/services/services.md b/docs/services/services.md index 8698d74..90aa341 100644 --- a/docs/services/services.md +++ b/docs/services/services.md @@ -15,7 +15,7 @@ | MeshAI | utility (CT 108) | 192.168.1.144:4403 | Internal | LLM-powered Meshtastic assistant (Docker, Gemini Flash, Google grounding) | | ARGUS | utility (CT 103) | 192.168.1.103 | Internal | OSINT intelligence gathering platform (Docker, SearXNG + local LLM analysis) | | Authentik | Contabo | 5.189.158.149:9000 | https://auth.echo6.co | SSO provider (Echo6 branded, custom CSS, dark theme) | -| Forge | Contabo | 5.189.158.149 | https://forge.echo6.co | Git server | +| Forge (Forgejo) | edge2 (CT 103) | 100.64.0.34:3001 HTTP / :2222 SSH (via Contabo DNAT) | https://forge.echo6.co | Git server — fronted by Contabo Caddy (reverse_proxy 100.64.0.34:3001); git SSH via iptables DNAT on Contabo (forgejo-ssh-dnat.service) — **migrated from Contabo 2026-06-16** | | Headscale | Contabo | 5.189.158.149 | https://vpn.echo6.co | Tailscale coordination (OIDC enabled) | | Headplane | Contabo | 127.0.0.1:3100 | https://vpn.echo6.co/admin | Headscale web UI (OIDC via Authentik) | | Mailcow | Contabo | 5.189.158.149 | https://mail.echo6.co | Email server | @@ -254,6 +254,18 @@ - Static MAC: A7:A1:30:79:BB:BB - Tailscale registered on IdahoMesh Headscale (vpn.idahomesh.com) under malice user +### edge2 - CT 103 (10.10.10.21 / Tailscale: 100.64.0.34, node 46 `forgejo`) +- Forgejo git server (https://forge.echo6.co — **migrated from Contabo 2026-06-16**) + - Headscale node id 46, name `forgejo`, user `echo6` + - Compose path: `/opt/forgejo/docker-compose.yml` + - Containers: `forgejo` (codeberg.org/forgejo/forgejo:14) + `forgejo-db` (postgres:16-alpine) + - Volumes: `forgejo-data` (git repos, app.ini, SSH host keys) + `forgejo-db` + - HTTP: binds to `100.64.0.34:3001`; Contabo Caddy proxies here over tailnet (`reverse_proxy 100.64.0.34:3001`) + - SSH: `git@forge.echo6.co:2222` → Contabo iptables DNAT (systemd unit `forgejo-ssh-dnat.service`, enabled, reboot-persistent) → `100.64.0.34:2222` → container port 22; SSH host keys preserved (no client warning) + - Login via Authentik OIDC (auth.echo6.co) — unchanged + - DB: PostgreSQL 16 (forgejo_db container); 9 repos, 1 user verified at migration + - Source (Contabo `/opt/forgejo`) STOPPED but intact as cold rollback; rollback = `systemctl disable --now forgejo-ssh-dnat` + restore `/etc/caddy/Caddyfile.bak-preforge` + `systemctl restart caddy` + `docker compose up -d` on Contabo + ### edge2 - CT 102 (10.10.10.20 / Tailscale: 100.64.0.33, node 45 `vaultwarden`) - Vaultwarden password manager (port 8086, https://vault.echo6.co, Docker) - Headscale node id 45, name `vaultwarden`, user `echo6` @@ -265,7 +277,7 @@ ### Contabo VPS (5.189.158.149 / Tailscale: 100.64.0.1) - Authentik (SSO, Echo6 branded — custom CSS, dark theme, logo, favicon, flow titles) -- Forge (Git) +- Forge (Git) — **migrated to edge2 CT 103 on 2026-06-16** (Caddy now proxies to 100.64.0.34:3001; SSH DNAT via forgejo-ssh-dnat.service) - Headscale (mesh VPN) - Mailcow (email) - Vaultwarden — **migrated to edge2 CT 102 on 2026-06-16** (Caddy now proxies to 100.64.0.33:8086) diff --git a/runbooks/lxc-service-migration.md b/runbooks/lxc-service-migration.md index 0c74683..1351fbc 100644 --- a/runbooks/lxc-service-migration.md +++ b/runbooks/lxc-service-migration.md @@ -1,6 +1,6 @@ # LXC Service Migration — Contabo → edge2 -> Proven pilot: **Vaultwarden → edge2 CT 102** (2026-06-16). This runbook generalizes that pattern into a reusable template for evacuating any Contabo-Caddy-fronted service to an edge2 LXC. +> Proven pilots: **Vaultwarden → edge2 CT 102** (SQLite, 2026-06-16) and **Forgejo → edge2 CT 103** (PostgreSQL + non-Caddy SSH port, 2026-06-16). This runbook generalizes both patterns into a reusable template for evacuating any Contabo-Caddy-fronted service to an edge2 LXC. --- @@ -151,10 +151,11 @@ This phase is entirely service-specific. Common patterns: |-----------|--------| | Stateless (no persistent data) | Skip this phase | | SQLite (WAL mode) | Stop source → `sqlite3 .backup` → verify integrity → transfer | -| PostgreSQL | `pg_dump` → transfer → `psql` restore | -| File data | `rsync` or `tar` | +| PostgreSQL | Stop app only (keep DB up) → `pg_dump -Fc` → transfer → restore on target | +| Multi-volume (file data + keys) | `tar` the data volume → transfer → extract into target volume | +| File data only | `rsync` or `tar` | -**Zero-loss for single-user SQLite:** +**Variant A — Zero-loss for single-container SQLite:** ```bash # Stop source (keeps container + data intact as rollback baseline) ssh root@100.64.0.1 'cd /opt/ && docker compose stop' @@ -169,10 +170,70 @@ ssh edge2 "sudo pct push /tmp/svc-snap.sqlite3 /opt//data/db.sql Do NOT copy live `-wal`/`-shm` files alongside the snapshot. +**Variant B — PostgreSQL dump/restore (proven with Forgejo):** + +Stop the application container only — keep the DB container running so `pg_dump` can connect cleanly: + +```bash +# Stop app only; DB stays up +ssh root@100.64.0.1 'cd /opt/ && docker compose stop ' + +# Dump to custom format (compressed, schema + data) +ssh root@100.64.0.1 'docker exec pg_dump -U -d -Fc -f /tmp/.dump' +ssh root@100.64.0.1 'docker cp :/tmp/.dump /tmp/.dump' + +# Transfer to cortex, then push into CT +scp root@100.64.0.1:/tmp/.dump /tmp/ +ssh edge2 "sudo pct push /tmp/.dump /tmp/.dump" + +# On target: start DB container only, wait for it to be healthy, then restore +ssh edge2 "sudo pct exec -- bash -c 'cd /opt/ && docker compose up -d '" +# Wait ~10s for postgres to initialise +ssh edge2 "sudo pct exec -- bash -c 'docker cp /tmp/.dump :/tmp/.dump'" +ssh edge2 "sudo pct exec -- bash -c 'docker exec pg_restore -U -d --no-owner /tmp/.dump'" +``` + +**Integrity gate — compare key table row counts source vs target:** + +```bash +# Source (still stopped app, DB still up) +ssh root@100.64.0.1 'docker exec psql -U -d -c "SELECT COUNT(*) FROM ;"' + +# Target +ssh edge2 "sudo pct exec -- bash -c 'docker exec psql -U -d -c \"SELECT COUNT(*) FROM ;\"'" +``` + +Row counts must match exactly before proceeding. Do not proceed to Phase 6 if they differ. + +**Variant C — Multi-volume tar (file data + SSH host keys, proven with Forgejo):** + +When a service has a data volume containing binary or structured data (e.g., git repos, SSH host keys, app config) alongside a database, tar the entire named Docker volume and extract it into a freshly created target volume: + +```bash +# On source: tar the named volume via a throwaway container +ssh root@100.64.0.1 'docker run --rm -v :/source -v /tmp:/target alpine \ + tar -czf /target/.tar.gz -C /source .' + +# Transfer +scp root@100.64.0.1:/tmp/.tar.gz /tmp/ +ssh edge2 "sudo pct push /tmp/.tar.gz /tmp/.tar.gz" + +# On target: create volume + extract +ssh edge2 "sudo pct exec -- bash -c ' + docker volume create + docker run --rm -v :/target -v /tmp:/source alpine \ + tar -xzf /source/.tar.gz -C /target +'" +``` + +SSH host keys MUST travel in the data volume (see G11) so clients see no key-change warning. + **Rollback:** destroy CT + `ssh root@100.64.0.1 'cd /opt/ && docker compose up -d'` **Service-specific (example: Vaultwarden):** Also transferred `rsa_key.pem` byte-for-byte (JWT signing key — see G9). Did NOT transfer `db.sqlite3-wal`, `db.sqlite3-shm`, `tmp/`. +**Service-specific (example: Forgejo):** Used Variant B (PostgreSQL) + Variant C (forgejo-data volume with git repos + SSH host keys). Stopped only the `forgejo` app container; kept `forgejo-db` running for dump. Integrity gate: `SELECT COUNT(*) FROM repository` == 9 source and target. SSH host keys in forgejo-data volume transferred intact — no client warning on reconnect. + --- ### Phase 6 — Start + health gate `[G/S]` @@ -234,6 +295,71 @@ vault.echo6.co { --- +#### Phase 7a — Non-Caddy public TCP port (iptables DNAT) `[S]` *(Forgejo SSH variant)* + +When a service exposes a raw TCP port that is NOT handled by Caddy (e.g., Forgejo git SSH on port 2222), the one-token Caddy edit in Phase 7 is insufficient. Use a reboot-persistent **iptables DNAT systemd unit** on the Contabo host to forward the port to the target tailnet IP. + +**Do NOT apply by rebooting the Contabo host** — it runs production services. Create the unit directly and start it without a reboot. + +Template unit (`/etc/systemd/system/forgejo-ssh-dnat.service` — parameterize PORT and TARGET_TS_IP): + +```ini +[Unit] +Description=Forgejo SSH DNAT — forward : to edge2 LXC +After=network.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/sbin/iptables -t nat -A PREROUTING -p tcp --dport -j DNAT --to-destination : +ExecStart=/sbin/iptables -t nat -A OUTPUT -p tcp --dport -j DNAT --to-destination : +ExecStop=/sbin/iptables -t nat -D PREROUTING -p tcp --dport -j DNAT --to-destination : +ExecStop=/sbin/iptables -t nat -D OUTPUT -p tcp --dport -j DNAT --to-destination : + +[Install] +WantedBy=multi-user.target +``` + +Deploy and enable: + +```bash +# Write unit to Contabo +ssh root@100.64.0.1 'cat > /etc/systemd/system/-ssh-dnat.service' << 'EOF' +[Unit] +Description= SSH DNAT — forward : to edge2 LXC +After=network.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/sbin/iptables -t nat -A PREROUTING -p tcp --dport -j DNAT --to-destination : +ExecStart=/sbin/iptables -t nat -A OUTPUT -p tcp --dport -j DNAT --to-destination : +ExecStop=/sbin/iptables -t nat -D PREROUTING -p tcp --dport -j DNAT --to-destination : +ExecStop=/sbin/iptables -t nat -D OUTPUT -p tcp --dport -j DNAT --to-destination : + +[Install] +WantedBy=multi-user.target +EOF + +ssh root@100.64.0.1 'systemctl daemon-reload && systemctl enable --now -ssh-dnat.service' +``` + +Verify DNAT is live (before restarting caddy): + +```bash +ssh -p git@5.189.158.149 # Should land on target service, not source +``` + +**Rollback:** + +```bash +ssh root@100.64.0.1 'systemctl disable --now -ssh-dnat.service && rm /etc/systemd/system/-ssh-dnat.service && systemctl daemon-reload' +``` + +**Service-specific (example: Forgejo):** PORT=2222, TARGET_TS_IP=100.64.0.34. Unit: `forgejo-ssh-dnat.service`. Verified: `ssh -p 2222 git@forge.echo6.co` arrived at edge2 CT 103 with preserved host keys (no client warning). + +--- + ### Phase 8 — End-to-end verification `[G/S]` - `.echo6.co` DNS still resolves to `5.189.158.149` (unchanged) @@ -270,13 +396,16 @@ vault.echo6.co { | G8 | Reboot survival requires all three: `--onboot 1`, `restart: unless-stopped`, and the Tailscale DNS-bootstrap drop-in. | | G9 | JWT signing keys (e.g. `rsa_key.pem` for Vaultwarden) **must travel byte-for-byte**. If missing, existing sessions are invalidated (forced re-login everywhere) but data remains intact. Check startup logs for "generating new key" messages. | | G10 | Contabo Caddy runs with `admin off` → `caddy reload` and `systemctl reload caddy` FAIL. Always use `caddy validate` then `systemctl restart caddy`. Restart drops all Contabo-fronted sites for ~1–2 s. | +| G11 | SSH host keys must travel inside the data volume (Variant C tar). If they are missing or regenerated on the target, every git client gets a host-key-changed warning and must manually clear `~/.ssh/known_hosts`. Transfer the full data volume; verify on target with `docker exec cat /data/ssh/forgejo.rsa.pub` or equivalent before cutover. | +| G12 | DB row-count integrity gate is mandatory for PostgreSQL migrations. Never skip it — a silent pg_restore failure (wrong role, encoding mismatch) leaves the DB empty or partially populated while `pg_restore` exits 0. Compare at least one key business table. | +| G13 | iptables DNAT must be made reboot-persistent via a systemd `oneshot`/`RemainAfterExit` unit (Phase 7a). Do NOT rely on iptables-persistent packages or manual rules — they require package installs (forbidden) or do not survive all reboot paths. Do NOT apply by rebooting the Contabo host (production). Create the unit file, `daemon-reload`, `enable --now`. | | N | The composed **Contabo-Caddy → edge2-LXC tailnet** path is unexercised for each new service. Keep the Phase 6 HTTP `/alive` 200 gate as a HARD pre-cutover requirement (use `curl`, not ICMP). | --- ## Template Summary -**Generic phases (identical for every service):** 0 (recon) → 1 (LXC provision) → 2 (Docker) → 3 (Tailscale + DNS-bootstrap + reachability pre-gate) → 7 (one-token Caddy cutover + backup + validate + restart) → 8 (end-to-end + reboot survival) → 9 (deferred decommission). +**Generic phases (identical for every service):** 0 (recon) → 1 (LXC provision) → 2 (Docker) → 3 (Tailscale + DNS-bootstrap + reachability pre-gate) → 7 (one-token Caddy cutover + backup + validate + restart) → 7a (if service exposes non-Caddy TCP port: iptables DNAT systemd unit on Contabo) → 8 (end-to-end + reboot survival) → 9 (deferred decommission). **Service-specific phases:** 0a (pre-migration gate), 4 (compose/config — copy from live host), 5 (data migration method depends on storage type), 6 (health gates — service-specific checks before cutover). @@ -287,4 +416,4 @@ vault.echo6.co { --- -*Last updated: 2026-06-17* +*Last updated: 2026-06-17 — Added Forgejo pilot (PostgreSQL + multi-volume + iptables DNAT); updated Phase 5 Variants B/C, Phase 7a, Gotchas G11-G13, Template Summary*