From fa34ab930e447c333aa6ed6cbc65a87fda1c1e34 Mon Sep 17 00:00:00 2001 From: echo6-autocommit Date: Sat, 20 Jun 2026 18:00:05 +0000 Subject: [PATCH] auto: docs sync 2026-06-20T18:00:05+00:00 Files changed: engine/lint-report.md vault/.obsidian/workspace.json vault/projects/fleet-patch-audit.md --- engine/lint-report.md | 2 +- vault/.obsidian/workspace.json | 6 +-- vault/projects/fleet-patch-audit.md | 66 ++++++++++++++++++++++++++++- 3 files changed, 69 insertions(+), 5 deletions(-) diff --git a/engine/lint-report.md b/engine/lint-report.md index 84b5412..8f81018 100644 --- a/engine/lint-report.md +++ b/engine/lint-report.md @@ -1,6 +1,6 @@ # Vault Lint Report -Generated: 2026-06-20T06:00:08Z | Docs scanned: 92 | Elapsed: 0.0s +Generated: 2026-06-20T12:00:06Z | Docs scanned: 92 | Elapsed: 0.0s ## Summary diff --git a/vault/.obsidian/workspace.json b/vault/.obsidian/workspace.json index f8d8b1a..f1015ce 100644 --- a/vault/.obsidian/workspace.json +++ b/vault/.obsidian/workspace.json @@ -199,6 +199,9 @@ }, "active": "8d53cdb6c257e685", "lastOpenFiles": [ + "projects/fleet-patch-audit.md.tmp.1493418.9a000c8ac5aa", + "projects/fleet-patch-audit.md.tmp.1493418.c361bd10be1b", + "projects/fleet-patch-audit.md.tmp.1493418.83a57813be85", "projects/fleet-patch-audit.md.tmp.1493418.c4ba031d7df6", "projects/fleet-patch-audit.md.tmp.1493418.d191aac142fc", "projects/fleet-patch-audit.md.tmp.1493418.6777b1d5dff6", @@ -207,9 +210,6 @@ "projects/fleet-patch-audit.md.tmp.1493418.425c00fcc97b", "projects/fleet-patch-audit.md.tmp.1493418.61ad7f49d904", "projects/fleet-patch-audit.md.tmp.1493418.f6bb16e19902", - "projects/fleet-patch-audit.md.tmp.1493418.9f995983b41f", - "projects/fleet-patch-audit.md.tmp.1493418.e2a6a1d2c739", - "projects/fleet-patch-audit.md.tmp.1493418.26b5280af8d9", "projects/nominatim-v5-reimport.md", "2026-06-19.md", "Untitled.canvas", diff --git a/vault/projects/fleet-patch-audit.md b/vault/projects/fleet-patch-audit.md index ba73437..a4e64f6 100644 --- a/vault/projects/fleet-patch-audit.md +++ b/vault/projects/fleet-patch-audit.md @@ -5,7 +5,7 @@ tags: - proxmox - ai related: [] -updated: 2026-06-19 +updated: 2026-06-20 status: active --- @@ -192,6 +192,66 @@ Lowest-risk changes first; everything reboot-bearing deferred to scheduled windo --- +## Phase 1 — Execution Log (2026-06-20) + +**Status: COMPLETE.** 26 guests patched (security-pocket only), every service validated back up, ~600+ security packages cleared, zero data loss. No host reboots (guests only). Method: snapshot-or-DB-dump → security-only apt (`apt-get install --only-upgrade` from the security pocket) → `pct reboot` → service-by-service validation. + +### Guests patched + +- **utility:** CT100, CT101, CT102, CT103, CT104, CT106, CT107, CT108, CT109, CT112, CT118, CT119 +- **cloud:** CT120, CT121 +- **media:** VM105, CT110, CT111 +- **data:** VM1130 +- **edge2:** CT100, CT101, CT102, CT103, CT104, CT105, CT106, CT107 + +(CT107 = headscale — see Critical Finding 1 below.) + +### DB dumps / rollback points captured (reusable in Phase 2) + +| Guest | Location | Notes | +|---|---|---| +| authentik CT105 | `/root/authentik-db-20260620.sql` (126 MB) | — | +| matrix CT106 | `/root/matrix-db-20260620.sql` (60 MB) | — | +| forgejo CT103 | `/root/forgejo-db-20260620.sql` | — | +| wordpress CT101 | dump taken on CT101 | — | +| peertube CT110 | DB dump | — | +| opentakserver CT109 | PG dump + LVM snapshot | — | +| central CT104 | PG dump + snapshot | — | +| immich CT120 | `/root/immich-predates-20260620.sql` | — | +| headscale CT107 | `/root/headscale-db-20260620.sqlite` | — | + +**Note:** many edge2 and cloud CTs' storage does NOT support `pct snapshot` — DB dump or package-reinstall is the rollback model for those. + +### Incidental fixes made during Phase 1 + +- **(a) CT110 peertube — immutable `/etc/resolv.conf` blocked reboot.** The file had `chattr +i` set (intentional NordVPN DNS protection). Cleared the immutable flag to allow the reboot, then verified the flag was restored and DNS remained healthy after boot. +- **(b) CT111 mcc — DNS hijacked to unreachable MagicDNS.** Tailscale `accept-dns` was redirecting DNS to a MagicDNS address that was not reachable from this CT. Disabled `tailscale accept-dns`, set `1.1.1.1` / `8.8.8.8` persistently. +- **(c) PostgreSQL on central CT104 moved 16.13→16.14** as part of the security-pocket apt pass. +- **(d) Fleet-wide stale `/etc/hosts` fix** — see Critical Finding 2. + +### 🔴 Critical Finding 1 — CT107 headscale does NOT survive a reboot + +On `pct reboot 107`, the fleet headscale coordinator crash-looped and required approximately 15 minutes of manual recovery. Two root causes: + +- **(a) Docker compose bridge `headscale_default` came up `linkdown`** after the unprivileged-LXC reboot. +- **(b) Bootstrap chicken-and-egg:** Docker port-binds headscale to `100.64.0.38` (tailscale0), but tailscale0 needs headscale (the coordinator) to come up first. + +Recovery required: `compose down` → `docker restart` → manually add `100.64.0.38/32` to tailscale0 → `compose up` → re-auth CT107's own tailscale node with a fresh preauthkey → temporarily DNAT `vpn.echo6.co` through CT107's internal IP to bootstrap, then revert Caddy config and clean up iptables. + +**IMPACT: This is now the #1 Phase-3 blocker.** edge2 must NOT be rebooted (and CT107 must not be restarted) until a permanent boot-survival fix exists — for example, a systemd oneshot that recreates the bridge and seeds the tailscale0 IP before headscale starts, or removing headscale's bind-to-tailnet-IP dependency. A naive edge2 reboot takes the whole fleet coordinator down. + +### 🔴 Critical Finding 2 — fleet-wide stale /etc/hosts broke coordinator connectivity + +7 fleet nodes — data, cloud, media, utility hosts, plus caddy CT101, cobalt CT112, and peertube CT110 — had a stale `5.189.158.149 vpn.echo6.co` line in `/etc/hosts` left over from before the 2026-06-19 headscale migration to edge2. This pinned `vpn.echo6.co` to edge1 (now mail-only), so tailscaled hit Mailcow's TLS cert and could never reach the real coordinator — affected nodes showed OFFLINE in headscale while coasting on persistent WireGuard tunnels (still SSH-reachable, masking the problem). + +**Fixed 2026-06-20:** removed the stale line and ran `tailscale up` on all 7; all confirmed ONLINE in the coordinator. `/etc/hosts.bak-20260620` backups left on each host. + +**IMPACT on Phase 3:** this class of bug means a reboot of an affected host while its control connection was stale would have failed to rejoin the tailnet — the primary lockout risk. Now that all nodes have valid control connections, reboots should re-register cleanly. However: **verify each node is ONLINE in headscale before AND after any Phase-3 reboot.** + +Headscale stale-node cleanup: deleted dead nodes `mailcow` (destroyed CT108) and a stale peertube duplicate, both on 2026-06-20. + +--- + ## Execution Runbook (Meticulous) Step-by-step plan to bring every application and package current. Per-app target versions live in **Application Currency** above; this section is the *how/when/order*. Execution model: scoped Sonnet task per target → Opus verifies output → report to Matt; **approval gate before every load-bearing or reboot step.** @@ -303,6 +363,10 @@ Vaultwarden, PDM, WordPress, Synapse/Element/MAS, obsidian, recon-vm Postgres ### Phase 3 — Platform & reboot windows (coordinated, approval-gated) +> **🔴 BLOCKED — edge2 reboot is NOT permitted until the CT107 headscale boot-survival fix is in place (see Critical Finding 1 in the Phase 1 Execution Log).** A naive edge2 reboot crash-loops the fleet coordinator and requires ~15 min manual recovery. Do not schedule or start any edge2 reboot until a permanent fix exists (systemd oneshot to recreate the bridge / seed tailscale0 IP, or equivalent). + +> **🔴 GATE for every Phase-3 host reboot (Critical Finding 2):** Before AND after rebooting any node, confirm that node is ONLINE in the headscale coordinator (`headscale nodes list`). Stale `/etc/hosts` entries previously masked coordinator disconnects behind coasting WireGuard tunnels — the stale entries have been removed fleet-wide, but verify ONLINE status at each reboot step to catch any regression. This gate is IN ADDITION TO the corosync-quorum gate below. + **Corosync cluster rules (apply to EVERY node reboot in this section):** - The five PVE nodes (data, utility, cloud, media, toc) are ONE `echo6-cluster`, quorum = 3 of 5. Losing quorum makes `/etc/pve` read-only fleet-wide and freezes all VM/CT start/stop/config on every surviving node. - Before EACH node reboot: `pvecm status` must show `Quorate: Yes` and `Expected votes: 5`. Also run `ha-manager status` — if any guest is HA-managed, the reboot triggers fencing/auto-migration rather than a clean local stop/start.