From 9b8c0fe0d99bd399a14ff7baf05ef0d51ef95d7a Mon Sep 17 00:00:00 2001 From: echo6-autocommit Date: Sat, 20 Jun 2026 12:00:06 +0000 Subject: [PATCH] auto: docs sync 2026-06-20T12:00:06+00:00 Files changed: engine/changelog.md engine/lint-report.md vault/.obsidian/workspace.json vault/projects/fleet-patch-audit.md --- engine/changelog.md | 2 ++ engine/lint-report.md | 5 ++--- vault/.obsidian/workspace.json | 22 +++++++++++----------- vault/projects/fleet-patch-audit.md | 22 +++++++++++----------- 4 files changed, 26 insertions(+), 25 deletions(-) diff --git a/engine/changelog.md b/engine/changelog.md index 90e8c3d..249b8f6 100644 --- a/engine/changelog.md +++ b/engine/changelog.md @@ -117,3 +117,5 @@ - UPDATE .obsidian/graph.json — colorGroups by folder, nodeSizeMultiplier=2, showTags=false ## 2026-06-19T09:00:02Z — sweep deferred (competing GPU process: 2932 node /usr/bin/peertube-runner server --enable-job vod-hls-transcoding --enable-job vod-audio-merge-transcoding --enable-job live-rtmp-hls-transcoding --enable-job video-studio-transcoding --enable-job video-transcription) + +## 2026-06-20T09:00:01Z — sweep deferred (competing GPU process: 2932 node /usr/bin/peertube-runner server --enable-job vod-hls-transcoding --enable-job vod-audio-merge-transcoding --enable-job live-rtmp-hls-transcoding --enable-job video-studio-transcoding --enable-job video-transcription) diff --git a/engine/lint-report.md b/engine/lint-report.md index 947b14a..84b5412 100644 --- a/engine/lint-report.md +++ b/engine/lint-report.md @@ -1,6 +1,6 @@ # Vault Lint Report -Generated: 2026-06-20T00:00:07Z | Docs scanned: 92 | Elapsed: 0.0s +Generated: 2026-06-20T06:00:08Z | Docs scanned: 92 | Elapsed: 0.0s ## Summary @@ -8,7 +8,7 @@ Generated: 2026-06-20T00:00:07Z | Docs scanned: 92 | Elapsed: 0.0s |----------|-------| | ERROR (dead links) | 0 | | WARN (schema) | 1 | -| INFO (orphans) | 40 | +| INFO (orphans) | 39 | ### WARN breakdown - Missing frontmatter block: 1 @@ -52,7 +52,6 @@ _None. All wikilinks resolve._ - no incoming links: runbooks/meshmonitor-password-reset.md - no incoming links: runbooks/meshtastic-sidecar-node.md - no incoming links: runbooks/meshtasticd-sim-nodes-runbook.md -- no incoming links: projects/nominatim-v5-reimport.md - no incoming links: runbooks/nordvpn-lxc.md - no incoming links: runbooks/pg-backup.md - no incoming links: runbooks/pi-nas-omv-runbook.md diff --git a/vault/.obsidian/workspace.json b/vault/.obsidian/workspace.json index 750c959..f8d8b1a 100644 --- a/vault/.obsidian/workspace.json +++ b/vault/.obsidian/workspace.json @@ -199,18 +199,18 @@ }, "active": "8d53cdb6c257e685", "lastOpenFiles": [ - "projects/fleet-patch-audit.md.tmp.1493418.e48160886375", - "projects/fleet-patch-audit.md.tmp.1493418.e4f1f5df5445", + "projects/fleet-patch-audit.md.tmp.1493418.c4ba031d7df6", + "projects/fleet-patch-audit.md.tmp.1493418.d191aac142fc", + "projects/fleet-patch-audit.md.tmp.1493418.6777b1d5dff6", + "projects/fleet-patch-audit.md.tmp.1493418.731843387669", + "projects/fleet-patch-audit.md.tmp.1493418.b5aeb852fc62", + "projects/fleet-patch-audit.md.tmp.1493418.425c00fcc97b", + "projects/fleet-patch-audit.md.tmp.1493418.61ad7f49d904", + "projects/fleet-patch-audit.md.tmp.1493418.f6bb16e19902", + "projects/fleet-patch-audit.md.tmp.1493418.9f995983b41f", + "projects/fleet-patch-audit.md.tmp.1493418.e2a6a1d2c739", + "projects/fleet-patch-audit.md.tmp.1493418.26b5280af8d9", "projects/nominatim-v5-reimport.md", - "projects/nominatim-v5-reimport.md.tmp.1493418.334c5df83cc6", - "projects/fleet-patch-audit.md.tmp.1493418.c5907f7438c0", - "projects/fleet-patch-audit.md.tmp.1493418.06cd7896c26f", - "projects/fleet-patch-audit.md.tmp.1493418.38b71845197f", - "projects/fleet-patch-audit.md.tmp.1493418.f77ca61905a7", - "projects/fleet-patch-audit.md.tmp.1493418.07bf87858a4f", - "projects/fleet-patch-audit.md.tmp.1493418.dff12cfa9a42", - "projects/fleet-patch-audit.md.tmp.1493418.73d7194d7615", - "projects/fleet-patch-audit.md.tmp.1493418.b6f7dc14e8ae", "2026-06-19.md", "Untitled.canvas", "docs/hardware/environment.md", diff --git a/vault/projects/fleet-patch-audit.md b/vault/projects/fleet-patch-audit.md index 269729b..ba73437 100644 --- a/vault/projects/fleet-patch-audit.md +++ b/vault/projects/fleet-patch-audit.md @@ -13,7 +13,7 @@ status: active Read-only audit snapshot as of 2026-06-19. **Nothing has been applied — this is a planning document to build the patch plan from.** -**Topology note:** the old Contabo VPS has been rebuilt as **edge1 (mail-only)**; **edge2 is now the front door for everything else**. edge1 is excluded from this audit (mid-rebuild/maintenance). **Headscale:** edge2 CT107 is the main fleet tailnet (34 nodes, `vpn.echo6.co`, self-hosted Headscale 0.28.0); utility CT106 is a separate IdahoMesh sub-tailnet (`vpn.idahomesh.com`, 3 nodes, low-risk). No services route through old-Contabo. **Mailcow CT108:** confirmed decommissioned — MX/A for mail.echo6.co point to edge1 (5.189.158.149, active), CT108 is stopped, backup at `/opt/mailcow-backup/mailcow-2026-06-19-03-59-45/`. +**Topology note:** the old Contabo VPS has been rebuilt as **edge1 (mail-only)**; **edge2 is now the front door for everything else**. edge1 is excluded from this audit (mid-rebuild/maintenance). **Headscale:** edge2 CT107 is the main fleet tailnet (34 nodes, `vpn.echo6.co`, self-hosted Headscale 0.28.0); utility CT106 is a separate IdahoMesh sub-tailnet (`vpn.idahomesh.com`, 3 nodes, low-risk). No services route through old-Contabo. **Mailcow CT108:** destroyed 2026-06-20 (`pct destroy 108 --purge`); backup preserved durably on pi-nas (`…/contabo-prewipe-2026-06/mailcow/`, sha256-verified); live mail on edge1 (MX/A for mail.echo6.co → 5.189.158.149). --- @@ -44,7 +44,7 @@ Updates where the application or its Docker images have drifted from current ups | edge2 | CT103 | forgejo 14.0.5 → 15.0.3 | **14.x EOL 2026-04-30** — migrate branch, not just patch | | edge2 | CT106 | Synapse 1.155.0 / Element / MAS | Image drift + pending OS apt security updates | | edge2 | CT104 | livesync couchdb:3.4 | Docker image drift | -| edge2 | CT108 | mailcow (18 containers) | Upgrade via `update.sh` only | +| edge2 | CT108 | ~~mailcow (18 containers)~~ | ✅ **Decommissioned 2026-06-20** — superseded by edge1; no longer an update target | | cloud | CT120 | immich — server/ml/valkey:9/postgres(14-vectorchord) | 4 images drifted | | cloud | CT121 | nextcloud AIO — mastercontainer + NC app 32.0.4 | Mastercontainer behind; 12-container stack | | cortex | VM150 | ollama / tei(1.7) / qdrant / open-webui / obsidian | 5 AI containers drifted | @@ -117,7 +117,7 @@ Running application version vs latest stable upstream, per app — the "is every | Item | Where | Finding | |------|-------|---------| -| **Mailcow** | edge2 CT108 | **STOPPED** — very likely superseded by the new **edge1 mail-only** node. Confirm and decommission rather than update. | +| **Mailcow** | edge2 CT108 | ✅ **DECOMMISSIONED 2026-06-20** — destroyed (`pct destroy 108 --purge`); backup on pi-nas, live mail on edge1. | | **Host kernel** | **edge2 host** | Agent flagged DirtyFrag (CVE-2026-43284/-43500) + copy.fail (CVE-2026-31431, claimed CISA KEV) as host-kernel LPE. **Tension:** the host audit showed edge2 fully patched (0 upgradable) on its repo — **verify** whether these need a kernel newer than the no-subscription repo provides. | ### Current / already past the fix (no action) @@ -148,14 +148,14 @@ Lowest-risk changes first; everything reboot-bearing deferred to scheduled windo | Phase | Scope | Reboot? | Notes | |------|-------|---------|-------| -| **1 — Guest/VM security apt** | utility CT100,101,102,103,104,106,107,108,109,112,118,119 · cloud CT120,121 · media VM105,CT110,CT111 · data VM1130 · edge2 CT100–108 | No | Lowest blast radius. Worst-first: CT119, CT108, immich/nextcloud guest-OS, peertube. | +| **1 — Guest/VM security apt** | utility CT100,101,102,103,104,106,107,108,109,112,118,119 · cloud CT120,121 · media VM105,CT110,CT111 · data VM1130 · edge2 CT100–107 | No | Lowest blast radius. Worst-first: CT119, CT108, immich/nextcloud guest-OS, peertube. | | **2 — Hypervisor host OS security** | data, utility, cloud, media host OSes (**not toc**) | No | One node at a time; restarts hypervisor-side daemons (smbd etc.). edge2 host already patched. | | **3 — App / container updates (Tier 2)** | per-app, native updater each | Per-app | See "special handling" below — not a generic `docker pull`. | | **4 — Reboot windows (Tier 3)** | PVE 9.2 + QEMU 11 + LXC 7 + kernel on the 5 PVE-9 nodes; pi-nas kernel + OMV; cortex NVIDIA/DKMS | **Yes** | Schedule deliberately; toc+cortex coordinated. | | **Cross-cutting** | Tailscale 1.94→1.98 fleet-wide | No | Can ride along Phase 1/2. | **Special handling — do NOT bulk-patch these; use the native updater** -- **mailcow** (edge2 CT108) → `./update.sh` (upgrades all 18 components in lockstep) +- **mailcow** (edge2 CT108) → ✅ decommissioned 2026-06-20, no action - **nextcloud AIO** (cloud CT121) → update mastercontainer, then in-UI update button (port 8080) - **immich** (cloud CT120) → `docker compose pull && up -d` from its compose dir - **authentik** (edge2 CT105) → sequential version upgrades with migrations; 2025.12.4 → 2026.2.0 cannot skip releases @@ -185,7 +185,7 @@ Lowest-risk changes first; everything reboot-bearing deferred to scheduled windo - **PDM break-glass** — ✅ **DONE**: local `admin@pam` (Administrator) created on PDM CT100, login verified via API, Authentik realm untouched; cred in `credentials`, config backup at CT100 `/root/access.bak-2026-06-20`. - **CT118 archivist rpcbind** — ✅ **DONE**: nftables rule restricts port 111 to source `192.168.1.240` (NFS server) only; NFS mount healthy, ruleset persisted. Now 100% LAN-internal. -- **mailcow CT108** — backup now **durable on pi-nas** (`…/contabo-prewipe-2026-06/mailcow/`, 3 files, sha256-verified); edge1 confirmed live mail. **Pending: `pct destroy 108` on explicit approval** (edge2 `/tmp` sources retained until then). +- **mailcow CT108** — ✅ **DONE**: backup durable on pi-nas (`…/contabo-prewipe-2026-06/mailcow/`, sha256-verified), live mail confirmed on edge1; **CT108 destroyed (`pct destroy 108 --purge`, 2026-06-20)** — config + disk image purged. edge2 now hosts CT100–107. - **data disk (92%)** — ⏏️ **DE-SCOPED**: not a gate. OS package updates are roll-back-able (reinstall prior version); the migration-heavy apps that need real rollback (Authentik, Forgejo, Nextcloud, PeerTube) live on cloud/edge2, not `data`. Optional cleanup only (~6 GB obviously-safe: stale ISO, zimit temp) if ever wanted. **Rollback model (corrected):** OS packages → reinstall the prior version (no VM snapshot needed). App DB-migration upgrades (Authentik/Forgejo/Nextcloud/PeerTube) → restore a quiesced DB dump (cheap), since reinstalling the old binary won't unwind a migrated schema. @@ -226,7 +226,7 @@ Step-by-step plan to bring every application and package current. Per-app target **STEP 6: Decision gates.** Close genuinely-open decisions before Phase 1/2/3 can start: daemon-restart tolerance sign-off (Open Dec #7); verify post-cutoff CVE claims against primary advisories with owner+URL per claim; resolve edge2 host-kernel CVE question fully (pin `uname -r`, check repo kernel availability, decide reboot yes/no — not left conditional); define and announce maintenance windows in America/Boise naming user-facing blips. -**STEP 7: edge2 CT108 mailcow — decommission (CONFIRMED safe).** CT108 is stopped, edge1 is actively serving SMTP (MX/A → 5.189.158.149), backup exists at `/opt/mailcow-backup/mailcow-2026-06-19-03-59-45/`. Action: verify that backup is stored durably OFF CT108 (copy off-host), confirm no MX points at edge2, then `pct destroy 108` ON APPROVAL. Removes CT108 from all later scope. +**STEP 7: edge2 CT108 mailcow — ✅ DONE.** CT108 destroyed 2026-06-20 (`pct destroy 108 --purge`); config + disk image purged. Backup verified durable on pi-nas (`…/contabo-prewipe-2026-06/mailcow/`, sha256-verified); edge1 confirmed serving mail (MX/A → 5.189.158.149). edge2 now hosts CT100–107. ### Phase 1 — Guest OS security packages (no reboot; hosts folded into Phase 3) @@ -354,7 +354,7 @@ Vaultwarden, PDM, WordPress, Synapse/Element/MAS, obsidian, recon-vm Postgres | 0 | Off-host restorable backups (stateful guests) | Verify last-good vzdump/PBS off the node for central, OTS, forgejo, matrix, nextcloud, edge2 livesync CouchDB | N/A | Read-only verification + on-demand `pg_dump`/app backup copied off-host | ≥1 backup off the changing node, restore-testable | N/A | Precedes all mutating stateful steps | | 0 | Baseline HEALTH capture (all targets) | Record up/down, container states, endpoint 200s, `apt upgradable`/security counts; note oddities | None | `pct/qm status`, `docker ps`, curl probes, `apt list --upgradable` | Baseline recorded; oddities logged (jellyseerr dev tag, CT108 stopped, nominatim pin, CT118 rpcbind) | N/A | Precedes Phase 1 | | 0 | Decision gates | Close daemon-restart tolerance sign-off (Open Dec #7); verify post-cutoff CVE claims vs primary advisories w/ owner+URL; resolve edge2 kernel question (uname -r vs repo availability); define+announce Boise maintenance windows with user-facing blip list | None | Read-only / sign-off | Each decision recorded before Phase 1/2/3 can start | N/A | Gates Phase 1/2/3 | -| 0 | edge2 CT108 mailcow (STOPPED) | Confirm superseded by edge1, back up, decommission | `pct snapshot 108 predecommission` + vzdump + mailcow native backup | Confirm edge1 mail live + no MX at edge2; `pct stop`/`pct destroy 108` ON APPROVAL | CT108 gone/archived; edge1 mail in+out works; vzdump restorable | `pct restore 108` + start; re-point MX | edge1 confirmed; approval (Open Dec #9) | +| 0 | ~~edge2 CT108 mailcow~~ | ✅ **DONE — destroyed 2026-06-20** (`pct destroy 108 --purge`); backup durable on pi-nas (sha256-verified); edge1 confirmed serving mail | N/A | N/A | CT108 purged; edge1 mail in+out confirmed | N/A | Complete | | 0 | edge2 host kernel CVE question | Decide if DirtyFrag/copy.fail require a reboot | None (record `pveversion -v`) | Compare installed proxmox-kernel vs verified-advisory fixed versions; escalate if no-sub repo lacks fix (no repo changes w/o approval) | Decision (reboot yes/no) recorded | N/A | Gates edge2 Phase 3 row | | 1 | Procedure canary — utility CT112 cobalt (idle) or CT102 searxng | Prove apt→snapshot→security upgrade→needrestart→verify on low-stakes guest | `pct snapshot` pre-phase1 | `apt-get update`; security pocket only; `needrestart -r l` then deliberate | Security-upgradable=0; service healthy; needrestart clear | `pct rollback` | Phase 0; runs BEFORE worst-first guests | | 1 | utility CT119 mesh-territory (179/91, never patched) | Security-pocket apt | `pct snapshot 119` | `pct exec`; security pocket; `needrestart -r a` | sec count=0; meshwars up | rollback snapshot | After canary | @@ -400,7 +400,7 @@ Vaultwarden, PDM, WordPress, Synapse/Element/MAS, obsidian, recon-vm Postgres | 2 | cortex Qdrant 1.16.3→1.18.2 / TEI 1.7.4→1.9.3 | Lower-urgency image bumps | `qm snapshot 150` per app | bump tag; pull/up | health endpoints ok; collections/embeddings intact; docs engine works | re-pin prior; snapshot | After cortex Docker+toolkit; verify engines | | 2 | cortex obsidian-remote v1.12.7 | Verify-current / no-op | None | none | container up; UI reachable | N/A | — | | 2 | utility CT118 archivist rpcbind 0.0.0.0:111 | Remediate exposure (network change — APPROVAL) | `pct snapshot 118` | per approved option: disable rpcbind / bind localhost+TS / firewall | port 111 not on 0.0.0.0; app functions; external scan closed | re-enable / `pct rollback` | After CT118 Phase 1; approval-gated (Open Dec #6) | -| 2 | edge2 CT108 mailcow `./update.sh` — CONDITIONAL | Only if RETAINED; else SKIP | `pct snapshot 108` + mailcow backup | `./update.sh` (native lockstep) | 18 containers up; mail in/out | `./update.sh restore`; `pct rollback` | GATED on Phase 0 decision (likely skipped) | +| 2 | ~~edge2 CT108 mailcow~~ | **REMOVED — CT108 destroyed 2026-06-20 (not retained)** | N/A | N/A | N/A | N/A | N/A | | 3 | Per-node corosync/HA pre-flight (data/utility/cloud/media/toc) | Before EACH reboot | None | `pvecm status` (Quorate:Yes, 5 votes); `ha-manager status` | quorate + expected votes=5; HA implications known | abort if not quorate | Gates each Phase 3 node reboot | | 3 | Canary node (lowest blast + headroom; media, NOT data until disk freed) | Full 9.2/QEMU11/LXC7/kernel reboot to prove the path | vzdump all guests to EXTERNAL target + record `pveversion -v` | `apt update && apt dist-upgrade` → reboot; NO live-migration in mixed window | `pveversion`=9.2/QEMU11/LXC7/kernel 6.17.13; all guests `onboot` return; quorate | boot prior kernel (GRUB); restore guests from vzdump | Phase 1+2 done; canary before high-stakes nodes | | 3 | media host PVE 9.1.1→9.2 (+QEMU11/LXC7/kernel) | Platform + reboot | vzdump VM105/CT110/CT111 + snapshots | `dist-upgrade` → reboot | guests return; arr/peertube/caddy healthy; quorate | GRUB prior kernel; vzdump restore | After media Phase 1+2; one node at a time | @@ -509,7 +509,7 @@ Complete point-in-time state of every node, guest, and container service. ### edge2 (PVE 8.4.19) — Host Fully Patched -- 9 LXC guests +- 8 LXC guests (CT108 mailcow decommissioned 2026-06-20) | CT | Name | Services / Status | |----|------|-------------------| @@ -521,7 +521,7 @@ Complete point-in-time state of every node, guest, and container service. | CT105 | authentik | 2025.12.4 (server/worker/postgres) → upgrade to 2026.2.0 | | CT106 | matrix | Synapse 1.155.0 / Element / MAS + mautrix-signal + postgres; OS apt security updates pending; no Tailscale client | | CT107 | headscale | 0.28.0 → 0.29.0 + headplane; no Tailscale client | -| CT108 | mailcow | 18 containers (postfix/nginx/dovecot/rspamd/clamd/sogo/php-fpm/mariadb/redis/memcached/unbound/acme/olefy/watchdog/netfilter/ofelia/dockerapi/tlspol) — upgrade via `update.sh` only | +| ~~CT108~~ | ~~mailcow~~ | **Decommissioned 2026-06-20** — destroyed (`pct destroy 108 --purge`); superseded by edge1, backup on pi-nas | ---