From e47eaa93496a5c7c7c63894ac8782c89af036104 Mon Sep 17 00:00:00 2001 From: echo6-autocommit Date: Fri, 19 Jun 2026 12:00:06 +0000 Subject: [PATCH] auto: docs sync 2026-06-19T12:00:06+00:00 Files changed: engine/changelog.md engine/lint-report.md vault/.obsidian/workspace.json vault/projects/fleet-patch-audit.md --- engine/changelog.md | 2 + engine/lint-report.md | 2 +- vault/.obsidian/workspace.json | 10 +- vault/projects/fleet-patch-audit.md | 224 ++++++++++++++++++++++++++++ 4 files changed, 232 insertions(+), 6 deletions(-) create mode 100644 vault/projects/fleet-patch-audit.md diff --git a/engine/changelog.md b/engine/changelog.md index 935b26c..90e8c3d 100644 --- a/engine/changelog.md +++ b/engine/changelog.md @@ -115,3 +115,5 @@ - LINK+FM idahomesh-bridge-setup.md — 11 link candidates - LINK+FM add-peertube-channel.md — 4 link candidates - UPDATE .obsidian/graph.json — colorGroups by folder, nodeSizeMultiplier=2, showTags=false + +## 2026-06-19T09:00:02Z — sweep deferred (competing GPU process: 2932 node /usr/bin/peertube-runner server --enable-job vod-hls-transcoding --enable-job vod-audio-merge-transcoding --enable-job live-rtmp-hls-transcoding --enable-job video-studio-transcoding --enable-job video-transcription) diff --git a/engine/lint-report.md b/engine/lint-report.md index dcb2787..25c65b5 100644 --- a/engine/lint-report.md +++ b/engine/lint-report.md @@ -1,6 +1,6 @@ # Vault Lint Report -Generated: 2026-06-19T01:31:34Z | Docs scanned: 89 | Elapsed: 0.0s +Generated: 2026-06-19T06:00:06Z | Docs scanned: 89 | Elapsed: 0.0s ## Summary diff --git a/vault/.obsidian/workspace.json b/vault/.obsidian/workspace.json index 5011701..a222f3b 100644 --- a/vault/.obsidian/workspace.json +++ b/vault/.obsidian/workspace.json @@ -199,6 +199,11 @@ }, "active": "17bd4a6166f789d0", "lastOpenFiles": [ + "projects/fleet-patch-audit.md.tmp.1493418.a3b58c4003b3", + "projects/fleet-patch-audit.md.tmp.1493418.be6293479cb1", + "projects/fleet-patch-audit.md.tmp.1493418.b9b7f884075a", + "projects/fleet-patch-audit.md", + "projects/fleet-patch-audit.md.tmp.1493418.dc1460743661", "runbooks/lxc-service-migration.md.tmp.40509.b1b8250f733f", "runbooks/lxc-service-migration.md.tmp.40509.54d74cb07251", "runbooks/lxc-service-migration.md.tmp.40509.582e829e6d06", @@ -206,10 +211,6 @@ "docs/hardware/ip-allocation.md.tmp.40509.3dbface99681", "docs/hardware/ip-allocation.md.tmp.40509.9d0602266cad", "docs/hardware/ip-allocation.md.tmp.40509.978ea5421117", - "docs/hardware/ip-allocation.md.tmp.40509.437c60d52192", - "docs/hardware/environment.md.tmp.40509.0a6cfde6a808", - "docs/hardware/environment.md.tmp.40509.e7c9d394362e", - "docs/hardware/environment.md.tmp.40509.47371c0bb7f5", "rules/proxmox.md", "docs/services/services.md", "docs/software/central.md", @@ -235,7 +236,6 @@ "concepts/knowledge-extraction.md", "concepts/firewall.md", "entities/mesh-bridge.md", - "entities/meshtastic-hs.md", "assets/echo6yellow_logo_422x422_square.png", "assets/echo6yellow_logo_422x81.png", "assets/echo6_logo.png", diff --git a/vault/projects/fleet-patch-audit.md b/vault/projects/fleet-patch-audit.md new file mode 100644 index 0000000..3216c8b --- /dev/null +++ b/vault/projects/fleet-patch-audit.md @@ -0,0 +1,224 @@ +--- +title: Fleet Patch Audit — 2026-06-19 +type: project +tags: + - proxmox + - ai +related: [] +updated: 2026-06-19 +status: active +--- + +# Fleet Patch Audit — 2026-06-19 + +Read-only audit snapshot as of 2026-06-19. **Nothing has been applied — this is a planning document to build the patch plan from.** + +**Topology note:** the old Contabo VPS has been rebuilt as **edge1 (mail-only)**; **edge2 is now the front door for everything else**. edge1 is excluded from this audit (mid-rebuild/maintenance). The `CLAUDE.md` cheat-sheet still lists the old Contabo layout and is **stale** — refreshing it is a follow-up task (see Open Decisions). + +--- + +## Prioritized Backlog + +### Tier 1 — Security-Urgent Guest OS + +These containers have the highest raw security-update counts and have not been patched recently (or never). Address before any platform work. + +| Host | Guest | Upgradable / Security | Notes | +|------|-------|-----------------------|-------| +| utility | CT119 mesh-territory | 179 / 91 sec | Never patched | +| utility | CT108 meshai | 109 / 79 | — | +| cloud | CT120 immich guest-OS | 191 / 101 | — | +| cloud | CT121 nextcloud guest-OS | 98 / 75 | — | +| media | CT110 peertube | 81 / 37 | — | +| utility | CT109 opentakserver | 34 / 31 | — | +| utility | CT104 central | 50 / 38 | Includes PostgreSQL 16.13 → 16.14 | + +### Tier 2 — App / Container Updates + +Updates where the application or its Docker images have drifted from current upstream, ordered roughly by operational risk. + +| Scope | Guest | Item | Notes | +|-------|-------|------|-------| +| edge2 | CT105 | authentik 2025.12.4 → 2026.2.0 | Sequential upgrade required; read changelogs before applying | +| edge2 | CT107 | headscale 0.28.0 → 0.29.0 | — | +| edge2 | CT103 | forgejo:14 + postgres:16-alpine | Docker image drift | +| edge2 | CT106 | Synapse 1.155.0 / Element / MAS | Image drift + pending OS apt security updates | +| edge2 | CT104 | livesync couchdb:3.4 | Docker image drift | +| edge2 | CT108 | mailcow (18 containers) | Upgrade via `update.sh` only | +| cloud | CT120 | immich — server/ml/valkey:9/postgres(14-vectorchord) | 4 images drifted | +| cloud | CT121 | nextcloud AIO — mastercontainer + NC app 32.0.4 | Mastercontainer behind; 12-container stack | +| cortex | VM150 | ollama / tei(1.7) / qdrant / open-webui / obsidian | 5 AI containers drifted | +| media | VM105 | arr stack — 8 containers | jellyfin/sonarr/radarr/prowlarr/sabnzbd/lidarr/navidrome/jellyseerr all :latest | + +### Tier 3 — Platform / Reboot Windows + +Reboot-bearing updates. Coordinate maintenance windows carefully. cortex and toc are PROTECTED hosts. + +| Scope | Item | Detail | +|-------|------|--------| +| data, utility, cloud, media, toc (PVE 9 nodes) | PVE 9.1.1 → 9.2 | Reboot required | +| data, utility, cloud, media, toc | QEMU 10 → 11 | Reboot required | +| data, utility, cloud, media, toc | LXC 6 → 7 | Reboot required | +| data, utility, cloud, media, toc | Kernel 6.17.2 → 6.17.13 | Reboot required | +| edge2 | PVE 8.4.19 | Already fully patched — no action needed | +| cortex VM150 (PROTECTED) | NVIDIA driver 580.159 → 580.167 + DKMS | Reboot required | +| cortex VM150 (PROTECTED) | nvidia-container-toolkit 1.18 → 1.19 | — | +| pi-nas | Kernel 6.12 → 6.18 | Reboot required | +| pi-nas | OMV 8.1 → 8.4 | — | + +### Cross-Cutting (All / Most Hosts) + +| Item | Detail | +|------|--------| +| Tailscale | 1.94 → 1.98 nearly everywhere | +| Docker CE | → 29.6 wherever Docker is installed | + +--- + +## Non-Update Flags + +Issues noted that are not package/image updates but warrant attention. + +| Host / Guest | Flag | Detail | +|---|---|---| +| data | Disk 92% full | ~73 GB / 938 GB free; address before patching | +| utility CT118 archivist | rpcbind on 0.0.0.0:111 | No Tailscale client or firewall on this CT; exposed port | +| media VM105 jellyseerr | Non-stable image | Running preview-OIDC tag, not a stable release | +| data VM1130 nominatim | Stale image (14 months) | nominatim:4.5, pinned; confirm intentional | +| edge2 CT106 matrix / CT107 headscale | No Tailscale client | Ingress via Caddy; verify internal routing before patching | + +--- + +## Proposed Patch Approach (NOT executed — for planning) + +Lowest-risk changes first; everything reboot-bearing deferred to scheduled windows. + +**Principles** +- **Security-pocket apt only** in the first pass (openssl, openssh, gnutls, krb5, libc6, samba, nghttp2, etc.). No PVE 9.2 / QEMU / LXC / kernel, no Docker image pulls, no OMV/NVIDIA, no reboots. +- **Protected hosts (cortex, toc) never go in a bulk pass** — handled individually in their own window. Note: toc hosts cortex (VM150), so a toc reboot drops cortex — the two must be coordinated together. +- **edge1 (mail)** excluded while it's mid-rebuild. +- `needrestart` will bounce affected daemons after glibc/openssl upgrades — seconds of blip per guest, no data risk. + +**Phased plan** + +| Phase | Scope | Reboot? | Notes | +|------|-------|---------|-------| +| **1 — Guest/VM security apt** | utility CT100,101,102,103,104,106,107,108,109,112,118,119 · cloud CT120,121 · media VM105,CT110,CT111 · data VM1130 · edge2 CT100–108 | No | Lowest blast radius. Worst-first: CT119, CT108, immich/nextcloud guest-OS, peertube. | +| **2 — Hypervisor host OS security** | data, utility, cloud, media host OSes (**not toc**) | No | One node at a time; restarts hypervisor-side daemons (smbd etc.). edge2 host already patched. | +| **3 — App / container updates (Tier 2)** | per-app, native updater each | Per-app | See "special handling" below — not a generic `docker pull`. | +| **4 — Reboot windows (Tier 3)** | PVE 9.2 + QEMU 11 + LXC 7 + kernel on the 5 PVE-9 nodes; pi-nas kernel + OMV; cortex NVIDIA/DKMS | **Yes** | Schedule deliberately; toc+cortex coordinated. | +| **Cross-cutting** | Tailscale 1.94→1.98 fleet-wide | No | Can ride along Phase 1/2. | + +**Special handling — do NOT bulk-patch these; use the native updater** +- **mailcow** (edge2 CT108) → `./update.sh` (upgrades all 18 components in lockstep) +- **nextcloud AIO** (cloud CT121) → update mastercontainer, then in-UI update button (port 8080) +- **immich** (cloud CT120) → `docker compose pull && up -d` from its compose dir +- **authentik** (edge2 CT105) → sequential version upgrades with migrations; 2025.12.4 → 2026.2.0 cannot skip releases +- **pi-nas OMV** → OMV's own update path, not raw apt +- **cortex / toc** → manual, protected, own window + +--- + +## Open Decisions (for tomorrow's plan) + +1. **Phase 1 scope** — all non-protected guests at once, or staged worst-first? +2. **Phase 2** — patch the four hypervisor host OSes now (no reboot), or fold into the Phase 4 reboot window? +3. **Reboot-window scheduling** — order of the 5 PVE-9 nodes; **toc + cortex must be done together** (toc reboot drops cortex). edge2 needs none. +4. **Tier-2 app upgrades** — which to take on: authentik major (2025→2026, migration-heavy), Matrix/Synapse, Forgejo, headscale 0.29, mailcow. Each is its own task. +5. **data disk at 92%** — remediate before/independently of patching (operational risk regardless). +6. **CT118 archivist** rpcbind on `0.0.0.0:111` with no Tailscale/firewall — treat as a separate exposure fix. +7. **Refresh the stale `CLAUDE.md` cheat-sheet** to the edge1/edge2 topology — separate doc task. +8. **Daemon-restart tolerance** — confirm brief blips are acceptable for the stateful services (central PG16/NATS, opentakserver, peertube) during Phase 1/2. + +--- + +## Full Inventory + +Complete point-in-time state of every node, guest, and container service. + +### data (PVE 9.1.1) + +- Host: 126 upgradable / 51 security; no Docker installed; roles: NAS, NFS, Samba +- **VM1130 recon-vm** (Ubuntu 24.04) — 34 upgradable / 10 security + - PostgreSQL 16.14 (DBs: overture, padus), photon, kiwix, recon.py, 7x navi-backend, nginx, Apache, Samba + - Docker: valhalla:latest, nominatim:4.5 (stale 14 months), zimit:latest (not running) + +### utility (PVE 9.1.1) + +- Host: 148 upgradable / 38 security; PVE 9.2 platform update pending; 12 LXC guests + +| CT | Name | Upgradable / Sec | Services | +|----|------|-----------------|----------| +| CT100 | meshmonitor | 62 / 15 | ghcr.io/yeraze/meshmonitor:latest | +| CT101 | caddy (home ingress) | 59 / 18 | — | +| CT102 | searxng | 54 / 15 | searxng/searxng:latest + valkey/valkey:8-alpine | +| CT103 | argus | 29 / 26 | RF capture / transcribe / viewer | +| CT104 | central | 50 / 38 | PostgreSQL 16 + NATS/MQTT/JetStream | +| CT106 | meshtastic-hs | 54 / 17 | headscale control plane | +| CT107 | mesh-bridge | 53 / 15 | dual tailscaled | +| CT108 | meshai | 109 / 79 | work-meshai local build | +| CT109 | opentakserver | 34 / 31 | nginx / PG16 / rabbitmq / mumble / mediamtx / CoT | +| CT112 | cobalt | 35 / 29 | build/CI (idle) | +| CT118 | archivist | 62 / 17 | archivist + rpcbind (FLAG: port 111 on 0.0.0.0) | +| CT119 | mesh-territory | 179 / 91 | meshwars:latest (never patched) | + +### cloud (PVE 9.1.1) + +- Host: 113 upgradable / 38 security; 2 LXC guests + +| CT | Name | Upgradable / Sec | Services | +|----|------|-----------------|----------| +| CT120 | immich | 191 / 101 | immich_server, immich_machine_learning, valkey/valkey:9, immich postgres (14-vectorchord) — all drifted | +| CT121 | nextcloud AIO | 98 / 75 | 12 containers: mastercontainer + apache + nextcloud + postgresql + redis + collabora + clamav + imaginary + fulltextsearch + notify-push + whiteboard + docker-socket-proxy; NC 32.0.4; mastercontainer behind | + +### media (PVE 9.1.1) + +- Host: 107 upgradable / 36 security; 3 guests + +| Guest | Name | Upgradable / Sec | Services | +|-------|------|-----------------|----------| +| VM105 | arr | 43 / 9 (Ubuntu 24.04) | jellyfin / sonarr / radarr / prowlarr / sabnzbd / lidarr / navidrome / jellyseerr (preview-OIDC) all :latest + Samba | +| CT110 | peertube | 81 / 37 | v8.0.2; nginx / PG16 / redis / peertube / pt-downloader + importer + monitor / nordvpn | +| CT111 | mcc | 51 / 13 | caddy + postfix | + +### toc (PVE 9.1.1) — PROTECTED + +- Host: 189 upgradable / 43 security; PVE 9.2 platform update pending; reboot required +- Hosts only VM150 cortex — coordinate any toc work with cortex maintenance window + +### cortex (VM150, Ubuntu 24.04) — PROTECTED GPU / Claude Code host + +- 90 apt upgradable +- NVIDIA driver 580.159 → 580.167 + DKMS (reboot required) +- nvidia-container-toolkit 1.18 → 1.19 +- Docker: ollama / tei 1.7 / qdrant / open-webui / obsidian — all drifted + +### pi-nas (Debian 13, arm64, RPi + OMV) + +- 130 apt upgradable; kernel 6.12 → 6.18 (reboot required); OMV 8.1 → 8.4 +- Docker engine installed; 0 containers running + +### edge2 (PVE 8.4.19) — Host Fully Patched + +- 9 LXC guests + +| CT | Name | Services / Status | +|----|------|-------------------| +| CT100 | pdm | PDM 1.1.4, current (native) | +| CT101 | wordpress | Apache 2.4.67 / PHP 8.4 / MariaDB 11.8.6 / WP core 7.0 — plugin/theme status needs WP-CLI | +| CT102 | vaultwarden | vaultwarden/server:latest — drift indeterminate | +| CT103 | forgejo | 14.0.5 (forgejo:14 + postgres:16-alpine drifted) | +| CT104 | livesync | couchdb:3.4 drifted + local provisioner | +| CT105 | authentik | 2025.12.4 (server/worker/postgres) → upgrade to 2026.2.0 | +| CT106 | matrix | Synapse 1.155.0 / Element / MAS + mautrix-signal + postgres; OS apt security updates pending; no Tailscale client | +| CT107 | headscale | 0.28.0 → 0.29.0 + headplane; no Tailscale client | +| CT108 | mailcow | 18 containers (postfix/nginx/dovecot/rspamd/clamd/sogo/php-fpm/mariadb/redis/memcached/unbound/acme/olefy/watchdog/netfilter/ofelia/dockerapi/tlspol) — upgrade via `update.sh` only | + +--- + +## Coverage Notes + +- **Complete:** all five PVE-9 nodes (data, utility, cloud, media, toc), edge2, cortex, pi-nas, and all discovered guests/containers. +- **Excluded:** edge1 (the rebuilt ex-Contabo, mail-only) — mid-rebuild/maintenance at time of audit; re-audit once it settles. +- **Incomplete:** WordPress (CT101) plugin and theme status — requires WP-CLI; not assessed. +- **Method:** read-only throughout — SSH / `pct exec`, `apt list --upgradable`, `docker manifest inspect` for same-tag drift. Floating/pinned-tag caveats noted inline where drift could not be confirmed.