From 54652f31ea4711e6f76dcbc16edcec310370362c Mon Sep 17 00:00:00 2001 From: echo6-autocommit Date: Mon, 22 Jun 2026 12:00:09 +0000 Subject: [PATCH] auto: docs sync 2026-06-22T12:00:09+00:00 Files changed: engine/changelog.md engine/lint-report.md vault/.obsidian/workspace.json vault/projects/fleet-patch-audit.md vault/projects/fleet-platform-baseline.md --- engine/changelog.md | 2 + engine/lint-report.md | 7 ++- vault/.obsidian/workspace.json | 16 ++--- vault/projects/fleet-patch-audit.md | 42 +++++++++++-- vault/projects/fleet-platform-baseline.md | 76 +++++++++++++++++++++++ 5 files changed, 127 insertions(+), 16 deletions(-) create mode 100644 vault/projects/fleet-platform-baseline.md diff --git a/engine/changelog.md b/engine/changelog.md index 7424996..4780c6e 100644 --- a/engine/changelog.md +++ b/engine/changelog.md @@ -121,3 +121,5 @@ ## 2026-06-20T09:00:01Z — sweep deferred (competing GPU process: 2932 node /usr/bin/peertube-runner server --enable-job vod-hls-transcoding --enable-job vod-audio-merge-transcoding --enable-job live-rtmp-hls-transcoding --enable-job video-studio-transcoding --enable-job video-transcription) ## 2026-06-21T09:00:01Z — sweep deferred (competing GPU process: 2932 node /usr/bin/peertube-runner server --enable-job vod-hls-transcoding --enable-job vod-audio-merge-transcoding --enable-job live-rtmp-hls-transcoding --enable-job video-studio-transcoding --enable-job video-transcription) + +## 2026-06-22T09:00:01Z — sweep deferred (competing GPU process: 4201 node /usr/bin/peertube-runner server --enable-job vod-hls-transcoding --enable-job vod-audio-merge-transcoding --enable-job live-rtmp-hls-transcoding --enable-job video-studio-transcoding --enable-job video-transcription) diff --git a/engine/lint-report.md b/engine/lint-report.md index 8bd3fa0..fb3ebcf 100644 --- a/engine/lint-report.md +++ b/engine/lint-report.md @@ -1,6 +1,6 @@ # Vault Lint Report -Generated: 2026-06-22T00:00:08Z | Docs scanned: 92 | Elapsed: 0.0s +Generated: 2026-06-22T06:00:09Z | Docs scanned: 93 | Elapsed: 0.0s ## Summary @@ -8,7 +8,7 @@ Generated: 2026-06-22T00:00:08Z | Docs scanned: 92 | Elapsed: 0.0s |----------|-------| | ERROR (dead links) | 0 | | WARN (schema) | 1 | -| INFO (orphans) | 39 | +| INFO (orphans) | 40 | ### WARN breakdown - Missing frontmatter block: 1 @@ -63,6 +63,7 @@ _None. All wikilinks resolve._ - no incoming links: session-resume/SESSION-HANDOFF-meshai-test.md - no incoming links: docs/matrix/synapse_retention_discovery.md - no incoming links: runbooks/syncthing-add-node.md +- no incoming links: runbooks/toc-cortex-pve9.2-update.md - no incoming links: plans/vaultwarden-plan.md ## Gaps & suggestions @@ -106,5 +107,5 @@ Matt decides whether to create a real doc — when he does, future sweeps will l | `nextcloud` | 11 | | `vaultwarden` | 11 | | `headplane` | 10 | +| `livesync` | 10 | | `jellyfin` | 10 | -| `livesync` | 9 | diff --git a/vault/.obsidian/workspace.json b/vault/.obsidian/workspace.json index 2ee273d..a49f73a 100644 --- a/vault/.obsidian/workspace.json +++ b/vault/.obsidian/workspace.json @@ -199,18 +199,18 @@ }, "active": "17bd4a6166f789d0", "lastOpenFiles": [ + "projects/fleet-platform-baseline.md", + "projects/fleet-platform-baseline.md.tmp.5281.0e8c5dd2a601", + "projects/fleet-patch-audit.md.tmp.5281.827293ce77a8", + "projects/fleet-patch-audit.md.tmp.5281.3206bbe8ba4d", + "projects/fleet-patch-audit.md.tmp.5281.5d6a0ab80754", + "projects/fleet-patch-audit.md.tmp.5281.8edd2d841378", + "projects/fleet-patch-audit.md.tmp.5281.bc624a70230f", + "projects/fleet-patch-audit.md.tmp.5281.22de50591aae", "projects/fleet-patch-audit.md.tmp.1493418.7b411319326d", "projects/fleet-patch-audit.md.tmp.1493418.c97c38bf6380", "projects/fleet-patch-audit.md.tmp.1493418.36abe7516917", - "projects/fleet-patch-audit.md.tmp.1493418.7b690ad51bff", - "projects/fleet-patch-audit.md.tmp.1493418.5eb5cd85b0aa", - "projects/fleet-patch-audit.md.tmp.1493418.bfbc901d3ddb", - "projects/fleet-patch-audit.md.tmp.1493418.6000f6878a47", - "projects/fleet-patch-audit.md.tmp.1493418.59313ebf21bb", - "projects/fleet-patch-audit.md.tmp.1493418.c07599dde22a", - "projects/fleet-patch-audit.md.tmp.1493418.91a25bd2bfcb", "runbooks/toc-cortex-pve9.2-update.md", - "runbooks/toc-cortex-pve9.2-update.md.tmp.1493418.2185e6a8a44d", "projects/nominatim-v5-reimport.md", "2026-06-19.md", "Untitled.canvas", diff --git a/vault/projects/fleet-patch-audit.md b/vault/projects/fleet-patch-audit.md index 86e72f3..2e02bed 100644 --- a/vault/projects/fleet-patch-audit.md +++ b/vault/projects/fleet-patch-audit.md @@ -6,7 +6,7 @@ tags: - ai related: [] updated: 2026-06-22 -status: active +status: complete --- # Fleet Patch Audit — 2026-06-19 @@ -17,6 +17,22 @@ Read-only audit snapshot as of 2026-06-19. **Nothing has been applied — this i --- +## Campaign complete (2026-06-22) + +**Phases 1–3 are fully done. The entire fleet is on the current platform.** + +- **Phase 1** (guest/VM security apt) — COMPLETE 2026-06-20. 26 guests patched, ~600+ security packages cleared, zero data loss. +- **Phase 2** (app/container updates) — COMPLETE 2026-06-21. All app upgrades done: Authentik 2025.12.4→2026.5.3 (sequential), Forgejo 14→15, Headscale 0.28→0.29.1 (both instances), Immich 2.5.6→2.7.5, Nextcloud AIO→NC 33.0.5, media stack (Jellyfin/SABnzbd/arr), cortex AI stack (Ollama/TEI/Qdrant/Open-WebUI), and the low-urgency batch. +- **Phase 3** (platform/reboot windows) — COMPLETE 2026-06-22. All 5 PVE nodes on 9.2.3/kernel 7.0.12-1-pve (including toc+cortex); pi-nas on OMV 8.4/kernel 6.18; cortex NVIDIA driver 580.167.08 + DKMS + nvidia-container-toolkit 1.19.1; GPU passthrough (vfio) survived the 7.0 kernel; cluster 5/5 quorate. + +**Intentionally deferred / out of scope (not failures):** + +- **RabbitMQ** — left on 3.12 by decision; OTS does not support 4.x and exposure is localhost-bound. Revisit only if/when OTS officially supports RabbitMQ 4.x. +- **Nominatim v5** — full re-import project, spun off to [[nominatim-v5-reimport]]. +- **Optional cosmetic cleanups:** navidrome cert renewal (`navidrome.echo6.co` expired unmanaged cert); MediaMTX deprecated config param rename (`protocols`→`rtspTransports`, `encryption`→`rtspEncryption`); retained `/root` rollback artifacts (DB dumps, binary backups on their CTs); vestigial utility exit-node route (0.0.0.0/0, harmless); `rpi-eeprom` still held on pi-nas (SPI bootloader, intentionally untouched). + +--- + ## Prioritized Backlog ### Tier 1 — Security-Urgent Guest OS @@ -151,7 +167,7 @@ Lowest-risk changes first; everything reboot-bearing deferred to scheduled windo | **1 — Guest/VM security apt** | utility CT100,101,102,103,104,106,107,108,109,112,118,119 · cloud CT120,121 · media VM105,CT110,CT111 · data VM1130 · edge2 CT100–107 | No | Lowest blast radius. Worst-first: CT119, CT108, immich/nextcloud guest-OS, peertube. | | **2 — Hypervisor host OS security** | data, utility, cloud, media host OSes (**not toc**) | No | One node at a time; restarts hypervisor-side daemons (smbd etc.). edge2 host already patched. | | **3 — App / container updates (Tier 2)** | per-app, native updater each | Per-app | **COMPLETE 2026-06-21** — all app upgrades done (security-critical, low-urgency batch, and cortex AI stack). See Phase 2 Execution Log. | -| **4 — Reboot windows (Tier 3)** | PVE 9.2 + QEMU 11 + LXC 7 + kernel on the 5 PVE-9 nodes; pi-nas kernel + OMV; cortex NVIDIA/DKMS | **Yes** | **IN PROGRESS (2026-06-21/22)** — 4-node cluster (media/data/utility/cloud) ✅ + pi-nas fully ✅ (OMV 8.4 + kernel 6.18); only toc+cortex remains (matt-desktop). See Phase 3 Execution Log. | +| **4 — Reboot windows (Tier 3)** | PVE 9.2 + QEMU 11 + LXC 7 + kernel on the 5 PVE-9 nodes; pi-nas kernel + OMV; cortex NVIDIA/DKMS | **Yes** | **COMPLETE (2026-06-22)** — all 5 PVE nodes on 9.2.3/kernel 7.0.12; pi-nas on OMV 8.4/kernel 6.18; cortex NVIDIA 580.167/DKMS/toolkit 1.19.1; GPU passthrough verified. See Phase 3 Execution Log. | | **Cross-cutting** | Tailscale 1.94→1.98 fleet-wide | No | Can ride along Phase 1/2. | **Special handling — do NOT bulk-patch these; use the native updater** @@ -352,7 +368,7 @@ Empirically confirmed during the OTS update: updating OTS to 1.7.12 does **not** ## Phase 3 — Execution Log (2026-06-21/22) -**Status: IN PROGRESS — 4-node cluster ✅ + pi-nas fully ✅ (OMV 8.4 + kernel 6.18); only toc+cortex remains (handed to matt-desktop).** +**Status: COMPLETE (2026-06-22)** — all five PVE nodes on 9.2.3/kernel 7.0.12, pi-nas on OMV 8.4/kernel 6.18, cortex NVIDIA driver + DKMS + container-toolkit updated, GPU passthrough verified. Phase 3 has no open items. ### Pre-flight findings @@ -377,9 +393,25 @@ One at a time; cluster stayed 5/5 quorate throughout. **Important architecture note recorded:** pi-nas (2.8 TB RAID1) is the NFS storage backend for immich's photo library (644 GB), nextcloud files, peertube, and the *arr library — so a pi-nas reboot stalls those services' storage. -### Remaining (handed off) +### toc + cortex ✅ DONE (2026-06-22, run from matt-desktop) -- **toc + cortex** — deliberately NOT done by the cortex-based automation (rebooting toc drops cortex). Handed to a **matt-desktop** Claude Code session (Windows, now has Claude Code 2.1.185). Runbook + ready-to-paste prompt saved at `runbooks/toc-cortex-pve9.2-update.md`. Covers: cortex NVIDIA driver 580.159→580.167 + DKMS + container-toolkit + reboot, then toc PVE 9.2 + reboot. **Highest-risk check: GPU passthrough (vfio) surviving toc's new kernel 7.0.** Safe to run now (other 4 nodes up/quorate → toc reboot keeps 4/5). +- **cortex:** NVIDIA driver 580.159.03 → 580.167.08 + DKMS (built for running kernel) + nvidia-container-toolkit 1.19.1; all AI containers healthy on GPU — vault-tagger generates (Ollama), TEI bge-m3 embeds (200 OK from localhost:8090), Qdrant healthy. +- **toc:** PVE 9.1 → 9.2.3 / kernel 7.0.12-1-pve. **GPU passthrough (vfio) survived the 7.0 kernel** — the key risk, verified. +- Cluster 5/5 quorate with toc rejoined; cortex VM150 auto-started; no leftover snapshot. + +**Phase 3 has no open items.** + +### Wrap-up (2026-06-22) + +**Rollback-artifact sweep done.** Approximately 10.4 GB of campaign DB dumps and binary/config backups removed fleet-wide. Largest single item: a 7.79 GB central Postgres dump on utility CT104. **Retained intentionally:** `/root/boot-backup-pre6.18-20260622.tar.gz` on pi-nas (kernel rollback artifact; 154 MB) and the durable mailcow backup on pi-nas (`…/contabo-prewipe-2026-06/mailcow/`). + +**Accepted final state — operator-acknowledged decisions (2026-06-22).** These are closed decisions, not TODOs. + +- **Guest OS = security-only.** LXC containers and VMs received security-pocket apt updates only (the intentional Phase 1 scope). Non-security package drift (e.g. Docker CE versions, miscellaneous libs) was deliberately not swept with a full `apt full-upgrade`. The PVE hosts, pi-nas, and cortex did receive full upgrades. Operator accepted this state. A full guest `apt full-upgrade` ("Phase 1.5") remains an option if ever wanted. +- **Apps capped by external factors (decisions, not failures):** RabbitMQ 3.12 left by decision — OTS depends on it and 4.x would break it; Lidarr v2 — the lidarr-on-steroids image maintainer has not shipped v3, would require an image swap; Jellyseerr on `preview-OIDC` — kept because stable 3.3.0 lacks OIDC/SSO support; Mumble 1.5.517 — newest version in the Ubuntu 24.04 repo, upstream 1.5.901 is not available without going off-distro. +- **Deferred project:** Nominatim v5 re-import — spun off to [[nominatim-v5-reimport]]. +- **Optional cosmetic follow-ups (non-urgent, non-blocking):** navidrome.echo6.co expired unmanaged cert; MediaMTX deprecated config param names (`protocols`/`encryption`); `rpi-eeprom` held on pi-nas; vestigial utility exit-node route (0.0.0.0/0). +- **Kernel summary:** all 5 PVE hosts on kernel 7.0.12-1-pve; LXC containers share the host kernel (7.0); VM guests (recon-vm, arr VM105) have their own Ubuntu kernels (security-patched during Phase 1, not necessarily absolute-latest upstream); pi-nas on 6.18.34+rpt-rpi-2712; cortex kernel updated as part of the toc+cortex window. --- diff --git a/vault/projects/fleet-platform-baseline.md b/vault/projects/fleet-platform-baseline.md new file mode 100644 index 0000000..b8b83a4 --- /dev/null +++ b/vault/projects/fleet-platform-baseline.md @@ -0,0 +1,76 @@ +--- +title: Fleet Platform Baseline — post-patch 2026-06-22 +type: reference +tags: + - proxmox + - ai +related: + - projects/fleet-patch-audit +updated: 2026-06-22 +status: current +--- + +# Fleet Platform Baseline — post-patch 2026-06-22 + +Point-in-time platform state after the 2026-06-19/22 patch campaign. Full campaign record and accepted caveats in [[fleet-patch-audit]]. + +--- + +## Proxmox cluster (`echo6-cluster`) + +5 nodes, quorum 3/5, no HA configured. + +| Node | Platform | +|---|---| +| data, utility, cloud, media, toc | PVE 9.2.3 / kernel 7.0.12-1-pve (QEMU 11, LXC 7) | + +All 5 nodes on `pve-no-subscription` (trixie). data, cloud, and media had the repo added during the campaign (utility and toc already had it). LXC containers share the host kernel (7.0); VM guests carry their own Ubuntu kernels. + +--- + +## Other hosts + +**pi-nas** +OMV 8.4.0-3 / kernel 6.18.34+rpt-rpi-2712 (arm64, Raspberry Pi 4). `rpi-eeprom` held (SPI bootloader, intentionally untouched). RAID1 2.8 TB NFS backend serving `/export/{arr,immich,nextcloud,peertube,data}`. Retained artifact: `/root/boot-backup-pre6.18-20260622.tar.gz` (kernel rollback). Durable mailcow backup also on pi-nas (`…/contabo-prewipe-2026-06/mailcow/`, sha256-verified). + +**cortex (VM150 on toc, GPU)** +Ubuntu 24.04. NVIDIA driver 580.167.08, nvidia-container-toolkit 1.19.1. AI stack: Ollama 0.30.10 (vault-tagger, GPU), TEI 1.9 / bge-m3 (embeddings, GPU), Qdrant 1.18.2, Open-WebUI 0.9.6. These power the `.ref` vault engine. See [[toc-cortex-pve9.2-update]] for the toc+cortex upgrade procedure. + +**recon-vm (VM1130 on data)** +Ubuntu 24.04 (security-patched). PostgreSQL 16, Valhalla, Nominatim 4.5 (v5 deferred — see [[nominatim-v5-reimport]]), Photon, Kiwix. + +--- + +## Key application versions (as of 2026-06-22) + +| App | Host | Version | Notes | +|---|---|---|---| +| Authentik | edge2 CT105 | 2026.5.3 | Fleet SSO | +| Forgejo | edge2 CT103 | 15.0.3 | — | +| Headscale | edge2 CT107 | 0.29.1 | Fleet tailnet coordinator at `vpn.echo6.co`; boot-survival fixed (ports bound to 10.10.10.25, not tailscale IP) | +| Headscale (IdahoMesh) | utility CT106 | 0.29.1 | Separate IdahoMesh mesh at `vpn.idahomesh.com` | +| Nextcloud | cloud CT121 | 33.0.5 | AIO; data on pi-nas NFS | +| Immich | cloud CT120 | 2.7.5 | Photos on pi-nas NFS | +| PeerTube | media CT110 | 8.2.1 | — | +| OpenTAKServer | utility CT109 | 1.7.12 | RabbitMQ stays 3.12 by decision; MediaMTX 1.19.1; Mumble 1.5.517 | +| Matrix/Synapse | edge2 CT106 | 1.155.0 | — | +| Vaultwarden | edge2 CT102 | 1.36.0 | — | +| PDM | edge2 CT100 | 1.1.4 | — | +| CouchDB/LiveSync | edge2 CT104 | 3.5.2 | — | +| Jellyfin | media VM105 | 10.11.11 | — | +| Sonarr | media VM105 | 4.0.17 | — | +| Radarr | media VM105 | 6.2.1 | — | +| Prowlarr | media VM105 | 2.4.0 | — | +| SABnzbd | media VM105 | 5.0.4 | — | +| Navidrome | media VM105 | 0.62.0 | — | +| Lidarr | media VM105 | v2 | Image-capped — lidarr-on-steroids maintainer has not shipped v3 | +| Jellyseerr | media VM105 | preview-OIDC | Kept — stable 3.3.0 lacks OIDC/SSO support | + +--- + +## Accepted caveats (decisions, not TODOs) + +- **Guest OS security-only:** LXC containers and VMs received security-pocket apt only (Phase 1 scope). Non-security package drift was not swept. A full `apt full-upgrade` ("Phase 1.5") is an option if ever wanted. +- **RabbitMQ 3.12 (EOL):** left by decision — OTS does not support 4.x. AMQP/MQTT ports are localhost-bound; low exposure. Revisit if/when OTS officially supports RabbitMQ 4.x. +- **Nominatim 4.5:** v5 re-import is a separate project; see [[nominatim-v5-reimport]]. +- **Optional cosmetic follow-ups:** navidrome.echo6.co expired cert; MediaMTX deprecated config param names; `rpi-eeprom` held; vestigial utility exit-node route (0.0.0.0/0).