diff --git a/engine/lint-report.md b/engine/lint-report.md index 9d07148..e52aedd 100644 --- a/engine/lint-report.md +++ b/engine/lint-report.md @@ -1,6 +1,6 @@ # Vault Lint Report -Generated: 2026-08-14T18:00:06Z | Docs scanned: 112 | Elapsed: 0.0s +Generated: 2026-08-15T05:04:38Z | Docs scanned: 115 | Elapsed: 0.0s ## Summary @@ -8,7 +8,7 @@ Generated: 2026-08-14T18:00:06Z | Docs scanned: 112 | Elapsed: 0.0s |----------|-------| | ERROR (dead links) | 24 | | WARN (schema) | 0 | -| INFO (orphans) | 35 | +| INFO (orphans) | 36 | ### WARN breakdown - Missing frontmatter block: 0 @@ -69,6 +69,7 @@ Generated: 2026-08-14T18:00:06Z | Docs scanned: 112 | Elapsed: 0.0s - no incoming links: runbooks/meshai-prod-compose-override.md - no incoming links: runbooks/meshmonitor-password-reset.md - no incoming links: runbooks/meshtasticd-sim-nodes-runbook.md +- no incoming links: runbooks/navi-lift-to-media.md - no incoming links: runbooks/nordvpn-lxc.md - no incoming links: runbooks/pg-backup.md - no incoming links: runbooks/pipeline-patterns.md @@ -105,18 +106,18 @@ Matt decides whether to create a real doc — when he does, future sweeps will l | Term | Docs mentioning it | |------|--------------------| -| `tailscale` | 40 | -| `docker` | 35 | +| `tailscale` | 42 | +| `docker` | 38 | | `proxmox` | 34 | -| `headscale` | 24 | -| `peertube` | 24 | +| `headscale` | 25 | +| `peertube` | 25 | | `meshtastic` | 19 | | `mailcow` | 17 | -| `immich` | 15 | -| `nextcloud` | 14 | +| `immich` | 16 | +| `nextcloud` | 15 | +| `forgejo` | 14 | | `element` | 14 | -| `forgejo` | 13 | -| `aida-nebra` | 12 | -| `livesync` | 12 | -| `jellyfin` | 12 | -| `vaultwarden` | 12 | +| `livesync` | 13 | +| `jellyfin` | 13 | +| `vaultwarden` | 13 | +| `meshmonitor` | 13 | diff --git a/vault/.obsidian/workspace.json b/vault/.obsidian/workspace.json index b08c316..d7755a1 100644 --- a/vault/.obsidian/workspace.json +++ b/vault/.obsidian/workspace.json @@ -199,22 +199,25 @@ }, "active": "17bd4a6166f789d0", "lastOpenFiles": [ - "docs/services/services.md.tmp.2730138.e216805f7b3b", - "docs/hardware/environment.md.tmp.2730138.982fd86dec7c", - "docs/services/services.md.tmp.2730138.e01d11c7a6b5", - "docs/hardware/environment.md.tmp.2730138.c76433dc40b4", + "projects/navi-recon-separation.md.tmp.2730138.1a9ee67ba5ca", + "runbooks/navi-lift-to-media.md", + "runbooks/navi-lift-to-media.md.tmp.2730138.56b7ce0eca2e", + "docs/hardware/environment.md.tmp.2730138.237b7072261a", + "docs/hardware/environment.md.tmp.2730138.58738aa3e22c", + "docs/hardware/environment.md.tmp.2730138.e8527bb48e1d", + "docs/hardware/environment.md.tmp.2730138.0fcdc6bee9f6", + "runbooks/edge2-access-reference.md.tmp.3701844.be7e6cfc6226", + "runbooks/peertube-sitemap-redis-oom.md.tmp.3701844.88863a688de8", + "projects/fleet-storage-memory-upgrade.md.tmp.3701844.7763e1bbbff5", + "projects/fleet-storage-memory-upgrade.md.tmp.3701844.9037775c0789", + "runbooks/edge2-access-reference.md.tmp.3701844.a8a5a00eaf49", + "runbooks/edge2-boot-recovery.md", + "runbooks/corescope-ingest-stall-oom.md", "projects/navi-recon-separation.md", - "projects/navi-recon-separation.md.tmp.2730138.fb2f5474a2bc", "projects/fleet-storage-memory-upgrade.md", - "projects/fleet-storage-memory-upgrade.md.tmp.2730138.f2694abef97e", "runbooks/peertube-sitemap-redis-oom.md", - "runbooks/peertube-sitemap-redis-oom.md.tmp.2730138.acfcfa86751d", "runbooks/omv-add-nfs-share.md", - "runbooks/omv-add-nfs-share.md.tmp.2730138.04a2d2fb1ec7", "runbooks/pve-guest-park-and-adopt.md", - "runbooks/pve-guest-park-and-adopt.md.tmp.2730138.216ea270cee7", - "docs/services/services.md.tmp.1912821.a9007ef6f645", - "projects/fleet-patch-audit.md.tmp.3598161.730170652bdf", "runbooks/conduit-operations.md", "docs/software/conduit.md", "archive/projects/vaultwarden-plan.md", @@ -232,10 +235,7 @@ "runbooks/fleet-magicdns-resolved-migration.md", "projects/fleet-platform-baseline.md", "runbooks/toc-cortex-pve9.2-update.md", - "projects/nominatim-v5-reimport.md", - "2026-06-19.md", "Untitled.canvas", - "docs/hardware/environment.md", "assets/echo6yellow_logo_422x422_square.png", "assets/echo6yellow_logo_422x81.png", "assets/echo6_logo.png", diff --git a/vault/docs/hardware/environment.md b/vault/docs/hardware/environment.md index e1031f1..d287875 100644 --- a/vault/docs/hardware/environment.md +++ b/vault/docs/hardware/environment.md @@ -23,7 +23,7 @@ Five nodes running Proxmox VE: | data | 192.168.1.240 | 100.64.0.6 | AMD Ryzen 7 PRO 5750GE, 1TB NVMe + 1TB SATA SSD | 32GB DDR4-3200 | Database [[services]] | | utility | 192.168.1.241 | 100.64.0.5 | AMD Ryzen 7 PRO 5750GE, 512GB NVMe | 32GB DDR4-3200 | Utility [[services]], monitoring | | cloud | 192.168.1.242 | 100.64.0.4 | Intel i7-12700T, 512GB NVMe | 32GB DDR4-3200 | Cloud storage, personal [[services]] | -| media | 192.168.1.243 | 100.64.0.3 | Intel i7-14700T, 2x 512GB NVMe | 32GB DDR5-4800 | Media server, *arr stack | +| media | 192.168.1.243 | 100.64.0.3 | Intel i7-14700T, 512GB + 2TB NVMe | 64GB DDR5-4800 | Media server, *arr stack | | toc | 192.168.1.244 | 100.64.0.13 | Workstation (i9-10900X) | 64GB DDR4 | GPU compute, AI/ML workloads | ### Node Storage Details @@ -32,9 +32,11 @@ Five nodes running Proxmox VE: | data | Samsung SSD 980 1TB (NVMe) | SanDisk SDSSDH3 1TB (SATA SSD) | | utility | WD PC SN740 512GB (NVMe) | — | | cloud | SK Hynix HFS512GEJ9X164N 512GB (NVMe) | — | -| media | 2x Intel SSDPEKNU512GZH 512GB (NVMe) | — | +| media | Intel SSDPEKNU512GZH 512GB (NVMe, boot/`pve`) | WD Green SN350 2TB (NVMe, VG `tank`) | | toc | 512GB NVMe | — | +**media upgraded 2026-08-15:** 32 → 64 GB DDR5-4800 (2 × 32 GB, board maximum reached), and the second M.2 — which held a leftover BitLocker Windows install — replaced with a 2 TB WD Green SN350. That drive is VG `tank` as a **single 1.82 TB volume** — `tank/data`, ext4, mounted `/mnt/nvme2tb`, registered as Proxmox dir storage `media-nvme` (`nodes media`-scoped). Deliberately not split into separate pools: navi's geo data is the intended consumer and it needs one large contiguous filesystem it can grow into. The displaced Windows data was archived to pi-nas first — see [[fleet-storage-memory-upgrade]]. + **Free positions (verified live 2026-08-14):** data has none — only three external PCIe root ports exist and all are populated (NVMe, NIC, Wi-Fi), and the Wi-Fi M.2 is E-keyed so it cannot take a storage drive. utility and cloud each have an empty 2.5" SATA bay. media's second M.2 holds a leftover BitLocker Windows install (serial `PHKA142402U8512A`; the Proxmox drive is `PHKA142504HP512A`). toc has 8 unpopulated SATA ports and 4 free PCIe slots. pi-nas has one free SATA port (`ata5`). Placement plan is [[fleet-storage-memory-upgrade]]. **Memory ceilings:** data and utility are hard-capped at 32 GB and already there. cloud and media max at 64 GB. toc has 2 of 8 DIMM slots free but takes full-size DDR4 DIMMs, not SODIMM, and currently runs all modules at 2133 MT/s because rated speeds are mixed (3200/2666/2133). diff --git a/vault/projects/fleet-storage-memory-upgrade.md b/vault/projects/fleet-storage-memory-upgrade.md index d27d703..f063351 100644 --- a/vault/projects/fleet-storage-memory-upgrade.md +++ b/vault/projects/fleet-storage-memory-upgrade.md @@ -10,7 +10,7 @@ related: - [[toc-cortex-pve9.2-update]] - [[fleet-patch-audit]] - [[pve-guest-park-and-adopt]] -updated: 2026-08-14 +updated: 2026-08-15 --- # Fleet Storage and Memory Upgrade @@ -86,6 +86,34 @@ Open question: whether any of the spare 512 GB drives are 2.5" SATA rather than --- +## Candidate node — Dell OptiPlex 3080 Micro (tag D9NPZB3) + +Decoded from the Dell factory configuration 2026-08-14. A 2020-era 1-litre machine, same class as the ThinkCentres running utility and data. + +| | As shipped | +|---|---| +| CPU | Core i5-10500T — 10th gen Comet Lake, 6C/12T, 12 MB cache, 2.3 → 3.8 GHz, **35 W** | +| RAM | 8 GB (2 x 4 GB) DDR4-2666 non-ECC SODIMM, 1Rx16 | +| Storage | 256 GB WD SN730 NVMe (M.2 2280) | +| Wireless | Intel Wi-Fi 6 AX200 2x2 + BT 5, internal antennas | +| Other | Discrete TPM enabled, 65 W adapter, Windows 10 Pro | + +**Expansion:** 2 SODIMM slots, **64 GB** maximum, DDR4-2666 non-ECC. Both slots ship filled with 4 GB sticks, so any upgrade is a replacement, not an addition. One M.2 2230/2280 slot at **PCIe Gen3 x4** (up to 2 TB) — a Gen4 drive buys nothing here. One free 2.5" SATA bay, up to 2 TB. + +With 32 GB and a 1 TB NVMe it becomes a legitimate Proxmox node, roughly on par with utility. + +Three caveats before committing it: + +- **Non-vPro** (`2RC2N : INFO,INTEL,N-VPRO,BASE`). No Intel AMT, so no out-of-band management — it lands in the same hole as the rest of the fleet, where a wedged node means a physical visit. Only in-band management is configured, and `817-BBSI` shows system monitoring was not selected either. +- **The 2.5" bay needs parts that did not ship.** The config carries `379-BBCY : No Additional Cable`, so the SATA cable and drive bracket are absent. Source them at purchase time; they are annoying to find later. +- **Single 1 GbE Realtek NIC** (`K7F14 : SRV,DRVR,REALTEK,LOM`). Not a concern in practice — utility already loads `r8169` and runs fine. + +Dell service-tag lookups cannot be automated from here: their support site is behind Akamai bot protection and returns `Access Denied` to plain fetches, to their own JSON product-selector endpoints, and to a real headless Chromium with a full browser fingerprint. Their TechDirect API would work but needs an OAuth key we do not hold. Either read the tag in a browser, or skip Dell entirely and run `dmidecode` on the machine, which yields more than the support page does. + +Service tags already read off the fleet: utility `MJ0LZNYT` and data `MZ010LPV` (both Lenovo 11JN002RUS), cloud `MJ0LQCGJ` (Lenovo 11T3000RUS), media `58SM6X3` (Dell OptiPlex Micro 7020). + +--- + ## Ceph — evaluated, not adopted Goal was migration freedom: move guests between nodes at will and rebuild nodes as needed. diff --git a/vault/projects/navi-recon-separation.md b/vault/projects/navi-recon-separation.md index 338d9bb..7c9b307 100644 --- a/vault/projects/navi-recon-separation.md +++ b/vault/projects/navi-recon-separation.md @@ -94,3 +94,14 @@ A 250-mile disc is 80,400² ≈ 6.5 billion cells; full-grid is quadratic in rad Physical note: with 4/3-Earth refraction, `d(km) = 4.12(√h₁ + √h₂)`. A 250-mile path needs ~2,380 m at both ends — summit-to-summit only. Realistic siting at 1,500 m gives about 200 miles station-to-station. This budget does not fit on data, which is capped at 32 GB and already runs the whole geo stack. It fits comfortably on media at 64 GB. See [[fleet-storage-memory-upgrade]]. + +--- + +## Execution plan + +Decided 2026-08-15: **lift the whole VM and strip each copy**, rather than rebuilding navi's ~20 services from scratch. Step-by-step procedure with rollback points is [[navi-lift-to-media]]. + +Two findings that made this tractable, both verified rather than assumed: + +- **The front door is a single upstream.** Caddy on utility CT 101 sends all of `navi.echo6.co` to `100.64.0.24:8440` (nginx inside recon-vm, which fans out to the gunicorns, Valhalla, Nominatim and Photon). Cutover is four `reverse_proxy` lines in one Caddyfile block — no DNS change, no Authentik change. `apache2` on the VM is only a default vhost and fronts nothing. +- **recon has no dependency on navi.** Grepping `/opt/recon` for every navi port (84xx, 8002, 8010, 2322) and hostname returns nothing, so splitting them creates no cross-host call path. diff --git a/vault/runbooks/corescope-ingest-stall-oom.md b/vault/runbooks/corescope-ingest-stall-oom.md new file mode 100644 index 0000000..4a53776 --- /dev/null +++ b/vault/runbooks/corescope-ingest-stall-oom.md @@ -0,0 +1,136 @@ +--- +title: CoreScope Silent Ingest Stall — Diagnosis and Fix +type: runbook +tags: + - mesh + - proxmox +aliases: [] +related: + - [[services]] + - [[ct-runbook]] + - [[peertube-sitemap-redis-oom]] + - [[environment]] + - [[meshcore-transport]] +updated: 2026-08-15 +--- +# CoreScope Silent Ingest Stall — Diagnosis and Fix + +CoreScope stops recording MeshCore packets while the website stays up and looks perfectly healthy. It wedged twice this way (2026-08-07 and 2026-08-08), the second time for five days before anyone noticed. Root cause found and fixed 2026-08-13. + +CoreScope is a MeshCore packet analyser running as a Docker container on **CT 100 on utility**, alongside MeshMonitor and PotatoMesh. Public at `corescope.idahomesh.com` and embedded at `idahomesh.com/corescope`. Contributors feed it over MQTT. + +--- + +## Signature + +The dangerous part is that nothing looks broken. The container reports `Up N days`, the web UI serves data, WebSocket clients connect, and the five-minute stats line keeps printing — **with identical numbers forever**. + +```bash +pct exec 100 -- docker logs --since 30m corescope 2>&1 | grep '\[stats\]' +``` + +Frozen `tx_inserted` / `tx_dupes` / `obs_inserted` across consecutive ticks is the one reliable tell. Judge whether the counter is **moving**, not whether it is non-zero. + +Corroborate from outside: at `https://corescope.idahomesh.com/api/observers`, every observer shares the same `last_seen` and reports `packetsLastHour: 0`. If all of them stop at the same instant, the fault is ours, not the contributors'. + +Earlier in the log you will find the MQTT client giving up: + +``` +MQTT [local] disconnected from tcp://localhost:1883: pingresp not received, disconnecting +MQTT [local] WATCHDOG: client reports connected but no messages received for 5m0s +MQTT [local] WATCHDOG forcing reconnect (half-open TCP suspected) +MQTT [local] WATCHDOG reconnect attempt issued + <- then silence, and no "subscribed to meshcore/#" ever again +``` + +The watchdog falling quiet is **not** recovery. It tried three times and stopped. + +### Prove the feed is fine before blaming contributors + +Subscribe to the broker directly inside the container. If packets are arriving here while `tx_inserted` is frozen, the broker, the edge2 relay and the contributors are all healthy and the ingestor alone is wedged. + +```bash +pct exec 100 -- docker exec corescope \ + timeout 25 mosquitto_sub -h 127.0.0.1 -p 1883 -t 'meshcore/#' -v +``` + +--- + +## Cause + +Not MQTT, and not the disk. + +`corescope-server` keeps a seven-day packet store **in memory**. Upstream's default budget is `packetStore.maxMemoryMB: 1024`, from which it derives a Go soft limit of 1.5x = **1536 MiB**. CT 100 was a **2 GB** container also running MeshMonitor, two PotatoMesh containers, Mosquitto and the ingestor. That budget never fit. + +The server creeps to roughly 1.8 GB RSS, the cgroup falls into reclaim thrash, and everything inside CT 100 crawls for about 1h45m. Measured during the second event: a single SQLite `InsertTransmission` took **6,449 seconds**, and a neighbour-build tick took 1h46m. While that is happening the ingestor cannot service its MQTT keepalive, so the connection drops and the reconnects cannot complete either. + +Then the kernel kills the largest process: + +``` +Aug 06 23:48:55 Memory cgroup out of memory: Killed process (corescope-serve) anon-rss:1776728kB +Aug 08 17:41:15 Memory cgroup out of memory: Killed process (corescope-serve) anon-rss:1840364kB +``` + +(Host journal is in local time; those are 08-07 05:48 and 08-08 23:41 UTC, matching both stalls to the second.) + +**Why it never self-heals:** supervisord runs `corescope-server` and `corescope-ingestor` as two separate programs. The OOM killer takes the *server*, and supervisord dutifully restarts only that one. The ingestor is never killed, so it is never restarted — it sits wedged while the UI comes back looking healthy. + +**Why it stays dead for days:** once ingest stops, memory stops growing, so there is no second OOM to force the issue. The failure is self-limiting and silent. Time to failure was about eight hours of ingest after a fresh start. + +The recurring `[db] WAL checkpoint error: disk I/O error (778)` messages are **swap thrash, not failing storage**. Do not chase the disk. There are no vzdump jobs on utility. + +--- + +## Fix + +Raise the container's memory. LXC applies this **live** — no restart, nothing drops. + +```bash +pct set 100 -memory 6144 +pct exec 100 -- free -m # expect 6144 total +``` + +Check host headroom first: utility has 30 GB, and although CT 104 is allocated 12 GB it is retired and idle. + +CoreScope has **no Docker-level memory cap** (`HostConfig.Memory=0`, cgroup `max`), so it inherits the container's limit — which is why raising the CT is what actually moves its ceiling. + +```bash +pct exec 100 -- docker inspect corescope --format 'Memory={{.HostConfig.Memory}}' +pct exec 100 -- docker exec corescope cat /sys/fs/cgroup/memory.max +``` + +**`docker stats` still prints "/2GiB" for uncapped containers** — that is a stale daemon reading of host RAM, purely cosmetic. Verify with `docker inspect` and `memory.max`, never with `docker stats`. + +### Clearing a stall that has already happened + +```bash +pct exec 100 -- docker restart corescope +``` + +Plain `docker run` container with `--restart unless-stopped` and no compose project, so restart rather than recreate is correct. Recovery is immediate. Look for `Running — 1 MQTT source(s) connected` and, critically, **`MQTT [local] subscribed to meshcore/#`** — that subscribe line is the real proof, and it is what was missing the whole time. + +Counters reset to zero because they are per-process. **Climbing from zero is success, not data loss.** + +--- + +## Verify + +```bash +pct exec 100 -- docker logs --since 20m corescope 2>&1 | grep '\[stats\]' | tail -3 +pct exec 100 -- free -m +journalctl -k --since '4 hours ago' | grep -c 'Memory cgroup out of memory' +``` + +Healthy means `tx_inserted` increasing across consecutive ticks and an OOM count of zero. + +Monitored every three hours for 26 hours after the fix: ingest never stalled, memory stayed flat at about 1.87 GB of 6 GB, zero OOM kills. Against a failure that previously arrived within eight hours, that is the fix holding rather than the clock being reset. + +--- + +## If it ever recurs + +More memory raises the ceiling; it does not cap growth. The next lever is to bound CoreScope itself in `/opt/corescope-data/config.json` — `packetStore.maxMemoryMB` around 320, `retentionHours` 48, and `runtime.maxMemoryMB` around 512 (that last one applies to **both** the server and the ingestor). The config is a **partial override**, so adding keys will not disturb `mqttSources`. Upstream documents this deployment case in issues #836 and #1010. + +A flat-`tx_inserted` self-heal watchdog on CT 100 is worth building if the pattern returns, since the container's own watchdog demonstrably stops trying. + +Same family of failure as [[peertube-sitemap-redis-oom]] — a cgroup ceiling far below what the application assumes, presenting as something else entirely. diff --git a/vault/runbooks/edge2-access-reference.md b/vault/runbooks/edge2-access-reference.md index f5af62a..c66003d 100644 --- a/vault/runbooks/edge2-access-reference.md +++ b/vault/runbooks/edge2-access-reference.md @@ -10,31 +10,42 @@ related: - [[lxc-service-migration]] - [[authentik]] - [[headscale-onboard-node]] -updated: 2026-07-13 + - [[edge2-boot-recovery]] +updated: 2026-08-15 --- # edge2 Access Reference ## SSH Access -edge2 is hardened differently from home-cluster Proxmox nodes. +edge2 is hardened differently from home-cluster Proxmox nodes. If you are here because edge2 rebooted and its services are returning 502, go to [[edge2-boot-recovery]] instead. | Property | edge2 | Home Proxmox (data, utility, etc.) | |----------|-------|------------------------------------| | SSH user | `admin` (not root) | `root` or `zvx` | | Auth method | Key-only | Key or password | -| PasswordAuthentication | `no` (hardened) | varies | -| Root login | Disabled | Allowed | +| PasswordAuthentication | **effectively `yes`** — see below | varies | +| Root login | Key-only (`without-password`) | Allowed | | Sudo | Passwordless for admin | N/A (already root) | | SSH alias | `ssh edge2` | `ssh zvx@` | ### SSH config entry +On cortex there are two aliases — `edge2` goes over the public IP, `ts-edge2` over the tailnet. Both use the contabo2 key. + ``` Host edge2 + HostName 184.174.35.153 + User admin + IdentityFile ~/.ssh/contabo2_ed25519 + +Host ts-edge2 HostName 100.64.0.26 User admin + IdentityFile ~/.ssh/contabo2_ed25519 ``` +`ssh root@100.64.0.26` does **not** work — root login is key-only and root has no authorized key. Use the alias. + ### Authorized keys Located at `/home/admin/.ssh/authorized_keys` on edge2: @@ -46,13 +57,26 @@ Located at `/home/admin/.ssh/authorized_keys` on edge2: | matt-desktop-wsl2 | WSL2 default key | WSL/Claude sessions | | cookie-sync | Windows SSH key | Native Windows terminal | -### Known gotcha: cloud-init conflict +### Known gotcha: cloud-init conflict — the drop-in WINS (corrected 2026-08-15) -`/etc/ssh/sshd_config.d/50-cloud-init.conf` has `PasswordAuthentication yes`, which contradicts the hardened `no` in the main config. The main config wins (loaded first), but this inconsistency should be cleaned up: +`/etc/ssh/sshd_config.d/50-cloud-init.conf` sets `PasswordAuthentication yes`, contradicting the hardened `no` in the main config. **An earlier version of this page claimed the main config wins. It does not.** sshd takes the *first* value it obtains, and `Include` sits at **line 12** while `PasswordAuthentication no` is at **line 57** — so the drop-in is read first and its `yes` wins. + +Verified on edge2: + +``` +$ sudo sshd -T | grep -iE 'passwordauthentication|permitrootlogin' +permitrootlogin without-password +passwordauthentication yes +``` + +So password authentication is live on edge2's public IP. Root is still key-only (`without-password`), which is why the constant internet brute-force against root in `journalctl -u ssh` never succeeds — but non-root accounts are exposed to password guessing. Always confirm with `sshd -T` (effective config) rather than reading `sshd_config` and assuming. + +Fix: ```bash ssh edge2 sudo rm /etc/ssh/sshd_config.d/50-cloud-init.conf +sudo sshd -T | grep -i passwordauthentication # expect: no sudo systemctl reload sshd ``` @@ -68,6 +92,15 @@ The PVE API is useful when SSH is unavailable (e.g., before keys are authorized) ### Container management via SSH +**`pct` needs sudo, and the failure is not an obvious permission error.** Running it as `admin` fails with pmxcfs IPC noise that reads like a broken cluster: + +``` +ipcc_send_rec[1] failed: Unknown error -1 +Unable to load access control list: Unknown error -1 +``` + +That means "you are not root", not "PVE is broken". Use `sudo -n` for every `pct` and in-container `systemctl` call. + ```bash # List CTs ssh edge2 'sudo pct list' diff --git a/vault/runbooks/edge2-boot-recovery.md b/vault/runbooks/edge2-boot-recovery.md new file mode 100644 index 0000000..b0e8eb5 --- /dev/null +++ b/vault/runbooks/edge2-boot-recovery.md @@ -0,0 +1,155 @@ +--- +title: edge2 Boot Recovery — compose-reconcile Tailscale Race +type: runbook +tags: + - proxmox + - auth +aliases: [] +related: + - [[edge2-access-reference]] + - [[expose-service-edge2]] + - [[authentik]] + - [[caddy]] + - [[ct-runbook]] + - [[deploy-livesync]] + - [[matrix-synapse-deployment]] +updated: 2026-08-15 +--- +# edge2 Boot Recovery — compose-reconcile Tailscale Race + +edge2 reboots and most of its services never come back. Every vhost behind it returns 502 while all nine containers report `running`. Hit on 2026-08-14 for 2h05m. Root cause found and fixed the same day. + +The June 2026 `compose-reconcile.service` was built to prevent exactly this and had never been tested by a real reboot. This was its first, and it lost. + +--- + +## Signature + +``` +auth 502 forge 502 notes 502 vault 502 matrix 502 element 502 +vpn 200 <- headscale is not Docker, so it is unaffected +``` + +Home services through the utility [[caddy]] (ai, jellyfin, immich, nextcloud) stay healthy — the blast radius is edge2 only. + +`pct list` shows every CT `running`, which is misleading. Look inside them: + +```bash +ssh edge2 'sudo -n pct exec 105 -- docker ps -a --format "{{.Names}} | {{.Status}}"' +``` + +- CT 102 vaultwarden, CT 103 forgejo, CT 104 livesync: containers stuck in **`Created`**, never started +- CT 105 authentik, CT 106 matrix: app containers crash-looping on `Temporary failure in name resolution` for their compose-sibling database hostnames (`postgresql`, `matrix-postgres`) + +**[[authentik]] being down 502s everything behind forward-auth**, including services that are themselves perfectly healthy. On 2026-08-14 this is how the outage surfaced: `mesh.echo6.co` returned 502, but MeshMonitor was fine the entire time — only its login was broken. Check `auth.echo6.co` before diagnosing any individual service. + +--- + +## Cause + +The reconcile unit fired in every container and failed in every one: + +``` +failed to bind host port 100.64.0.36:9000/tcp: cannot assign requested address (authentik) +failed to bind host port 100.64.0.33:8086/tcp: cannot assign requested address (vaultwarden) +``` + +The compose files publish ports on each container's **Tailscale** address. The unit's `ExecStartPre` waited only for `docker info` — not for `tailscale0` to have its address. It ran about two minutes after boot, the address did not exist yet, the bind failed, and with **no retry** the oneshot exited 1 and gave up permanently. + +```bash +ssh edge2 'sudo -n pct exec 105 -- systemctl status compose-reconcile.service' +ssh edge2 'sudo -n pct exec 105 -- journalctl -u compose-reconcile.service -b' +``` + +--- + +## Fix (applied 2026-08-15) + +Deployed to all six Docker CTs — 102 vaultwarden, 103 forgejo, 104 livesync, 105 authentik, 106 matrix, 107 headscale. Originals backed up in place as `compose-reconcile.service.bak-`. + +- `After=` / `Wants=` now include **`tailscaled.service`** +- A second `ExecStartPre` waits for `tailscale0` to hold an IPv4, **bounded at 180 tries** so it can never hang a boot +- `ExecStart` retries `docker compose up -d` **six times at ten seconds**, giving roughly four minutes of total tolerance +- `TimeoutStartSec=900` + +```ini +ExecStartPre=/bin/sh -c 'until docker info >/dev/null 2>&1; do sleep 1; done' +ExecStartPre=/bin/sh -c 'n=0; until ip -4 addr show tailscale0 2>/dev/null | grep -q "inet "; do n=$$((n+1)); if [ $$n -ge 180 ]; then break; fi; sleep 1; done' +ExecStart=/bin/sh -c 'docker compose down --remove-orphans || true; n=0; until docker compose up -d; do n=$$((n+1)); if [ $$n -ge 6 ]; then exit 1; fi; sleep 10; done' +``` + +Two traps worth knowing before editing this unit: + +**`Restart=on-failure` is not usable.** systemd forbids `Restart=` on `Type=oneshot`. That is why the retry lives inside `ExecStart` instead. Do not "fix" it by adding `Restart=`. + +**A `$` in a unit file must be written `$$`** to reach the shell literally. Get it wrong and the loop counter silently expands to nothing. + +`WorkingDirectory` differs per container (`/opt/authentik`, `/opt/vaultwarden`, and so on) — preserve each one when rewriting. + +--- + +## Manual recovery + +If it wedges again before the unit is trusted, re-run the reconcile. Once Tailscale is up the binds succeed and it heals in seconds. + +```bash +ssh edge2 'sudo -n pct exec 105 -- systemctl restart compose-reconcile.service' # authentik FIRST +# then 102, 103, 104, 106 +``` + +**Do CT 105 first** — clearing [[authentik]] lifts the 502 cascade off everything else. + +The underlying heal is `docker compose down --remove-orphans && docker compose up -d`. A plain `up -d` does **not** work: compose sees the containers "Up" and the network "present" and concludes state has converged. **Never use `-v`** — it would wipe the CouchDB, Postgres and Vault volumes. + +--- + +## Testing this without rebooting the host + +Reboot a single container. That exercises the real race — `tailscaled` against Docker at container boot — with no risk to the host. + +```bash +ssh edge2 'sudo -n pct reboot 103' +ssh edge2 'sudo -n pct exec 103 -- systemctl show compose-reconcile.service -p Result -p ExecMainStatus' +``` + +Proven this way on 2026-08-15: `Result=success`, `ExecMainStatus=0`, containers up, `forge.echo6.co` 200. **Never reboot the edge2 host to test this.** + +--- + +## Verify + +```bash +for u in auth forge notes vault matrix element vpn; do + printf '%-8s ' "$u"; curl -s -o /dev/null -m 15 -w 'HTTP %{http_code}\n' "https://$u.echo6.co/" +done +``` + +Healthy is `auth` 302 (login redirect), `notes` 401 (CouchDB auth), and 200 for the rest. A 302 from a forward-auth vhost is correct behaviour, not a fault. + +--- + +## What triggered it: the 2026-08-14 hard reset + +edge2 died mid-operation at **01:12:45 UTC** after 46 days of uptime and returned 26 seconds later. Nothing inside the guest asked for it or recorded a fault. Ruled out with evidence: + +- **No reboot issued** — no shutdown record in `last -x`; the PVE task log has no `stopall`, only a boot-time `startall` +- **No clean shutdown** — zero shutdown markers in the final five minutes; the journal stops mid-stream in routine sshd and tailscaled traffic +- **No panic, oops, OOM, I/O error, hung task or soft lockup** anywhere in the previous boot; the last kernel message was a routine hourly `drop_caches` 56 minutes earlier +- **Not the watchdog** — edge2 is not clustered, so there is no HA and `/dev/watchdog` was never armed +- **No resource pressure** — last full sample before death: load 3.38, CPU 15.9%, iowait 2.43%, memory 6.2 GB of 25 GB +- **Not datacenter-wide** — edge1, the other Contabo VPS, had been up since 2026-06-19 and sailed straight through +- **No migration signature** — identical CPU (AMD EPYC family 0x17 model 0x1 stepping 0x2) and memory map before and after + +The boot was a full cold SeaBIOS re-init, so the VM was destroyed and recreated at hypervisor level. Contabo's status page listed no incident, but they do not publish single-node resets. + +Honest caveat: a guest kernel panic cannot be fully excluded. `kernel.panic = 10` is set and the 26-second gap fits panic then reset. A hard panic can lose the journal tail. Nothing supports it, and the only discriminator is console output, which only Contabo's panel retains. + +One suggestive detail: about two minutes of external network flakiness immediately beforehand — tailscaled's IPv6 flapped at 01:10:52, a logtail upload failed at 01:11:26 and took 55 seconds to recover, then death at 01:12:45. + +Closing it needs Contabo's server history, or a ticket asking why `184.174.35.153` was reset at 01:12:45 UTC on 2026-08-14. + +**Gap this exposed:** edge2 has no sysstat or atop, so only 30-minute PVE RRD samples exist for that window — too coarse to catch a short spike. `rrdtool` CLI is not installed either; use the API instead, which needs no install: + +```bash +ssh edge2 'sudo -n pvesh get /nodes/edge2/rrddata --timeframe day --output-format json' +``` diff --git a/vault/runbooks/navi-lift-to-media.md b/vault/runbooks/navi-lift-to-media.md new file mode 100644 index 0000000..6ff5021 --- /dev/null +++ b/vault/runbooks/navi-lift-to-media.md @@ -0,0 +1,194 @@ +--- +title: Lift navi off recon-vm onto media +type: runbook +tags: + - proxmox + - recon +aliases: [] +related: [] +updated: 2026-08-15 +--- +# Lift navi off recon-vm onto media + +**Status: planned, not executed.** Nothing below has been run. + +[[navi]] and [[recon]] share one VM (`recon-vm`, VMID 1130 on data), so neither can move without the other — see [[navi-recon-separation]]. This lifts the whole VM to media as a clone, then strips each copy down to one platform. No services are rebuilt. + +Target is media, which now has 64 GB RAM and a 1.82 TB NVMe volume — see [[fleet-storage-memory-upgrade]]. + +--- + +## Why lift rather than rebuild + +navi is ~20 interlocking services: seven `navi-*` gunicorns, nginx:8440, Valhalla and Nominatim in Docker, Photon (Java), and PostgreSQL 16 whose `data_directory` sits at `/mnt/nav/overture/pgdata`. Standing that up fresh means reproducing every version, path and config. Cloning carries all of it intact; you then delete what doesn't belong. + +Cost: the clone inherits recon's files too, which you remove afterwards, and it inherits the source VM's identity, which **must** be fixed before it ever touches the network. + +--- + +## Facts this plan relies on + +| Item | Value | +|---|---| +| Source | `recon-vm` VMID 1130 on data, 4 cores, 24 GB, 180 GB disk | +| New VMID | **1131** (next free; 1130 is the highest in use) | +| navi data | `/mnt/data/nav` — **625 GB** | +| Front door | Caddy on utility CT 101 → `100.64.0.24:8440` (nginx in recon-vm) | +| Auth | Authentik forward_auth at the Caddy layer — unchanged by this | +| Space needed on media | ~180 GB system + 625 GB data = **~805 GB** of 1.82 TB | + +**recon does not call navi.** Grepping `/opt/recon` for navi ports (84xx, 8002, 8010, 2322) and hostnames returns nothing. The split creates no cross-host dependency. + +**Stays on data:** `recon.service` (8420), `recon-watchdog`, `kiwix-serve` (8430), and the `library` + `kiwix` shares. + +--- + +## The three traps + +**1. Identity collision.** The clone is byte-identical: same static IP `192.168.1.130`, same hostname, same `machine-id`, same Tailscale node key. Booting it on the network unfixed will fight the original for its IP and its tailnet identity. **Fix identity before first network boot** (step 3). + +**2. The virtiofs `args:` will not start on media.** `recon-vm` carries raw QEMU args wiring `vhost-user-fs-pci` devices to `/run/virtiofsd-{nav,kiwix,library}.sock`, plus a `memory-backend-memfd` object that virtiofs requires. Those sockets do not exist on media, so the clone fails to start as-is. + +Rather than recreate virtiofsd on media, **give the new VM a real virtual data disk**. That makes it self-contained — no host-side sockets, no raw args — and therefore movable with [[pve-guest-park-and-adopt]], which the current VM is not. + +**3. PostgreSQL moves with the data.** `pgdata` lives inside `/mnt/nav`, so it travels with the 625 GB. Nothing separate to dump and restore, but the copy must be consistent — stop postgres before the final sync (step 5). + +--- + +## 1. Back up the source + +```bash +ssh ts-data +vzdump 1130 --mode snapshot --storage pinas-backup --compress zstd +``` + +`--mode snapshot` runs live — no downtime. This is both the rollback point and the clone source. Note the archive name. + +## 2. Restore as VMID 1131 on media + +```bash +ssh ts-media +qmrestore pinas-backup:backup/vzdump-qemu-1130-.vma.zst 1131 --storage media-nvme +``` + +`media-nvme` is the 1.82 TB volume. **Do not start it yet.** + +## 3. Fix identity before it ever boots on the network + +```bash +qm set 1131 --name navi-vm +qm set 1131 --ipconfig0 ip=192.168.1.132/24,gw=192.168.1.1 +qm set 1131 --memory 32768 # navi + viewshed headroom +qm set 1131 --delete args # drop virtiofs wiring +``` + +Removing `args` also removes the `memory-backend-memfd` object, which is only needed for virtiofs. + +Then boot **with the NIC detached**, and from the console clear the inherited identity: + +```bash +qm set 1131 --net0 virtio,bridge=vmbr0,link_down=1 +qm start 1131 +# on the console: +tailscale logout && systemctl stop tailscaled +rm -f /var/lib/tailscale/tailscaled.state +hostnamectl set-hostname navi-vm +rm -f /etc/machine-id && systemd-machine-id-setup +``` + +Only then reattach the NIC (`qm set 1131 --net0 virtio,bridge=vmbr0`) and re-register Tailscale as a new node. + +## 4. Attach and prepare the data disk + +```bash +qm set 1131 --scsi1 media-nvme:700,format=qcow2 +# inside the guest: +mkfs.ext4 -L nav /dev/sdb +mkdir -p /mnt/nav +echo "LABEL=nav /mnt/nav ext4 defaults,noatime,nofail 0 2" >> /etc/fstab +mount /mnt/nav +``` + +700 GB gives the 625 GB room to grow. Also remove the three virtiofs lines from the guest's `/etc/fstab`, which will otherwise fail on boot. + +## 5. Copy the 625 GB + +First pass live, while navi keeps serving from data: + +```bash +rsync -aHAX --info=progress2 zvx@100.64.0.24:/mnt/nav/ /mnt/nav/ +``` + +Expect 2–3 hours over 1 GbE. Then stop the writers on the source and take a final delta: + +```bash +# on recon-vm +systemctl stop navi-geo navi-landclass navi-offroute navi-places \ + navi-contacts navi-admin navi-config photon argus-resolver +docker stop nominatim-v5 valhalla +systemctl stop postgresql@16-main +# on navi-vm +rsync -aHAX --delete zvx@100.64.0.24:/mnt/nav/ /mnt/nav/ +``` + +The postgres stop is what makes `pgdata` consistent. Do not skip it. + +## 6. Strip each side + +On **navi-vm** (1131) — remove recon and kiwix: + +```bash +systemctl disable --now recon recon-watchdog kiwix +rm -rf /opt/recon +``` + +On **recon-vm** (1130) — remove navi and the geo stack: + +```bash +systemctl disable --now navi-geo navi-landclass navi-offroute navi-places \ + navi-contacts navi-admin navi-config photon argus-resolver postgresql@16-main +docker rm -f nominatim-v5 valhalla +``` + +Leave `/mnt/data/nav` on data untouched until step 9. + +## 7. Cutover — one line + +In the Caddyfile on utility CT 101, inside the `navi.echo6.co` block, repoint every `reverse_proxy` from `100.64.0.24:8440` to the new VM's tailnet address on 8440. There are four occurrences (`@authed_api`, `@public_api`, `@valhalla`, and the fallthrough `handle`). + +```bash +ssh ts-utility +pct exec 101 -- caddy validate --config /etc/caddy/Caddyfile +pct exec 101 -- systemctl reload caddy +``` + +DNS and Authentik are untouched — the hostname and auth flow do not change. + +## 8. Verify + +Confirm navi answers through the front door, not just that the VM runs — check the map loads, a geocode returns, and a route computes. Follow [[headless-browser-page-verification]] for the UI. Then confirm recon is still healthy on 8420 and kiwix on 8430. + +## 9. Reclaim — a week later, not the same day + +Once navi has been healthy on media for several days: + +```bash +# on data +rm -rf /mnt/data/nav +systemctl disable --now virtiofsd-nav +qm set 1130 --memory 8192 # recon alone needs far less than 24 GB +``` + +That drops `/mnt/data` from 93% to roughly 21% and frees ~16 GB of RAM on data — which matters, since data is hard-capped at 32 GB. + +--- + +## Rollback + +| Stage | Recovery | +|---|---| +| Any point before step 7 | Nothing changed for users — navi still serves from data | +| After cutover | Revert the four Caddy lines to `100.64.0.24:8440`, restart navi's services on recon-vm | +| Catastrophic | `qmrestore` the step 1 archive from `pinas-backup` | + +The rollback path stays open until step 9 deletes the source data. Do not run step 9 early. diff --git a/vault/runbooks/peertube-sitemap-redis-oom.md b/vault/runbooks/peertube-sitemap-redis-oom.md index eb7bc2a..4139325 100644 --- a/vault/runbooks/peertube-sitemap-redis-oom.md +++ b/vault/runbooks/peertube-sitemap-redis-oom.md @@ -141,3 +141,5 @@ Post-fix: load 136 → 2.6, io.pressure 99% → 5%, swap 0, D-state 0, OOM kills 8 GB raises the ceiling; it does not stop growth. At ~136K videos and climbing, revisit if the library grows substantially. Fetching `/sitemap.xml` to measure it mints a fresh 402 MB redis entry, so do not casually curl it. Related pipeline failure modes: [[peertube-remote-runner]], [[add-peertube-channel]]. + +The same shape — a cgroup ceiling well below what the application assumes, presenting as something other than memory — bit CoreScope on utility. See [[corescope-ingest-stall-oom]]. Worth checking any container whose app keeps a large in-memory cache against its `memory.max`.