From 4cfebfc47be07432ad4e4eba663a736b57a5908a Mon Sep 17 00:00:00 2001 From: echo6-autocommit Date: Mon, 22 Jun 2026 18:00:09 +0000 Subject: [PATCH] auto: docs sync 2026-06-22T18:00:09+00:00 Files changed: engine/lint-report.md vault/.obsidian/workspace.json vault/runbooks/fleet-magicdns-resolved-migration.md --- engine/lint-report.md | 26 +- vault/.obsidian/workspace.json | 24 +- .../fleet-magicdns-resolved-migration.md | 464 ++++++++++++++++++ 3 files changed, 489 insertions(+), 25 deletions(-) create mode 100644 vault/runbooks/fleet-magicdns-resolved-migration.md diff --git a/engine/lint-report.md b/engine/lint-report.md index fb3ebcf..378cc8e 100644 --- a/engine/lint-report.md +++ b/engine/lint-report.md @@ -1,6 +1,6 @@ # Vault Lint Report -Generated: 2026-06-22T06:00:09Z | Docs scanned: 93 | Elapsed: 0.0s +Generated: 2026-06-22T12:00:09Z | Docs scanned: 94 | Elapsed: 0.0s ## Summary @@ -38,6 +38,7 @@ _None. All wikilinks resolve._ - no incoming links: runbooks/expose-service-contabo.md - no incoming links: runbooks/expose-service-edge2.md - no incoming links: runbooks/expose-service-home.md +- no incoming links: projects/fleet-platform-baseline.md - no incoming links: docs/software/geo-tools.md - no incoming links: glossary.md - no incoming links: runbooks/headscale-onboard-node.md @@ -63,7 +64,6 @@ _None. All wikilinks resolve._ - no incoming links: session-resume/SESSION-HANDOFF-meshai-test.md - no incoming links: docs/matrix/synapse_retention_discovery.md - no incoming links: runbooks/syncthing-add-node.md -- no incoming links: runbooks/toc-cortex-pve9.2-update.md - no incoming links: plans/vaultwarden-plan.md ## Gaps & suggestions @@ -94,18 +94,18 @@ Matt decides whether to create a real doc — when he does, future sweeps will l | Term | Docs mentioning it | |------|--------------------| -| `tailscale` | 33 | +| `tailscale` | 34 | | `docker` | 31 | -| `proxmox` | 30 | -| `headscale` | 22 | -| `peertube` | 16 | +| `proxmox` | 31 | +| `headscale` | 23 | +| `peertube` | 17 | +| `mailcow` | 16 | | `meshtastic` | 15 | -| `mailcow` | 15 | | `element` | 13 | -| `forgejo` | 11 | -| `immich` | 11 | -| `nextcloud` | 11 | -| `vaultwarden` | 11 | +| `forgejo` | 12 | +| `immich` | 12 | +| `nextcloud` | 12 | +| `vaultwarden` | 12 | +| `livesync` | 11 | +| `jellyfin` | 11 | | `headplane` | 10 | -| `livesync` | 10 | -| `jellyfin` | 10 | diff --git a/vault/.obsidian/workspace.json b/vault/.obsidian/workspace.json index a49f73a..30e1feb 100644 --- a/vault/.obsidian/workspace.json +++ b/vault/.obsidian/workspace.json @@ -199,17 +199,19 @@ }, "active": "17bd4a6166f789d0", "lastOpenFiles": [ + "runbooks/fleet-magicdns-resolved-migration.md.tmp.5281.32d6e661d1df", + "runbooks/fleet-magicdns-resolved-migration.md.tmp.5281.e72f1c289265", + "runbooks/fleet-magicdns-resolved-migration.md.tmp.5281.5b1645835a01", + "runbooks/fleet-magicdns-resolved-migration.md.tmp.5281.d0e7c45a9ac5", + "runbooks/fleet-magicdns-resolved-migration.md.tmp.5281.96dd988d42a7", + "runbooks/fleet-magicdns-resolved-migration.md.tmp.5281.285d3829268c", + "runbooks/fleet-magicdns-resolved-migration.md.tmp.5281.3ff04dc232a4", + "runbooks/fleet-magicdns-resolved-migration.md.tmp.5281.cebede70f052", + "runbooks/fleet-magicdns-resolved-migration.md.tmp.5281.597a5c023d8a", + "runbooks/fleet-magicdns-resolved-migration.md.tmp.5281.dada4d7c2ef2", + "runbooks/fleet-magicdns-resolved-migration.md.tmp.5281.852dd5a9449f", + "runbooks/fleet-magicdns-resolved-migration.md", "projects/fleet-platform-baseline.md", - "projects/fleet-platform-baseline.md.tmp.5281.0e8c5dd2a601", - "projects/fleet-patch-audit.md.tmp.5281.827293ce77a8", - "projects/fleet-patch-audit.md.tmp.5281.3206bbe8ba4d", - "projects/fleet-patch-audit.md.tmp.5281.5d6a0ab80754", - "projects/fleet-patch-audit.md.tmp.5281.8edd2d841378", - "projects/fleet-patch-audit.md.tmp.5281.bc624a70230f", - "projects/fleet-patch-audit.md.tmp.5281.22de50591aae", - "projects/fleet-patch-audit.md.tmp.1493418.7b411319326d", - "projects/fleet-patch-audit.md.tmp.1493418.c97c38bf6380", - "projects/fleet-patch-audit.md.tmp.1493418.36abe7516917", "runbooks/toc-cortex-pve9.2-update.md", "projects/nominatim-v5-reimport.md", "2026-06-19.md", @@ -234,8 +236,6 @@ "glossary.md", "concepts/youtube.md", "concepts/tailscale-ssh.md", - "concepts/reverse-proxy.md", - "concepts/raspberry-pi.md", "assets/echo6yellow_logo_422x422_square.png", "assets/echo6yellow_logo_422x81.png", "assets/echo6_logo.png", diff --git a/vault/runbooks/fleet-magicdns-resolved-migration.md b/vault/runbooks/fleet-magicdns-resolved-migration.md new file mode 100644 index 0000000..e440ec9 --- /dev/null +++ b/vault/runbooks/fleet-magicdns-resolved-migration.md @@ -0,0 +1,464 @@ +--- +title: Fleet MagicDNS / systemd-resolved Migration +type: runbook +tags: [dns, vpn] +related: [] +updated: 2026-06-22 +--- + +# Fleet MagicDNS / systemd-resolved Migration + +## Context / why + +Fleet guests on Tailscale had a fragile DNS setup. On LXC CTs without systemd-resolved, Tailscale owns `/etc/resolv.conf` and points **all** queries (public included) at the MagicDNS proxy `100.100.100.100`. If tailscaled loses its link, all DNS dies — including the lookup of the coordinator `vpn.echo6.co` needed to reconnect. That circular dependency is a hard brick. + +This triggered during the June 2026 patch campaign: a CT went completely offline when tailscaled dropped and couldn't resolve its way back. + +MagicDNS itself already works (Headscale `base_domain: echo6.mesh`, upstream 1.1.1.1) — the problem was fragility and lack of public-DNS fallback, not the feature itself. + +**Goal of this migration:** bring every Tailscale guest to the robust split-DNS state the two VMs (recon-vm, arr) already had: + +- systemd-resolved owns `/etc/resolv.conf` in stub mode +- Tailscale registers as a per-domain resolver for `echo6.mesh` +- A resolved global upstream (1.1.1.1/8.8.8.8) guarantees public DNS even when Tailscale is fully down +- Every guest uniform and documented + +--- + +## Gold-standard target state + +Verified on recon-vm (VM 1130). After migration, each guest must satisfy all of these: + +| Check | Expected value | +|---|---| +| `systemctl is-active systemd-resolved` | `active` | +| `systemctl is-enabled systemd-resolved` | `enabled` | +| `/etc/resolv.conf` | symlink → `../run/systemd/resolve/stub-resolv.conf` | +| `resolvectl` Global `DNSMode` | `stub` | +| `/etc/systemd/resolved.conf.d/upstream.conf` | `DNS=1.1.1.1 8.8.8.8` | +| `journalctl -u tailscaled \| grep "dns: using"` | `dns: using "systemd-resolved" mode` | +| `journalctl -u tailscaled \| grep "rc="` | `rc=resolved ... ret=systemd-resolved` | +| `resolvectl status tailscale0` DNS | `100.100.100.100` | +| `resolvectl status tailscale0` Domains | `echo6.mesh ~.` | +| `tailscale status --json \| jq .Self.CorpDNS` | `true` | +| Docker CTs: `/etc/docker/daemon.json` | `"dns": ["1.1.1.1","8.8.8.8"]` | + +Net result: `*.echo6.mesh` resolves via MagicDNS; public DNS is forwarded through Tailscale when it is up, and falls back to resolved's 1.1.1.1/8.8.8.8 when it is down. + +--- + +## Per-guest pre-flight checks + +Run these before starting. They determine which steps apply. + +### 1. PVE nameserver entry + +```bash +# On the PVE host: +grep -i nameserver /etc/pve/lxc/.conf +``` + +If a `nameserver:` line is present, clear it **before** anything else: + +```bash +sed -i "/^nameserver:/d" /etc/pve/lxc/.conf +``` + +> **Note:** `pct set --delete nameserver` silently no-ops on PVE 9.x. Use the `sed` directly on the conf file. + +### 2. Docker + +```bash +pct exec -- bash -lc "which dockerd && cat /etc/docker/daemon.json 2>/dev/null || echo MISSING" +``` + +If `dockerd` is present and `/etc/docker/daemon.json` has no `dns` pin → apply **Step 0** first. + +### 3. NordVPN + +```bash +pct exec -- bash -lc "which nordvpn && nordvpn settings 2>/dev/null || echo NOT_PRESENT" +``` + +If nordvpn is present, its kill-switch can block tailscaled from reaching the coordinator after a restart. Allowlist edge2 first: + +```bash +pct exec -- bash -lc " +nordvpn allowlist add subnet 184.174.35.153/32 +nordvpn allowlist add port 443 +" +``` + +### 4. Bucket assignment + +```bash +pct exec -- bash -lc "systemctl is-active systemd-resolved" +``` + +- `active` → **Bucket R** (already installed; reconfigure only — skip Step 1) +- `inactive` / `not-found` → **Bucket I** (needs install — run Step 1) + +--- + +## The recipe + +Commands that run inside the CT are wrapped in `pct exec -- bash -lc '...'`; commands on the PVE host are noted explicitly. + +### Step 0 — Docker DNS pin *(Docker CTs only)* + +If `/etc/docker/daemon.json` does not exist: + +```bash +pct exec -- bash -lc " +cat > /etc/docker/daemon.json <<'EOF' +{\"dns\": [\"1.1.1.1\", \"8.8.8.8\"]} +EOF +systemctl restart docker +" +``` + +If it already exists with other keys, merge the `dns` field manually — do not overwrite blindly. + +After `systemctl restart docker`, running containers will briefly restart. Expected and safe. + +> **Note on container DNS:** containers on bridge networks will show `127.0.0.11` in their own `/etc/resolv.conf`. That is Docker's embedded relay which forwards to the pinned upstream — expected behavior, not a stub. + +--- + +### Step 1 — Install systemd-resolved *(Bucket I only)* + +```bash +pct exec -- bash -lc " +apt-get update && +DEBIAN_FRONTEND=noninteractive apt-get install -y systemd-resolved && +systemctl enable --now systemd-resolved +" +``` + +This pulls 3 new packages plus a routine systemd point-release bump. It does not disrupt DNS — Tailscale still owns `/etc/resolv.conf` at this point. + +--- + +### Step 2 — Resolved upstream drop-in + +```bash +pct exec -- bash -lc " +mkdir -p /etc/systemd/resolved.conf.d +cat > /etc/systemd/resolved.conf.d/upstream.conf <<'EOF' +[Resolve] +DNS=1.1.1.1 8.8.8.8 +EOF +systemctl restart systemd-resolved +" +``` + +--- + +### Step 3 — Guard #1: tailscaled ordering + +This guard stops tailscaled racing systemd-resolved's D-Bus at boot. Without it, tailscaled starts before resolved is ready and falls back to "direct" mode — breaking the split-DNS config. + +```bash +pct exec -- bash -lc " +mkdir -p /etc/systemd/system/tailscaled.service.d +cat > /etc/systemd/system/tailscaled.service.d/resolved-ordering.conf <<'EOF' +[Unit] +Wants=systemd-resolved.service +After=systemd-resolved.service + +[Service] +ExecStartPre=/bin/bash -c \"c=0; while ! busctl status org.freedesktop.resolve1 >/dev/null 2>&1; do c=\$((c+1)); [ \$c -ge 30 ] && break; sleep 0.5; done; true\" +EOF +systemctl daemon-reload +" +``` + +--- + +### Step 4 — Guard #2: PVE resolv.conf overwrite protection + +PVE rewrites `/etc/resolv.conf` from the host at `pct start` — and may mark it immutable with `chattr +i`. This service restores the stub symlink before any container services run. + +```bash +pct exec -- bash -lc " +cat > /etc/systemd/system/fix-resolv-stub.service <<'EOF' +[Unit] +Description=Restore /etc/resolv.conf stub symlink (overwritten/immutable-flagged at container start) +DefaultDependencies=no +Before=systemd-resolved.service network-pre.target +After=local-fs-pre.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c \"mkdir -p /run/systemd/resolve && if [ ! -L /etc/resolv.conf ]; then chattr -i /etc/resolv.conf 2>/dev/null; rm -f /etc/resolv.conf && ln -s /run/systemd/resolve/stub-resolv.conf /etc/resolv.conf; fi\" + +[Install] +WantedBy=sysinit.target +EOF +systemctl enable fix-resolv-stub.service +" +``` + +> **Why both guards?** +> - Guard #1 (Step 3): tailscaled races resolved's D-Bus on boot → would silently pick "direct" mode. The `ExecStartPre` probe waits for D-Bus to be ready. +> - Guard #2 (Step 4): PVE restores its own static resolv.conf (possibly `chattr +i`) at every `pct start`, before systemd-resolved runs. Without this, the stub symlink is always gone on boot. Both were proven required on a clean guest. + +--- + +### Step 5 — Clear PVE nameserver (host, if still present) + +```bash +# On the PVE host: +sed -i "/^nameserver:/d" /etc/pve/lxc/.conf +``` + +Safe to re-run if already done in pre-flight. + +--- + +### Step 6 — Live stub swap + +Stop tailscaled **first** — while it is running it instantly rewrites `/etc/resolv.conf` the moment the symlink is removed. + +```bash +pct exec -- bash -lc " +systemctl stop tailscaled +chattr -i /etc/resolv.conf 2>/dev/null +rm -f /etc/resolv.conf +ln -s /run/systemd/resolve/stub-resolv.conf /etc/resolv.conf +systemctl restart systemd-resolved +" +``` + +--- + +### Step 7 — Restart tailscaled and verify mode + +```bash +pct exec -- bash -lc "systemctl restart tailscaled" +``` + +Wait ~8 seconds, then check: + +```bash +pct exec -- bash -lc " +journalctl -b -u tailscaled | grep -E 'rc=|dns: using' +" +``` + +Expected output includes both: +- `dns: using "systemd-resolved" mode` +- `rc=resolved ... ret=systemd-resolved` + +**Known pattern:** if `resolvectl status tailscale0` shows no DNS domain scopes after the first restart, restart tailscaled once more. The second restart reliably pushes the `echo6.mesh ~.` scope config. This was observed on all 3 baseline guests. + +```bash +pct exec -- bash -lc "systemctl restart tailscaled && sleep 8 && resolvectl status tailscale0" +``` + +--- + +### Step 8 — Enable accept-dns + +```bash +pct exec -- bash -lc " +tailscale set --accept-dns=true +" +``` + +Verify: + +```bash +pct exec -- bash -lc " +resolvectl status tailscale0 +tailscale status --json | grep -i corpDNS +" +``` + +Expected: `tailscale0` shows `Domains: echo6.mesh ~.` and `CorpDNS: true`. + +--- + +## Validation + +Run after every guest. All checks must pass before moving on. + +### 1. Resolver state + +```bash +pct exec -- bash -lc " +resolvectl status +" +``` + +Confirm: +- Global `DNSMode: stub` +- `tailscale0` DNS: `100.100.100.100` +- `tailscale0` Domains: `echo6.mesh ~.` + +### 2. Resolution checks + +```bash +pct exec -- bash -lc " +getent hosts meshmonitor.echo6.mesh +getent hosts github.com +getent hosts mirror.gcr.io +" +``` + +- `meshmonitor.echo6.mesh` → a `100.64.x.x` tailnet IP (MagicDNS working) +- `github.com` and `mirror.gcr.io` → public IPs (public DNS working) + +### 3. Kill-test — the point of this whole migration + +```bash +# Bring Tailscale down +pct exec -- bash -lc "tailscale down" + +# Public DNS must still resolve (resolved fallback) +pct exec -- bash -lc "getent hosts github.com" + +# Coordinator must still resolve — no circular brick +pct exec -- bash -lc "getent hosts vpn.echo6.co" + +# Bring Tailscale back +pct exec -- bash -lc "tailscale up" + +# After ~8s, MagicDNS must work again +sleep 8 +pct exec -- bash -lc "getent hosts meshmonitor.echo6.mesh" +``` + +All four must succeed. + +### 4. App health check + +Verify the guest's primary application is healthy (method is per-guest). + +### 5. Docker CTs + +```bash +pct exec -- bash -lc " +docker exec getent hosts github.com +" +``` + +Must resolve. Proves the DNS pin insulates containers from any Tailscale-related resolver changes. + +### 6. Reboot persistence (mandatory) + +```bash +# From PVE host: +pct reboot +``` + +After ~85 seconds: + +```bash +pct exec -- bash -lc " +ls -la /etc/resolv.conf +resolvectl status | head -20 +journalctl -b -u tailscaled | grep -E 'rc=|dns: using' +getent hosts meshmonitor.echo6.mesh +getent hosts github.com +" +``` + +All must pass. This is the check the two boot-ordering guards exist to satisfy: stub symlink intact, `mode: stub`, tailscaled in `systemd-resolved` mode, both DNS paths working. + +--- + +## Rollback + +Per guest — returns to the known-good static state (public DNS works, no MagicDNS). No reboot required. + +**On the PVE host:** + +```bash +pct set --nameserver "1.1.1.1 8.8.8.8" +``` + +**Inside the CT:** + +```bash +pct exec -- bash -lc " +chattr -i /etc/resolv.conf 2>/dev/null +rm -f /etc/resolv.conf +printf 'nameserver 1.1.1.1\nnameserver 8.8.8.8\n' > /etc/resolv.conf +tailscale set --accept-dns=false +systemctl restart tailscaled +" +``` + +To also disable the guards (optional): + +```bash +pct exec -- bash -lc " +systemctl disable fix-resolv-stub.service +rm -f /etc/systemd/system/tailscaled.service.d/resolved-ordering.conf +systemctl daemon-reload +" +``` + +--- + +## How to use MagicDNS + +Any tailnet node resolves at `.echo6.mesh` → its `100.64.0.x` tailnet IP. + +Examples: +- `meshmonitor.echo6.mesh` +- `recon-vm.echo6.mesh` +- `cortex.echo6.mesh` + +List node names: + +```bash +# On edge2 (CT 107, mesh-bridge): +headscale nodes list +``` + +Base domain `echo6.mesh` is set in Headscale config. Use these names instead of hardcoding tailnet IPs in service configs and cron jobs. + +--- + +## Fleet rollout status + +### Completed — 2026-06-22 + +| CT | Host | Bucket | Docker | Notes | +|---|---|---|---|---| +| CT108 meshai | utility | R | yes | — | +| CT110 peertube | media | I | no | NordVPN allowlist applied | +| CT102 searxng | utility | I | yes | — | +| CT111 mcc | media | I | no | Caddy on :80; healthy before+after | +| CT112 cobalt | utility | R | yes | No containers deployed; daemon.json DNS pin applied | +| CT100 meshmonitor | utility | I | yes | meshai (CT108) stayed healthy throughout; MagicDNS canary validated via media.echo6.mesh | +| CT120 immich | cloud | R | yes | daemon.json pre-pinned; machine_learning unhealthy pre-existing (self-cleared after reboot) | +| CT103 argus | cloud | R | yes | daemon.json merged (had runtime keys, no dns pin); argus-app stack (grafana/postgres) healthy; pre-existing: stack lacks restart-always policy — needs manual `docker compose up -d` after host reboot (not a DNS issue) | +| CT121 nextcloud | cloud | R | yes | daemon.json created (missing); 12-container AIO stack (apache/app/db/redis/collabora/…) all healthy before+after; CorpDNS null (expected for this build) — validated via resolvectl tailscale0 echo6.mesh ~.; reboot-persistent | + +### Already compliant (gold-standard VMs) + +| Guest | Notes | +|---|---| +| recon-vm (VM 1130) | Reference state; validated | +| arr | Compliant | + +### Pending + +| CT | Host | Notes | +|---|---|---| +| CT101 caddy | utility | — | +| CT104 central | utility | — | +| CT107 mesh-bridge | utility | — | +| CT100 pdm | edge2 | Production front-door — do last, one at a time | +| CT101 wordpress | edge2 | Production front-door — do last, one at a time | +| CT102 vaultwarden | edge2 | Production front-door — do last, one at a time | +| CT103 forgejo | edge2 | Production front-door — do last, one at a time | +| CT104 livesync | edge2 | Production front-door — do last, one at a time | +| CT105 authentik | edge2 | Production front-door — do last, one at a time | +| CT106 matrix | edge2 | Production front-door — do last, one at a time | + +> **edge2 CTs are the production front door** (auth, forge, notes, vpn, vault, matrix, element). Migrate them last, strictly one at a time, with a full validation pass between each.