From bde8af22adfeedf3bc03aa12bccd2c51b24558da Mon Sep 17 00:00:00 2001 From: echo6-autocommit Date: Tue, 30 Jun 2026 18:00:04 +0000 Subject: [PATCH] auto: docs sync 2026-06-30T18:00:04+00:00 Files changed: engine/lint-report.md vault/.obsidian/workspace.json vault/runbooks/headscale-oidc-boot-order.md --- engine/lint-report.md | 2 +- vault/.obsidian/workspace.json | 4 +- vault/runbooks/headscale-oidc-boot-order.md | 80 +++++++++++++++++++++ 3 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 vault/runbooks/headscale-oidc-boot-order.md diff --git a/engine/lint-report.md b/engine/lint-report.md index 6f2acb3..30350d7 100644 --- a/engine/lint-report.md +++ b/engine/lint-report.md @@ -1,6 +1,6 @@ # Vault Lint Report -Generated: 2026-06-30T06:00:04Z | Docs scanned: 97 | Elapsed: 0.0s +Generated: 2026-06-30T12:00:07Z | Docs scanned: 97 | Elapsed: 0.0s ## Summary diff --git a/vault/.obsidian/workspace.json b/vault/.obsidian/workspace.json index 9c7ecc2..c33b283 100644 --- a/vault/.obsidian/workspace.json +++ b/vault/.obsidian/workspace.json @@ -199,6 +199,8 @@ }, "active": "17bd4a6166f789d0", "lastOpenFiles": [ + "runbooks/headscale-oidc-boot-order.md", + "runbooks/headscale-oidc-boot-order.md.tmp.246153.e2a5642e10dd", "runbooks/headscale-onboard-node.md.tmp.246153.9adf70459e77", "runbooks/headscale-onboard-node.md.tmp.246153.0132e66f616c", "runbooks/headscale-onboard-node.md.tmp.246153.52329ab2810a", @@ -209,8 +211,6 @@ "runbooks/headless-browser-page-verification.md.tmp.3603785.4eb1d115aefe", "runbooks/headless-browser-page-verification.md.tmp.3603785.f017d6928577", "runbooks/headless-browser-page-verification.md", - "runbooks/headless-browser-page-verification.md.tmp.3603785.012d08be6da5", - "docs/software/central.md.tmp.3603785.6c9205f08816", "runbooks/central-deploy-cutover.md", "runbooks/fleet-magicdns-resolved-migration.md", "projects/fleet-platform-baseline.md", diff --git a/vault/runbooks/headscale-oidc-boot-order.md b/vault/runbooks/headscale-oidc-boot-order.md new file mode 100644 index 0000000..02aee48 --- /dev/null +++ b/vault/runbooks/headscale-oidc-boot-order.md @@ -0,0 +1,80 @@ +--- +title: Headscale — OIDC Disabled at Boot (Authentik Boot-Order Dependency) +type: runbook +tags: + - vpn + - auth +related: + - [[headscale-onboard-node]] + - [[authentik-oidc-application]] + - [[edge2-access-reference]] +updated: 2026-06-30 +--- +# Headscale — OIDC Disabled at Boot (Authentik Boot-Order Dependency) + +Headscale wires up its OIDC provider **once, at process startup**, by fetching Authentik's discovery document. If Authentik is unreachable at that moment, Headscale silently falls back to CLI-only auth and runs **OIDC-disabled until it is restarted**. This is a boot-ordering hazard: the fleet Headscale (edge2 **CT107**) and Authentik (edge2 **CT105**, `auth.echo6.co`) live on the same host, so an edge2 reboot — or the DNS-bootstrap window after one — can bring Headscale up before Authentik is serving. + +## Symptom + +- A client (especially the **Tailscale Android app on a GrapheneOS phone**) hits **Connect** and lands on Headscale's **manual node-registration page** (the "run `headscale nodes register ...`" screen) instead of being redirected to **Authentik** for SSO login. +- New nodes can't self-register via SSO; only the preauth-key / CLI path works. +- `vpn.echo6.co/health` still returns `pass` — the service is *up*, it's just running without OIDC. The failure is silent. + +## Root cause + +The Headscale config sets `only_start_if_oidc_is_available: false`, so Headscale boots even when OIDC discovery fails. When Authentik was unreachable at startup, the log shows: + +``` +WRN failed to set up OIDC provider, falling back to CLI based authentication + error="creating OIDC provider from issuer config: 502 Bad Gateway" +``` + +From that point the running process serves only manual/CLI registration — even after Authentik recovers — because OIDC is initialized only at startup, never retried. + +## Diagnosis + +Access path: `ssh edge2` (admin) then `sudo pct exec 107 -- docker exec headscale headscale `. See [[edge2-access-reference]]. + +```bash +# 1. Is OIDC currently broken? Look for the 502 / fallback warning since the container started: +ssh edge2 "sudo pct exec 107 -- docker logs headscale --since 48h 2>&1 | grep -iE 'oidc|502|fallback'" + +# 2. Is Authentik discovery healthy NOW (from inside CT107)? +ssh edge2 "sudo pct exec 107 -- curl -fsS --max-time 8 \ + https://auth.echo6.co/application/o/headscale/.well-known/openid-configuration \ + -o /dev/null -w 'HTTP %{http_code}\n'" +``` + +If step 1 shows the fallback warning AND step 2 returns `HTTP 200`, OIDC is recoverable by a restart. + +## Fix + +Restart the Headscale container so it re-initializes the OIDC provider against now-healthy Authentik: + +```bash +ssh edge2 "sudo pct exec 107 -- docker restart headscale" +``` + +Verify the fix: + +```bash +# Fallback warning should be GONE this start (only the benign 'listening without TLS' WRN is expected): +ssh edge2 "sudo pct exec 107 -- docker logs headscale --since 3m 2>&1 | grep -iE 'oidc|502|fallback|warn'" + +# Health + fleet intact: +curl -fsS https://vpn.echo6.co/health # -> {"status":"pass"} +ssh edge2 "sudo pct exec 107 -- docker exec headscale headscale nodes list | grep -c 100.64.0" # ~34+ nodes +``` + +Then have the client hit **Connect** again — it should now redirect to Authentik (silently, if the device's browser already holds a live `auth.echo6.co` SSO session). + +> **Impact of the restart:** low. Existing WireGuard tunnels stay up (no node drops off the tailnet); only control-plane coordination pauses for a few seconds. Headscale is lockout-critical — drive the restart only from the edge2 out-of-band path above, never over `vpn.echo6.co` itself. + +## Prevention / follow-up + +- The failure is **silent** (health stays green), so it recurs on any future edge2/Authentik restart-ordering hiccup. Candidate hardening: a health-gated watchdog that restarts Headscale when the logs show the OIDC fallback warning *while* Authentik discovery returns 200; or a systemd boot-order/`ExecStartPre` gate so Headscale waits for Authentik discovery before starting. +- After any **edge2 reboot**, treat "can a fresh client SSO-register?" as a post-reboot check — not just `vpn.echo6.co/health`. + +## Reference incident + +- **2026-06-30** — GrapheneOS phone reauth landed on the manual-registration page. Headscale had booted ~11h earlier (during an edge2 reboot / DNS-bootstrap window) while Authentik was 502'ing, so OIDC was disabled. Restarting the `headscale` container restored OIDC; the phone then registered via Authentik (silent SSO) as user `matt`. Onboarding control-server path also corrected in [[headscale-onboard-node]] the same day.