From e9cb7777fe7a81a5f25489b24d0bfb56180592d7 Mon Sep 17 00:00:00 2001 From: echo6-autocommit Date: Wed, 17 Jun 2026 00:00:14 +0000 Subject: [PATCH] auto: docs sync 2026-06-17T00:00:14+00:00 Files changed: .obsidian/workspace.json credentials docs/hardware/environment.md plans/vaultwarden-plan.md runbooks/edge2-access-reference.md runbooks/expose-service-edge2.md --- .obsidian/workspace.json | 12 +- credentials | 11 ++ docs/hardware/environment.md | 20 ++- plans/vaultwarden-plan.md | 202 +++++++++++++++++++++++++++++ runbooks/edge2-access-reference.md | 102 +++++++++++++++ runbooks/expose-service-edge2.md | 180 +++++++++++++++++++++++++ 6 files changed, 519 insertions(+), 8 deletions(-) create mode 100644 plans/vaultwarden-plan.md create mode 100644 runbooks/edge2-access-reference.md create mode 100644 runbooks/expose-service-edge2.md diff --git a/.obsidian/workspace.json b/.obsidian/workspace.json index bc8bc60..50e973b 100644 --- a/.obsidian/workspace.json +++ b/.obsidian/workspace.json @@ -189,6 +189,12 @@ }, "active": "ea4cc678c44e8b67", "lastOpenFiles": [ + "credentials.tmp.3861660.701c90c30e54", + "runbooks/edge2-access-reference.md", + "runbooks/expose-service-edge2.md", + "plans/vaultwarden-plan.md", + "plans/vaultwarden-migration.md", + "plans", "rules/radio.md", "rules/tak.md", "archive/matrix/PHASE6_DECISION.md", @@ -213,18 +219,12 @@ "archive/matrix/archivist_discovery.md", "archive/matrix/appservices.md", "archive/matrix/archive_receiver_discovery.md", - "archive/matrix/hookshot_room_targeting_discovery.md", - "archive/matrix/hookshot_payload_discovery.md", - "archive/matrix/hookshot_e2ee_discovery.md", - "archive/matrix/hookshot_deployment_discovery.md", - "docs/matrix/synapse_retention_discovery.md", "notes", "archive/matrix", "docs/matrix", "archive/projects/mmud/mmud-prompts/mmud-prompts", "archive/projects/mmud/mmud-prompts.tar.gz", "archive/projects/mmud/mmud-prompts", - "archive/projects/mmud/last-ember.html", "assets/echo6yellow_logo_422x422_square.png", "assets/echo6yellow_logo_422x81.png", "assets/echo6_logo.png", diff --git a/credentials b/credentials index 161fcc5..5d8a6ec 100755 --- a/credentials +++ b/credentials @@ -131,6 +131,9 @@ MESHMONITOR_ADMIN_PASS="7redditGold" # Vaultwarden (Password Manager) # Deployed: 2026-02-05 on Contabo VPS # URL: https://vault.echo6.co +# Local-login account (verified Phase 0a 2026-06-16): matt@echo6.co — master password is SET; Matt has confirmed he knows it. +# SSO_ONLY=false → local email + master-password login works independent of Authentik. +# Master password is Matt's own secret — NOT stored here (Vaultwarden keeps only the PBKDF2/Argon2 hash; unrecoverable). VAULTWARDEN_URL="https://vault.echo6.co" VAULTWARDEN_ADMIN_TOKEN="lHYCnZvETLirtRYCTICM6i6s2yYbMFjW3CkHwxnO1Brqe6vIq0zKNwF01IWB01" VAULTWARDEN_ADMIN_URL="https://vault.echo6.co/admin" @@ -345,3 +348,11 @@ IDAHO_511_API_KEY=5af12fb5fdc743e0ae0b041bc4c47f2d # edge2 PVE host root@pam password (set 2026-06-16 — for PDM remote-add + break-glass) EDGE2_ROOT_PASSWORD=rAWdarLXCsTVnEQCmGxX + +# intermountainmesh.com WordPress (edge2 CT 101, 10.10.10.11) +# Deployed: 2026-06-16 +WP_IMESH_DB_ROOT_PW=BenH5nw7Y25k2xtrXfXcapt4xLzsQC2W +WP_IMESH_DB_USER=wp_user +WP_IMESH_DB_PASSWORD=9cnwfhQzkdyfU8C3yodIXu_IA1PUOawr +WP_IMESH_DB_NAME=wordpress +# WordPress admin credentials: set during browser setup wizard at https://intermountainmesh.com diff --git a/docs/hardware/environment.md b/docs/hardware/environment.md index ab9fc47..31b6d94 100644 --- a/docs/hardware/environment.md +++ b/docs/hardware/environment.md @@ -85,9 +85,9 @@ Five nodes running Proxmox VE: | pi-nas | 192.168.1.245 | 100.64.0.21 | Raspberry Pi NAS | | matt-desktop | 192.168.1.111 | 100.64.0.10 | Personal workstation (Windows, your PC) | | Contabo Server | 5.189.158.149 | 100.64.0.1 | External VPS: Mail, Authentik, Headscale, Forge, Matrix | -| edge2 | 184.174.35.153 | 100.64.0.26 | Contabo Cloud VPS 30 NVMe — Proxmox VE 8.4.19 (LXC-only), 8c/24GB/400GB, role TBD | +| edge2 | 184.174.35.153 | 100.64.0.26 | Contabo Cloud VPS 30 NVMe — Proxmox VE 8.4.19 (LXC-only), 8c/24GB/400GB | -*Last updated: 2026-06-16 — Added edge2 (Contabo Cloud VPS 30 NVMe, Proxmox PVE 8.4.19, 100.64.0.26)* +*Last updated: 2026-06-16 — Added edge2 CTs (pdm CT 100, wordpress CT 101), Caddy reverse proxy on edge2 host* ## LXC Containers @@ -104,6 +104,8 @@ Five nodes running Proxmox VE: | archivist | utility (CT 118) | 192.168.1.118 | — | Archivist knowledge pipeline | | argus | utility (CT 103) | 192.168.1.103 | 100.64.0.25 | ARGUS - OSINT intelligence gathering platform | | peertube | media (CT 110) | 192.168.1.170 | 100.64.0.23 | PeerTube video streaming | +| pdm | edge2 (CT 100) | 10.10.10.10 | 100.64.0.28 | Proxmox Datacenter Manager | +| wordpress | edge2 (CT 101) | 10.10.10.11 | 100.64.0.31 | WordPress for intermountainmesh.com | ## IP Allocation Scheme @@ -149,6 +151,7 @@ Current registered nodes (26 total): | meshmonitor-dev | 100.64.0.27 | LXC | | gl-a1300 | 100.64.0.29 | Router | | bluefin | 100.64.0.30 | Desktop | +| wordpress | 100.64.0.31 | LXC | | meshai | 100.64.0.32 | LXC | ## IdahoMesh Headscale Node List @@ -179,8 +182,21 @@ ssh zvx@192.168.1.150 # cortex VM ssh zvx@192.168.1.241 # utility Proxmox ssh root@100.64.0.1 # Contabo (via Tailscale) ssh zvx@cortex # cortex via Tailscale hostname +ssh edge2 # edge2 via SSH config alias (admin@100.64.0.26) ``` +### edge2 SSH access + +edge2 is hardened: `PasswordAuthentication no`, root login disabled. SSH user is `admin` with passwordless sudo. The SSH config alias `edge2` maps to `admin@100.64.0.26`. + +Authorized keys on edge2 (`/home/admin/.ssh/authorized_keys`): +- `echo6-contabo2-184.174.35.153` — the original provisioning key (cortex `~/.ssh/contabo2_ed25519`) +- `cortex` — cortex default `id_ed25519` +- `matt-desktop-wsl2` — WSL2 default key +- `cookie-sync` — Windows native SSH key + +To access CTs on edge2: `ssh edge2` then `sudo pct exec -- bash`. + ### Password-auth hosts These require password authentication (no SSH keys installed): diff --git a/plans/vaultwarden-plan.md b/plans/vaultwarden-plan.md new file mode 100644 index 0000000..c19e610 --- /dev/null +++ b/plans/vaultwarden-plan.md @@ -0,0 +1,202 @@ +# Vaultwarden → edge2 LXC — Migration Pilot (+ reusable LXC-migration runbook) — v2 + +## Changes from v1 (what was corrected and why) + +- **CT ID 101 → 102.** CT 101 is the **live, running `wordpress` container** on edge2 (`pct list`: 100=pdm, 101=wordpress, both running, /etc/pve/lxc/101.conf exists). v1's `pct create 101` would error on ID-in-use, and a careless rollback `pct destroy 101` would **destroy WordPress**. New ID = **102** (`pvesh get /cluster/nextid` → 102; confirmed free, no 102.conf, no qemu config). IP `10.10.10.20/24` is still free, so only the CTID changes. +- **edge2 access path pinned.** `root@100.64.0.26` over the tailnet is **REFUSED**. All edge2 `pct`/`pvesm`/`pveam` commands run as `ssh admin@184.174.35.153` (alias `edge2`, key `~/.ssh/contabo2_ed25519`) then **`sudo`**. (Contabo/headscale commands stay `root@100.64.0.1` — different host, unchanged.) +- **Debian 12 template must be downloaded first.** Only `debian-13-standard_13.1-2` is cached locally; Debian 12 is in the catalog but **not present**. Added explicit `pveam download local debian-12-standard_12.12-1_amd64.tar.zst` step (volid `local:vztmpl/debian-12-standard_12.12-1_amd64.tar.zst`). +- **Right-sized RAM/disk.** v1 copied pdm's `memory 2048 / rootfs local:16`. Vaultwarden is a single Rust binary (idle RSS ~30–60 MB) over ~5.3 MB of SQLite. Downsized to **`memory 1024 / swap 512 / rootfs local:8`** — generous (8 GB rootfs holds Debian 12 + Docker + image + data with ~5 GB free). +- **`--ostype debian` added** to `pct create` (CT 100 has it; v1 omitted it). +- **Live-WAL data-consistency handling made explicit & zero-loss.** Source DB is in WAL mode with a **live 2.4 MB `db.sqlite3-wal`** — a bare `cp` of `db.sqlite3` loses ~2.4 MB of changes. Verified that `sqlite3 .backup` produces **one self-contained file with NO `-wal`/`-shm` side files**. For this single-user instance we **stop the source container before `.backup`** (quiescent DB, zero post-snapshot writes), then keep the source intact as rollback baseline. +- **Compose port edit corrected.** Source binds `127.0.0.1:8086:80` (container listens on **80**). The edit is `127.0.0.1:8086:80` → **`VW_TS_IP:8086:80`** (keep the container-side `:80`). v1's `…→ VW_TS_IP:8086` dropped the target port and would break the mapping. +- **Cutover line corrected — NO scheme.** v1 said `reverse_proxy http://VW_TS_IP:8086`. Vaultwarden's backend is **plain HTTP**; bare `host:port` already defaults to http. Adding `http://` is non-idiomatic and risks a validate failure. The minimal edit is **one token**: `127.0.0.1:8086` → `VW_TS_IP:8086` (the `proxmox.echo6.co` block uses a scheme only because *that* backend is HTTPS). +- **G9 wording softened (accurate failure mode).** `rsa_key.pem` signs JWTs (session/device/push tokens); it is **not** the vault-data encryption key. Missing it = mass session invalidation (forced re-login on every client) + push breakage, **not** undecryptable data. Carrying it is still required; the "generating new RSA key" log gate stays. +- **G10 blast radius noted.** `systemctl restart caddy` momentarily drops **all** Contabo-Caddy-fronted sites (auth/forge/mail/vault…) for ~1–2 s, not just vault. Acceptable, but stated. Also: `systemctl reload caddy` / `caddy reload` **fail** under `admin off` (ExecReload uses `caddy reload --force`) — restart is hard-required. +- **Stale-archive guard added.** `/home/zvx/projects/.ref/archive/projects/vaultwarden-deployment.md` is STALE (shows nonexistent `:3012` port, `/notifications/hub` route, dnsmasq entry, `/oidc-signin` URI). Phase 4 copies **only from the live `/opt/vaultwarden` host**, never the archive. Phase 9 retires the archive doc. +- **Composed cutover path treated as unproven → hard gate.** Contabo-Caddy → edge2-LXC tailnet path is new (precedent proxies to the *data* node, not an edge2 LXC). Phase 3/6 keep a **hard HTTP `curl …/alive` pre-cutover gate** from Contabo. + +--- + +## Context + +We're consolidating both Contabo VPSes onto Proxmox. edge2 is up and proven (PVE 8 + PDM). The next phase is evacuating services off the **main Contabo** so it can be rebuilt as `edge1` — moving each service into an LXC on edge2, with the live Contabo as rollback, mail handled last on its original IP. + +**Vaultwarden is the pilot** because it's the lowest-risk service: a single container, ~5.3 MB of SQLite data, no external DB, and `SSO_ONLY=false` so local master-password login works even if Authentik is down. This run **doubles as the reusable per-service LXC-migration template** — phases are tagged `[G]` generic vs `[S]` service-specific. + +The cutover uses a **proven, low-risk pattern**: public DNS never changes; we stand up the service on edge2, then re-point *one token* of Contabo's Caddy backend to the new instance over the tailnet (exactly how `proxmox.echo6.co` already routes to `100.64.0.6:8006`). Rollback = revert that one token + restart source (~1–2 s). + +**edge2 access (pinned):** `ssh admin@184.174.35.153` (alias `edge2`, key `~/.ssh/contabo2_ed25519`), then `sudo` for every `pct`/`pvesm`/`pveam` command. `root@100.64.0.26` is refused — do not use it. + +## Approach & decisions (defaults baked in) + +- **Zero data loss for this single-user vault.** We **stop the source container immediately before** the SQLite hot `.backup`, so the snapshot is taken from a quiescent DB and no post-snapshot writes can occur. The source container + its `data/` stay **intact and become the rollback baseline** (do not delete). WAL mtime shows ~0 write activity (only WS keepalives), so this is the simplest, safest path. (If true zero-downtime were ever required: accept a brief no-write window, or do a WAL-aware delta re-snapshot at cutover.) +- **CT 102**, hostname `vaultwarden`, **static `10.10.10.20/24` gw `10.10.10.1` ns `1.1.1.1`** (edge2 `vmbr0` has no DHCP), storage `local` (dir, ~368 GB free; no local-lvm exists), `--ostype debian`, unprivileged, `nesting=1,keyctl=1`, `--onboot 1`, **right-sized `memory 1024 / swap 512 / rootfs local:8`**. +- **Bind Vaultwarden to the LXC's tailnet IP:8086** (tight — only Contabo Caddy reaches it over the tailnet), not `0.0.0.0`. +- **Source NOT deleted in the pilot** — stopped at Phase 5, kept as a cold backup + final tarball before any eventual removal. +- **Local-login assurance:** `SSO_ONLY=false` preserved (carried in the untouched `.env`); immediately after cutover, test **email + master-password (local) login** in the browser — instant rollback if it fails. + +## Phases + +### Phase 0 — Recon & baseline `[G]` +- On Contabo (`ssh root@100.64.0.1`): capture the **verbatim** `vault.echo6.co` Caddy block (rollback baseline — see Phase 7) and grep the whole Caddyfile for any `vault`/`notifications`/`3012` sub-route (none exists — confirm; resolves **G1**). +- Re-read live `/opt/vaultwarden/docker-compose.yml` + `.env` + `ls -la data/` **from the running host** (never the stale `.ref/archive` doc). +- On edge2 (`ssh admin@184.174.35.153 'sudo …'`): + - Confirm **CT 102** and **10.10.10.20** are free: `sudo pct list` (expect 100=pdm@.10, 101=wordpress@.11), `sudo pvesh get /cluster/nextid` (→ 102), `sudo pct status 102` (→ does not exist). **Do not touch 101 — it is live WordPress.** + - Download the Debian 12 template (NOT cached yet): `sudo pveam download local debian-12-standard_12.12-1_amd64.tar.zst`, then `sudo pvesm list local | grep debian-12`. + - Confirm `local` storage free space: `sudo pvesm status`. +- **Rollback:** n/a (read-only except the template download). + +### Phase 0a — Validate a LOCAL user exists + record to credentials `[S]` ← user-requested GATE +- Take a read-only hot snapshot of the live DB and query `users`: confirm **≥1 account has a non-empty `password_hash`** (= master password set → local email+password login works, independent of SSO). Recon already confirms: **one account `matt@echo6.co`, password_hash non-empty, email verified, enabled → GATE PASS.** Record email(s), `login_count`, verified status. +- **Record to `/home/zvx/projects/.ref/credentials`** under a `# Vaultwarden (vault.echo6.co)` section: the account email(s), `SSO_ONLY=false` (local login enabled), and a note that the **master password is Matt's own secret — not stored anywhere in plaintext** (Vaultwarden keeps only the PBKDF2/Argon2 hash). The `ADMIN_TOKEN` (from `.env`) goes here as admin-panel break-glass. +- **GATE:** if NO account has a master password set, **STOP and surface it** — the local-login safety net wouldn't exist. Also confirm with Matt he knows the master password for at least one listed account (we can't recover it, only verify the account exists). + +### Phase 1 — Provision the LXC `[G]` +- Create the container (right-sized, ID 102, ostype debian): + ``` + sudo pct create 102 local:vztmpl/debian-12-standard_12.12-1_amd64.tar.zst \ + --hostname vaultwarden --cores 2 --memory 1024 --swap 512 --rootfs local:8 \ + --net0 name=eth0,bridge=vmbr0,ip=10.10.10.20/24,gw=10.10.10.1 \ + --nameserver 1.1.1.1 --ostype debian \ + --features nesting=1,keyctl=1 --unprivileged 1 --onboot 1 + ``` +- Append the TUN device lines to `/etc/pve/lxc/102.conf` (verbatim, matching CT 100): + ``` + printf 'lxc.cgroup2.devices.allow: c 10:200 rwm\nlxc.mount.entry: /dev/net/tun dev/net/tun none bind,create=file\n' | sudo tee -a /etc/pve/lxc/102.conf + ``` + (Resulting lines: `lxc.cgroup2.devices.allow: c 10:200 rwm` and `lxc.mount.entry: /dev/net/tun dev/net/tun none bind,create=file`.) +- `sudo pct start 102`; verify Debian 12, internet via NAT masquerade, DNS resolution (same health checks as the pdm CT). +- **Rollback:** `sudo pct stop 102 && sudo pct destroy 102`. (Production untouched. Never `destroy 101`.) + +### Phase 2 — Docker baseline `[G]` +- Inside CT 102 (`sudo pct exec 102 -- …`): install Docker + compose plugin (`curl -fsSL get.docker.com | sh`), `systemctl enable --now docker`. (`nesting=1`+`keyctl=1` from Phase 1 make Docker-in-LXC work.) +- **Rollback:** destroy CT 102. + +### Phase 3 — Tailscale onboard `[G]` +- Create preauth key on Contabo (user 1 = `echo6`, the infra user): + ``` + ssh root@100.64.0.1 'docker exec headscale headscale preauthkeys create --user 1 --reusable --expiration 1h' + ``` +- In CT 102 (`sudo pct exec 102 -- …`): install tailscale; add the **DNS-bootstrap drop-in** so `vpn.echo6.co` resolves at boot before MagicDNS is up (CT uses ns `1.1.1.1`): + ``` + mkdir -p /etc/systemd/system/tailscaled.service.d + cat > /etc/systemd/system/tailscaled.service.d/dns-bootstrap.conf <<'EOF' + [Service] + Environment=TS_DEBUG_RESOLV_CONF=1 + EOF + systemctl daemon-reload && systemctl restart tailscaled + ``` + Then bring it up: + ``` + tailscale up --login-server=https://vpn.echo6.co --auth-key= --hostname=vaultwarden + ``` +- **Capture `VW_TS_IP`:** `tailscale ip -4`. (Do NOT hardcode — allocation has gaps, e.g. `.31` free, highest is `.32`; the next IP is assigned dynamically at registration.) Confirm the node appears in `ssh root@100.64.0.1 'docker exec headscale headscale nodes list'` (no existing `vaultwarden` node — clean onboard). +- **HARD pre-cutover reachability gate (run again after Phase 6):** the composed Contabo-Caddy → edge2-LXC tailnet path is new, so prove it over HTTP (not ICMP — edge2↔Contabo may not ICMP-ping although both online): + ``` + ssh root@100.64.0.1 "curl -sS -o /dev/null -w '%{http_code}' http://VW_TS_IP:8086/alive" + ``` + Will return `000` until Phase 6 (nothing listening yet) — that only proves the tailnet route; the **200** gate is enforced in Phase 6. +- **Note (G10/systemd):** the bundled `caddy.service` `ExecReload` runs `caddy reload --force`, which **errors under `admin off`**. Do not run `systemctl reload caddy` anywhere in this runbook — **restart** is required (see Phase 7). +- **Rollback:** destroy CT 102 + `ssh root@100.64.0.1 'docker exec headscale headscale nodes delete -i '`. + +### Phase 4 — Stage compose + .env `[S]` +- Copy `/opt/vaultwarden/docker-compose.yml` + `.env` **byte-for-byte FROM the live running Contabo host** into the CT (e.g. `/opt/vaultwarden/`). **Never copy from `/home/zvx/projects/.ref/archive/projects/vaultwarden-deployment.md`** — it is stale (shows a nonexistent `:3012` port + `/notifications/hub` route). +- `.env` stays **untouched** → `SSO_ONLY=false`, `DOMAIN=https://vault.echo6.co`, `ADMIN_TOKEN`, `SIGNUPS_ALLOWED=false`, `WEBSOCKET_ENABLED=true`, `SSO_ENABLED=true`, `SSO_CLIENT_ID/SECRET`, `SSO_AUTHORITY=https://auth.echo6.co/application/o/vaultwarden/`, `SSO_PKCE=true` all preserved. +- **The ONE compose edit (G3):** change the single `ports` line `127.0.0.1:8086:80` → **`VW_TS_IP:8086:80`** (keep the container-side `:80`; the container listens on 80, 8086 is only the host-published port). Everything else stays byte-for-byte: `image: vaultwarden/server:latest`, `volume ./data:/data`, `env_file .env`, `restart: unless-stopped`, `TZ`. Confirm there is **exactly one** ports line and **NO `3012` line**. +- **Rollback:** destroy CT 102. + +### Phase 5 — Migrate data (zero-loss snapshot) `[S]` +- **Stop the source** so the DB is quiescent (zero post-snapshot writes): on Contabo `cd /opt/vaultwarden && docker compose stop`. (Keep the container + `data/` intact as rollback baseline — do **not** `down`/delete.) +- **Hot consistent snapshot** on the source host (`/usr/bin/sqlite3` v3.45.1 confirmed present): + ``` + sqlite3 /opt/vaultwarden/data/db.sqlite3 ".backup /tmp/vw-snap.sqlite3" + sqlite3 /tmp/vw-snap.sqlite3 "PRAGMA integrity_check" # must == ok BEFORE transfer + ``` + `.backup` uses the online backup API and writes **one self-contained file with NO `-wal`/`-shm` side files** (verified). The output's `journal_mode=wal` readout is just a persisted header setting — it is fully self-contained; do not be alarmed. +- **File manifest:** + - **MUST travel:** `vw-snap.sqlite3` → target `data/db.sqlite3`; **`rsa_key.pem`** byte-for-byte (JWT signing key — G9). + - **OPTIONAL:** `icon_cache/` (cosmetic favicon cache, auto-rebuilds). + - **MUST NOT travel:** `db.sqlite3-wal`, `db.sqlite3-shm`, the bare live `db.sqlite3`, `tmp/`. (Copying a live `-wal` alongside a separately-snapshotted db risks recovery corruption.) +- Verify in CT 102: `sqlite3 data/db.sqlite3 "PRAGMA integrity_check"` == ok; `rsa_key.pem` present (header `-----BEGIN RSA PRIVATE KEY-----`, ~1679 B). +- **Note:** when Vaultwarden opens the migrated db it will create **fresh** `db.sqlite3-wal`/`-shm` on the target automatically — expected and correct; do not pre-create or copy them. +- **Rollback:** destroy CT 102; restart source: `cd /opt/vaultwarden && docker compose up -d`. + +### Phase 6 — Start + health gate `[S]` +- `cd /opt/vaultwarden && docker compose up -d` in CT 102. +- **Health gates (all must pass before cutover):** + 1. **Pre-start integrity:** `sqlite3 data/db.sqlite3 "PRAGMA integrity_check"` == ok (done in Phase 5). + 2. **RSA key loaded:** container logs **must NOT** contain `generating new RSA key` (would mean `rsa_key.pem` didn't travel/load → mass session invalidation), and should show normal startup. + 3. **`/alive` == 200** over the tailnet from Contabo (the HARD composed-path gate from Phase 3): + ``` + ssh root@100.64.0.1 "curl -sS -o /dev/null -w '%{http_code}' http://VW_TS_IP:8086/alive" + ``` + 4. **Row-count gate:** `sqlite3 data/db.sqlite3 "SELECT count(*) FROM users"` == **1** (matches source `matt@echo6.co`). +- **Rollback:** destroy CT 102; restart source (`docker compose up -d` on Contabo). + +### Phase 7 — Cutover `[G]` ← the primary, instant-rollback step +- On Contabo, capture a verbatim on-box baseline first: + ``` + cp /etc/caddy/Caddyfile /etc/caddy/Caddyfile.bak-prevault + ``` +- **The ONE edit:** in `/etc/caddy/Caddyfile`, replace **only the upstream token** on the `vault.echo6.co` backend line — `127.0.0.1:8086` → `VW_TS_IP:8086` (**no scheme** — bare host:port defaults to http; the proxmox block uses a scheme only because that backend is HTTPS). Keep `header_up X-Real-IP {remote_host}` and the braces untouched (G5). Resulting block must read exactly: + ``` + vault.echo6.co { + reverse_proxy VW_TS_IP:8086 { + header_up X-Real-IP {remote_host} + } + } + ``` +- Validate (works under `admin off`), then **restart** (G10 — `caddy reload` and `systemctl reload caddy` both FAIL under `admin off`): + ``` + caddy validate --config /etc/caddy/Caddyfile + systemctl restart caddy + ``` +- **Blast-radius note (G10):** `systemctl restart caddy` briefly drops **all** Contabo-Caddy-fronted sites (auth/forge/mail/vault…) for ~1–2 s, not just vault. Acceptable; validate-before-restart prevents restarting into a broken config. +- **Rollback (primary, ~1–2 s, public DNS never moved):** either + ``` + cp /etc/caddy/Caddyfile.bak-prevault /etc/caddy/Caddyfile && systemctl restart caddy + ``` + OR revert the line to exactly: + ``` + vault.echo6.co { + reverse_proxy 127.0.0.1:8086 { + header_up X-Real-IP {remote_host} + } + } + ``` + then `caddy validate --config /etc/caddy/Caddyfile && systemctl restart caddy`, and ensure the source container is running (`cd /opt/vaultwarden && docker compose up -d`). + +### Phase 8 — End-to-end verification `[G/S]` +- `vault.echo6.co` resolves unchanged → `5.189.158.149`; `curl -I https://vault.echo6.co` 200, valid cert. +- **Local login:** log in at `https://vault.echo6.co` with **email + master password** (non-SSO) → vault items decrypt + list (proves local-login safety net + data integrity). ← user-requested assurance. +- **SSO login:** the "Log in with SSO" path round-trips through Authentik and lands back **logged-in** on the **unchanged** `https://vault.echo6.co`. Authentik needs **no** change (DNS + `DOMAIN` unchanged → registered redirect URI stays valid) (G6). Verify only that the round-trip completes logged-in — do **not** assert a specific redirect path (the archive's `/oidc-signin` vs `/callback` is stale/inconsistent and irrelevant since Authentik is untouched). +- **Websocket sync:** browser-extension live sync works (101 Upgrade through Caddy) (G7). No `/notifications/hub` or `:3012` route exists or is needed — WS is in-process on `:8086`, so the single upstream carries the Upgrade. +- **Reboot survival:** `sudo pct reboot 102` → CT comes back, Docker + container auto-start (`--onboot 1` + `restart: unless-stopped`), tailscale reconnects (DNS-bootstrap drop-in), `/alive` 200 (G8). +- **Rollback:** Phase 7 revert. + +### Phase 9 — Decommission (deferred) `[G]` +- Source stays **installed + stopped** as a cold rollback for a soak period. Final `tar` of `/opt/vaultwarden` before any eventual deletion. +- **Retire the stale archive doc** `/home/zvx/projects/.ref/archive/projects/vaultwarden-deployment.md` (it misdescribes ports/routes/URI). +- Update `.ref` (services.md, environment.md, ip-allocation.md → CT 102 @ 10.10.10.20, the new headscale node). Promote this file to `.ref/runbooks/lxc-service-migration.md`. + +## Gotchas +- **G1** Stale `caddy.md`/archive mentions a `:3012`/`notifications/hub` WS route — live Caddy has **none** (in-process WS on `:8086`). Confirmed in Phase 0. +- **G2** edge2 `vmbr0` has **no DHCP** and no `local-lvm` → static IP `10.10.10.20/24` + `local` (dir) storage. (Overrides ct-runbook defaults.) +- **G3** Source binds loopback-only (`127.0.0.1:8086:80`); LXC must bind the tailnet IP (`VW_TS_IP:8086:80`) so Caddy reaches it. Keep the container-side `:80`. +- **G4** SQLite is in WAL mode with a **live 2.4 MB WAL** → use hot `.backup` (single self-contained file, no `-wal`/`-shm`); never copy a live `db.sqlite3`+WAL naively. Here we also **stop the source first** for zero-loss. +- **G5** Preserve `header_up X-Real-IP {remote_host}` in the new Caddy backend line (edit only the upstream token). +- **G6** SSO redirect derives from the unchanged `DOMAIN`/DNS → **do not touch Authentik**. Verify the round-trip lands logged-in; don't assert a specific redirect path. +- **G7** Caddy passes websocket upgrades by default; no WS-specific directive needed (in-process WS on `:8086`). +- **G8** Reboot survival needs all three: `--onboot 1`, `restart: unless-stopped`, and the DNS-bootstrap drop-in (so `vpn.echo6.co` resolves at boot). +- **G9** `rsa_key.pem` is the **JWT signing key** — must travel byte-for-byte. **Accurate failure mode:** if missing, Vaultwarden generates a new one and **all existing device/session tokens are invalidated (forced re-login everywhere) + push breaks** — vault DATA still decrypts on master-password login (data is client-side encrypted). The "generating new RSA key" log gate (Phase 6) catches this. +- **G10** Contabo Caddy admin API is **off** → `caddy validate` then **`systemctl restart caddy`**. Both `caddy reload` and `systemctl reload caddy` FAIL (`ExecReload` uses `caddy reload --force`). Restart briefly drops all Contabo-fronted sites (~1–2 s). +- **N (new):** the composed **Contabo-Caddy → edge2-LXC tailnet** path is unexercised → keep the Phase 6 HTTP `/alive` 200 gate as a HARD pre-cutover requirement (use curl/http, not ICMP). + +## Generalize for future services +Template = phases **0→1→2→3→ (4,5,6 swap per service) →7→8→9**. Generic, identical for every Contabo-Caddy-fronted service: free-CTID pick (`pvesh get /cluster/nextid`) + LXC provision (right-sized per service), Docker, Tailscale onboard + DNS-bootstrap drop-in, the one-token Caddy cutover + its verbatim rollback. Service-specific only: the compose/`.env` (Phase 4 — copy from the **live host**, never archives), what "data" is + its consistency method (Phase 5 — stateless services collapse this; SQLite uses stop-then-`.backup`; Postgres-backed services dump/restore instead of file-copy), and the login/functional checks (Phases 6/8). Always pin edge2 access to `admin@184.174.35.153` + `sudo`, and always download the right template first. + +## What I'll need to do (permissions) +- SSH to **Contabo** as `root@100.64.0.1` (Caddy edit, headscale preauth/node, source sqlite3 `.backup`, source `docker compose stop/up`). +- SSH to **edge2** as **`admin@184.174.35.153`** (key `~/.ssh/contabo2_ed25519`, alias `edge2`) then **`sudo`** for all `pct`/`pvesm`/`pveam` commands. (NOT `root@100.64.0.26` — refused.) +- `pveam download` one Debian 12 template; `pct`/`docker`/`tailscale` inside CT 102; create one Headscale preauth key + node; edit **one token** of Contabo's Caddyfile; copy ~5 MB of data. +- **No public DNS changes, no Authentik changes, no dnsmasq changes; source left intact (stopped) as rollback.** Never touch CT 101 (live WordPress). diff --git a/runbooks/edge2-access-reference.md b/runbooks/edge2-access-reference.md new file mode 100644 index 0000000..28e0737 --- /dev/null +++ b/runbooks/edge2-access-reference.md @@ -0,0 +1,102 @@ +# edge2 Access Reference + +## SSH Access + +edge2 is hardened differently from home-cluster Proxmox nodes. + +| Property | edge2 | Home Proxmox (data, utility, etc.) | +|----------|-------|------------------------------------| +| SSH user | `admin` (not root) | `root` or `zvx` | +| Auth method | Key-only | Key or password | +| PasswordAuthentication | `no` (hardened) | varies | +| Root login | Disabled | Allowed | +| Sudo | Passwordless for admin | N/A (already root) | +| SSH alias | `ssh edge2` | `ssh zvx@` | + +### SSH config entry + +``` +Host edge2 + HostName 100.64.0.26 + User admin +``` + +### Authorized keys + +Located at `/home/admin/.ssh/authorized_keys` on edge2: + +| Key name | Source | Purpose | +|----------|--------|---------| +| echo6-contabo2-184.174.35.153 | cortex `~/.ssh/contabo2_ed25519` | Original provisioning key | +| cortex (id_ed25519) | cortex default key | Normal automation | +| matt-desktop-wsl2 | WSL2 default key | WSL/Claude sessions | +| cookie-sync | Windows SSH key | Native Windows terminal | + +### Known gotcha: cloud-init conflict + +`/etc/ssh/sshd_config.d/50-cloud-init.conf` has `PasswordAuthentication yes`, which contradicts the hardened `no` in the main config. The main config wins (loaded first), but this inconsistency should be cleaned up: + +```bash +ssh edge2 +sudo rm /etc/ssh/sshd_config.d/50-cloud-init.conf +sudo systemctl reload sshd +``` + +## PVE API Access + +The PVE web UI and REST API use a separate auth system (`root@pam`) with its own password (see credentials file: `EDGE2_ROOT_PASSWORD`). + +- **Web UI:** https://100.64.0.26:8006 (or https://184.174.35.153:8006) +- **API base:** `https://100.64.0.26:8006/api2/json/` +- **Auth:** `POST /access/ticket` with `username=root@pam&password=` + +The PVE API is useful when SSH is unavailable (e.g., before keys are authorized). It can create/start/stop CTs, read node status, and manage storage. It cannot directly execute arbitrary commands on the host — for that, use SSH or the PVE web shell. + +### Container management via SSH + +```bash +# List CTs +ssh edge2 'sudo pct list' + +# Exec into a CT +ssh edge2 'sudo pct exec -- bash' + +# Push files into a CT +scp file.txt edge2:/tmp/ +ssh edge2 'sudo pct push /tmp/file.txt /tmp/file.txt' + +# Start/stop CTs +ssh edge2 'sudo pct start ' +ssh edge2 'sudo pct stop ' +``` + +## Lessons Learned (2026-06-16 deployment) + +### Problem: SSH "Permission denied" to edge2 + +**Symptoms:** `ssh root@100.64.0.26` → `Permission denied (publickey,password)` + +**Root cause:** edge2 was hardened at provisioning: +- SSH user is `admin`, not `root` — root login is disabled +- `PasswordAuthentication no` — only key auth works +- Only one key was authorized: `echo6-contabo2-184.174.35.153`, which is cortex's `~/.ssh/contabo2_ed25519` (not the default `id_ed25519`) + +**Why it was confusing:** +1. The SSH error shows `publickey,password` as available methods — this is misleading because `PasswordAuthentication no` is enforced, but the SSH banner still lists both +2. We tried `root@` (wrong user) and the default `id_ed25519` (wrong key) +3. The environment docs didn't document the `admin` user or the specific key requirement + +**Resolution:** Added cortex's default `id_ed25519`, WSL2 key, and Windows key to admin's `authorized_keys`. Added SSH config alias `edge2` → `admin@100.64.0.26`. + +**Prevention:** +- Always document the SSH user + required key for hardened hosts in environment.md +- Add SSH config aliases immediately when onboarding new hosts +- For Contabo VPS instances: check cloud-init config for hardening applied at provisioning + +### Problem: PVE API vs system passwords + +**Symptoms:** PVE API login works with `EDGE2_ROOT_PASSWORD`, but SSH with same password fails. + +**Root cause:** PVE `root@pam` password and the system root SSH password are managed separately. On edge2, the system root password was set by cloud-init at provisioning and may differ. Additionally, root SSH login is disabled entirely. + +**Prevention:** Document both auth paths (SSH user + PVE API) separately in credentials and environment docs. diff --git a/runbooks/expose-service-edge2.md b/runbooks/expose-service-edge2.md new file mode 100644 index 0000000..87e3428 --- /dev/null +++ b/runbooks/expose-service-edge2.md @@ -0,0 +1,180 @@ +# Expose Service on edge2 (Contabo Cloud VPS) + +## Context + +edge2 is a Proxmox VE 8 node (184.174.35.153 / 100.64.0.26) running LXC containers on an internal bridge (`vmbr0`, subnet `10.10.10.0/24`, gateway `10.10.10.1`). Services run inside unprivileged LXC containers. Caddy on the edge2 host terminates TLS and reverse-proxies to the container's internal IP. + +## Prerequisites + +- SSH access to edge2: `ssh edge2` (admin@100.64.0.26, key auth, passwordless sudo) +- Debian 13 CT template cached: `local:vztmpl/debian-13-standard_13.1-2_amd64.tar.zst` +- DNS provider access (Cloudflare, GoDaddy, etc.) + +## Steps + +### 1. Create the LXC container + +Pick the next available CTID (current: 100=pdm, 101=wordpress). All CTs use static IPs on the `10.10.10.0/24` subnet. + +```bash +ssh edge2 + +sudo pct create local:vztmpl/debian-13-standard_13.1-2_amd64.tar.zst \ + --hostname \ + --rootfs local: \ + --memory \ + --swap 512 \ + --cores \ + --net0 name=eth0,bridge=vmbr0,ip=10.10.10./24,gw=10.10.10.1 \ + --nameserver 1.1.1.1 \ + --unprivileged 1 \ + --features nesting=1 \ + --onboot 1 \ + --start 1 + +# Verify +sudo pct list +sudo pct exec -- ping -c1 1.1.1.1 +``` + +### 2. Install the service inside the container + +```bash +sudo pct exec -- bash +# ... install service, bind to port on 0.0.0.0 or 10.10.10. +# ... the host will reach the CT via the internal bridge +``` + +For file transfer into the CT: +```bash +# From edge2 host +sudo pct push /path/on/host /path/in/ct +``` + +### 3. Add Caddy site block on edge2 host + +Caddy runs on the edge2 host and terminates TLS. + +**For Cloudflare-proxied domains** (orange cloud / Full SSL mode): +```bash +# tls internal generates a self-signed cert — Cloudflare "Full" mode +# encrypts transit without needing a publicly-trusted cert +sudo tee -a /etc/caddy/Caddyfile << 'EOF' + + { + tls internal + reverse_proxy 10.10.10.: +} +EOF + +sudo caddy validate --config /etc/caddy/Caddyfile +sudo systemctl reload caddy +``` + +**For non-Cloudflare domains** (needs real cert): +```bash +# Caddy will auto-provision a Let's Encrypt cert +sudo tee -a /etc/caddy/Caddyfile << 'EOF' + + { + reverse_proxy 10.10.10.: +} +EOF + +sudo caddy validate --config /etc/caddy/Caddyfile +sudo systemctl reload caddy +``` + +### 4. Configure DNS + +**GoDaddy domains:** +```bash +# On cortex/toc +source /home/zvx/projects/.ref/credentials +godaddy-dns.py add-a 184.174.35.153 +``` + +**Cloudflare domains:** +Manual via Cloudflare dashboard (no API token): +1. A record → 184.174.35.153 (proxied / orange cloud) +2. SSL/TLS mode → Full (not Strict — since `tls internal` uses self-signed) +3. Optional: www CNAME → domain (proxied) + +### 5. Verify + +```bash +# From edge2 (proper SNI required for tls internal) +curl -sk --resolve :443:127.0.0.1 -o /dev/null -w "%{http_code}\n" https:/// + +# From cortex (direct to origin) +curl -sk --resolve :443:184.174.35.153 -o /dev/null -w "%{http_code}\n" https:/// + +# Via DNS (after propagation) +curl -I https:/// +``` + +### 6. Update docs + +- Add CT to `environment.md` LXC Containers table +- Add credentials to `.ref/credentials` +- Add Tailscale IP if Tailscale is installed in the CT + +## Checklist + +``` +□ CT created, started, networking verified (ping 1.1.1.1) +□ Service installed and running inside CT +□ Caddy site block added and reloaded on edge2 host +□ DNS record pointing to 184.174.35.153 +□ SSL mode correct (Full for Cloudflare, auto for others) +□ HTTPS verified from edge2 + externally +□ Docs updated (environment.md, credentials) +``` + +## Network Reference + +| Resource | Value | +|----------|-------| +| edge2 public IP | 184.174.35.153 | +| edge2 Tailscale | 100.64.0.26 | +| Internal bridge | vmbr0, 10.10.10.0/24 | +| Gateway | 10.10.10.1 (edge2 host) | +| DNS in CTs | 1.1.1.1 | +| CT IP range | 10.10.10.10+ (10=pdm, 11=wordpress) | + +## CT Creation via PVE API (alternative) + +When SSH is unavailable, CTs can be created via the PVE REST API: + +```python +import urllib.request, urllib.parse, json, ssl + +ctx = ssl.create_default_context() +ctx.check_hostname = False +ctx.verify_mode = ssl.CERT_NONE +base = "https://100.64.0.26:8006/api2/json" + +# Authenticate +data = urllib.parse.urlencode({"username": "root@pam", "password": ""}).encode() +req = urllib.request.Request(f"{base}/access/ticket", data=data) +resp = urllib.request.urlopen(req, context=ctx) +auth = json.loads(resp.read())["data"] +ticket, csrf = auth["ticket"], auth["CSRFPreventionToken"] + +# Create CT +params = urllib.parse.urlencode({ + "vmid": , "hostname": "", + "ostemplate": "local:vztmpl/debian-13-standard_13.1-2_amd64.tar.zst", + "cores": 2, "memory": 2048, "swap": 512, "rootfs": "local:16", + "net0": "name=eth0,bridge=vmbr0,ip=10.10.10./24,gw=10.10.10.1", + "nameserver": "1.1.1.1", "unprivileged": 1, "features": "nesting=1", + "onboot": 1, "start": 1, +}).encode() +req = urllib.request.Request(f"{base}/nodes/edge2/lxc", data=params, method="POST") +req.add_header("Cookie", f"PVEAuthCookie={ticket}") +req.add_header("CSRFPreventionToken", csrf) +resp = urllib.request.urlopen(req, context=ctx) +print(json.loads(resp.read())["data"]) # UPID of creation task +``` + +Note: The PVE API uses `root@pam` auth. This is separate from the system SSH user (`admin`). See credentials file for the PVE password.