diff --git a/engine/.githooks/pre-commit b/engine/.githooks/pre-commit new file mode 100755 index 0000000..cda7c77 --- /dev/null +++ b/engine/.githooks/pre-commit @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# .githooks/pre-commit — Vault lint report hook +# +# Runs lint.py in REPORT mode (never --strict) so it writes lint-report.md +# and always exits 0. This hook MUST NOT block commits — the root autocommit +# cron (echo6-agent) depends on commits succeeding unconditionally. +# +# To wire: git config core.hooksPath engine/.githooks + +set -euo pipefail + +REPO_ROOT="$(git rev-parse --show-toplevel)" +LINT="${REPO_ROOT}/engine/lib/lint.py" + +if [[ ! -f "${LINT}" ]]; then + echo "[lint] WARNING: lint.py not found at ${LINT}, skipping." >&2 + exit 0 +fi + +echo "[lint] Running vault lint (report mode)..." >&2 +python3 "${LINT}" >&2 || true # || true: never fail the commit + +# Always exit 0 — do NOT change this to exit 1 or add --strict. +exit 0 diff --git a/engine/Modelfile b/engine/Modelfile new file mode 100644 index 0000000..6e25e4a --- /dev/null +++ b/engine/Modelfile @@ -0,0 +1,9 @@ +# Echo6 Vault Tagger — pinned, reproducible. Build: ollama create vault-tagger -f Modelfile +FROM qwen2.5:7b-instruct-q8_0 + +PARAMETER temperature 0.1 +PARAMETER top_p 0.9 +PARAMETER num_ctx 8192 + +# System prompt is kept in prompts/system.md (source of truth). Keep this in sync on bootstrap. +SYSTEM """You are the Echo6 vault tagger. Given a markdown document and a controlled vocabulary (topic categories + an entity lexicon of this user's hosts, services, containers, projects, and acronyms), you classify the document and extract entities. You output ONLY valid JSON matching the requested schema. You never invent tags outside the provided vocabulary. You never expand acronyms you are unsure about — you flag them instead.""" diff --git a/engine/README.md b/engine/README.md new file mode 100644 index 0000000..341fcda --- /dev/null +++ b/engine/README.md @@ -0,0 +1,68 @@ +# Echo6 Vault Engine + +Self-contained maintenance engine for the Obsidian knowledge vault at +`/home/zvx/projects/.ref/vault/`. Runs entirely on cortex (RTX A4000). + +## What it does + +| Layer | Tool | Job | +|-------|------|-----| +| Tagger | Qwen2.5-7B-Instruct (Ollama) | Classifies docs; assigns topic tags, type, entity refs | +| Embeddings | bge-m3 via TEI (reused service) | Embeds docs into Qdrant for similarity / related links | +| Lint | `lib/lint.py` (deterministic) | Enforces frontmatter schema; fixes safe violations automatically | +| Agent | `lib/agent.py` | Orchestrates tagger + embeddings over changed docs | +| Sweep | `sweep.sh` | Daily entry point; GPU guard, lint, agent, changelog | + +## File layout + +``` +engine/ + config.yaml — single source of truth (endpoints, vocab, schema, schedule) + Modelfile — pinned vault-tagger build (qwen2.5:7b-instruct-q8_0, temp 0.1) + bootstrap.sh — idempotent setup: verify services, build model, install cron + sweep.sh — daily maintenance sweep (called by cron) + prompts/ + system.md — canonical system prompt (source of truth; synced into Modelfile) + fewshot.md — tagged examples for prompt engineering (added in Step 5) + lib/ + vocab_gen.py — generates vocab.json from live infra inventory (Step 3) + lint.py — deterministic frontmatter linter (Step 4) + agent.py — tagger + embeddings agent (Step 5) + vocab.json — generated entity lexicon (not committed; built by vocab_gen.py) + changelog.md — append-only audit log of all automated changes +``` + +## How it runs + +- **Daily cron** (`0 9 * * *` UTC): `sweep.sh` checks GPU VRAM, runs lint, runs agent + over docs changed since last sweep, appends to `changelog.md`. +- **Git pre-commit hook** (Step 4): runs lint against staged vault docs before commit. +- **Manual**: `./sweep.sh` or `python3 lib/agent.py --full` to reprocess all docs. + +## Setup + +```bash +./bootstrap.sh +``` + +`bootstrap.sh` is idempotent and documents every step. Base model weights (~8 GB) are +pulled by bootstrap — they are not committed to the repo. Rebuilding from scratch: + +```bash +./bootstrap.sh # pulls qwen2.5:7b-instruct-q8_0, builds vault-tagger, installs cron +``` + +## Vocabulary + +- **Tier 1 — topic tags**: stable list in `config.yaml` under `topic_categories` +- **Tier 2 — entity lexicon**: generated into `vocab.json` by `lib/vocab_gen.py` + from live Proxmox, Docker, and Headscale inventory. Regenerate anytime with: + `python3 lib/vocab_gen.py` + +## Configuration + +All tunables are in `config.yaml`. Key settings: + +- `behavior.auto_apply` — write changes directly (true) or dry-run only (false) +- `behavior.confidence_threshold` — below this, changes are flagged not applied +- `schedule.defer_if_gpu_busy_mib` — skip sweep if GPU is already under load diff --git a/engine/bootstrap.sh b/engine/bootstrap.sh new file mode 100755 index 0000000..f0dc7b1 --- /dev/null +++ b/engine/bootstrap.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# bootstrap.sh — Echo6 Vault Engine setup +# +# Idempotent setup script. Safe to re-run; each step is guarded. +# Run from any directory. Script locates itself via SCRIPT_DIR. +# +# Steps: +# 1. Verify services reachable (ollama, TEI, qdrant) +# 2. [DISABLED] Pull base model (~8 GB) — enable when ready +# 3. [DISABLED] Build vault-tagger modelfile — depends on step 2 +# 4. TODO: Install Python deps for lib/ +# 5. TODO: Generate initial vocab (lib/vocab_gen.py) +# 6. TODO: Install git pre-commit hook +# 7. TODO: Install cron job from config.yaml schedule + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CONFIG="${SCRIPT_DIR}/config.yaml" +MODELFILE="${SCRIPT_DIR}/Modelfile" + +echo "==> Echo6 Vault Engine bootstrap" +echo " engine_dir : ${SCRIPT_DIR}" +echo " config : ${CONFIG}" +echo "" + +# --------------------------------------------------------------------------- +# Step 1 — Verify services reachable +# --------------------------------------------------------------------------- +echo "[1/7] Checking service endpoints..." + +check_endpoint() { + local name="$1" + local url="$2" + if curl -sf --max-time 5 "$url" > /dev/null 2>&1; then + echo " OK ${name} (${url})" + else + echo " FAIL ${name} (${url}) — is the service running?" + return 1 + fi +} + +check_endpoint "ollama" "http://localhost:11434" +check_endpoint "TEI/bge-m3" "http://localhost:8090/health" +check_endpoint "qdrant" "http://localhost:6333" + +echo "" + +# --------------------------------------------------------------------------- +# Step 2 — Pull base model (DISABLED — enable when ready to pull ~8 GB) +# --------------------------------------------------------------------------- +# Uncomment the following block together with Step 3 when ready: +# +# echo "[2/7] Pulling base model qwen2.5:7b-instruct-q8_0..." +# ollama pull qwen2.5:7b-instruct-q8_0 +# echo "" + +echo "[2/7] SKIPPED — base model pull disabled. Uncomment in bootstrap.sh when ready." +echo "" + +# --------------------------------------------------------------------------- +# Step 3 — Build vault-tagger from Modelfile (DISABLED — depends on step 2) +# --------------------------------------------------------------------------- +# Uncomment together with Step 2: +# +# echo "[3/7] Building vault-tagger model from Modelfile..." +# ollama create vault-tagger -f "${MODELFILE}" +# echo "" + +echo "[3/7] SKIPPED — vault-tagger build disabled. Uncomment after pulling base model." +echo "" + +# --------------------------------------------------------------------------- +# Step 4 — Python dependencies for lib/ +# --------------------------------------------------------------------------- +# TODO (Step 2): Install Python deps once requirements.txt is written. +# DO NOT run pip install without explicit approval from Matt. +# +# Example (do NOT uncomment without approval): +# pip install -r "${SCRIPT_DIR}/requirements.txt" + +echo "[4/7] TODO — Python deps not yet defined. See lib/ stubs. Do not pip install without approval." +echo "" + +# --------------------------------------------------------------------------- +# Step 5 — Generate initial vocab (entity lexicon) +# --------------------------------------------------------------------------- +# TODO (Step 3): Calls lib/vocab_gen.py to query proxmox/docker/headscale +# and write engine/vocab.json. +# +# Example: +# python3 "${SCRIPT_DIR}/lib/vocab_gen.py" + +echo "[5/7] TODO — vocab_gen.py not yet implemented (Step 3)." +echo "" + +# --------------------------------------------------------------------------- +# Step 6 — Install git pre-commit hook +# --------------------------------------------------------------------------- +# TODO (Step 4): Symlink or copy a pre-commit hook that runs lint.py +# against staged vault docs before commit. +# +# Example: +# HOOK="${SCRIPT_DIR}/../../.git/hooks/pre-commit" +# ln -sf "${SCRIPT_DIR}/hooks/pre-commit" "${HOOK}" + +echo "[6/7] TODO — git pre-commit hook not yet implemented (Step 4)." +echo "" + +# --------------------------------------------------------------------------- +# Step 7 — Install cron job +# --------------------------------------------------------------------------- +# TODO (Step 6): Install cron from schedule.cron in config.yaml. +# Cron entry should call sweep.sh with appropriate guards. +# +# config.yaml schedule.cron: "0 9 * * *" +# Example crontab line: +# 0 9 * * * /home/zvx/projects/.ref/engine/sweep.sh >> /home/zvx/projects/.ref/engine/sweep.log 2>&1 + +echo "[7/7] TODO — cron job not yet installed (Step 6)." +echo "" + +echo "==> bootstrap.sh complete (partial — disabled steps noted above)." diff --git a/engine/config.yaml b/engine/config.yaml new file mode 100644 index 0000000..4784c54 --- /dev/null +++ b/engine/config.yaml @@ -0,0 +1,54 @@ +# Echo6 Vault Engine — configuration (single source of truth for the engine) +vault_dir: /home/zvx/projects/.ref/vault +engine_dir: /home/zvx/projects/.ref/engine + +models: + tagger: + ollama_endpoint: http://localhost:11434 + model: vault-tagger # built by bootstrap from Modelfile + base_model: qwen2.5:7b-instruct-q8_0 + temperature: 0.1 + format: json + embeddings: + tei_endpoint: http://localhost:8090 # existing bge-m3 service (reuse) + model: bge-m3 + qdrant_endpoint: http://localhost:6333 + qdrant_collection: vault_docs + +# Stable topic-tag vocabulary (tier 1). The entity lexicon (tier 2) is GENERATED into vocab.json. +topic_categories: + - mesh + - matrix + - recon + - media + - auth + - dns + - vpn + - storage + - proxmox + - ai + - mail + +# Read-only inventory sources for the generated entity lexicon (tier 2) +inventory: + proxmox_nodes: [data, utility, cloud, media, toc] # pct list / qm list + proxmox_vps: [contabo, edge2] # pct list + docker_hosts: [cortex, utility, media, contabo] # docker ps + headscale_host: contabo # headscale nodes list + ssh_user: zvx + +# Frontmatter property schema enforced by lint +frontmatter_schema: + required: [title, type, tags, updated] + optional: [aliases, related, status] + types: [reference, runbook, project, note, index, session] + +behavior: + auto_apply: true # write changes directly... + log_changes: true # ...but log every change for audit/revert + changelog: /home/zvx/projects/.ref/engine/changelog.md + confidence_threshold: 0.6 # below this, flag in changelog instead of silent + +schedule: + cron: "0 9 * * *" # 09:00 UTC daily (off-peak); guard checks GPU before running + defer_if_gpu_busy_mib: 6000 # skip/defer if >this much VRAM already in use diff --git a/engine/lib/__init__.py b/engine/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/engine/lib/__pycache__/vocab_gen.cpython-312.pyc b/engine/lib/__pycache__/vocab_gen.cpython-312.pyc new file mode 100644 index 0000000..e9121b9 Binary files /dev/null and b/engine/lib/__pycache__/vocab_gen.cpython-312.pyc differ diff --git a/engine/lib/agent.py b/engine/lib/agent.py new file mode 100644 index 0000000..fdfe72d --- /dev/null +++ b/engine/lib/agent.py @@ -0,0 +1,82 @@ +""" +agent.py — Vault Tagger + Embeddings Agent + +Job: + Run the vault-tagger LLM (Qwen2.5-7B via Ollama) over new or changed vault documents, + update their frontmatter with corrected tags/type/entities, then re-embed them into + Qdrant via the existing bge-m3 TEI service. + +Pipeline per document: + 1. Read doc + current frontmatter + 2. Load vocab: topic_categories from config.yaml + entity lexicon from vocab.json + 3. Call vault-tagger via Ollama /api/generate (JSON mode, temp 0.1) + 4. Parse JSON response; validate fields against vocabulary + 5. If confidence >= threshold: apply tags/type to frontmatter (auto_apply) + Else: flag in changelog, do not modify file + 6. Re-embed via TEI bge-m3 and upsert into Qdrant (collection: vault_docs) + 7. Update related wikilinks in frontmatter.related if embedding similarity > 0.85 + 8. Append changelog entry (file, old tags, new tags, confidence, timestamp) + +State tracking: + Maintains engine/.last_sweep (ISO timestamp) to process only docs modified since last run. + Pass --full to reprocess all docs. + +Implemented in: Step 5 +""" + +# TODO (Step 5): imports — pathlib, json, yaml, httpx or requests, datetime, argparse, logging + + +def load_config(config_path: str) -> dict: + """Load and return parsed config.yaml.""" + raise NotImplementedError("implemented in step 5") + + +def load_vocab(engine_dir: str) -> dict: + """Load topic_categories from config + entity lexicon from vocab.json.""" + raise NotImplementedError("implemented in step 5") + + +def get_changed_docs(vault_dir: str, since: str) -> list: + """Return list of .md paths modified after `since` (ISO timestamp).""" + raise NotImplementedError("implemented in step 5") + + +def call_tagger(doc_text: str, vocab: dict, config: dict) -> dict: + """ + POST to Ollama /api/generate with vault-tagger model. + Returns parsed JSON response dict. + """ + raise NotImplementedError("implemented in step 5") + + +def embed_document(doc_text: str, tei_endpoint: str) -> list[float]: + """POST to TEI bge-m3 endpoint; return embedding vector.""" + raise NotImplementedError("implemented in step 5") + + +def upsert_qdrant(doc_id: str, vector: list[float], payload: dict, config: dict) -> None: + """Upsert a document vector + metadata into Qdrant vault_docs collection.""" + raise NotImplementedError("implemented in step 5") + + +def find_related(doc_id: str, vector: list[float], config: dict, threshold: float = 0.85) -> list[str]: + """Query Qdrant for nearest neighbours above threshold; return doc ids.""" + raise NotImplementedError("implemented in step 5") + + +def apply_tagger_result(path, result: dict, config: dict) -> dict: + """ + Write tagger output back to doc frontmatter if confidence >= threshold. + Returns summary of changes made. + """ + raise NotImplementedError("implemented in step 5") + + +def main() -> None: + """Entry point. Parse args, load state, process changed docs, update state.""" + raise NotImplementedError("implemented in step 5") + + +if __name__ == "__main__": + main() diff --git a/engine/lib/lint.py b/engine/lib/lint.py new file mode 100644 index 0000000..6a2d092 --- /dev/null +++ b/engine/lib/lint.py @@ -0,0 +1,697 @@ +#!/usr/bin/env python3 +""" +lint.py — Deterministic Vault Lint (no LLM, stdlib only) + +Checks performed (all deterministic): + 1. Frontmatter schema — required keys, valid type, tags is a list + 2. Tag vocabulary — tags must be in topic_categories + {meta} + 3. Dead wikilinks — [[target]] resolves against note basenames + entity vocab + 4. Orphans — notes with zero incoming wikilinks (INFO, capped at 40) + +Severity: + ERROR — dead wikilinks + WARN — schema violations, unknown tags + INFO — orphan notes + +Usage: + python3 engine/lib/lint.py # report mode, always exit 0 + python3 engine/lib/lint.py --strict # exit 1 if any ERROR findings + +Writes: engine/lint-report.md +""" + +from __future__ import annotations + +import json +import os +import re +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import NamedTuple + +# --------------------------------------------------------------------------- +# Config / vocab loading (stdlib-only minimal YAML parser) +# --------------------------------------------------------------------------- + +def _parse_simple_yaml(text: str) -> dict: + """ + Minimal YAML parser: handles only the scalar/list constructs in config.yaml. + Supports: key: value, key: [a, b], - item under a key, # comments. + Does NOT handle nested dicts beyond top-level — enough for config.yaml. + """ + result: dict = {} + current_key = None + current_list: list | None = None + + for raw_line in text.splitlines(): + line = raw_line.rstrip() + stripped = line.lstrip() + if not stripped or stripped.startswith("#"): + # Flush pending list if indent drops + if current_list is not None and not line.startswith(" "): + result[current_key] = current_list + current_list = None + current_key = None + continue + + # List item under current key + if stripped.startswith("- ") and current_list is not None: + current_list.append(stripped[2:].strip().strip('"').strip("'")) + continue + + # End of list block + if current_list is not None and not stripped.startswith("-"): + result[current_key] = current_list + current_list = None + current_key = None + + if ":" in stripped: + key, _, value = stripped.partition(":") + key = key.strip() + value = value.strip() + if value.startswith("[") and value.endswith("]"): + # Inline list: [a, b, c] + inner = value[1:-1] + result[key] = [v.strip().strip('"').strip("'") for v in inner.split(",") if v.strip()] + elif value == "": + # May start a list block + current_key = key + current_list = [] + elif value.startswith("#"): + result[key] = "" + else: + # Scalar — strip inline comment + value = value.split(" #")[0].strip().strip('"').strip("'") + result[key] = value + + if current_list is not None and current_key: + result[current_key] = current_list + + return result + + +def load_config(engine_dir: Path) -> dict: + """Load config.yaml using minimal YAML parser.""" + config_path = engine_dir / "config.yaml" + text = config_path.read_text(encoding="utf-8") + return _parse_simple_yaml(text) + + +def load_vocab(engine_dir: Path) -> dict: + """Load vocab.json.""" + vocab_path = engine_dir / "vocab.json" + return json.loads(vocab_path.read_text(encoding="utf-8")) + + +# --------------------------------------------------------------------------- +# Frontmatter parsing +# --------------------------------------------------------------------------- + +_FM_FENCE = re.compile(r"^---\s*$") + + +def parse_frontmatter(path: Path) -> tuple[dict | None, str]: + """ + Parse YAML frontmatter block from a markdown file. + Returns (frontmatter_dict_or_None, body_text). + body_text is the full file text if no frontmatter. + """ + text = path.read_text(encoding="utf-8", errors="replace") + lines = text.splitlines(keepends=True) + + if not lines or not _FM_FENCE.match(lines[0].rstrip()): + return None, text + + end_idx = None + for i in range(1, len(lines)): + if _FM_FENCE.match(lines[i].rstrip()): + end_idx = i + break + + if end_idx is None: + return None, text + + fm_text = "".join(lines[1:end_idx]) + body = "".join(lines[end_idx + 1 :]) + fm = _parse_yaml_frontmatter(fm_text) + return fm, body + + +def _parse_yaml_frontmatter(text: str) -> dict: + """ + Parse simple flat YAML frontmatter. Handles: + key: scalar + key: [a, b, c] + key: + - a + - b + tags: [a, b] + tags: + - a + """ + result: dict = {} + current_key: str | None = None + current_list: list | None = None + + for raw_line in text.splitlines(): + line = raw_line.rstrip() + stripped = line.lstrip() + + if not stripped: + continue + + # List item + if stripped.startswith("- ") and current_list is not None: + current_list.append(stripped[2:].strip().strip('"').strip("'")) + continue + + # New key — flush pending list + if current_list is not None: + result[current_key] = current_list + current_list = None + current_key = None + + if ":" in stripped: + key, _, value = stripped.partition(":") + key = key.strip() + value = value.strip() + + if value.startswith("[") and value.endswith("]"): + inner = value[1:-1] + result[key] = [ + v.strip().strip('"').strip("'") + for v in inner.split(",") + if v.strip() + ] + elif value == "": + current_key = key + current_list = [] + else: + result[key] = value.strip('"').strip("'") + + if current_list is not None and current_key: + result[current_key] = current_list + + return result + + +# --------------------------------------------------------------------------- +# Check 1: Frontmatter schema +# --------------------------------------------------------------------------- + +VALID_TYPES = {"reference", "runbook", "project", "note", "index", "session"} + + +def check_frontmatter(fm: dict | None, config: dict) -> list[dict]: + """ + Returns list of findings dicts: {severity, check, message} + WARN severity for all schema violations. + """ + findings = [] + + required_keys = ["title", "type", "tags", "updated"] + + if fm is None: + findings.append( + { + "severity": "WARN", + "check": "frontmatter", + "message": "missing frontmatter block entirely", + } + ) + return findings + + for key in required_keys: + if key not in fm or fm[key] is None or fm[key] == "": + findings.append( + { + "severity": "WARN", + "check": "frontmatter", + "message": f"missing required key: {key!r}", + } + ) + + if "type" in fm and fm["type"]: + if fm["type"] not in VALID_TYPES: + findings.append( + { + "severity": "WARN", + "check": "frontmatter", + "message": f"invalid type {fm['type']!r} — must be one of {sorted(VALID_TYPES)}", + } + ) + + if "tags" in fm and fm["tags"] is not None: + if not isinstance(fm["tags"], list): + findings.append( + { + "severity": "WARN", + "check": "frontmatter", + "message": f"tags must be a list, got {type(fm['tags']).__name__}: {fm['tags']!r}", + } + ) + + return findings + + +# --------------------------------------------------------------------------- +# Check 2: Tag vocabulary +# --------------------------------------------------------------------------- + +ALLOWED_EXTRA_TAGS = {"meta"} + + +def check_tags(fm: dict | None, allowed_tags: set[str]) -> list[dict]: + """Returns WARN findings for tags outside the allowed vocabulary.""" + if fm is None: + return [] + tags = fm.get("tags") + if not tags or not isinstance(tags, list): + return [] + + findings = [] + for tag in tags: + if tag not in allowed_tags: + findings.append( + { + "severity": "WARN", + "check": "tag-vocab", + "message": f"unknown tag {tag!r} (not in topic_categories or allowed extras)", + } + ) + return findings + + +# --------------------------------------------------------------------------- +# Check 3: Dead wikilinks +# --------------------------------------------------------------------------- + +# Matches [[target]], [[target|alias]], [[target#heading]] +# But NOT bash [[ ... ]] test syntax. +# Strategy: require the inner text to look like a note name: +# - no spaces at start/end of target +# - may contain word chars, hyphens, dots, spaces, #, | — but NOT operators like +# -n, -f, ==, !, &&, ||, etc. +# We detect shell syntax by checking for space-separated words starting with - or +# containing shell operators. + +_WIKILINK_RE = re.compile( + r""" + \[\[ # opening [[ + ([^\[\]\n]+?) # capture: link target (non-greedy, no newlines) + (?:\|[^\[\]\n]*)? # optional |alias + \]\] # closing ]] + """, + re.VERBOSE, +) + +_FENCED_CODE_RE = re.compile(r"```.*?```", re.DOTALL) +_INLINE_CODE_RE = re.compile(r"`[^`\n]+`") + + +def _looks_like_shell(inner: str) -> bool: + """Return True if the wikilink inner text looks like bash test syntax.""" + # Shell: -n "$VAR", ! -f "$FILE", "$CODEC" == "value", -z, etc. + s = inner.strip() + # Contains bash operators or variable expansions + if re.search(r'\$[{(A-Za-z_]', s): + return True + # Starts with - (flag) or ! (negation), or contains == / != / && / || + if re.match(r'^\s*[!-]', s): + return True + if re.search(r'==|!=|&&|\|\|', s): + return True + # Multiple space-separated tokens that look like args + tokens = s.split() + if len(tokens) > 1 and tokens[0].startswith("-"): + return True + return False + + +def _is_note_like(inner: str) -> bool: + """ + Return True if inner text looks like a note name: + word chars, hyphens, dots, spaces, but no shell operators. + """ + if _looks_like_shell(inner): + return False + # Must match a reasonable note-name pattern + # Allow: letters, digits, hyphens, underscores, dots, spaces + return bool(re.match(r'^[\w\s.\-/]+$', inner.strip())) + + +def _strip_code_blocks(text: str) -> str: + """Remove fenced code blocks and inline code to avoid false wikilink matches.""" + text = _FENCED_CODE_RE.sub("", text) + text = _INLINE_CODE_RE.sub("", text) + return text + + +def _extract_wikilinks(body: str) -> list[str]: + """Extract note targets from wikilinks in body text, ignoring code blocks and shell syntax.""" + clean = _strip_code_blocks(body) + targets = [] + for m in _WIKILINK_RE.finditer(clean): + raw = m.group(1) + # Strip heading: [[target#heading|alias]] -> target + # Strip alias already handled by regex (group 1 = before |) + target = raw.split("|")[0].split("#")[0].strip() + if target and _is_note_like(target): + targets.append(target) + return targets + + +def _note_basename(path: Path) -> str: + """Return the basename (without extension) of a vault note, lowercased.""" + return path.stem.lower() + + +def _normalize_name(name: str) -> str: + """Normalize: lowercase, spaces↔hyphens.""" + return name.lower().replace(" ", "-").replace("_", "-") + + +def build_note_index(vault_dir: Path) -> dict[str, Path]: + """ + Build a mapping of normalized name → Path for all vault notes. + One entry per file, keyed by normalized basename. + """ + index: dict[str, Path] = {} + for p in vault_dir.rglob("*.md"): + key = _normalize_name(p.stem) + index[key] = p + return index + + +def build_entity_names(vocab: dict) -> set[str]: + """ + Build a set of normalized entity names + aliases from vocab.json. + These count as valid wikilink targets (they're in the knowledge graph). + """ + names: set[str] = set() + for entity in vocab.get("entities", []): + names.add(_normalize_name(entity["name"])) + names.add(_normalize_name(entity["display"])) + for alias in entity.get("aliases", []): + names.add(_normalize_name(alias)) + for cat in vocab.get("topic_categories", []): + names.add(_normalize_name(cat)) + return names + + +def check_dead_links( + path: Path, + body: str, + note_index: dict[str, Path], + entity_names: set[str], +) -> list[dict]: + """ + Returns ERROR findings for wikilinks that resolve to neither a vault note + nor a known entity name. + """ + findings = [] + for target in _extract_wikilinks(body): + norm = _normalize_name(target) + if norm in note_index or norm in entity_names: + continue + findings.append( + { + "severity": "ERROR", + "check": "dead-link", + "message": f"dead wikilink [[{target}]]", + } + ) + return findings + + +# --------------------------------------------------------------------------- +# Check 4: Orphans (incoming wikilink count) +# --------------------------------------------------------------------------- + +ORPHAN_CAP = 40 + + +def build_backlink_counts( + vault_dir: Path, + note_index: dict[str, Path], + entity_names: set[str], +) -> dict[str, int]: + """ + Count how many other notes link TO each note. + Returns {normalized_stem: count}. + """ + counts: dict[str, int] = {k: 0 for k in note_index} + + for p in vault_dir.rglob("*.md"): + try: + _, body = parse_frontmatter(p) + except Exception: + continue + for target in _extract_wikilinks(body): + norm = _normalize_name(target) + if norm in counts: + counts[norm] += 1 + + return counts + + +def find_orphans( + vault_dir: Path, + note_index: dict[str, Path], + backlink_counts: dict[str, int], +) -> list[dict]: + """ + Return INFO findings for notes with zero incoming wikilinks. + Capped at ORPHAN_CAP entries. + """ + findings = [] + for key, path in sorted(note_index.items()): + if backlink_counts.get(key, 0) == 0: + rel = path.relative_to(vault_dir) + findings.append( + { + "severity": "INFO", + "check": "orphan", + "message": f"no incoming links: {rel}", + "_path": path, + } + ) + return findings[:ORPHAN_CAP] + + +# --------------------------------------------------------------------------- +# Reporting +# --------------------------------------------------------------------------- + +class LintResult(NamedTuple): + path: Path + findings: list[dict] + + +def format_report( + results: list[LintResult], + vault_dir: Path, + total_docs: int, + elapsed_s: float, +) -> str: + """Format a human-readable + markdown report.""" + ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + errors = [f for r in results for f in r.findings if f["severity"] == "ERROR"] + warns = [f for r in results for f in r.findings if f["severity"] == "WARN"] + infos = [f for r in results for f in r.findings if f["severity"] == "INFO"] + + # Count specific warn subcategories + fm_missing = sum( + 1 + for r in results + for f in r.findings + if f["severity"] == "WARN" + and f["check"] == "frontmatter" + and "missing frontmatter block" in f["message"] + ) + fm_invalid = sum( + 1 + for r in results + for f in r.findings + if f["severity"] == "WARN" + and f["check"] == "frontmatter" + and "missing frontmatter block" not in f["message"] + ) + tag_warns = sum( + 1 for r in results for f in r.findings if f["severity"] == "WARN" and f["check"] == "tag-vocab" + ) + + lines = [ + f"# Vault Lint Report", + f"", + f"Generated: {ts} | Docs scanned: {total_docs} | Elapsed: {elapsed_s:.1f}s", + f"", + f"## Summary", + f"", + f"| Severity | Count |", + f"|----------|-------|", + f"| ERROR (dead links) | {len(errors)} |", + f"| WARN (schema) | {len(warns)} |", + f"| INFO (orphans) | {len(infos)} |", + f"", + f"### WARN breakdown", + f"- Missing frontmatter block: {fm_missing}", + f"- Invalid/missing frontmatter fields: {fm_invalid}", + f"- Unknown tags: {tag_warns}", + f"", + ] + + if errors: + lines.append("## ERROR — Dead Wikilinks") + lines.append("") + for r in results: + errs = [f for f in r.findings if f["severity"] == "ERROR"] + if errs: + rel = r.path.relative_to(vault_dir) + for f in errs: + lines.append(f"- `{rel}` — {f['message']}") + lines.append("") + else: + lines.append("## ERROR — Dead Wikilinks") + lines.append("") + lines.append("_None. All wikilinks resolve._") + lines.append("") + + if warns: + lines.append("## WARN — Schema & Tag Violations") + lines.append("") + for r in results: + ws = [f for f in r.findings if f["severity"] == "WARN"] + if ws: + rel = r.path.relative_to(vault_dir) + for f in ws: + lines.append(f"- `{rel}` — {f['message']}") + lines.append("") + + if infos: + lines.append(f"## INFO — Orphan Notes (no incoming links, capped at {ORPHAN_CAP})") + lines.append("") + for f in infos: + lines.append(f"- {f['message']}") + lines.append("") + + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Public API (importable by sweep.sh or other scripts) +# --------------------------------------------------------------------------- + +def run_lint(vault_dir: Path, engine_dir: Path) -> tuple[list[LintResult], dict]: + """ + Run all lint checks. Returns (results, stats_dict). + Importable entry point. + """ + import time + + config = load_config(engine_dir) + vocab = load_vocab(engine_dir) + + # Build allowed tag set + topic_cats = set(vocab.get("topic_categories", [])) + topic_cats |= ALLOWED_EXTRA_TAGS + + note_index = build_note_index(vault_dir) + entity_names = build_entity_names(vocab) + + # Build backlink counts (single pass over all docs) + backlink_counts = build_backlink_counts(vault_dir, note_index, entity_names) + + all_paths = sorted(vault_dir.rglob("*.md")) + results: list[LintResult] = [] + + t0 = time.monotonic() + + for path in all_paths: + findings: list[dict] = [] + try: + fm, body = parse_frontmatter(path) + except Exception as exc: + findings.append( + {"severity": "WARN", "check": "parse-error", "message": f"could not parse: {exc}"} + ) + results.append(LintResult(path=path, findings=findings)) + continue + + findings.extend(check_frontmatter(fm, config)) + findings.extend(check_tags(fm, topic_cats)) + findings.extend(check_dead_links(path, body, note_index, entity_names)) + results.append(LintResult(path=path, findings=findings)) + + # Orphan check (uses backlink counts already computed) + orphan_findings = find_orphans(vault_dir, note_index, backlink_counts) + # Attach orphan findings as a synthetic "vault" result + if orphan_findings: + results.append(LintResult(path=vault_dir / "_orphans_", findings=orphan_findings)) + + elapsed = time.monotonic() - t0 + + stats = { + "total_docs": len(all_paths), + "errors": sum(1 for r in results for f in r.findings if f["severity"] == "ERROR"), + "warns": sum(1 for r in results for f in r.findings if f["severity"] == "WARN"), + "infos": sum(1 for r in results for f in r.findings if f["severity"] == "INFO"), + "elapsed_s": elapsed, + } + return results, stats + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def main() -> None: + import time + + strict = "--strict" in sys.argv + + # Locate engine/vault dirs relative to this file + this_file = Path(__file__).resolve() + engine_dir = this_file.parent.parent # engine/lib/lint.py → engine/ + config = load_config(engine_dir) + + vault_str = config.get("vault_dir", "") + if not vault_str: + # Fallback: sibling of engine_dir named "vault" + vault_dir = engine_dir.parent / "vault" + else: + vault_dir = Path(vault_str) + + if not vault_dir.exists(): + print(f"ERROR: vault_dir not found: {vault_dir}", file=sys.stderr) + sys.exit(1) + + results, stats = run_lint(vault_dir, engine_dir) + + report_text = format_report( + results, vault_dir, stats["total_docs"], stats["elapsed_s"] + ) + + # Write report + report_path = engine_dir / "lint-report.md" + report_path.write_text(report_text, encoding="utf-8") + + # Print to stdout + print(report_text) + print(f"--- Wrote: {report_path} ---") + + # Summary line + print( + f"\nSummary: {stats['total_docs']} docs | " + f"{stats['errors']} ERRORs | {stats['warns']} WARNs | {stats['infos']} INFOs" + ) + + if strict and stats["errors"] > 0: + sys.exit(1) + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/engine/lib/vocab_gen.py b/engine/lib/vocab_gen.py new file mode 100644 index 0000000..8cac505 --- /dev/null +++ b/engine/lib/vocab_gen.py @@ -0,0 +1,1175 @@ +#!/usr/bin/env python3 +""" +vocab_gen.py — Entity Lexicon + Acronym Vocabulary Generator (Tier 2) + +Primary source: vault docs (reliable, complete) +Secondary source: live infra inventory (best-effort, SSH, read-only) + +Outputs: + engine/vocab.json — structured vocabulary + vault/glossary.md — browsable Obsidian note +""" + +import json +import os +import re +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + + +# --------------------------------------------------------------------------- +# Config loader (stdlib-only YAML parser for the keys we need) +# --------------------------------------------------------------------------- + +def load_config(config_path: str) -> dict: + """Minimal YAML parser for flat / simple-list config.yaml.""" + config = { + "vault_dir": "/home/zvx/projects/.ref/vault", + "engine_dir": "/home/zvx/projects/.ref/engine", + "topic_categories": [ + "mesh", "matrix", "recon", "media", "auth", + "dns", "vpn", "storage", "proxmox", "ai", "mail", + ], + "inventory": { + "proxmox_nodes": ["data", "utility", "cloud", "media", "toc"], + "proxmox_vps": ["contabo", "edge2"], + "docker_hosts": ["cortex", "utility", "media", "contabo"], + "headscale_host": "contabo", + "ssh_user": "zvx", + }, + } + try: + with open(config_path) as f: + lines = f.readlines() + + # Parse vault_dir / engine_dir + for line in lines: + m = re.match(r'^(vault_dir|engine_dir):\s*(.+)', line) + if m: + config[m.group(1)] = m.group(2).strip() + + # Parse topic_categories list + in_topics = False + topics = [] + for line in lines: + if re.match(r'^topic_categories:', line): + in_topics = True + continue + if in_topics: + m = re.match(r'^\s+-\s+(\S+)', line) + if m: + topics.append(m.group(1).strip()) + elif line.strip() and not line.startswith(' '): + in_topics = False + if topics: + config["topic_categories"] = topics + + # Parse inventory.proxmox_nodes (inline list) + for line in lines: + m = re.match(r'\s*proxmox_nodes:\s*\[(.+)\]', line) + if m: + config["inventory"]["proxmox_nodes"] = [ + x.strip().strip("'\"") for x in m.group(1).split(',') + ] + m2 = re.match(r'\s*proxmox_vps:\s*\[(.+)\]', line) + if m2: + config["inventory"]["proxmox_vps"] = [ + x.strip().strip("'\"") for x in m2.group(1).split(',') + ] + m3 = re.match(r'\s*docker_hosts:\s*\[(.+)\]', line) + if m3: + config["inventory"]["docker_hosts"] = [ + x.strip().strip("'\"") for x in m3.group(1).split(',') + ] + m4 = re.match(r'\s*headscale_host:\s*(\S+)', line) + if m4: + config["inventory"]["headscale_host"] = m4.group(1).strip() + m5 = re.match(r'\s*ssh_user:\s*(\S+)', line) + if m5: + config["inventory"]["ssh_user"] = m5.group(1).strip() + + except Exception as e: + print(f"[warn] Could not fully parse {config_path}: {e}; using defaults", file=sys.stderr) + + return config + + +# --------------------------------------------------------------------------- +# Host IP map (for enrichment during doc parsing) +# --------------------------------------------------------------------------- + +HOST_IPS = { + "data": ("192.168.1.240", "100.64.0.6"), + "utility": ("192.168.1.241", "100.64.0.5"), + "cloud": ("192.168.1.242", "100.64.0.4"), + "media": ("192.168.1.243", "100.64.0.3"), + "toc": ("192.168.1.244", "100.64.0.13"), + "cortex": ("192.168.1.150", "100.64.0.14"), + "recon-vm": ("192.168.1.130", "100.64.0.24"), + "arr": ("192.168.1.160", "100.64.0.18"), + "aida-nebra": ("192.168.1.253", "100.64.0.9"), + "mt-isr": ("192.168.1.141", "100.100.0.5"), + "mt-burleybutte": ("192.168.1.185", None), + "pi-nas": ("192.168.1.245", "100.64.0.21"), + "matt-desktop": ("192.168.1.111", "100.64.0.10"), + "contabo": ("5.189.158.149", "100.64.0.1"), + "edge2": ("184.174.35.153", "100.64.0.26"), +} + + +# --------------------------------------------------------------------------- +# Fallback English-word stoplist (~300 most common words). +# Used when /usr/share/dict/words is absent. +# A candidate token whose .lower() is in this set is rejected as a plain +# English word and NOT treated as a domain acronym. +# --------------------------------------------------------------------------- + +_COMMON_ENGLISH_WORDS = { + # articles / determiners + "a", "an", "the", "this", "that", "these", "those", "its", "our", "your", + "their", "my", "his", "her", "all", "both", "each", "every", "few", "more", + "most", "other", "some", "such", "no", "nor", "not", "only", "same", "so", + "than", "too", "very", + # pronouns + "i", "me", "we", "us", "you", "he", "she", "it", "they", "them", "who", + "whom", "which", "what", "one", + # conjunctions / prepositions + "and", "but", "or", "yet", "for", "nor", "so", "at", "by", "in", "of", + "on", "to", "up", "as", "be", "do", "if", "is", "it", "no", "was", + "are", "had", "has", "have", "how", "may", "per", "did", "due", "out", + "via", "any", "can", "could", "from", "into", "over", "then", "they", + "with", "will", "been", "down", "also", "when", "where", "while", + "after", "about", "above", "below", "under", "until", "before", + "between", "through", "during", "without", "within", "against", + "around", "despite", + # verbs + "run", "get", "set", "put", "use", "let", "see", "say", "add", "act", + "ask", "cut", "end", "hit", "log", "map", "mix", "off", "own", "pass", + "read", "send", "stop", "test", "try", "wait", "want", "work", "show", + "load", "save", "move", "list", "boot", "call", "copy", "drop", "edit", + "exit", "fail", "find", "give", "help", "hold", "keep", "kill", "link", + "lock", "make", "mark", "must", "need", "note", "open", "pick", "ping", + "push", "pull", "quit", "skip", "take", "tell", "used", "view", "warn", + "check", "clean", "clear", "close", "count", "cover", "create", "defer", + "delete", "deploy", "enable", "export", "fetch", "flush", "force", + "grant", "import", "inject", "insert", "launch", "manage", "mount", + "output", "parse", "patch", "print", "prove", "query", "raise", + "reboot", "reload", "remove", "rename", "reset", "restart", "return", + "rotate", "select", "should", "signal", "start", "stream", "submit", + "switch", "sync", "update", "upload", "verify", "write", + # adjectives / adverbs + "new", "old", "big", "low", "high", "raw", "hot", "cold", "fast", + "slow", "free", "full", "good", "best", "bad", "next", "last", "long", + "main", "same", "true", "false", "empty", "first", "local", "valid", + "basic", "clean", "close", "direct", "extra", "final", "fixed", + "global", "human", "large", "light", "minor", "never", "plain", + "quick", "ready", "right", "short", "small", "smart", "solid", + "spare", "super", "total", "unique", "upper", "valid", "whole", + "active", "actual", "custom", "failed", "hidden", "modern", "native", + "normal", "public", "random", "recent", "remote", "simple", "single", + "stable", "static", "strong", "target", + # common nouns (non-technical) + "way", "day", "man", "end", "top", "yes", "now", "ago", + "area", "base", "case", "date", "door", "edge", "else", "face", + "fact", "file", "form", "game", "home", "host", "idea", "info", + "item", "kind", "lack", "lane", "life", "line", "link", "list", + "mode", "name", "node", "note", "page", "part", "path", "plan", + "plug", "plus", "port", "post", "rate", "rule", "side", "site", + "size", "slot", "spec", "step", "tag", "task", "term", "text", + "time", "type", "user", "word", "year", + "admin", "agent", "alias", "array", "audit", "batch", "block", + "board", "build", "cache", "chain", "chart", "chunk", "class", + "cloud", "codec", "color", "count", "debug", "delta", "depth", + "error", "event", "field", "flags", "floor", "frame", "front", + "graph", "group", "guard", "guide", "image", "index", "input", + "issue", "label", "layer", "lease", "level", "limit", "match", + "media", "model", "mount", "mutex", "order", "owner", "panel", + "phase", "place", "point", "power", "queue", "quota", "range", + "relay", "reply", "retry", "route", "round", "scale", "scope", + "score", "shard", "share", "shift", "skill", "slice", "space", + "stack", "stage", "state", "stats", "store", "suite", "table", + "theme", "token", "trace", "track", "trait", "trunk", "tuple", + "union", "unit", "value", "vault", "watch", "wheel", "world", + # words that look like acronyms when uppercased + "ward", "warp", "wrap", "wire", "wars", "star", "mark", "mars", + "arc", + # common uppercase bash/config tokens that are not acronyms + "eof", "ok", "yes", "true", "false", "none", "null", + "enabled", "disabled", "missing", "blocked", "forward", "nopasswd", + "before", "after", "begin", "end", "stop", "start", "pass", "fail", + "skip", "done", "running", "stopped", "exited", "created", + "password", "hostname", "logfile", "timestamp", "database", + "metric", "metrics", "redacted", "domain", + # standard computing abbreviations that are generic (not Echo6-specific) + "http", "https", "dns", "ssh", "api", "url", "json", "yaml", + "cpu", "gpu", "ram", "ip", "tcp", "udp", "ssl", "tls", + "cors", "nvme", "usb", "lan", "vpn", "sso", "nas", "mqtt", + "html", "css", "nfs", "mac", "tun", "uri", "wal", + "pve", "rss", "sim", "vram", "vps", "lxc", "cli", "pdf", + "smtp", "jwt", "jwks", "rsa", "bbs", "web", "llm", "rag", + "pem", "aio", "crud", "uuid", "hls", "wsl", "tsv", + "esp", "gps", "cuda", "epub", "zim", "npc", "hmac", + "nat", "utc", "dhcp", "sql", "vm", "ct", "ai", + "ddr", "rtx", "ssd", "nvidia", "amd", "arm", "gnu", "iso", + "xml", "csv", "svg", "png", "mp3", "zip", "ini", "toml", + "ttl", "irq", "nic", "imap", "smtp", "scp", "sftp", "ftp", + "rgb", "utf", "utf8", "ascii", "hex", "eof", "bom", + "iommu", "vfio", "acl", "cname", "mx", "txt", "ptr", + "icmp", "arp", "bgp", "ospf", "vlan", "mpls", "gre", + "tun", "tap", "psk", "rsa", "ecdsa", "aes", "hmac", + "jwt", "oidc", # oidc is in entity_upper_tokens so rule 4 will re-admit if needed + "rs256", "es512", # crypto alg names + "orm", "spa", "dom", "ajax", "mvc", "api", + "uefi", "bios", "grub", "raid", "lvm", "xfs", "ext4", + "hevc", "h264", "nvenc", "cuda", "opencl", + "sdr", "fm", "am", "uhf", "vhf", "hf", + "pii", "gdpr", "opsec", + "ml", "ai", "nlp", "ocr", # generic AI/ML terms + "tbd", "tbr", "wip", "poc", "mvp", "rfc", "pr", "mr", + "ui", "ux", "seo", "cms", + # shell / scripting env vars and common uppercase tokens + "path", "home", "user", "term", "lang", "shell", "editor", + "pager", "display", "tmpdir", "config", "debug", "verbose", + "output", "input", "timeout", + # 2-char garbage tokens (generic abbreviations, not Echo6-specific) + "db", "id", "mb", "gb", "kb", "tb", "hp", "ui", "ux", + "cc", "ad", "pk", "vk", "fk", "js", "ts", "ws", "wl", + "de", "en", "uk", "us", "ca", "eu", "au", "nz", "sf", + "tx", "fl", "pr", "bx", "la", "cs", "ml", "dl", "rl", + "ve", "ia", "os", "re", "io", "ok", "ps", "ls", "rm", + "mv", "cp", "ln", "cd", "bg", "fg", "mm", "em", "px", + "ms", "ns", "hz", "mhz", "ghz", "tb", "pb", + # generic ALL-CAPS words that aren't acronyms + "client", "server", "binary", "script", "memory", "cores", + "router", "bridge", "backup", "latest", "stale", "broken", + "health", "login", "works", "found", "exists", "nodes", + "scopes", "phases", "height", "weight", "depth", "width", + "accept", "reject", "allow", "deny", "block", "permit", + "pragma", "pragma", "upsert", "revert", "regen", "comms", + "apikey", "pubkey", "dbpass", "ssheof", "pyeof", "xxxx", + "ctid", "vmid", "puid", "pgid", "uid", "gid", "pid", "oid", + "cid", "rid", "sid", "tid", "mid", "bid", "vid", "fid", + "dest", "src", "dst", "tmp", "var", "buf", "ptr", "ref", + "max", "min", "avg", "sum", "cnt", "num", "idx", "pos", + "key", "val", "str", "int", "obj", "arr", "len", "cap", + "lts", "oom", "tui", "isp", "acl", "crl", "csr", "der", + "pem", "ber", "asn", "oid", "dn", "cn", "ou", "dc", + "sata", "nvme", "pcie", "usb", "hdmi", "vga", "dvi", + "ddr4", "ecc", "efi", "gpt", "mbr", + "xxx", "xxxx", "xxxxxx", + "tz", "bbs", "dcrg", "embr", + "lure", "gate", "shop", "lord", "dead", "bank", "rap", + "spd", "def", "ff", "rpt", "ys", "bk", + # generic tech + project acronyms that are NOT Echo6-specific + "rest", "smb", "cifs", "nfs", "omv", "obs", "tui", "gui", + "diy", "blm", "usfs", "arrl", "csec", "wspr", # ham/org abbreviations + "tv", "pro", "yt", "mp4", "mkv", "dvd", "blu", "hdr", + "dcrg", "embr", # already in but restate + "cannot", "plenty", "works", "stale", "broken", "hard", + "two", "three", "four", "five", "six", "seven", "eight", "nine", + "yyyy", "mmdd", "hhmm", "utc", "gmt", + "xx", "xxx", "xxxx", + "glob", "role", "scope", "claim", "grant", "token", + "login", "logout", "signup", "register", + "atak", # Android Team Awareness Kit — covered as TAK + "mpeg", "mpeg4", "hevc", "h264", "h265", "avc", "av1", "vp9", + "wsl2", # Windows Subsystem — generic + "derp", "stun", "turn", # Tailscale/DERP networking — generic + "amqp", "mqtt", "stomp", # messaging protocols — generic + "hnsw", # vector index algo — generic tech + "jwk", "jwe", "jws", # JWT variants — generic + "ec", "rsa", "dsa", "dh", # crypto primitives + "cmp", "sub", "add", "xor", "and", "div", "mod", + "nbr", "lp", "lp", + "sut", # System Under Test + "fm10", "g10", "g9", "g8", "g7", "g6", "g5", "g3", "g2", "g1", + "g11", "g13", "g12", # camera model series + "n2", "n3", "n4", "n5", # Raspberry Pi / Compute Module designators + "cm3", "cm4", "cm5", # Compute Module designators + "a4000", "a5000", "a6000", "rtx4090", "rx580", # GPU model numbers + "mv51", "mv52", # hardware model numbers + "sx1262", "sx1268", "sx1276", # LoRa chip designators + "tsip", "tsop", # chip/protocol designators + "nebra", # Nebra is a specific brand entity covered as aida-nebra + "sigint", # signals intelligence — generic intel term + "dnat", "snat", "masq", # iptables — generic networking + "opsec", "pii", # generic security terms + "gdal", "geos", "proj", # geospatial libs — generic + "jsonl", "toml", "yaml", "msgpack", # data formats + "grst", # unknown but generic-looking + "mrn", "trvl", "j51b", # unclear short codes + "f0d848", # hex color / hash + "sata", "pcie", "nvme", "ddr4", "ddr5", # hardware bus standards + "e6", "e7", "e8", # generic model/version numbers + "claude", # Anthropic AI — not an Echo6 acronym + "bible", # not an acronym + "spd", "rpm", "rps", # metrics + # common English words missed above + "rest", "stop", "hold", "pass", "fail", "warn", "info", + "send", "recv", "read", "write", "open", "close", "seek", + "two", "one", "ten", "max", "min", "sum", "avg", + "hard", "soft", "fast", "slow", "long", "short", "wide", "thin", + "new", "old", "raw", "hot", "cold", "full", "empty", +} + + +# Curated acronym expansions — these OVERRIDE auto-derived expansions. +# Add entries here for Echo6-specific acronyms that need a fixed, authoritative expansion. +CURATED_EXPANSIONS: dict[str, str] = { + "AIDA": "Autonomous Idaho Assistant", + "MMUD": "Mesh Multi-User Dungeon", +} + +# Generic/googleable acronyms — excluded; glossary is for Echo6-specific terms only. Add to this list as needed. +GENERIC_ACRONYM_BLOCKLIST: set[str] = { + # Seeded from identified generic/googleable terms + "MAS", "TAK", "OTS", "OSINT", "DEM", "DM", "E2EE", "E2BE", + # Common tech/web/protocol acronyms + "OIDC", "SAML", "SSO", "JWT", "OAUTH", + "NVENC", "DNAT", "SNAT", "ACL", "CIDR", "DHCP", "NTP", + "SMTP", "IMAP", "LDAP", "REST", "GRPC", "CORS", "MQTT", + "RAID", "ZFS", "LVM", "VLAN", "NAT", "WAF", "CDN", "UPS", "PSU", + # Military / ATAK / comms + "ATAK", "COT", + # Generic networking + "RTT", "TTL", "MTU", "RPC", + # Hardware / SBC + "SBC", + # Web / app + "PWA", +} + + +# --------------------------------------------------------------------------- +# A. Parse vault docs +# --------------------------------------------------------------------------- + +def parse_environment_md(path: Path) -> list[dict]: + """Extract entities from environment.md tables.""" + entities = [] + seen_names = set() + + try: + text = path.read_text() + except Exception as e: + print(f"[warn] Cannot read {path}: {e}", file=sys.stderr) + return entities + + def add(name, display, etype, aliases, runs_on=None, extra=None): + canonical = name.lower().replace(' ', '-').replace('_', '-') + if canonical in seen_names: + return + seen_names.add(canonical) + entry = { + "name": canonical, + "display": display, + "type": etype, + "aliases": [a for a in aliases if a], + "runs_on": runs_on, + "source": "docs", + } + if extra: + entry.update(extra) + entities.append(entry) + + # --- Proxmox cluster nodes table --- + # | Node | Local IP | Tailscale | Hardware | RAM | Purpose | + node_pat = re.compile( + r'^\|\s*(data|utility|cloud|media|toc)\s*\|\s*(\d+\.\d+\.\d+\.\d+)\s*\|\s*(\S+)\s*\|', + re.MULTILINE + ) + for m in node_pat.finditer(text): + node, lip, ts = m.group(1), m.group(2), m.group(3) + add(node, node.capitalize(), "host", [lip, ts]) + + # --- VMs table --- + # | VM | Host | VMID | Local IP | Tailscale | Purpose | + vm_pat = re.compile( + r'^\|\s*(\S+)\s*\|\s*(\S+)\s*\|\s*(\d+)\s*\|\s*(\d+\.\d+\.\d+\.\d+)\s*\|\s*(\S+)\s*\|', + re.MULTILINE + ) + for m in vm_pat.finditer(text): + name, host, vmid, lip, ts = m.group(1), m.group(2), m.group(3), m.group(4), m.group(5) + if name.startswith('--') or name == 'VM': + continue + aliases = [lip] + if ts and ts not in ('—', '-', '|'): + aliases.append(ts) + aliases.append(f"VM {vmid}") + add(name, name, "vm", aliases, runs_on=host) + + # --- Key Servers table --- + # | Server | Local IP | Tailscale | Purpose | + server_pat = re.compile( + r'^\|\s*([a-zA-Z][a-zA-Z0-9\-\.]+)\s*\|\s*(\d+\.\d+\.\d+\.\d+)\s*\|\s*([^\|]+)\s*\|\s*([^\|]+)\s*\|', + re.MULTILINE + ) + for m in server_pat.finditer(text): + name, lip, ts_raw, purpose = ( + m.group(1).strip(), m.group(2).strip(), + m.group(3).strip(), m.group(4).strip() + ) + # Skip header rows and already-parsed nodes + if name in ('Node', 'VM', 'Container', 'Server', 'Node Storage'): + continue + if name.startswith('-'): + continue + ts = ts_raw if re.match(r'100\.\d+', ts_raw) else None + aliases = [lip] + if ts: + aliases.append(ts) + # Special: aida-nebra has !27780c47 + node_id_m = re.search(r'(![\da-f]+)', purpose) + if node_id_m: + aliases.append(node_id_m.group(1)) + + # Infer type + if 'Contabo' in name or 'VPS' in name.upper(): + etype = 'vps' + elif re.search(r'pi|Pi|raspberry', purpose, re.I) or name.startswith('pi-') or name.startswith('mt-') or name == 'aida-nebra': + etype = 'pi' + elif 'desktop' in name.lower() or 'Desktop' in purpose: + etype = 'desktop' + else: + etype = 'host' + + add(name, name, etype, aliases) + + # --- LXC Containers table --- + # | Container | Host | Local IP | Tailscale | Purpose | + ct_pat = re.compile( + r'^\|\s*([a-zA-Z][a-zA-Z0-9\-]+)\s*\|\s*([^\|]+?)\(CT (\d+)\)\s*\|\s*(\d+\.\d+\.\d+\.\d+)\s*\|\s*([^\|]*)\s*\|\s*([^\|]+)\s*\|', + re.MULTILINE + ) + for m in ct_pat.finditer(text): + cname, host_raw, ctid, lip, ts_raw, purpose = ( + m.group(1).strip(), m.group(2).strip(), m.group(3).strip(), + m.group(4).strip(), m.group(5).strip(), m.group(6).strip() + ) + if cname.startswith('-') or cname == 'Container': + continue + host_name = host_raw.split()[0].lower() + aliases = [lip, f"CT {ctid}"] + if ts_raw and re.match(r'100\.\d+', ts_raw): + aliases.append(ts_raw) + add(cname, cname, "ct", aliases, runs_on=host_name) + + # --- Headscale node list table --- + # | Node | Tailscale IP | Type | + hs_pat = re.compile( + r'^\|\s*([a-zA-Z][a-zA-Z0-9\-]+)\s*\|\s*(100\.\d+\.\d+\.\d+)\s*\|\s*([^\|]+)\s*\|', + re.MULTILINE + ) + for m in hs_pat.finditer(text): + node, ts_ip, ntype_raw = m.group(1).strip(), m.group(2).strip(), m.group(3).strip() + if node in ('Node', '--'): + continue + # These should already be in entities; just enrich aliases if missing + canonical = node.lower() + found = next((e for e in entities if e['name'] == canonical), None) + if found: + if ts_ip not in found['aliases']: + found['aliases'].append(ts_ip) + else: + # Map HS type to our type + nt = ntype_raw.split()[0].lower() + type_map = {'proxmox': 'host', 'lxc': 'ct', 'vm': 'vm', 'pi': 'pi', + 'vps': 'vps', 'desktop': 'desktop', 'mobile': 'mobile', + 'router': 'router'} + etype = type_map.get(nt, 'host') + if canonical not in seen_names: + seen_names.add(canonical) + entities.append({ + "name": canonical, + "display": node, + "type": etype, + "aliases": [ts_ip], + "runs_on": None, + "source": "docs", + }) + + return entities + + +def parse_services_md(path: Path, known_entities: list[dict]) -> list[dict]: + """Extract service entities from services.md table.""" + entities = [] + seen_names = set(e['name'] for e in known_entities) + + try: + text = path.read_text() + except Exception as e: + print(f"[warn] Cannot read {path}: {e}", file=sys.stderr) + return entities + + # Main table: | Service | Location | IP:Port | Access | Notes | + svc_pat = re.compile( + r'^\|\s*([^\|]+?)\s*\|\s*([^\|]+?)\s*\|\s*([^\|]+?)\s*\|\s*([^\|]+?)\s*\|', + re.MULTILINE + ) + + for m in svc_pat.finditer(text): + svc_raw, loc_raw, ipport_raw, access_raw = ( + m.group(1).strip(), m.group(2).strip(), + m.group(3).strip(), m.group(4).strip() + ) + # Skip headers and separators + if svc_raw.startswith('-') or svc_raw in ('Service', 'Container', 'Node', 'Server'): + continue + if '**Decommissioned' in svc_raw or 'Decommissioned' in loc_raw: + continue + + # Canonical name: lowercase-kebab of service name, strip ~~ + svc_clean = re.sub(r'~~', '', svc_raw) + svc_clean = re.sub(r'\*+', '', svc_clean).strip() + if not svc_clean or svc_clean.startswith('-'): + continue + + canonical = re.sub(r'[^a-z0-9]+', '-', svc_clean.lower()).strip('-') + if not canonical or canonical in seen_names: + continue + seen_names.add(canonical) + + # Extract subdomain alias from access URL + aliases = [svc_clean] + url_m = re.search(r'https?://([a-z0-9\-\.]+\.echo6\.co)', access_raw, re.I) + if url_m: + sub = url_m.group(1).split('.')[0] + if sub not in aliases: + aliases.append(sub) + + # runs_on: first word of location + runs_on = None + loc_clean = re.sub(r'\([^)]*\)', '', loc_raw).strip() + if loc_clean: + first_word = loc_clean.split()[0].lower() + # normalize host names + host_map = { + 'utility': 'utility', 'cloud': 'cloud', 'media': 'media', + 'data': 'data', 'toc': 'toc', 'cortex': 'cortex', + 'contabo': 'contabo', 'edge2': 'edge2', + 'aida-nebra': 'aida-nebra', 'pi-nas': 'pi-nas', + } + runs_on = host_map.get(first_word, first_word) + + entities.append({ + "name": canonical, + "display": svc_clean, + "type": "service", + "aliases": aliases, + "runs_on": runs_on, + "source": "docs", + }) + + return entities + + +def parse_projects(vault_dir: Path) -> list[dict]: + """Each *.md under vault/projects/ is a project entity.""" + entities = [] + projects_dir = vault_dir / "projects" + if not projects_dir.exists(): + return entities + for md in sorted(projects_dir.glob("*.md")): + stem = md.stem + canonical = re.sub(r'[^a-z0-9]+', '-', stem.lower()).strip('-') + entities.append({ + "name": canonical, + "display": stem, + "type": "project", + "aliases": [stem], + "runs_on": None, + "source": "docs", + }) + return entities + + +def _expansion_matches_acronym(full: str, acro: str) -> bool: + """ + Heuristic: does 'full' plausibly expand 'acro'? + Two strategies: + 1. Multi-word: initials of words map to acronym letters (standard expansion) + 2. CamelCase compound word: split on uppercase transitions (OpenTAKServer → O,T,S → OTS) + At minimum, must start with the same letter as the acronym. + """ + if not full or not acro: + return False + if full[0].upper() != acro[0]: + return False + + # Strategy 1: multi-word + words = [w for w in re.split(r'[\s\-]+', full) if w] + if len(words) >= 2: + initials = ''.join(w[0].upper() for w in words if len(w) > 1 or w[0].isupper()) + acro_letters = re.sub(r'[0-9]', '', acro) + matches = sum(1 for ch in acro_letters if ch in initials) + if matches >= max(1, len(acro_letters) - 1): + return True + + # Strategy 2: CamelCase single compound word (e.g. OpenTAKServer → OTS) + if len(words) == 1: + # Extract uppercase letters (treating runs of uppercase as separate tokens) + caps = re.findall(r'[A-Z][a-z]*|[A-Z]+(?=[A-Z]|$)', full) + initials = ''.join(c[0] for c in caps) + acro_letters = re.sub(r'[0-9]', '', acro) + if len(acro_letters) >= 2 and initials.upper().startswith(acro_letters[0]): + matches = sum(1 for ch in acro_letters if ch in initials.upper()) + if matches >= max(1, len(acro_letters) - 1): + return True + + return False + + +def _load_dict_words() -> tuple[set, bool]: + """Load system dictionary; fall back to built-in stoplist. Returns (word_set, used_system_dict).""" + dict_path = "/usr/share/dict/words" + try: + import os + if os.path.isfile(dict_path) and os.path.getsize(dict_path) > 1000: + words = set() + with open(dict_path) as f: + for line in f: + w = line.strip().lower() + if w: + words.add(w) + return words, True + except Exception: + pass + return set(_COMMON_ENGLISH_WORDS), False + + +def harvest_acronyms(vault_dir: Path, all_entities: list[dict]) -> list[dict]: + """ + Scan all vault/**/*.md for acronym tokens and expansion patterns. + Returns list of {acronym, expansion, source}. + + Qualification rules — ALL must pass: + 1. Shape: 2–6 chars, uppercase letters/digits only, at least one letter. + 2. NOT a plain English word (checked against /usr/share/dict/words or + built-in stoplist — case-insensitive). + 3. NOT already an entity name or alias (those are covered as entities). + 4. At least ONE of: + a. Has an expansion found via adjacency patterns in these docs. + b. Referenced by / maps to an entity (appears in entity display name + or aliases — e.g. AIDA↔aida-nebra). + c. Appears as a standalone all-caps token ≥3 times across all docs. + """ + # ---- shape regex: 2–6 chars, A-Z0-9, at least one letter ---- + # We harvest everything matching [A-Z][A-Z0-9]{1,5} and filter below. + acro_pat = re.compile(r'\b([A-Z][A-Z0-9]{1,5})\b') + + # Expansion patterns + # "Full Name (ACRO)" or "Full Name — ACRO" or "Full Name: ACRO" + expand_pat = re.compile( + r'((?:[A-Z][A-Za-z0-9]+(?:[ \-][A-Za-z][A-Za-z0-9]*){1,6})|(?:[A-Z][A-Za-z0-9]{4,}))' + r'(?:\s*[\(—:]\s*)([A-Z][A-Z0-9]{1,5})(?:[\)\s]|$)' + ) + # "ACRO (Full Name)" or "ACRO — Full Name" or "ACRO: Full Name" + expand_pat2 = re.compile( + r'\b([A-Z][A-Z0-9]{1,5})\s+(?:\(([A-Z][A-Za-z0-9 \-]{3,50})\)|(?:—\s*|:\s*)([A-Z][A-Za-z0-9 \-]{3,50}))' + ) + + # ---- build entity lookup sets ---- + entity_names: set[str] = set() + entity_aliases_upper: set[str] = set() + for ent in all_entities: + entity_names.add(ent['name'].lower()) + if ent.get('display'): + entity_names.add(ent['display'].lower()) + for al in ent.get('aliases', []): + entity_names.add(str(al).lower()) + + # Build set of uppercase tokens that are entity names/aliases + # (for rule 3: reject if token.lower() in entity_names) + # We also build a set of uppercase strings that appear in entity display names + # (for rule 4b: token is referenced by an entity) + entity_upper_tokens: set[str] = set() + for ent in all_entities: + # Split display name and aliases into words, collect ≥2-char uppercase-looking words + for text_val in [ent.get('display', ''), ent['name']] + list(ent.get('aliases', [])): + for word in re.split(r'[\s\-_]+', str(text_val)): + if re.match(r'[A-Z][A-Z0-9]{1,5}$', word): + entity_upper_tokens.add(word) + + # ---- load dictionary ---- + dict_words, used_system_dict = _load_dict_words() + + # ---- scan docs ---- + acro_counts: dict[str, int] = {} + expansions: dict[str, str] = {} + + for md in vault_dir.rglob("*.md"): + try: + doc_text = md.read_text(errors='replace') + except Exception: + continue + + # Harvest raw tokens + for m in acro_pat.finditer(doc_text): + acro = m.group(1) + # Rule 1: must have at least one letter (not pure digits) + if not re.search(r'[A-Z]', acro): + continue + acro_counts[acro] = acro_counts.get(acro, 0) + 1 + + # Harvest expansions "Full Name (ACRO)" and "Full Name — ACRO" + for m in expand_pat.finditer(doc_text): + full, acro = m.group(1).strip(), m.group(2) + if not re.search(r'[A-Z]', acro): + continue + if not _expansion_matches_acronym(full, acro): + continue + if acro not in expansions: + expansions[acro] = full + acro_counts[acro] = acro_counts.get(acro, 0) + 1 + + # Harvest expansions "ACRO (Full Name)" and "ACRO — Full Name" + for m in expand_pat2.finditer(doc_text): + acro = m.group(1) + full = (m.group(2) or m.group(3) or '').strip() + if not full or not re.search(r'[A-Z]', acro): + continue + if acro not in expansions and _expansion_matches_acronym(full, acro): + expansions[acro] = full + acro_counts[acro] = acro_counts.get(acro, 0) + 1 + + # ---- apply filtering rules ---- + result = [] + for acro, count in sorted(acro_counts.items(), key=lambda x: (-x[1], x[0])): + # Rule 1: shape already enforced by regex + letter check above + # Extra length check just to be safe + if not (2 <= len(acro) <= 6): + continue + if not re.match(r'[A-Z][A-Z0-9]{1,5}$', acro): + continue + if not re.search(r'[A-Z]', acro): + continue + + # Rule 2: reject plain English words + if acro.lower() in dict_words: + continue + + # Rule 2b: reject generic/googleable acronyms (not Echo6-specific) + if acro in GENERIC_ACRONYM_BLOCKLIST: + continue + + # Rule 3: reject if it IS an entity name / alias + if acro.lower() in entity_names: + continue + + # Rule 4: must qualify via at least one signal + has_expansion = acro in expansions + maps_to_entity = acro in entity_upper_tokens + high_freq = count >= 3 + + if not (has_expansion or maps_to_entity or high_freq): + continue + + # Curated expansion takes priority over auto-derived + if acro in CURATED_EXPANSIONS: + exp = CURATED_EXPANSIONS[acro] + else: + exp = expansions.get(acro) + if exp: + exp = re.sub(r'\\s+', ' ', exp).strip() + exp = re.sub(r'[,;:]+$', '', exp) + if len(exp) > 60 or len(exp.split()) > 8: + exp = None + result.append({ + "acronym": acro, + "expansion": exp, + "source": "docs", + }) + + return result + + +# --------------------------------------------------------------------------- +# B. Live inventory (best-effort) +# --------------------------------------------------------------------------- + +def _ssh_run(host: str, user: str, cmd: str, timeout: int = 5) -> str | None: + """Run a remote command; return stdout or None on any failure.""" + try: + r = subprocess.run( + ["ssh", "-o", "BatchMode=yes", "-o", f"ConnectTimeout={timeout}", + "-o", "StrictHostKeyChecking=no", f"{user}@{host}", cmd], + capture_output=True, text=True, timeout=timeout + 2 + ) + if r.returncode == 0: + return r.stdout + return None + except Exception: + return None + + +def _proxmox_ip_for_name(name: str) -> str | None: + """Return the local or Tailscale IP for a known Proxmox host name.""" + ip_map = { + "data": "192.168.1.240", + "utility": "192.168.1.241", + "cloud": "192.168.1.242", + "media": "192.168.1.243", + "toc": "192.168.1.244", + "contabo": "5.189.158.149", + "edge2": "184.174.35.153", + } + return ip_map.get(name) + + +def query_proxmox_node(host_name: str) -> list[dict]: + """Try pct list + qm list on a Proxmox host; return entity dicts.""" + ip = _proxmox_ip_for_name(host_name) + if not ip: + return [] + entities = [] + + # Try root SSH (Proxmox hosts use root) + out_pct = _ssh_run(ip, "root", "pct list 2>/dev/null") + out_qm = _ssh_run(ip, "root", "qm list 2>/dev/null") + + if out_pct: + for line in out_pct.splitlines(): + m = re.match(r'^\s*(\d+)\s+(\S+)', line) + if m and m.group(1) != 'VMID': + ctid, name = m.group(1), m.group(2).lower() + entities.append({ + "name": name, "display": name, "type": "ct", + "aliases": [f"CT {ctid}"], + "runs_on": host_name, "source": "live", + }) + + if out_qm: + for line in out_qm.splitlines(): + m = re.match(r'^\s*(\d+)\s+(\S+)', line) + if m and m.group(1) != 'VMID': + vmid, name = m.group(1), m.group(2).lower() + entities.append({ + "name": name, "display": name, "type": "vm", + "aliases": [f"VM {vmid}"], + "runs_on": host_name, "source": "live", + }) + + return entities + + +def query_docker_host(host_name: str, user: str) -> list[dict]: + """Try docker ps on a host; return container name entities.""" + ip_map = { + "cortex": "192.168.1.150", + "utility": "192.168.1.241", + "media": "192.168.1.160", + "contabo": "5.189.158.149", + } + ip = ip_map.get(host_name) + if not ip: + return [] + + out = _ssh_run(ip, user, "docker ps --format '{{.Names}}' 2>/dev/null") + if not out: + out = _ssh_run(ip, "root", "docker ps --format '{{.Names}}' 2>/dev/null") + + if not out: + return [] + + entities = [] + for name in out.splitlines(): + name = name.strip() + if not name: + continue + canonical = re.sub(r'[^a-z0-9]+', '-', name.lower()).strip('-') + entities.append({ + "name": canonical, "display": name, "type": "service", + "aliases": [name], + "runs_on": host_name, "source": "live", + }) + return entities + + +def query_headscale(headscale_host: str) -> list[dict]: + """Try headscale nodes list on contabo; return node entities.""" + ip = _proxmox_ip_for_name(headscale_host) or headscale_host + out = _ssh_run(ip, "root", "headscale nodes list 2>/dev/null") + if not out: + return [] + + entities = [] + for line in out.splitlines(): + # headscale output: ID | Name | Prefix | IPs | Ephemeral | Last seen | ... + parts = [p.strip() for p in line.split('|')] + if len(parts) >= 3 and parts[1] and not parts[1].startswith('Name'): + name = parts[1].strip().lower() + if name: + canonical = re.sub(r'[^a-z0-9]+', '-', name).strip('-') + entities.append({ + "name": canonical, "display": name, "type": "host", + "aliases": [], + "runs_on": None, "source": "live", + }) + return entities + + +def run_live_inventory(config: dict) -> tuple[list[dict], list[str]]: + """ + Run all live inventory queries. + Returns (entities, list_of_sources_that_succeeded). + """ + inv = config.get("inventory", {}) + ssh_user = inv.get("ssh_user", "zvx") + live_entities: list[dict] = [] + succeeded: list[str] = [] + + # Proxmox nodes + for node_name in inv.get("proxmox_nodes", []): + try: + ents = query_proxmox_node(node_name) + if ents: + live_entities.extend(ents) + succeeded.append(f"proxmox:{node_name}") + except Exception: + pass + + for node_name in inv.get("proxmox_vps", []): + try: + ents = query_proxmox_node(node_name) + if ents: + live_entities.extend(ents) + succeeded.append(f"proxmox:{node_name}") + except Exception: + pass + + # Docker hosts + for host_name in inv.get("docker_hosts", []): + try: + ents = query_docker_host(host_name, ssh_user) + if ents: + live_entities.extend(ents) + succeeded.append(f"docker:{host_name}") + except Exception: + pass + + # Headscale + try: + ents = query_headscale(inv.get("headscale_host", "contabo")) + if ents: + live_entities.extend(ents) + succeeded.append("headscale:contabo") + except Exception: + pass + + return live_entities, succeeded + + +# --------------------------------------------------------------------------- +# C. Build vocab.json +# --------------------------------------------------------------------------- + +def merge_entities(doc_entities: list[dict], live_entities: list[dict]) -> list[dict]: + """Merge live entities that aren't already present from docs.""" + existing_names = {e['name'] for e in doc_entities} + result = list(doc_entities) + for le in live_entities: + if le['name'] not in existing_names: + result.append(le) + existing_names.add(le['name']) + return result + + +def build_vocab(config: dict) -> dict: + vault_dir = Path(config["vault_dir"]) + engine_dir = Path(config["engine_dir"]) + + print("[1/4] Parsing environment.md ...", file=sys.stderr) + env_md = vault_dir / "docs" / "hardware" / "environment.md" + host_entities = parse_environment_md(env_md) + print(f" → {len(host_entities)} entities from environment.md", file=sys.stderr) + + print("[2/4] Parsing services.md ...", file=sys.stderr) + svc_md = vault_dir / "docs" / "services" / "services.md" + svc_entities = parse_services_md(svc_md, host_entities) + print(f" → {len(svc_entities)} entities from services.md", file=sys.stderr) + + print("[3/4] Parsing vault/projects/ ...", file=sys.stderr) + project_entities = parse_projects(vault_dir) + print(f" → {len(project_entities)} project entities", file=sys.stderr) + + doc_entities = host_entities + svc_entities + project_entities + + print("[4/4] Harvesting acronyms from all vault docs ...", file=sys.stderr) + acronyms = harvest_acronyms(vault_dir, doc_entities) + print(f" → {len(acronyms)} acronyms ({sum(1 for a in acronyms if a['expansion'])} with expansions)", file=sys.stderr) + + print("[5/5] Running live inventory (best-effort) ...", file=sys.stderr) + live_entities, live_sources = run_live_inventory(config) + print(f" → {len(live_entities)} live entities, sources: {live_sources or ['none']}", file=sys.stderr) + + all_entities = merge_entities(doc_entities, live_entities) + + # Counts by type + type_counts: dict[str, int] = {} + for e in all_entities: + t = e['type'] + type_counts[t] = type_counts.get(t, 0) + 1 + + expansions_count = sum(1 for a in acronyms if a['expansion']) + + vocab = { + "_meta": { + "generated": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "generated_from": ["docs"] + live_sources, + "counts": { + "entities_total": len(all_entities), + "entities_by_type": type_counts, + "acronyms_total": len(acronyms), + "acronyms_with_expansion": expansions_count, + }, + }, + "topic_categories": config["topic_categories"], + "entities": all_entities, + "acronyms": acronyms, + } + return vocab + + +# --------------------------------------------------------------------------- +# D. Write outputs +# --------------------------------------------------------------------------- + +def write_vocab_json(vocab: dict, engine_dir: Path) -> Path: + out_path = engine_dir / "vocab.json" + out_path.write_text(json.dumps(vocab, indent=2, ensure_ascii=False)) + return out_path + + +def write_glossary_md(vocab: dict, vault_dir: Path) -> Path: + today = datetime.now(timezone.utc).strftime("%Y-%m-%d") + lines = [ + "---", + "title: Glossary & Vocabulary", + "type: reference", + "tags: [meta]", + f"updated: {today}", + "---", + "# Glossary & Vocabulary", + "", + "> Auto-generated by the vault engine (`engine/lib/vocab_gen.py`).", + "> Acronym expansions marked _(unconfirmed)_ need a human pass.", + "", + "## Topic categories", + "", + " · ".join(vocab["topic_categories"]), + "", + "## Acronyms", + "", + "| Acronym | Expansion |", + "|---|---|", + ] + for a in vocab["acronyms"]: + exp = a["expansion"] if a["expansion"] else "_(unconfirmed)_" + lines.append(f"| {a['acronym']} | {exp} |") + + lines += [ + "", + "## Entities", + "", + ] + + # Group by type + type_order = ["host", "vm", "ct", "pi", "vps", "desktop", "mobile", "router", "service", "project"] + type_labels = { + "host": "Hosts / Proxmox nodes", + "vm": "Virtual Machines", + "ct": "LXC Containers", + "pi": "Raspberry Pi / Edge nodes", + "vps": "VPS / External servers", + "desktop": "Desktops", + "mobile": "Mobile devices", + "router": "Routers / Network devices", + "service": "Services", + "project": "Projects", + } + by_type: dict[str, list[dict]] = {t: [] for t in type_order} + by_type["other"] = [] + for e in vocab["entities"]: + t = e.get("type", "other") + if t in by_type: + by_type[t].append(e) + else: + by_type.setdefault("other", []).append(e) + + for etype in type_order: + ents = by_type.get(etype, []) + if not ents: + continue + label = type_labels.get(etype, etype.title()) + lines.append(f"### {label}") + lines.append("") + for e in sorted(ents, key=lambda x: x['name']): + display = e.get('display', e['name']) + name_part = f"**{e['name']}**" + if display != e['name']: + name_part += f" ({display})" + alias_str = "" + if e.get('aliases'): + alias_str = " — aliases: " + ", ".join(e['aliases']) + runs_str = "" + if e.get('runs_on'): + runs_str = f" — on: {e['runs_on']}" + src = " _(live)_" if e.get('source') == 'live' else "" + lines.append(f"- {name_part}{alias_str}{runs_str}{src}") + lines.append("") + + out_path = vault_dir / "glossary.md" + out_path.write_text("\n".join(lines) + "\n") + return out_path + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main(): + config_path = Path(__file__).parent.parent / "config.yaml" + alt_path = Path("/home/zvx/projects/.ref/engine/config.yaml") + if not config_path.exists() and alt_path.exists(): + config_path = alt_path + + print(f"Loading config from {config_path}", file=sys.stderr) + config = load_config(str(config_path)) + + vault_dir = Path(config["vault_dir"]) + engine_dir = Path(config["engine_dir"]) + + if not vault_dir.exists(): + print(f"[error] vault_dir not found: {vault_dir}", file=sys.stderr) + sys.exit(1) + if not engine_dir.exists(): + print(f"[error] engine_dir not found: {engine_dir}", file=sys.stderr) + sys.exit(1) + + vocab = build_vocab(config) + + vocab_path = write_vocab_json(vocab, engine_dir) + print(f"\nWrote {vocab_path}", file=sys.stderr) + + glossary_path = write_glossary_md(vocab, vault_dir) + print(f"Wrote {glossary_path}", file=sys.stderr) + + # Summary + meta = vocab["_meta"] + print(f"\n=== vocab_gen complete ===", file=sys.stderr) + print(f" Entities : {meta['counts']['entities_total']} total", file=sys.stderr) + for t, c in sorted(meta['counts']['entities_by_type'].items()): + print(f" {t:12s}: {c}", file=sys.stderr) + print(f" Acronyms : {meta['counts']['acronyms_total']} total, " + f"{meta['counts']['acronyms_with_expansion']} with expansions", file=sys.stderr) + _, used_sys = _load_dict_words() + dict_src = "/usr/share/dict/words" if used_sys else "built-in stoplist (~300 words)" + print(f" Dict source : {dict_src}", file=sys.stderr) + print(f" Live sources: {meta['generated_from']}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/engine/lint-report.md b/engine/lint-report.md new file mode 100644 index 0000000..ed215b9 --- /dev/null +++ b/engine/lint-report.md @@ -0,0 +1,170 @@ +# Vault Lint Report + +Generated: 2026-06-18T05:41:36Z | Docs scanned: 98 | Elapsed: 0.0s + +## Summary + +| Severity | Count | +|----------|-------| +| ERROR (dead links) | 0 | +| WARN (schema) | 106 | +| INFO (orphans) | 38 | + +### WARN breakdown +- Missing frontmatter block: 95 +- Invalid/missing frontmatter fields: 3 +- Unknown tags: 8 + +## ERROR — Dead Wikilinks + +_None. All wikilinks resolve._ + +## WARN — Schema & Tag Violations + +- `CLAUDE-baseline.md` — missing frontmatter block entirely +- `INDEX.md` — missing required key: 'tags' +- `archive/AUDIT-2026-02-21.md` — missing frontmatter block entirely +- `archive/README.md` — missing frontmatter block entirely +- `archive/matrix/PHASE6_DECISION.md` — missing frontmatter block entirely +- `archive/matrix/PLAN.md` — missing frontmatter block entirely +- `archive/matrix/appservices.md` — missing frontmatter block entirely +- `archive/matrix/archive_receiver_discovery.md` — missing frontmatter block entirely +- `archive/matrix/archivist_discovery.md` — missing frontmatter block entirely +- `archive/matrix/hookshot_deployment_discovery.md` — missing frontmatter block entirely +- `archive/matrix/hookshot_e2ee_discovery.md` — missing frontmatter block entirely +- `archive/matrix/hookshot_payload_discovery.md` — missing frontmatter block entirely +- `archive/matrix/hookshot_room_targeting_discovery.md` — missing frontmatter block entirely +- `archive/projects/DEPLOY-API-KEYS-TAB.md` — missing frontmatter block entirely +- `archive/projects/arr-stack-runbook.md` — missing frontmatter block entirely +- `archive/projects/arr-wiring-runbook.md` — missing frontmatter block entirely +- `archive/projects/cc-deploy-watchtower-v2.md` — missing frontmatter block entirely +- `archive/projects/headscale-full-deployment.md` — missing frontmatter block entirely +- `archive/projects/last-ember-project.md` — missing frontmatter block entirely +- `archive/projects/mmud/mmud-phase5-prompt.md` — missing frontmatter block entirely +- `archive/projects/mmud/mmud-phase6-prompt.md` — missing frontmatter block entirely +- `archive/projects/mmud/mmud-prompts/mmud-prompts/01-update-planned.md` — missing frontmatter block entirely +- `archive/projects/mmud/mmud-prompts/mmud-prompts/02-npc-nodes.md` — missing frontmatter block entirely +- `archive/projects/mmud/mmud-prompts/mmud-prompts/03-darkcragg.md` — missing frontmatter block entirely +- `archive/projects/mmud/mmud-prompts/mmud-prompts/04-dcrg-node.md` — missing frontmatter block entirely +- `archive/projects/mmud/mmud-prompts/mmud-prompts/05-phase5.md` — missing frontmatter block entirely +- `archive/projects/mmud/mmud-prompts/mmud-prompts/06-phase6.md` — missing frontmatter block entirely +- `archive/projects/mmud/mmud-prompts/mmud-prompts/README.md` — missing frontmatter block entirely +- `archive/projects/mmud/mmud-prompts/mmud-prompts/mmud-project.md` — missing frontmatter block entirely +- `archive/projects/openwebui-theme-deploy.md` — missing frontmatter block entirely +- `archive/projects/peertube-phase2-project.md` — missing frontmatter block entirely +- `archive/projects/peertube-rebuild.md` — missing frontmatter block entirely +- `archive/projects/utility-caddy-initial-setup.md` — missing frontmatter block entirely +- `archive/projects/vaultwarden-deployment.md` — missing frontmatter block entirely +- `archive/reports/logistics_migration.md` — missing frontmatter block entirely +- `archive/reports/post_validation_report.md` — missing frontmatter block entirely +- `archive/reports/task_a_aurora_validation.md` — missing frontmatter block entirely +- `archive/reports/task_c_watchdog_test.md` — missing frontmatter block entirely +- `docs/hardware/environment.md` — missing frontmatter block entirely +- `docs/hardware/ip-allocation.md` — missing frontmatter block entirely +- `docs/matrix/archivist.md` — missing frontmatter block entirely +- `docs/matrix/matrix_host.md` — missing frontmatter block entirely +- `docs/matrix/mautrix_signal.md` — missing frontmatter block entirely +- `docs/matrix/synapse.md` — missing frontmatter block entirely +- `docs/matrix/synapse_retention_discovery.md` — missing frontmatter block entirely +- `docs/navi/cc-rules.md` — missing frontmatter block entirely +- `docs/navi/deployment.md` — missing frontmatter block entirely +- `docs/navi/themes.md` — missing frontmatter block entirely +- `docs/services/ots-setup.md` — missing frontmatter block entirely +- `docs/services/services.md` — missing frontmatter block entirely +- `docs/services/usenet.md` — missing frontmatter block entirely +- `docs/software/authentik.md` — missing frontmatter block entirely +- `docs/software/caddy.md` — missing frontmatter block entirely +- `docs/software/dns.md` — missing frontmatter block entirely +- `docs/software/geo-tools.md` — missing frontmatter block entirely +- `docs/software/recon.md` — missing frontmatter block entirely +- `docs/software/searxng.md` — missing frontmatter block entirely +- `notes/echo6-landing-page-data-export.md` — missing frontmatter block entirely +- `notes/ia-download-queue.md` — missing frontmatter block entirely +- `plans/vaultwarden-plan.md` — missing frontmatter block entirely +- `projects/advbbs-project.md` — missing frontmatter block entirely +- `projects/argus.md` — missing frontmatter block entirely +- `projects/deploy-livesync.md` — missing frontmatter block entirely +- `projects/matrix-synapse-deployment.md` — missing frontmatter block entirely +- `projects/meshtastic-headscale-runbook.md` — missing frontmatter block entirely +- `projects/mmud-project.md` — missing frontmatter block entirely +- `runbooks/add-peertube-channel.md` — missing frontmatter block entirely +- `runbooks/authentik-access-groups.md` — missing frontmatter block entirely +- `runbooks/authentik-create-invitation.md` — missing frontmatter block entirely +- `runbooks/authentik-oidc-application.md` — missing frontmatter block entirely +- `runbooks/authentik-upgrade.md` — missing frontmatter block entirely +- `runbooks/ct-runbook.md` — missing frontmatter block entirely +- `runbooks/edge2-access-reference.md` — missing frontmatter block entirely +- `runbooks/expose-service-contabo.md` — missing frontmatter block entirely +- `runbooks/expose-service-edge2.md` — missing frontmatter block entirely +- `runbooks/expose-service-home.md` — missing frontmatter block entirely +- `runbooks/headscale-onboard-node.md` — missing frontmatter block entirely +- `runbooks/ia-cli-reference.md` — missing frontmatter block entirely +- `runbooks/ia-download-mirror.md` — missing frontmatter block entirely +- `runbooks/idahomesh-bridge-setup.md` — missing frontmatter block entirely +- `runbooks/idahomesh-vpn-device-setup.md` — missing frontmatter block entirely +- `runbooks/lxc-service-migration.md` — missing frontmatter block entirely +- `runbooks/mailcow-create-mailbox.md` — missing frontmatter block entirely +- `runbooks/meshmonitor-password-reset.md` — missing frontmatter block entirely +- `runbooks/meshtastic-sidecar-node.md` — missing frontmatter block entirely +- `runbooks/meshtasticd-sim-nodes-runbook.md` — missing frontmatter block entirely +- `runbooks/nordvpn-lxc.md` — missing frontmatter block entirely +- `runbooks/peertube-remote-runner.md` — missing frontmatter block entirely +- `runbooks/pg-backup.md` — missing frontmatter block entirely +- `runbooks/pi-nas-omv-runbook.md` — missing frontmatter block entirely +- `runbooks/pipeline-patterns.md` — missing frontmatter block entirely +- `runbooks/proxmox-create-ubuntu-vm.md` — missing frontmatter block entirely +- `runbooks/proxmox-onboard-node.md` — missing frontmatter block entirely +- `runbooks/recon-operations.md` — missing frontmatter block entirely +- `runbooks/recon-service-integration.md` — missing frontmatter block entirely +- `runbooks/syncthing-add-node.md` — missing frontmatter block entirely +- `session-resume/SESSION-HANDOFF-meshai-test.md` — missing required key: 'updated' +- `session-resume/SESSION-HANDOFF-meshai-test.md` — invalid type 'session-resume' — must be one of ['index', 'note', 'project', 'reference', 'runbook', 'session'] +- `session-resume/SESSION-HANDOFF-meshai-test.md` — unknown tag 'session-resume' (not in topic_categories or allowed extras) +- `session-resume/SESSION-HANDOFF-meshai-test.md` — unknown tag 'handoff' (not in topic_categories or allowed extras) +- `session-resume/SESSION-HANDOFF-meshai-test.md` — unknown tag 'meshai' (not in topic_categories or allowed extras) +- `session-resume/SESSION-HANDOFF-meshai-test.md` — unknown tag 'meshtastic' (not in topic_categories or allowed extras) +- `session-resume/SESSION-HANDOFF-meshai-test.md` — unknown tag 'aida-nebra' (not in topic_categories or allowed extras) +- `session-resume/SESSION-HANDOFF-meshai-test.md` — unknown tag 'diagnostic' (not in topic_categories or allowed extras) +- `session-resume/SESSION-HANDOFF-meshai-test.md` — unknown tag 'resilience-test' (not in topic_categories or allowed extras) +- `session-resume/SESSION-HANDOFF-meshai-test.md` — unknown tag 'open' (not in topic_categories or allowed extras) + +## INFO — Orphan Notes (no incoming links, capped at 40) + +- no incoming links: archive/projects/mmud/mmud-prompts/mmud-prompts/01-update-planned.md +- no incoming links: archive/projects/mmud/mmud-prompts/mmud-prompts/02-npc-nodes.md +- no incoming links: archive/projects/mmud/mmud-prompts/mmud-prompts/03-darkcragg.md +- no incoming links: archive/projects/mmud/mmud-prompts/mmud-prompts/04-dcrg-node.md +- no incoming links: archive/projects/mmud/mmud-prompts/mmud-prompts/05-phase5.md +- no incoming links: archive/projects/mmud/mmud-prompts/mmud-prompts/06-phase6.md +- no incoming links: archive/matrix/appservices.md +- no incoming links: archive/matrix/archive_receiver_discovery.md +- no incoming links: archive/matrix/archivist_discovery.md +- no incoming links: archive/projects/arr-stack-runbook.md +- no incoming links: archive/projects/arr-wiring-runbook.md +- no incoming links: archive/AUDIT-2026-02-21.md +- no incoming links: archive/projects/cc-deploy-watchtower-v2.md +- no incoming links: archive/projects/DEPLOY-API-KEYS-TAB.md +- no incoming links: runbooks/edge2-access-reference.md +- no incoming links: runbooks/expose-service-edge2.md +- no incoming links: glossary.md +- no incoming links: archive/projects/headscale-full-deployment.md +- no incoming links: archive/matrix/hookshot_deployment_discovery.md +- no incoming links: archive/matrix/hookshot_e2ee_discovery.md +- no incoming links: archive/matrix/hookshot_payload_discovery.md +- no incoming links: archive/matrix/hookshot_room_targeting_discovery.md +- no incoming links: INDEX.md +- no incoming links: archive/projects/last-ember-project.md +- no incoming links: archive/reports/logistics_migration.md +- no incoming links: archive/projects/mmud/mmud-phase5-prompt.md +- no incoming links: archive/projects/mmud/mmud-phase6-prompt.md +- no incoming links: archive/projects/openwebui-theme-deploy.md +- no incoming links: archive/projects/peertube-phase2-project.md +- no incoming links: archive/projects/peertube-rebuild.md +- no incoming links: archive/matrix/PLAN.md +- no incoming links: archive/reports/post_validation_report.md +- no incoming links: archive/projects/mmud/mmud-prompts/mmud-prompts/README.md +- no incoming links: archive/reports/task_a_aurora_validation.md +- no incoming links: archive/reports/task_c_watchdog_test.md +- no incoming links: archive/projects/utility-caddy-initial-setup.md +- no incoming links: archive/projects/vaultwarden-deployment.md +- no incoming links: plans/vaultwarden-plan.md diff --git a/engine/prompts/fewshot.md b/engine/prompts/fewshot.md new file mode 100644 index 0000000..0707972 --- /dev/null +++ b/engine/prompts/fewshot.md @@ -0,0 +1,52 @@ +# Echo6 Vault Tagger — Few-Shot Examples + +These examples will be populated from real vault documents during the tagger implementation +step (Step 5). Each example should show an input document snippet and the exact JSON output +the tagger should produce, demonstrating correct vocabulary usage. + +TODO: Add 3–5 real examples drawn from actual vault docs (runbook, reference, project, note). + +--- + +## Template: Example Structure + +### Input (document snippet) + +```markdown +--- +title: Headscale Setup +type: runbook +tags: [mesh, vpn] +updated: 2025-03-10 +--- + +# Headscale Setup + +Steps to install and configure Headscale on the Contabo VPS... +``` + +### Expected JSON output + +```json +{ + "tags": ["mesh", "vpn"], + "entities": ["contabo", "headscale"], + "glossary_proposals": [], + "type": "runbook", + "confidence": 0.95 +} +``` + +### Notes on this example + +- `tags` uses only values from `topic_categories` +- `entities` matches names present in `entity_lexicon` (contabo is a proxmox_vps; headscale is a known service) +- `confidence` is high because type is explicitly set in frontmatter and tags are unambiguous +- If the frontmatter had said `type: guide` (not in the allowed list), the tagger would infer `runbook` from the content and note the discrepancy + +--- + +TODO: Add example 2 — reference doc (host description, multiple entities) +TODO: Add example 3 — note/session doc (low entity density, inferred type) +TODO: Add example 4 — ambiguous doc with glossary_proposals populated +TODO: Add example 5 — doc where confidence drops below 0.6 diff --git a/engine/prompts/system.md b/engine/prompts/system.md new file mode 100644 index 0000000..d5a9ff7 --- /dev/null +++ b/engine/prompts/system.md @@ -0,0 +1,67 @@ +# Echo6 Vault Tagger — System Prompt (Canonical) + +## Role + +You are the Echo6 vault tagger, a local AI assistant running on cortex (RTX A4000). +Your sole job is to classify Obsidian markdown documents and extract structured metadata +from them using a controlled vocabulary. You operate fully offline and deterministically. + +## Inputs (provided per call) + +- **document**: the full text of a markdown file (frontmatter + body) +- **topic_categories**: a stable list of tier-1 topic tags (e.g. mesh, auth, proxmox, ai) +- **entity_lexicon**: a generated JSON dictionary mapping known names to type + (hosts, services, containers, projects, acronyms) — tier 2 vocabulary + +## Output + +Respond with ONLY a single valid JSON object. No prose, no markdown fences, no explanation. + +```json +{ + "tags": [ "string", "..." ], + "entities": [ "string", "..." ], + "glossary_proposals": [ "string", "..." ], + "type": "string", + "confidence": 0.0 +} +``` + +Field definitions: +- **tags**: tier-1 topic tags drawn exclusively from topic_categories +- **entities**: known names matched from entity_lexicon +- **glossary_proposals**: unknown acronyms or terms worth adding to the lexicon +- **type**: one of reference | runbook | project | note | index | session +- **confidence**: float 0.0–1.0, your overall confidence in this classification + +## Rules — follow exactly + +1. **Only use provided vocabulary.** `tags` must be a subset of `topic_categories`. + `entities` must be a subset of the keys in `entity_lexicon`. Never invent new tags. + +2. **Strict JSON only.** The output must parse with `json.loads()` with no preprocessing. + No trailing commas. No comments. No markdown code fences around the JSON. + +3. **Low confidence — flag, do not guess.** If `confidence < 0.6`, still emit valid JSON + but keep `tags` and `entities` conservative — only include what you are sure of. + Add uncertain terms to `glossary_proposals` instead. + +4. **Never hallucinate expansions.** If you encounter an acronym not in `entity_lexicon`, + do NOT guess its expansion. Add the raw acronym to `glossary_proposals`. + +5. **Never fabricate wikilinks or related files.** You output metadata only. + +6. **Type inference.** Use the document frontmatter `type` field if present and valid. + Otherwise infer from content: runbooks have steps/commands; references describe systems; + projects track work; sessions are journal/meeting notes; index files link to others. + +7. **Tags are used as-is** from the vocab list — do not pluralize or alter them. + +## Confidence scoring guide + +| Range | Meaning | +|-----------|----------------------------------------------------------------------| +| 0.9–1.0 | Clear topic, entities all recognized, type obvious | +| 0.7–0.89 | Good confidence; minor ambiguity in one dimension | +| 0.6–0.69 | Borderline; result written but flagged in changelog | +| below 0.6 | Do not apply silently; flag for human review | diff --git a/engine/sweep.sh b/engine/sweep.sh new file mode 100755 index 0000000..f888701 --- /dev/null +++ b/engine/sweep.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# sweep.sh — Echo6 Vault Engine daily maintenance sweep +# +# Invoked by cron (schedule: "0 9 * * *" from config.yaml). +# Also callable manually: ./sweep.sh +# +# What this does (when fully implemented): +# 1. GPU-busy guard: check VRAM usage; defer if > defer_if_gpu_busy_mib (6000 MiB default) +# 2. Run lint (lib/lint.py) over all vault docs — fix or flag frontmatter issues +# 3. Run agent (lib/agent.py) over changed/new docs since last run — tag + embed +# 4. Append a summary entry to changelog.md +# +# Configuration is read from config.yaml (engine_dir, vault_dir, thresholds, changelog path). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CONFIG="${SCRIPT_DIR}/config.yaml" + +echo "==> Echo6 vault sweep — $(date -u '+%Y-%m-%dT%H:%M:%SZ')" + +# --------------------------------------------------------------------------- +# Step 1 — GPU-busy guard +# --------------------------------------------------------------------------- +# TODO: Query nvidia-smi for used VRAM; compare to defer_if_gpu_busy_mib from config.yaml. +# If busy, log a deferred entry to changelog and exit 0 (not an error, just deferred). +# +# Example skeleton: +# USED_MIB=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -1) +# THRESHOLD=6000 # read from config.yaml +# if [ "$USED_MIB" -gt "$THRESHOLD" ]; then +# echo "GPU busy (${USED_MIB} MiB > ${THRESHOLD} MiB threshold) — deferring sweep." +# exit 0 +# fi + +echo "[1/4] TODO — GPU-busy guard not yet implemented." + +# --------------------------------------------------------------------------- +# Step 2 — Run lint +# --------------------------------------------------------------------------- +# TODO: Call lib/lint.py to validate/fix frontmatter across vault docs. +# Lint should be idempotent and log all changes to changelog. +# +# python3 "${SCRIPT_DIR}/lib/lint.py" --config "${CONFIG}" + +echo "[2/4] TODO — lint.py not yet implemented (Step 2)." + +# --------------------------------------------------------------------------- +# Step 3 — Run agent over changed/new docs +# --------------------------------------------------------------------------- +# TODO: Call lib/agent.py to tag + embed documents modified since last sweep. +# Agent tracks last-run timestamp in a state file (e.g. engine/.last_sweep). +# +# python3 "${SCRIPT_DIR}/lib/agent.py" --config "${CONFIG}" + +echo "[3/4] TODO — agent.py not yet implemented (Step 5)." + +# --------------------------------------------------------------------------- +# Step 4 — Append changelog summary +# --------------------------------------------------------------------------- +# TODO: agent.py and lint.py both append to changelog.md directly. +# This step adds a sweep-level summary entry. + +echo "[4/4] TODO — changelog summary not yet implemented." + +echo "==> sweep.sh done." diff --git a/engine/vocab.json b/engine/vocab.json new file mode 100644 index 0000000..f4d2936 --- /dev/null +++ b/engine/vocab.json @@ -0,0 +1,1215 @@ +{ + "_meta": { + "generated": "2026-06-18T05:37:32Z", + "generated_from": [ + "docs", + "proxmox:data", + "proxmox:utility", + "proxmox:cloud", + "proxmox:media", + "proxmox:toc", + "docker:cortex", + "docker:media", + "docker:contabo" + ], + "counts": { + "entities_total": 111, + "entities_by_type": { + "host": 17, + "vm": 4, + "pi": 5, + "desktop": 1, + "ct": 10, + "router": 1, + "service": 67, + "project": 6 + }, + "acronyms_total": 2, + "acronyms_with_expansion": 1 + } + }, + "topic_categories": [ + "mesh", + "matrix", + "recon", + "media", + "auth", + "dns", + "vpn", + "storage", + "proxmox", + "ai", + "mail" + ], + "entities": [ + { + "name": "data", + "display": "Data", + "type": "host", + "aliases": [ + "192.168.1.240", + "100.64.0.6" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "utility", + "display": "Utility", + "type": "host", + "aliases": [ + "192.168.1.241", + "100.64.0.5" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "cloud", + "display": "Cloud", + "type": "host", + "aliases": [ + "192.168.1.242", + "100.64.0.4" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "media", + "display": "Media", + "type": "host", + "aliases": [ + "192.168.1.243", + "100.64.0.3" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "toc", + "display": "Toc", + "type": "host", + "aliases": [ + "192.168.1.244", + "100.64.0.13" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "cortex", + "display": "cortex", + "type": "vm", + "aliases": [ + "192.168.1.150", + "100.64.0.14", + "VM 150" + ], + "runs_on": "toc", + "source": "docs" + }, + { + "name": "recon-vm", + "display": "recon-vm", + "type": "vm", + "aliases": [ + "192.168.1.130", + "100.64.0.24", + "VM 1130" + ], + "runs_on": "data", + "source": "docs" + }, + { + "name": "arr", + "display": "arr", + "type": "vm", + "aliases": [ + "192.168.1.160", + "100.64.0.18", + "VM 105" + ], + "runs_on": "media", + "source": "docs" + }, + { + "name": "aida-nebra", + "display": "aida-nebra", + "type": "pi", + "aliases": [ + "192.168.1.253", + "100.64.0.9", + "!27780c47" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "mt-isr", + "display": "mt-isr", + "type": "pi", + "aliases": [ + "192.168.1.141", + "100.100.0.5 (IdahoMesh)", + "100.100.0.5" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "mt-burleybutte", + "display": "mt-burleybutte", + "type": "pi", + "aliases": [ + "192.168.1.185" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "pi-nas", + "display": "pi-nas", + "type": "pi", + "aliases": [ + "192.168.1.245", + "100.64.0.21" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "matt-desktop", + "display": "matt-desktop", + "type": "desktop", + "aliases": [ + "192.168.1.111", + "100.64.0.10" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "edge2", + "display": "edge2", + "type": "host", + "aliases": [ + "184.174.35.153", + "100.64.0.26" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "contabo", + "display": "contabo", + "type": "host", + "aliases": [ + "100.64.0.1" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "meshmonitor", + "display": "meshmonitor", + "type": "host", + "aliases": [ + "100.64.0.7" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "nextcloud", + "display": "nextcloud", + "type": "host", + "aliases": [ + "100.64.0.11" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "iphone-eud", + "display": "iphone-eud", + "type": "host", + "aliases": [ + "100.64.0.16" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "peertube", + "display": "peertube", + "type": "host", + "aliases": [ + "100.64.0.23" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "argus", + "display": "argus", + "type": "host", + "aliases": [ + "100.64.0.25" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "meshmonitor-dev", + "display": "meshmonitor-dev", + "type": "host", + "aliases": [ + "100.64.0.27" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "bluefin", + "display": "bluefin", + "type": "host", + "aliases": [ + "100.64.0.30" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "meshai", + "display": "meshai", + "type": "host", + "aliases": [ + "100.64.0.32" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "forgejo", + "display": "forgejo", + "type": "host", + "aliases": [ + "100.64.0.34" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "mesh-bridge", + "display": "mesh-bridge", + "type": "host", + "aliases": [ + "100.100.0.3", + "100.64.0.22" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "burley-butte", + "display": "burley-butte", + "type": "pi", + "aliases": [ + "100.100.0.1" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "caddy", + "display": "caddy", + "type": "ct", + "aliases": [ + "192.168.1.101", + "CT 101", + "100.64.0.8" + ], + "runs_on": "utility", + "source": "docs" + }, + { + "name": "searxng", + "display": "searxng", + "type": "ct", + "aliases": [ + "192.168.1.102", + "CT 102", + "100.64.0.15" + ], + "runs_on": "utility", + "source": "docs" + }, + { + "name": "immich", + "display": "immich", + "type": "ct", + "aliases": [ + "192.168.1.182", + "CT 120", + "100.64.0.2" + ], + "runs_on": "cloud", + "source": "docs" + }, + { + "name": "meshtastic-hs", + "display": "meshtastic-hs", + "type": "ct", + "aliases": [ + "192.168.1.106", + "CT 106" + ], + "runs_on": "utility", + "source": "docs" + }, + { + "name": "archivist", + "display": "archivist", + "type": "ct", + "aliases": [ + "192.168.1.118", + "CT 118" + ], + "runs_on": "utility", + "source": "docs" + }, + { + "name": "pdm", + "display": "pdm", + "type": "ct", + "aliases": [ + "10.10.10.10", + "CT 100", + "100.64.0.28" + ], + "runs_on": "edge2", + "source": "docs" + }, + { + "name": "wordpress", + "display": "wordpress", + "type": "ct", + "aliases": [ + "10.10.10.11", + "CT 101", + "100.64.0.31" + ], + "runs_on": "edge2", + "source": "docs" + }, + { + "name": "vaultwarden", + "display": "vaultwarden", + "type": "ct", + "aliases": [ + "10.10.10.20", + "CT 102", + "100.64.0.33" + ], + "runs_on": "edge2", + "source": "docs" + }, + { + "name": "livesync", + "display": "livesync", + "type": "ct", + "aliases": [ + "10.10.10.22", + "CT 104", + "100.64.0.35" + ], + "runs_on": "edge2", + "source": "docs" + }, + { + "name": "recon", + "display": "recon", + "type": "vm", + "aliases": [ + "100.64.0.24" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "gl-a1300", + "display": "gl-a1300", + "type": "router", + "aliases": [ + "100.64.0.29" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "utility-caddy", + "display": "Utility Caddy", + "type": "service", + "aliases": [ + "Utility Caddy" + ], + "runs_on": "utility", + "source": "docs" + }, + { + "name": "echo6-search-searxng", + "display": "Echo6 Search (SearXNG)", + "type": "service", + "aliases": [ + "Echo6 Search (SearXNG)" + ], + "runs_on": "utility", + "source": "docs" + }, + { + "name": "meshtasticd-aida-n2", + "display": "meshtasticd (AIDA-N2)", + "type": "service", + "aliases": [ + "meshtasticd (AIDA-N2)" + ], + "runs_on": "aida-nebra", + "source": "docs" + }, + { + "name": "meshtastic-cli", + "display": "Meshtastic CLI", + "type": "service", + "aliases": [ + "Meshtastic CLI" + ], + "runs_on": "mt-isr", + "source": "docs" + }, + { + "name": "meshtasticd", + "display": "meshtasticd", + "type": "service", + "aliases": [ + "meshtasticd" + ], + "runs_on": "mt-burleybutte", + "source": "docs" + }, + { + "name": "idahomesh-headscale", + "display": "IdahoMesh Headscale", + "type": "service", + "aliases": [ + "IdahoMesh Headscale" + ], + "runs_on": "utility", + "source": "docs" + }, + { + "name": "authentik", + "display": "Authentik", + "type": "service", + "aliases": [ + "Authentik", + "auth" + ], + "runs_on": "contabo", + "source": "docs" + }, + { + "name": "forge-forgejo", + "display": "Forge (Forgejo)", + "type": "service", + "aliases": [ + "Forge (Forgejo)", + "forge" + ], + "runs_on": "edge2", + "source": "docs" + }, + { + "name": "headscale", + "display": "Headscale", + "type": "service", + "aliases": [ + "Headscale", + "vpn" + ], + "runs_on": "contabo", + "source": "docs" + }, + { + "name": "headplane", + "display": "Headplane", + "type": "service", + "aliases": [ + "Headplane", + "vpn" + ], + "runs_on": "contabo", + "source": "docs" + }, + { + "name": "mailcow", + "display": "Mailcow", + "type": "service", + "aliases": [ + "Mailcow", + "mail" + ], + "runs_on": "contabo", + "source": "docs" + }, + { + "name": "syncthing", + "display": "Syncthing", + "type": "service", + "aliases": [ + "Syncthing" + ], + "runs_on": "contabo", + "source": "docs" + }, + { + "name": "proxmox-ve", + "display": "Proxmox VE", + "type": "service", + "aliases": [ + "Proxmox VE", + "proxmox" + ], + "runs_on": "data", + "source": "docs" + }, + { + "name": "jellyfin", + "display": "Jellyfin", + "type": "service", + "aliases": [ + "Jellyfin", + "jellyfin" + ], + "runs_on": "media", + "source": "docs" + }, + { + "name": "jellyseer", + "display": "Jellyseer", + "type": "service", + "aliases": [ + "Jellyseer", + "requests" + ], + "runs_on": "media", + "source": "docs" + }, + { + "name": "sonarr", + "display": "Sonarr", + "type": "service", + "aliases": [ + "Sonarr" + ], + "runs_on": "media", + "source": "docs" + }, + { + "name": "radarr", + "display": "Radarr", + "type": "service", + "aliases": [ + "Radarr" + ], + "runs_on": "media", + "source": "docs" + }, + { + "name": "prowlarr", + "display": "Prowlarr", + "type": "service", + "aliases": [ + "Prowlarr" + ], + "runs_on": "media", + "source": "docs" + }, + { + "name": "sabnzbd", + "display": "SABnzbd", + "type": "service", + "aliases": [ + "SABnzbd" + ], + "runs_on": "media", + "source": "docs" + }, + { + "name": "open-webui", + "display": "Open WebUI", + "type": "service", + "aliases": [ + "Open WebUI", + "ai" + ], + "runs_on": "cortex", + "source": "docs" + }, + { + "name": "qdrant", + "display": "Qdrant", + "type": "service", + "aliases": [ + "Qdrant" + ], + "runs_on": "cortex", + "source": "docs" + }, + { + "name": "tei", + "display": "TEI", + "type": "service", + "aliases": [ + "TEI" + ], + "runs_on": "cortex", + "source": "docs" + }, + { + "name": "files", + "display": "Files", + "type": "service", + "aliases": [ + "Files", + "files" + ], + "runs_on": "data", + "source": "docs" + }, + { + "name": "samba", + "display": "Samba", + "type": "service", + "aliases": [ + "Samba" + ], + "runs_on": "data", + "source": "docs" + }, + { + "name": "matrix-synapse", + "display": "Matrix Synapse", + "type": "service", + "aliases": [ + "Matrix Synapse", + "matrix" + ], + "runs_on": "contabo", + "source": "docs" + }, + { + "name": "element-web", + "display": "Element Web", + "type": "service", + "aliases": [ + "Element Web", + "element" + ], + "runs_on": "contabo", + "source": "docs" + }, + { + "name": "mautrix-signal", + "display": "mautrix-signal", + "type": "service", + "aliases": [ + "mautrix-signal" + ], + "runs_on": "contabo", + "source": "docs" + }, + { + "name": "opentakserver-ots", + "display": "OpenTAKServer (OTS)", + "type": "service", + "aliases": [ + "OpenTAKServer (OTS)" + ], + "runs_on": "utility", + "source": "docs" + }, + { + "name": "echo6-cortex-agent", + "display": "Echo6 Cortex Agent", + "type": "service", + "aliases": [ + "Echo6 Cortex Agent" + ], + "runs_on": "cortex", + "source": "docs" + }, + { + "name": "echo6-contabo-agent", + "display": "Echo6 Contabo Agent", + "type": "service", + "aliases": [ + "Echo6 Contabo Agent" + ], + "runs_on": "contabo", + "source": "docs" + }, + { + "name": "matrix-mas", + "display": "Matrix MAS", + "type": "service", + "aliases": [ + "Matrix MAS" + ], + "runs_on": "contabo", + "source": "docs" + }, + { + "name": "termix", + "display": "Termix", + "type": "service", + "aliases": [ + "Termix" + ], + "runs_on": "contabo", + "source": "docs" + }, + { + "name": "pt-transcoder", + "display": "pt-transcoder", + "type": "service", + "aliases": [ + "pt-transcoder" + ], + "runs_on": "cortex", + "source": "docs" + }, + { + "name": "recon-sparse", + "display": "recon-sparse", + "type": "service", + "aliases": [ + "recon-sparse" + ], + "runs_on": "cortex", + "source": "docs" + }, + { + "name": "tak-server", + "display": "TAK Server", + "type": "service", + "aliases": [ + "TAK Server" + ], + "runs_on": "2026-06-16", + "source": "docs" + }, + { + "name": "sigil", + "display": "SIGIL", + "type": "service", + "aliases": [ + "SIGIL" + ], + "runs_on": "2026-06-16", + "source": "docs" + }, + { + "name": "watchtower", + "display": "WATCHTOWER", + "type": "service", + "aliases": [ + "WATCHTOWER" + ], + "runs_on": "2026-06-16", + "source": "docs" + }, + { + "name": "echo6-agent", + "display": "echo6-agent", + "type": "service", + "aliases": [ + "echo6-agent" + ], + "runs_on": "2026-06-16", + "source": "docs" + }, + { + "name": "nexus-hub", + "display": "nexus-hub", + "type": "service", + "aliases": [ + "nexus-hub" + ], + "runs_on": "2026-06-16", + "source": "docs" + }, + { + "name": "nexus-agent", + "display": "nexus-agent", + "type": "service", + "aliases": [ + "nexus-agent" + ], + "runs_on": "2026-06-16", + "source": "docs" + }, + { + "name": "advbbs-project", + "display": "advbbs-project", + "type": "project", + "aliases": [ + "advbbs-project" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "argus", + "display": "argus", + "type": "project", + "aliases": [ + "argus" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "deploy-livesync", + "display": "deploy-livesync", + "type": "project", + "aliases": [ + "deploy-livesync" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "matrix-synapse-deployment", + "display": "matrix-synapse-deployment", + "type": "project", + "aliases": [ + "matrix-synapse-deployment" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "meshtastic-headscale-runbook", + "display": "meshtastic-headscale-runbook", + "type": "project", + "aliases": [ + "meshtastic-headscale-runbook" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "mmud-project", + "display": "mmud-project", + "type": "project", + "aliases": [ + "mmud-project" + ], + "runs_on": null, + "source": "docs" + }, + { + "name": "running", + "display": "running", + "type": "ct", + "aliases": [ + "CT 100" + ], + "runs_on": "utility", + "source": "live" + }, + { + "name": "obsidian-remote", + "display": "obsidian-remote", + "type": "service", + "aliases": [ + "obsidian-remote" + ], + "runs_on": "cortex", + "source": "live" + }, + { + "name": "ollama", + "display": "ollama", + "type": "service", + "aliases": [ + "ollama" + ], + "runs_on": "cortex", + "source": "live" + }, + { + "name": "navidrome", + "display": "navidrome", + "type": "service", + "aliases": [ + "navidrome" + ], + "runs_on": "media", + "source": "live" + }, + { + "name": "lidarr", + "display": "lidarr", + "type": "service", + "aliases": [ + "lidarr" + ], + "runs_on": "media", + "source": "live" + }, + { + "name": "authentik-server", + "display": "authentik-server", + "type": "service", + "aliases": [ + "authentik-server" + ], + "runs_on": "contabo", + "source": "live" + }, + { + "name": "authentik-worker", + "display": "authentik-worker", + "type": "service", + "aliases": [ + "authentik-worker" + ], + "runs_on": "contabo", + "source": "live" + }, + { + "name": "authentik-postgres", + "display": "authentik-postgres", + "type": "service", + "aliases": [ + "authentik-postgres" + ], + "runs_on": "contabo", + "source": "live" + }, + { + "name": "matrix-postgres", + "display": "matrix-postgres", + "type": "service", + "aliases": [ + "matrix-postgres" + ], + "runs_on": "contabo", + "source": "live" + }, + { + "name": "mailcowdockerized-watchdog-mailcow-1", + "display": "mailcowdockerized-watchdog-mailcow-1", + "type": "service", + "aliases": [ + "mailcowdockerized-watchdog-mailcow-1" + ], + "runs_on": "contabo", + "source": "live" + }, + { + "name": "mailcowdockerized-acme-mailcow-1", + "display": "mailcowdockerized-acme-mailcow-1", + "type": "service", + "aliases": [ + "mailcowdockerized-acme-mailcow-1" + ], + "runs_on": "contabo", + "source": "live" + }, + { + "name": "mailcowdockerized-nginx-mailcow-1", + "display": "mailcowdockerized-nginx-mailcow-1", + "type": "service", + "aliases": [ + "mailcowdockerized-nginx-mailcow-1" + ], + "runs_on": "contabo", + "source": "live" + }, + { + "name": "mailcowdockerized-ofelia-mailcow-1", + "display": "mailcowdockerized-ofelia-mailcow-1", + "type": "service", + "aliases": [ + "mailcowdockerized-ofelia-mailcow-1" + ], + "runs_on": "contabo", + "source": "live" + }, + { + "name": "mailcowdockerized-rspamd-mailcow-1", + "display": "mailcowdockerized-rspamd-mailcow-1", + "type": "service", + "aliases": [ + "mailcowdockerized-rspamd-mailcow-1" + ], + "runs_on": "contabo", + "source": "live" + }, + { + "name": "mailcowdockerized-dovecot-mailcow-1", + "display": "mailcowdockerized-dovecot-mailcow-1", + "type": "service", + "aliases": [ + "mailcowdockerized-dovecot-mailcow-1" + ], + "runs_on": "contabo", + "source": "live" + }, + { + "name": "mailcowdockerized-php-fpm-mailcow-1", + "display": "mailcowdockerized-php-fpm-mailcow-1", + "type": "service", + "aliases": [ + "mailcowdockerized-php-fpm-mailcow-1" + ], + "runs_on": "contabo", + "source": "live" + }, + { + "name": "mailcowdockerized-postfix-mailcow-1", + "display": "mailcowdockerized-postfix-mailcow-1", + "type": "service", + "aliases": [ + "mailcowdockerized-postfix-mailcow-1" + ], + "runs_on": "contabo", + "source": "live" + }, + { + "name": "mailcowdockerized-redis-mailcow-1", + "display": "mailcowdockerized-redis-mailcow-1", + "type": "service", + "aliases": [ + "mailcowdockerized-redis-mailcow-1" + ], + "runs_on": "contabo", + "source": "live" + }, + { + "name": "mailcowdockerized-mysql-mailcow-1", + "display": "mailcowdockerized-mysql-mailcow-1", + "type": "service", + "aliases": [ + "mailcowdockerized-mysql-mailcow-1" + ], + "runs_on": "contabo", + "source": "live" + }, + { + "name": "mailcowdockerized-postfix-tlspol-mailcow-1", + "display": "mailcowdockerized-postfix-tlspol-mailcow-1", + "type": "service", + "aliases": [ + "mailcowdockerized-postfix-tlspol-mailcow-1" + ], + "runs_on": "contabo", + "source": "live" + }, + { + "name": "mailcowdockerized-clamd-mailcow-1", + "display": "mailcowdockerized-clamd-mailcow-1", + "type": "service", + "aliases": [ + "mailcowdockerized-clamd-mailcow-1" + ], + "runs_on": "contabo", + "source": "live" + }, + { + "name": "mailcowdockerized-memcached-mailcow-1", + "display": "mailcowdockerized-memcached-mailcow-1", + "type": "service", + "aliases": [ + "mailcowdockerized-memcached-mailcow-1" + ], + "runs_on": "contabo", + "source": "live" + }, + { + "name": "mailcowdockerized-sogo-mailcow-1", + "display": "mailcowdockerized-sogo-mailcow-1", + "type": "service", + "aliases": [ + "mailcowdockerized-sogo-mailcow-1" + ], + "runs_on": "contabo", + "source": "live" + }, + { + "name": "mailcowdockerized-unbound-mailcow-1", + "display": "mailcowdockerized-unbound-mailcow-1", + "type": "service", + "aliases": [ + "mailcowdockerized-unbound-mailcow-1" + ], + "runs_on": "contabo", + "source": "live" + }, + { + "name": "mailcowdockerized-netfilter-mailcow-1", + "display": "mailcowdockerized-netfilter-mailcow-1", + "type": "service", + "aliases": [ + "mailcowdockerized-netfilter-mailcow-1" + ], + "runs_on": "contabo", + "source": "live" + }, + { + "name": "mailcowdockerized-dockerapi-mailcow-1", + "display": "mailcowdockerized-dockerapi-mailcow-1", + "type": "service", + "aliases": [ + "mailcowdockerized-dockerapi-mailcow-1" + ], + "runs_on": "contabo", + "source": "live" + }, + { + "name": "mailcowdockerized-olefy-mailcow-1", + "display": "mailcowdockerized-olefy-mailcow-1", + "type": "service", + "aliases": [ + "mailcowdockerized-olefy-mailcow-1" + ], + "runs_on": "contabo", + "source": "live" + }, + { + "name": "matrix-element", + "display": "matrix-element", + "type": "service", + "aliases": [ + "matrix-element" + ], + "runs_on": "contabo", + "source": "live" + } + ], + "acronyms": [ + { + "acronym": "MMUD", + "expansion": "Mesh Multi-User Dungeon", + "source": "docs" + }, + { + "acronym": "AIDA", + "expansion": null, + "source": "docs" + } + ] +} \ No newline at end of file diff --git a/.gitignore b/vault/.gitignore similarity index 100% rename from .gitignore rename to vault/.gitignore diff --git a/.obsidian/app.json b/vault/.obsidian/app.json similarity index 100% rename from .obsidian/app.json rename to vault/.obsidian/app.json diff --git a/.obsidian/appearance.json b/vault/.obsidian/appearance.json similarity index 100% rename from .obsidian/appearance.json rename to vault/.obsidian/appearance.json diff --git a/.obsidian/community-plugins.json b/vault/.obsidian/community-plugins.json similarity index 100% rename from .obsidian/community-plugins.json rename to vault/.obsidian/community-plugins.json diff --git a/.obsidian/core-plugins.json b/vault/.obsidian/core-plugins.json similarity index 100% rename from .obsidian/core-plugins.json rename to vault/.obsidian/core-plugins.json diff --git a/.obsidian/graph.json b/vault/.obsidian/graph.json similarity index 96% rename from .obsidian/graph.json rename to vault/.obsidian/graph.json index 3dda752..e21a18d 100644 --- a/.obsidian/graph.json +++ b/vault/.obsidian/graph.json @@ -17,6 +17,6 @@ "repelStrength": 10, "linkStrength": 1, "linkDistance": 250, - "scale": 2.25, + "scale": 1, "close": false } \ No newline at end of file diff --git a/.obsidian/plugins/obsidian-livesync/data.json b/vault/.obsidian/plugins/obsidian-livesync/data.json similarity index 100% rename from .obsidian/plugins/obsidian-livesync/data.json rename to vault/.obsidian/plugins/obsidian-livesync/data.json diff --git a/.obsidian/plugins/obsidian-livesync/main.js b/vault/.obsidian/plugins/obsidian-livesync/main.js similarity index 100% rename from .obsidian/plugins/obsidian-livesync/main.js rename to vault/.obsidian/plugins/obsidian-livesync/main.js diff --git a/.obsidian/plugins/obsidian-livesync/manifest.json b/vault/.obsidian/plugins/obsidian-livesync/manifest.json similarity index 100% rename from .obsidian/plugins/obsidian-livesync/manifest.json rename to vault/.obsidian/plugins/obsidian-livesync/manifest.json diff --git a/.obsidian/plugins/obsidian-livesync/styles.css b/vault/.obsidian/plugins/obsidian-livesync/styles.css similarity index 100% rename from .obsidian/plugins/obsidian-livesync/styles.css rename to vault/.obsidian/plugins/obsidian-livesync/styles.css diff --git a/.obsidian/workspace.json b/vault/.obsidian/workspace.json similarity index 86% rename from .obsidian/workspace.json rename to vault/.obsidian/workspace.json index 149ed9f..876d743 100644 --- a/.obsidian/workspace.json +++ b/vault/.obsidian/workspace.json @@ -11,10 +11,14 @@ "id": "ea4cc678c44e8b67", "type": "leaf", "state": { - "type": "graph", - "state": {}, - "icon": "lucide-git-fork", - "title": "Graph view" + "type": "markdown", + "state": { + "file": "glossary.md", + "mode": "source", + "source": false + }, + "icon": "lucide-file", + "title": "glossary" } } ] @@ -185,7 +189,18 @@ }, "active": "ea4cc678c44e8b67", "lastOpenFiles": [ - "docs/hardware/environment.md.tmp.40509.b6a12d701b09", + "credentials.tmp.40509.595364788ca8", + "runbooks/lxc-service-migration.md.tmp.40509.ac2c03680b76", + "runbooks/lxc-service-migration.md.tmp.40509.c3f4d9e4365e", + "runbooks/lxc-service-migration.md.tmp.40509.ae9f0d9aaaea", + "runbooks/lxc-service-migration.md.tmp.40509.f6c568f75061", + "docs/hardware/ip-allocation.md.tmp.40509.03b7ba9c244f", + "docs/hardware/ip-allocation.md.tmp.40509.a068767a4b20", + "docs/hardware/ip-allocation.md.tmp.40509.da0228cd9f66", + "docs/hardware/environment.md.tmp.40509.e6282fa31980", + "docs/hardware/environment.md.tmp.40509.4bb7ceaced2b", + "docs/hardware/environment.md.tmp.40509.a6b0a267d126", + "glossary.md", "docs/hardware/environment.md", "nodes/ots-ct.md", "mocs/mesh.md", @@ -210,17 +225,6 @@ "nodes/meshtastic-hs-ct.md", "nodes/meshai-ct.md", "nodes/meshmonitor-ct.md", - "nodes/utility.md", - "mocs", - "hardware", - "services", - "nodes", - "session-resume", - "runbooks/lxc-service-migration.md.tmp.40509.ede2941a9a27", - "runbooks/lxc-service-migration.md.tmp.40509.f5a73384917a", - "runbooks/lxc-service-migration.md.tmp.40509.0b5e082dca64", - "runbooks/lxc-service-migration.md.tmp.40509.1dd3d6e8eb64", - "docs/hardware/ip-allocation.md.tmp.40509.ca654fd93971", "assets/echo6yellow_logo_422x422_square.png", "assets/echo6yellow_logo_422x81.png", "assets/echo6_logo.png", diff --git a/CLAUDE-baseline.md b/vault/CLAUDE-baseline.md similarity index 100% rename from CLAUDE-baseline.md rename to vault/CLAUDE-baseline.md diff --git a/INDEX.md b/vault/INDEX.md similarity index 100% rename from INDEX.md rename to vault/INDEX.md diff --git a/archive/AUDIT-2026-02-21.md b/vault/archive/AUDIT-2026-02-21.md similarity index 100% rename from archive/AUDIT-2026-02-21.md rename to vault/archive/AUDIT-2026-02-21.md diff --git a/archive/README.md b/vault/archive/README.md similarity index 100% rename from archive/README.md rename to vault/archive/README.md diff --git a/archive/matrix/PHASE6_DECISION.md b/vault/archive/matrix/PHASE6_DECISION.md similarity index 100% rename from archive/matrix/PHASE6_DECISION.md rename to vault/archive/matrix/PHASE6_DECISION.md diff --git a/archive/matrix/PLAN.md b/vault/archive/matrix/PLAN.md similarity index 100% rename from archive/matrix/PLAN.md rename to vault/archive/matrix/PLAN.md diff --git a/archive/matrix/appservices.md b/vault/archive/matrix/appservices.md similarity index 100% rename from archive/matrix/appservices.md rename to vault/archive/matrix/appservices.md diff --git a/archive/matrix/archive_receiver_discovery.md b/vault/archive/matrix/archive_receiver_discovery.md similarity index 100% rename from archive/matrix/archive_receiver_discovery.md rename to vault/archive/matrix/archive_receiver_discovery.md diff --git a/archive/matrix/archivist_discovery.md b/vault/archive/matrix/archivist_discovery.md similarity index 100% rename from archive/matrix/archivist_discovery.md rename to vault/archive/matrix/archivist_discovery.md diff --git a/archive/matrix/hookshot_deployment_discovery.md b/vault/archive/matrix/hookshot_deployment_discovery.md similarity index 100% rename from archive/matrix/hookshot_deployment_discovery.md rename to vault/archive/matrix/hookshot_deployment_discovery.md diff --git a/archive/matrix/hookshot_e2ee_discovery.md b/vault/archive/matrix/hookshot_e2ee_discovery.md similarity index 100% rename from archive/matrix/hookshot_e2ee_discovery.md rename to vault/archive/matrix/hookshot_e2ee_discovery.md diff --git a/archive/matrix/hookshot_payload_discovery.md b/vault/archive/matrix/hookshot_payload_discovery.md similarity index 100% rename from archive/matrix/hookshot_payload_discovery.md rename to vault/archive/matrix/hookshot_payload_discovery.md diff --git a/archive/matrix/hookshot_room_targeting_discovery.md b/vault/archive/matrix/hookshot_room_targeting_discovery.md similarity index 100% rename from archive/matrix/hookshot_room_targeting_discovery.md rename to vault/archive/matrix/hookshot_room_targeting_discovery.md diff --git a/archive/projects/DEPLOY-API-KEYS-TAB.md b/vault/archive/projects/DEPLOY-API-KEYS-TAB.md similarity index 100% rename from archive/projects/DEPLOY-API-KEYS-TAB.md rename to vault/archive/projects/DEPLOY-API-KEYS-TAB.md diff --git a/archive/projects/arr-stack-runbook.md b/vault/archive/projects/arr-stack-runbook.md similarity index 100% rename from archive/projects/arr-stack-runbook.md rename to vault/archive/projects/arr-stack-runbook.md diff --git a/archive/projects/arr-wiring-runbook.md b/vault/archive/projects/arr-wiring-runbook.md similarity index 100% rename from archive/projects/arr-wiring-runbook.md rename to vault/archive/projects/arr-wiring-runbook.md diff --git a/archive/projects/cc-deploy-watchtower-v2.md b/vault/archive/projects/cc-deploy-watchtower-v2.md similarity index 100% rename from archive/projects/cc-deploy-watchtower-v2.md rename to vault/archive/projects/cc-deploy-watchtower-v2.md diff --git a/archive/projects/headscale-full-deployment.md b/vault/archive/projects/headscale-full-deployment.md similarity index 100% rename from archive/projects/headscale-full-deployment.md rename to vault/archive/projects/headscale-full-deployment.md diff --git a/archive/projects/last-ember-project.md b/vault/archive/projects/last-ember-project.md similarity index 100% rename from archive/projects/last-ember-project.md rename to vault/archive/projects/last-ember-project.md diff --git a/archive/projects/mmud/last-ember-chronicle.html b/vault/archive/projects/mmud/last-ember-chronicle.html similarity index 100% rename from archive/projects/mmud/last-ember-chronicle.html rename to vault/archive/projects/mmud/last-ember-chronicle.html diff --git a/archive/projects/mmud/last-ember-howto.html b/vault/archive/projects/mmud/last-ember-howto.html similarity index 100% rename from archive/projects/mmud/last-ember-howto.html rename to vault/archive/projects/mmud/last-ember-howto.html diff --git a/archive/projects/mmud/last-ember.html b/vault/archive/projects/mmud/last-ember.html similarity index 100% rename from archive/projects/mmud/last-ember.html rename to vault/archive/projects/mmud/last-ember.html diff --git a/archive/projects/mmud/mmud-phase5-prompt.md b/vault/archive/projects/mmud/mmud-phase5-prompt.md similarity index 100% rename from archive/projects/mmud/mmud-phase5-prompt.md rename to vault/archive/projects/mmud/mmud-phase5-prompt.md diff --git a/archive/projects/mmud/mmud-phase6-prompt.md b/vault/archive/projects/mmud/mmud-phase6-prompt.md similarity index 100% rename from archive/projects/mmud/mmud-phase6-prompt.md rename to vault/archive/projects/mmud/mmud-phase6-prompt.md diff --git a/archive/projects/mmud/mmud-prompts/mmud-prompts/01-update-planned.md b/vault/archive/projects/mmud/mmud-prompts/mmud-prompts/01-update-planned.md similarity index 100% rename from archive/projects/mmud/mmud-prompts/mmud-prompts/01-update-planned.md rename to vault/archive/projects/mmud/mmud-prompts/mmud-prompts/01-update-planned.md diff --git a/archive/projects/mmud/mmud-prompts/mmud-prompts/02-npc-nodes.md b/vault/archive/projects/mmud/mmud-prompts/mmud-prompts/02-npc-nodes.md similarity index 100% rename from archive/projects/mmud/mmud-prompts/mmud-prompts/02-npc-nodes.md rename to vault/archive/projects/mmud/mmud-prompts/mmud-prompts/02-npc-nodes.md diff --git a/archive/projects/mmud/mmud-prompts/mmud-prompts/03-darkcragg.md b/vault/archive/projects/mmud/mmud-prompts/mmud-prompts/03-darkcragg.md similarity index 100% rename from archive/projects/mmud/mmud-prompts/mmud-prompts/03-darkcragg.md rename to vault/archive/projects/mmud/mmud-prompts/mmud-prompts/03-darkcragg.md diff --git a/archive/projects/mmud/mmud-prompts/mmud-prompts/04-dcrg-node.md b/vault/archive/projects/mmud/mmud-prompts/mmud-prompts/04-dcrg-node.md similarity index 100% rename from archive/projects/mmud/mmud-prompts/mmud-prompts/04-dcrg-node.md rename to vault/archive/projects/mmud/mmud-prompts/mmud-prompts/04-dcrg-node.md diff --git a/archive/projects/mmud/mmud-prompts/mmud-prompts/05-phase5.md b/vault/archive/projects/mmud/mmud-prompts/mmud-prompts/05-phase5.md similarity index 100% rename from archive/projects/mmud/mmud-prompts/mmud-prompts/05-phase5.md rename to vault/archive/projects/mmud/mmud-prompts/mmud-prompts/05-phase5.md diff --git a/archive/projects/mmud/mmud-prompts/mmud-prompts/06-phase6.md b/vault/archive/projects/mmud/mmud-prompts/mmud-prompts/06-phase6.md similarity index 100% rename from archive/projects/mmud/mmud-prompts/mmud-prompts/06-phase6.md rename to vault/archive/projects/mmud/mmud-prompts/mmud-prompts/06-phase6.md diff --git a/archive/projects/mmud/mmud-prompts/mmud-prompts/README.md b/vault/archive/projects/mmud/mmud-prompts/mmud-prompts/README.md similarity index 100% rename from archive/projects/mmud/mmud-prompts/mmud-prompts/README.md rename to vault/archive/projects/mmud/mmud-prompts/mmud-prompts/README.md diff --git a/archive/projects/mmud/mmud-prompts/mmud-prompts/mmud-project.md b/vault/archive/projects/mmud/mmud-prompts/mmud-prompts/mmud-project.md similarity index 100% rename from archive/projects/mmud/mmud-prompts/mmud-prompts/mmud-project.md rename to vault/archive/projects/mmud/mmud-prompts/mmud-prompts/mmud-project.md diff --git a/archive/projects/openwebui-theme-deploy.md b/vault/archive/projects/openwebui-theme-deploy.md similarity index 100% rename from archive/projects/openwebui-theme-deploy.md rename to vault/archive/projects/openwebui-theme-deploy.md diff --git a/archive/projects/peertube-phase2-project.md b/vault/archive/projects/peertube-phase2-project.md similarity index 100% rename from archive/projects/peertube-phase2-project.md rename to vault/archive/projects/peertube-phase2-project.md diff --git a/archive/projects/peertube-rebuild.md b/vault/archive/projects/peertube-rebuild.md similarity index 100% rename from archive/projects/peertube-rebuild.md rename to vault/archive/projects/peertube-rebuild.md diff --git a/archive/projects/utility-caddy-initial-setup.md b/vault/archive/projects/utility-caddy-initial-setup.md similarity index 100% rename from archive/projects/utility-caddy-initial-setup.md rename to vault/archive/projects/utility-caddy-initial-setup.md diff --git a/archive/projects/vaultwarden-deployment.md b/vault/archive/projects/vaultwarden-deployment.md similarity index 100% rename from archive/projects/vaultwarden-deployment.md rename to vault/archive/projects/vaultwarden-deployment.md diff --git a/archive/reports/logistics_migration.md b/vault/archive/reports/logistics_migration.md similarity index 100% rename from archive/reports/logistics_migration.md rename to vault/archive/reports/logistics_migration.md diff --git a/archive/reports/post_validation_report.md b/vault/archive/reports/post_validation_report.md similarity index 100% rename from archive/reports/post_validation_report.md rename to vault/archive/reports/post_validation_report.md diff --git a/archive/reports/task_a_aurora_validation.md b/vault/archive/reports/task_a_aurora_validation.md similarity index 100% rename from archive/reports/task_a_aurora_validation.md rename to vault/archive/reports/task_a_aurora_validation.md diff --git a/archive/reports/task_c_watchdog_test.md b/vault/archive/reports/task_c_watchdog_test.md similarity index 100% rename from archive/reports/task_c_watchdog_test.md rename to vault/archive/reports/task_c_watchdog_test.md diff --git a/assets/echo6-custom.css b/vault/assets/echo6-custom.css similarity index 100% rename from assets/echo6-custom.css rename to vault/assets/echo6-custom.css diff --git a/assets/echo6-openwebui-theme.css b/vault/assets/echo6-openwebui-theme.css similarity index 100% rename from assets/echo6-openwebui-theme.css rename to vault/assets/echo6-openwebui-theme.css diff --git a/assets/echo6-theme-toggle.js b/vault/assets/echo6-theme-toggle.js similarity index 100% rename from assets/echo6-theme-toggle.js rename to vault/assets/echo6-theme-toggle.js diff --git a/assets/echo6_favicon.png b/vault/assets/echo6_favicon.png similarity index 100% rename from assets/echo6_favicon.png rename to vault/assets/echo6_favicon.png diff --git a/assets/echo6_favicon_32x32.png b/vault/assets/echo6_favicon_32x32.png similarity index 100% rename from assets/echo6_favicon_32x32.png rename to vault/assets/echo6_favicon_32x32.png diff --git a/assets/echo6_logo.png b/vault/assets/echo6_logo.png similarity index 100% rename from assets/echo6_logo.png rename to vault/assets/echo6_logo.png diff --git a/assets/echo6yellow_logo_150x29.png b/vault/assets/echo6yellow_logo_150x29.png similarity index 100% rename from assets/echo6yellow_logo_150x29.png rename to vault/assets/echo6yellow_logo_150x29.png diff --git a/assets/echo6yellow_logo_422x422_square.png b/vault/assets/echo6yellow_logo_422x422_square.png similarity index 100% rename from assets/echo6yellow_logo_422x422_square.png rename to vault/assets/echo6yellow_logo_422x422_square.png diff --git a/assets/echo6yellow_logo_422x81.png b/vault/assets/echo6yellow_logo_422x81.png similarity index 100% rename from assets/echo6yellow_logo_422x81.png rename to vault/assets/echo6yellow_logo_422x81.png diff --git a/assets/key_manager.py b/vault/assets/key_manager.py similarity index 100% rename from assets/key_manager.py rename to vault/assets/key_manager.py diff --git a/credentials b/vault/credentials similarity index 99% rename from credentials rename to vault/credentials index fc8df6c..0b382d6 100755 --- a/credentials +++ b/vault/credentials @@ -10,7 +10,7 @@ ROOT_PASSWORD=7redditGold GODADDY_API_KEY=dKiSoC24ZLTR_3KkMjFyYrzgChk1JHjNLnU GODADDY_API_SECRET=ExCGBASgSrN4A2mP4daU4G -# Authentik SSO Platform (https://auth.echo6.co) +# Authentik SSO Platform (https://auth.echo6.co) — migrated to edge2 CT 105 / 10.10.10.23 / tailnet 100.64.0.36 on 2026-06-18 AUTHENTIK_API_TOKEN=YG24Zu7c7JNhrfC564N2NvJt2HmIr6Jyi9BgV629XGAZC70hvGbyNz8i4l7w AUTHENTIK_URL=https://auth.echo6.co # Admin credentials for web UI/API access diff --git a/docs/hardware/environment.md b/vault/docs/hardware/environment.md similarity index 96% rename from docs/hardware/environment.md rename to vault/docs/hardware/environment.md index 295faf4..342e7b3 100644 --- a/docs/hardware/environment.md +++ b/vault/docs/hardware/environment.md @@ -86,7 +86,7 @@ Five nodes running Proxmox VE: | Contabo Server | 5.189.158.149 | 100.64.0.1 | External VPS: Mail, Authentik, Headscale, Forge, Matrix | | edge2 | 184.174.35.153 | 100.64.0.26 | Contabo Cloud VPS 30 NVMe — Proxmox VE 8.4.19 (LXC-only), 8c/24GB/400GB | -*Last updated: 2026-06-17 — Added edge2 CT 104 (livesync, 10.10.10.22, 100.64.0.35, migrated 2026-06-16); previously added CT 103 (forgejo), CT 102 (vaultwarden), pdm CT 100, wordpress CT 101* +*Last updated: 2026-06-18 — Added edge2 CT 105 (authentik, 10.10.10.23, 100.64.0.36, node 48, migrated 2026-06-18); previously added CT 104 (livesync), CT 103 (forgejo), CT 102 (vaultwarden), pdm CT 100, wordpress CT 101* ## LXC Containers @@ -108,6 +108,7 @@ Five nodes running Proxmox VE: | vaultwarden | edge2 (CT 102) | 10.10.10.20 | 100.64.0.33 | Vaultwarden password manager (migrated from Contabo 2026-06-16) | | forgejo | edge2 (CT 103) | 10.10.10.21 | 100.64.0.34 | Forgejo git server (migrated from Contabo 2026-06-16) | | livesync | edge2 (CT 104) | 10.10.10.22 | 100.64.0.35 | LiveSync Obsidian sync (CouchDB + provisioner; migrated from Contabo 2026-06-16) | +| authentik | edge2 (CT 105) | 10.10.10.23 | 100.64.0.36 | Authentik SSO platform (migrated from Contabo 2026-06-18) | ## IP Allocation Scheme @@ -158,6 +159,7 @@ Current registered nodes (26 total): | vaultwarden | 100.64.0.33 | LXC (edge2 CT 102) | | forgejo | 100.64.0.34 | LXC (edge2 CT 103) — node id 46 | | livesync | 100.64.0.35 | LXC (edge2 CT 104) — migrated 2026-06-16 | +| authentik | 100.64.0.36 | LXC (edge2 CT 105) — node id 48, migrated 2026-06-18 | ## IdahoMesh Headscale Node List diff --git a/docs/hardware/ip-allocation.md b/vault/docs/hardware/ip-allocation.md similarity index 92% rename from docs/hardware/ip-allocation.md rename to vault/docs/hardware/ip-allocation.md index 69f0e17..b674812 100755 --- a/docs/hardware/ip-allocation.md +++ b/vault/docs/hardware/ip-allocation.md @@ -58,6 +58,7 @@ edge2 (Contabo Cloud VPS 184.174.35.153 / Tailscale 100.64.0.26) uses a separate | 10.10.10.20 | vaultwarden | CT 102 | 100.64.0.33 | Vaultwarden password manager (migrated from Contabo 2026-06-16) | | 10.10.10.21 | forgejo | CT 103 | 100.64.0.34 | Forgejo git server (migrated from Contabo 2026-06-16) | | 10.10.10.22 | livesync | CT 104 | 100.64.0.35 | LiveSync Obsidian sync (CouchDB + provisioner; migrated from Contabo 2026-06-16) | +| 10.10.10.23 | authentik | CT 105 | 100.64.0.36 | Authentik SSO platform (migrated from Contabo 2026-06-18) | ### VMs (.150-.199) | IP | VM | Host | Purpose | @@ -110,6 +111,7 @@ edge2 (Contabo Cloud VPS 184.174.35.153 / Tailscale 100.64.0.26) uses a separate | 100.64.0.33 | vaultwarden (CT 102 on edge2) — node id 45 | 10.10.10.20 (vmbr0) | | 100.64.0.34 | forgejo (CT 103 on edge2) — node id 46 | 10.10.10.21 (vmbr0) | | 100.64.0.35 | livesync (CT 104 on edge2) — hostname `livesync` | 10.10.10.22 (vmbr0) | +| 100.64.0.36 | authentik (CT 105 on edge2) — node id 48 | 10.10.10.23 (vmbr0) | --- @@ -122,4 +124,4 @@ edge2 (Contabo Cloud VPS 184.174.35.153 / Tailscale 100.64.0.26) uses a separate --- -*Last updated: 2026-06-17 — Added edge2 CT 104 (livesync) at 10.10.10.22 / 100.64.0.35; previously added CT 103 (forgejo), CT 102 (vaultwarden)* +*Last updated: 2026-06-18 — Added edge2 CT 105 (authentik) at 10.10.10.23 / 100.64.0.36 (node id 48); previously added CT 104 (livesync), CT 103 (forgejo), CT 102 (vaultwarden)* diff --git a/docs/matrix/archivist.md b/vault/docs/matrix/archivist.md similarity index 100% rename from docs/matrix/archivist.md rename to vault/docs/matrix/archivist.md diff --git a/docs/matrix/matrix_host.md b/vault/docs/matrix/matrix_host.md similarity index 100% rename from docs/matrix/matrix_host.md rename to vault/docs/matrix/matrix_host.md diff --git a/docs/matrix/mautrix_signal.md b/vault/docs/matrix/mautrix_signal.md similarity index 100% rename from docs/matrix/mautrix_signal.md rename to vault/docs/matrix/mautrix_signal.md diff --git a/docs/matrix/synapse.md b/vault/docs/matrix/synapse.md similarity index 100% rename from docs/matrix/synapse.md rename to vault/docs/matrix/synapse.md diff --git a/docs/matrix/synapse_homeserver.yaml.sanitized b/vault/docs/matrix/synapse_homeserver.yaml.sanitized similarity index 100% rename from docs/matrix/synapse_homeserver.yaml.sanitized rename to vault/docs/matrix/synapse_homeserver.yaml.sanitized diff --git a/docs/matrix/synapse_retention_discovery.md b/vault/docs/matrix/synapse_retention_discovery.md similarity index 100% rename from docs/matrix/synapse_retention_discovery.md rename to vault/docs/matrix/synapse_retention_discovery.md diff --git a/docs/navi/cc-rules.md b/vault/docs/navi/cc-rules.md similarity index 100% rename from docs/navi/cc-rules.md rename to vault/docs/navi/cc-rules.md diff --git a/docs/navi/deployment.md b/vault/docs/navi/deployment.md similarity index 100% rename from docs/navi/deployment.md rename to vault/docs/navi/deployment.md diff --git a/docs/navi/themes.md b/vault/docs/navi/themes.md similarity index 100% rename from docs/navi/themes.md rename to vault/docs/navi/themes.md diff --git a/docs/services/ots-setup.md b/vault/docs/services/ots-setup.md similarity index 100% rename from docs/services/ots-setup.md rename to vault/docs/services/ots-setup.md diff --git a/docs/services/services.md b/vault/docs/services/services.md similarity index 93% rename from docs/services/services.md rename to vault/docs/services/services.md index 310353c..3cf85e4 100644 --- a/docs/services/services.md +++ b/vault/docs/services/services.md @@ -14,7 +14,7 @@ | mesh-bridge | utility (CT 107) | 192.168.1.107 | Internal | Dual-tailscaled bridge (echo6 ↔ idahomesh) | | MeshAI | utility (CT 108) | 192.168.1.144:4403 | Internal | LLM-powered Meshtastic assistant (Docker, Gemini Flash, Google grounding) | | ARGUS | utility (CT 103) | 192.168.1.103 | Internal | OSINT intelligence gathering platform (Docker, SearXNG + local LLM analysis) | -| Authentik | Contabo | 5.189.158.149:9000 | https://auth.echo6.co | SSO provider (Echo6 branded, custom CSS, dark theme) | +| Authentik | edge2 (CT 105) | 100.64.0.36:9000 | https://auth.echo6.co | SSO provider (Echo6 branded, custom CSS, dark theme) — fronted by Contabo Caddy (reverse_proxy 100.64.0.36:9000); **migrated from Contabo 2026-06-18** | | Forge (Forgejo) | edge2 (CT 103) | 100.64.0.34:3001 HTTP / :2222 SSH (via Contabo DNAT) | https://forge.echo6.co | Git server — fronted by Contabo Caddy (reverse_proxy 100.64.0.34:3001); git SSH via iptables DNAT on Contabo (forgejo-ssh-dnat.service) — **migrated from Contabo 2026-06-16** | | Headscale | Contabo | 5.189.158.149 | https://vpn.echo6.co | Tailscale coordination (OIDC enabled) | | Headplane | Contabo | 127.0.0.1:3100 | https://vpn.echo6.co/admin | Headscale web UI (OIDC via Authentik) | @@ -280,6 +280,21 @@ - Source on Contabo STOPPED but intact as cold rollback; `/etc/caddy/Caddyfile.bak-prelivesync` exists - **Resources:** 2 cores / 1024 MB RAM / 512 MB swap / 8 GB rootfs on `local`; unprivileged; onboot; Docker +### edge2 - CT 105 (10.10.10.23 / Tailscale: 100.64.0.36, node 48 `authentik`) +- Authentik SSO platform (https://auth.echo6.co — **migrated from Contabo 2026-06-18**) + - Headscale node id 48, hostname `authentik`, tailnet IP 100.64.0.36 + - Compose path: `/opt/authentik/docker-compose.yml` + - Containers: `authentik-server` + `authentik-worker` (ghcr.io/goauthentik/server:2025.12.4) + `authentik-postgres` (postgres:16); NO Redis + - Worker runs as user:root and has docker.sock bind-mount (manages embedded outposts) + - Binds to `100.64.0.36:9000`; Contabo Caddy proxies here over tailnet for both `auth.echo6.co` (catch-all + outpost path matcher) and `notes.echo6.co` outpost/forward_auth references + - `AUTHENTIK_SECRET_KEY` carried byte-for-byte (sessions stayed valid across cutover — users dropped straight in) + - Bind-mounts (data/media, branding, certs, custom-templates) migrated intact + - Email dep: mail.echo6.co (unchanged) + - DB: PostgreSQL 16 (`authentik-postgres` container); ~705 MB (~18 MB pg_dump) + - Source on Contabo STOPPED but intact as cold rollback; `/etc/caddy/Caddyfile.bak-pre-authentik` exists on Contabo + - Reboot-survival fix: systemd unit on CT 105 gates `docker compose up` on `tailscale-online` (Docker was racing Tailscale on boot, failing the bind to the tailnet IP) + - **Resources:** 2 cores / 4096 MB RAM / 512 MB swap / 20 GB rootfs on `local`; unprivileged; onboot; Docker + ### edge2 - CT 102 (10.10.10.20 / Tailscale: 100.64.0.33, node 45 `vaultwarden`) - Vaultwarden password manager (port 8086, https://vault.echo6.co, Docker) - Headscale node id 45, name `vaultwarden`, user `echo6` @@ -290,7 +305,7 @@ - **Migrated from Contabo to edge2 CT 102 on 2026-06-16** ### Contabo VPS (5.189.158.149 / Tailscale: 100.64.0.1) -- Authentik (SSO, Echo6 branded — custom CSS, dark theme, logo, favicon, flow titles) +- ~~Authentik~~ — **migrated to edge2 CT 105 on 2026-06-18** (Caddy now proxies auth.echo6.co + notes.echo6.co outpost/forward_auth → 100.64.0.36:9000; source STOPPED at `/opt/authentik`, intact as cold rollback; `/etc/caddy/Caddyfile.bak-pre-authentik` exists) - Forge (Git) — **migrated to edge2 CT 103 on 2026-06-16** (Caddy now proxies to 100.64.0.34:3001; SSH DNAT via forgejo-ssh-dnat.service) - Headscale (mesh VPN) - Mailcow (email) diff --git a/docs/services/usenet.md b/vault/docs/services/usenet.md similarity index 100% rename from docs/services/usenet.md rename to vault/docs/services/usenet.md diff --git a/docs/software/authentik.md b/vault/docs/software/authentik.md similarity index 100% rename from docs/software/authentik.md rename to vault/docs/software/authentik.md diff --git a/docs/software/caddy.md b/vault/docs/software/caddy.md similarity index 100% rename from docs/software/caddy.md rename to vault/docs/software/caddy.md diff --git a/docs/software/dns.md b/vault/docs/software/dns.md similarity index 100% rename from docs/software/dns.md rename to vault/docs/software/dns.md diff --git a/docs/software/geo-tools.md b/vault/docs/software/geo-tools.md similarity index 100% rename from docs/software/geo-tools.md rename to vault/docs/software/geo-tools.md diff --git a/docs/software/recon.md b/vault/docs/software/recon.md similarity index 100% rename from docs/software/recon.md rename to vault/docs/software/recon.md diff --git a/docs/software/searxng.md b/vault/docs/software/searxng.md similarity index 100% rename from docs/software/searxng.md rename to vault/docs/software/searxng.md diff --git a/vault/glossary.md b/vault/glossary.md new file mode 100644 index 0000000..ccabebe --- /dev/null +++ b/vault/glossary.md @@ -0,0 +1,159 @@ +--- +title: Glossary & Vocabulary +type: reference +tags: [meta] +updated: 2026-06-18 +--- +# Glossary & Vocabulary + +> Auto-generated by the vault engine (`engine/lib/vocab_gen.py`). +> Acronym expansions marked _(unconfirmed)_ need a human pass. + +## Topic categories + +mesh · matrix · recon · media · auth · dns · vpn · storage · proxmox · ai · mail + +## Acronyms + +| Acronym | Expansion | +|---|---| +| MMUD | Mesh Multi-User Dungeon | +| AIDA | _(unconfirmed)_ | + +## Entities + +### Hosts / Proxmox nodes + +- **argus** — aliases: 100.64.0.25 +- **bluefin** — aliases: 100.64.0.30 +- **cloud** (Cloud) — aliases: 192.168.1.242, 100.64.0.4 +- **contabo** — aliases: 100.64.0.1 +- **data** (Data) — aliases: 192.168.1.240, 100.64.0.6 +- **edge2** — aliases: 184.174.35.153, 100.64.0.26 +- **forgejo** — aliases: 100.64.0.34 +- **iphone-eud** — aliases: 100.64.0.16 +- **media** (Media) — aliases: 192.168.1.243, 100.64.0.3 +- **mesh-bridge** — aliases: 100.100.0.3, 100.64.0.22 +- **meshai** — aliases: 100.64.0.32 +- **meshmonitor** — aliases: 100.64.0.7 +- **meshmonitor-dev** — aliases: 100.64.0.27 +- **nextcloud** — aliases: 100.64.0.11 +- **peertube** — aliases: 100.64.0.23 +- **toc** (Toc) — aliases: 192.168.1.244, 100.64.0.13 +- **utility** (Utility) — aliases: 192.168.1.241, 100.64.0.5 + +### Virtual Machines + +- **arr** — aliases: 192.168.1.160, 100.64.0.18, VM 105 — on: media +- **cortex** — aliases: 192.168.1.150, 100.64.0.14, VM 150 — on: toc +- **recon** — aliases: 100.64.0.24 +- **recon-vm** — aliases: 192.168.1.130, 100.64.0.24, VM 1130 — on: data + +### LXC Containers + +- **archivist** — aliases: 192.168.1.118, CT 118 — on: utility +- **caddy** — aliases: 192.168.1.101, CT 101, 100.64.0.8 — on: utility +- **immich** — aliases: 192.168.1.182, CT 120, 100.64.0.2 — on: cloud +- **livesync** — aliases: 10.10.10.22, CT 104, 100.64.0.35 — on: edge2 +- **meshtastic-hs** — aliases: 192.168.1.106, CT 106 — on: utility +- **pdm** — aliases: 10.10.10.10, CT 100, 100.64.0.28 — on: edge2 +- **running** — aliases: CT 100 — on: utility _(live)_ +- **searxng** — aliases: 192.168.1.102, CT 102, 100.64.0.15 — on: utility +- **vaultwarden** — aliases: 10.10.10.20, CT 102, 100.64.0.33 — on: edge2 +- **wordpress** — aliases: 10.10.10.11, CT 101, 100.64.0.31 — on: edge2 + +### Raspberry Pi / Edge nodes + +- **aida-nebra** — aliases: 192.168.1.253, 100.64.0.9, !27780c47 +- **burley-butte** — aliases: 100.100.0.1 +- **mt-burleybutte** — aliases: 192.168.1.185 +- **mt-isr** — aliases: 192.168.1.141, 100.100.0.5 (IdahoMesh), 100.100.0.5 +- **pi-nas** — aliases: 192.168.1.245, 100.64.0.21 + +### Desktops + +- **matt-desktop** — aliases: 192.168.1.111, 100.64.0.10 + +### Routers / Network devices + +- **gl-a1300** — aliases: 100.64.0.29 + +### Services + +- **authentik** (Authentik) — aliases: Authentik, auth — on: contabo +- **authentik-postgres** — aliases: authentik-postgres — on: contabo _(live)_ +- **authentik-server** — aliases: authentik-server — on: contabo _(live)_ +- **authentik-worker** — aliases: authentik-worker — on: contabo _(live)_ +- **echo6-agent** — aliases: echo6-agent — on: 2026-06-16 +- **echo6-contabo-agent** (Echo6 Contabo Agent) — aliases: Echo6 Contabo Agent — on: contabo +- **echo6-cortex-agent** (Echo6 Cortex Agent) — aliases: Echo6 Cortex Agent — on: cortex +- **echo6-search-searxng** (Echo6 Search (SearXNG)) — aliases: Echo6 Search (SearXNG) — on: utility +- **element-web** (Element Web) — aliases: Element Web, element — on: contabo +- **files** (Files) — aliases: Files, files — on: data +- **forge-forgejo** (Forge (Forgejo)) — aliases: Forge (Forgejo), forge — on: edge2 +- **headplane** (Headplane) — aliases: Headplane, vpn — on: contabo +- **headscale** (Headscale) — aliases: Headscale, vpn — on: contabo +- **idahomesh-headscale** (IdahoMesh Headscale) — aliases: IdahoMesh Headscale — on: utility +- **jellyfin** (Jellyfin) — aliases: Jellyfin, jellyfin — on: media +- **jellyseer** (Jellyseer) — aliases: Jellyseer, requests — on: media +- **lidarr** — aliases: lidarr — on: media _(live)_ +- **mailcow** (Mailcow) — aliases: Mailcow, mail — on: contabo +- **mailcowdockerized-acme-mailcow-1** — aliases: mailcowdockerized-acme-mailcow-1 — on: contabo _(live)_ +- **mailcowdockerized-clamd-mailcow-1** — aliases: mailcowdockerized-clamd-mailcow-1 — on: contabo _(live)_ +- **mailcowdockerized-dockerapi-mailcow-1** — aliases: mailcowdockerized-dockerapi-mailcow-1 — on: contabo _(live)_ +- **mailcowdockerized-dovecot-mailcow-1** — aliases: mailcowdockerized-dovecot-mailcow-1 — on: contabo _(live)_ +- **mailcowdockerized-memcached-mailcow-1** — aliases: mailcowdockerized-memcached-mailcow-1 — on: contabo _(live)_ +- **mailcowdockerized-mysql-mailcow-1** — aliases: mailcowdockerized-mysql-mailcow-1 — on: contabo _(live)_ +- **mailcowdockerized-netfilter-mailcow-1** — aliases: mailcowdockerized-netfilter-mailcow-1 — on: contabo _(live)_ +- **mailcowdockerized-nginx-mailcow-1** — aliases: mailcowdockerized-nginx-mailcow-1 — on: contabo _(live)_ +- **mailcowdockerized-ofelia-mailcow-1** — aliases: mailcowdockerized-ofelia-mailcow-1 — on: contabo _(live)_ +- **mailcowdockerized-olefy-mailcow-1** — aliases: mailcowdockerized-olefy-mailcow-1 — on: contabo _(live)_ +- **mailcowdockerized-php-fpm-mailcow-1** — aliases: mailcowdockerized-php-fpm-mailcow-1 — on: contabo _(live)_ +- **mailcowdockerized-postfix-mailcow-1** — aliases: mailcowdockerized-postfix-mailcow-1 — on: contabo _(live)_ +- **mailcowdockerized-postfix-tlspol-mailcow-1** — aliases: mailcowdockerized-postfix-tlspol-mailcow-1 — on: contabo _(live)_ +- **mailcowdockerized-redis-mailcow-1** — aliases: mailcowdockerized-redis-mailcow-1 — on: contabo _(live)_ +- **mailcowdockerized-rspamd-mailcow-1** — aliases: mailcowdockerized-rspamd-mailcow-1 — on: contabo _(live)_ +- **mailcowdockerized-sogo-mailcow-1** — aliases: mailcowdockerized-sogo-mailcow-1 — on: contabo _(live)_ +- **mailcowdockerized-unbound-mailcow-1** — aliases: mailcowdockerized-unbound-mailcow-1 — on: contabo _(live)_ +- **mailcowdockerized-watchdog-mailcow-1** — aliases: mailcowdockerized-watchdog-mailcow-1 — on: contabo _(live)_ +- **matrix-element** — aliases: matrix-element — on: contabo _(live)_ +- **matrix-mas** (Matrix MAS) — aliases: Matrix MAS — on: contabo +- **matrix-postgres** — aliases: matrix-postgres — on: contabo _(live)_ +- **matrix-synapse** (Matrix Synapse) — aliases: Matrix Synapse, matrix — on: contabo +- **mautrix-signal** — aliases: mautrix-signal — on: contabo +- **meshtastic-cli** (Meshtastic CLI) — aliases: Meshtastic CLI — on: mt-isr +- **meshtasticd** — aliases: meshtasticd — on: mt-burleybutte +- **meshtasticd-aida-n2** (meshtasticd (AIDA-N2)) — aliases: meshtasticd (AIDA-N2) — on: aida-nebra +- **navidrome** — aliases: navidrome — on: media _(live)_ +- **nexus-agent** — aliases: nexus-agent — on: 2026-06-16 +- **nexus-hub** — aliases: nexus-hub — on: 2026-06-16 +- **obsidian-remote** — aliases: obsidian-remote — on: cortex _(live)_ +- **ollama** — aliases: ollama — on: cortex _(live)_ +- **open-webui** (Open WebUI) — aliases: Open WebUI, ai — on: cortex +- **opentakserver-ots** (OpenTAKServer (OTS)) — aliases: OpenTAKServer (OTS) — on: utility +- **prowlarr** (Prowlarr) — aliases: Prowlarr — on: media +- **proxmox-ve** (Proxmox VE) — aliases: Proxmox VE, proxmox — on: data +- **pt-transcoder** — aliases: pt-transcoder — on: cortex +- **qdrant** (Qdrant) — aliases: Qdrant — on: cortex +- **radarr** (Radarr) — aliases: Radarr — on: media +- **recon-sparse** — aliases: recon-sparse — on: cortex +- **sabnzbd** (SABnzbd) — aliases: SABnzbd — on: media +- **samba** (Samba) — aliases: Samba — on: data +- **sigil** (SIGIL) — aliases: SIGIL — on: 2026-06-16 +- **sonarr** (Sonarr) — aliases: Sonarr — on: media +- **syncthing** (Syncthing) — aliases: Syncthing — on: contabo +- **tak-server** (TAK Server) — aliases: TAK Server — on: 2026-06-16 +- **tei** (TEI) — aliases: TEI — on: cortex +- **termix** (Termix) — aliases: Termix — on: contabo +- **utility-caddy** (Utility Caddy) — aliases: Utility Caddy — on: utility +- **watchtower** (WATCHTOWER) — aliases: WATCHTOWER — on: 2026-06-16 + +### Projects + +- **advbbs-project** — aliases: advbbs-project +- **argus** — aliases: argus +- **deploy-livesync** — aliases: deploy-livesync +- **matrix-synapse-deployment** — aliases: matrix-synapse-deployment +- **meshtastic-headscale-runbook** — aliases: meshtastic-headscale-runbook +- **mmud-project** — aliases: mmud-project + diff --git a/notes/echo6-landing-page-data-export.md b/vault/notes/echo6-landing-page-data-export.md similarity index 100% rename from notes/echo6-landing-page-data-export.md rename to vault/notes/echo6-landing-page-data-export.md diff --git a/notes/ia-download-queue.md b/vault/notes/ia-download-queue.md similarity index 100% rename from notes/ia-download-queue.md rename to vault/notes/ia-download-queue.md diff --git a/plans/vaultwarden-plan.md b/vault/plans/vaultwarden-plan.md similarity index 100% rename from plans/vaultwarden-plan.md rename to vault/plans/vaultwarden-plan.md diff --git a/projects/advbbs-project.md b/vault/projects/advbbs-project.md similarity index 100% rename from projects/advbbs-project.md rename to vault/projects/advbbs-project.md diff --git a/projects/argus.md b/vault/projects/argus.md similarity index 100% rename from projects/argus.md rename to vault/projects/argus.md diff --git a/projects/deploy-livesync.md b/vault/projects/deploy-livesync.md similarity index 100% rename from projects/deploy-livesync.md rename to vault/projects/deploy-livesync.md diff --git a/projects/matrix-synapse-deployment.md b/vault/projects/matrix-synapse-deployment.md similarity index 100% rename from projects/matrix-synapse-deployment.md rename to vault/projects/matrix-synapse-deployment.md diff --git a/projects/meshtastic-headscale-runbook.md b/vault/projects/meshtastic-headscale-runbook.md similarity index 100% rename from projects/meshtastic-headscale-runbook.md rename to vault/projects/meshtastic-headscale-runbook.md diff --git a/projects/mmud-project.md b/vault/projects/mmud-project.md similarity index 100% rename from projects/mmud-project.md rename to vault/projects/mmud-project.md diff --git a/rules b/vault/rules similarity index 100% rename from rules rename to vault/rules diff --git a/runbooks/add-peertube-channel.md b/vault/runbooks/add-peertube-channel.md similarity index 100% rename from runbooks/add-peertube-channel.md rename to vault/runbooks/add-peertube-channel.md diff --git a/runbooks/authentik-access-groups.md b/vault/runbooks/authentik-access-groups.md similarity index 100% rename from runbooks/authentik-access-groups.md rename to vault/runbooks/authentik-access-groups.md diff --git a/runbooks/authentik-create-invitation.md b/vault/runbooks/authentik-create-invitation.md similarity index 100% rename from runbooks/authentik-create-invitation.md rename to vault/runbooks/authentik-create-invitation.md diff --git a/runbooks/authentik-oidc-application.md b/vault/runbooks/authentik-oidc-application.md similarity index 100% rename from runbooks/authentik-oidc-application.md rename to vault/runbooks/authentik-oidc-application.md diff --git a/runbooks/authentik-upgrade.md b/vault/runbooks/authentik-upgrade.md similarity index 100% rename from runbooks/authentik-upgrade.md rename to vault/runbooks/authentik-upgrade.md diff --git a/runbooks/ct-runbook.md b/vault/runbooks/ct-runbook.md similarity index 100% rename from runbooks/ct-runbook.md rename to vault/runbooks/ct-runbook.md diff --git a/runbooks/edge2-access-reference.md b/vault/runbooks/edge2-access-reference.md similarity index 100% rename from runbooks/edge2-access-reference.md rename to vault/runbooks/edge2-access-reference.md diff --git a/runbooks/expose-service-contabo.md b/vault/runbooks/expose-service-contabo.md similarity index 100% rename from runbooks/expose-service-contabo.md rename to vault/runbooks/expose-service-contabo.md diff --git a/runbooks/expose-service-edge2.md b/vault/runbooks/expose-service-edge2.md similarity index 100% rename from runbooks/expose-service-edge2.md rename to vault/runbooks/expose-service-edge2.md diff --git a/runbooks/expose-service-home.md b/vault/runbooks/expose-service-home.md similarity index 100% rename from runbooks/expose-service-home.md rename to vault/runbooks/expose-service-home.md diff --git a/runbooks/headscale-onboard-node.md b/vault/runbooks/headscale-onboard-node.md similarity index 100% rename from runbooks/headscale-onboard-node.md rename to vault/runbooks/headscale-onboard-node.md diff --git a/runbooks/ia-cli-reference.md b/vault/runbooks/ia-cli-reference.md similarity index 100% rename from runbooks/ia-cli-reference.md rename to vault/runbooks/ia-cli-reference.md diff --git a/runbooks/ia-download-mirror.md b/vault/runbooks/ia-download-mirror.md similarity index 100% rename from runbooks/ia-download-mirror.md rename to vault/runbooks/ia-download-mirror.md diff --git a/runbooks/idahomesh-bridge-setup.md b/vault/runbooks/idahomesh-bridge-setup.md similarity index 100% rename from runbooks/idahomesh-bridge-setup.md rename to vault/runbooks/idahomesh-bridge-setup.md diff --git a/runbooks/idahomesh-vpn-device-setup.md b/vault/runbooks/idahomesh-vpn-device-setup.md similarity index 100% rename from runbooks/idahomesh-vpn-device-setup.md rename to vault/runbooks/idahomesh-vpn-device-setup.md diff --git a/runbooks/lxc-service-migration.md b/vault/runbooks/lxc-service-migration.md similarity index 88% rename from runbooks/lxc-service-migration.md rename to vault/runbooks/lxc-service-migration.md index 899d93f..0c3fcca 100644 --- a/runbooks/lxc-service-migration.md +++ b/vault/runbooks/lxc-service-migration.md @@ -1,6 +1,6 @@ # LXC Service Migration — Contabo → edge2 -> Proven pilots: **Vaultwarden → edge2 CT 102** (SQLite, 2026-06-16), **Forgejo → edge2 CT 103** (PostgreSQL + non-Caddy SSH port, 2026-06-16), and **LiveSync (CouchDB) → edge2 CT 104** (cold named-volume tar + bind-mounted config, 2026-06-16). This runbook generalizes these patterns into a reusable template for evacuating any Contabo-Caddy-fronted service to an edge2 LXC. +> Proven pilots: **Vaultwarden → edge2 CT 102** (SQLite, 2026-06-16), **Forgejo → edge2 CT 103** (PostgreSQL + non-Caddy SSH port, 2026-06-16), **LiveSync (CouchDB) → edge2 CT 104** (cold named-volume tar + bind-mounted config, 2026-06-16), and **Authentik (PostgreSQL keystone) → edge2 CT 105** (SECRET_KEY-must-travel, multi-block Caddy cutover across 2 site blocks, reboot tailscale-before-docker race, 2026-06-18). This runbook generalizes these patterns into a reusable template for evacuating any Contabo-Caddy-fronted service to an edge2 LXC. --- @@ -318,6 +318,8 @@ vault.echo6.co { **Multi-token cutover example (LiveSync, 2026-06-16):** LiveSync exposes TWO upstream ports (5984 for CouchDB, 5985 for provisioner) within a single Caddy site block. Both tokens were changed from `127.0.0.1:598x` → `100.64.0.35:598x` in one edit. A third upstream in the same block — the Authentik outpost at `127.0.0.1:9000` (used for `forward_auth` on `/_provision`) — was left **untouched** because it stays on Contabo. Change only the tokens that move; never touch the Authentik outpost address. +**Multi-block cutover example (Authentik, 2026-06-18):** Authentik appeared in 4 places across 2 site blocks — `auth.echo6.co` (outpost path matcher + catch-all both pointing to `127.0.0.1:9000`) and `notes.echo6.co` (outpost path matcher + `forward_auth` directive both pointing to `127.0.0.1:9000`). All 4 occurrences were updated to `100.64.0.36:9000` in one edit. Grep the entire Caddyfile for the service's port before cutting over — do not assume a service lives in only one block. See also G15 (dnsmasq must NOT be repointed) and G16 (SECRET_KEY must travel). + --- #### Phase 7a — Non-Caddy public TCP port (iptables DNAT) `[S]` *(Forgejo SSH variant)* @@ -424,6 +426,9 @@ ssh root@100.64.0.1 'systemctl disable --now -ssh-dnat.service && rm /e | G11 | SSH host keys must travel inside the data volume (Variant C tar). If they are missing or regenerated on the target, every git client gets a host-key-changed warning and must manually clear `~/.ssh/known_hosts`. Transfer the full data volume; verify on target with `docker exec cat /data/ssh/forgejo.rsa.pub` or equivalent before cutover. | | G12 | DB row-count integrity gate is mandatory for PostgreSQL migrations. Never skip it — a silent pg_restore failure (wrong role, encoding mismatch) leaves the DB empty or partially populated while `pg_restore` exits 0. Compare at least one key business table. | | G13 | iptables DNAT must be made reboot-persistent via a systemd `oneshot`/`RemainAfterExit` unit (Phase 7a). Do NOT rely on iptables-persistent packages or manual rules — they require package installs (forbidden) or do not survive all reboot paths. Do NOT apply by rebooting the Contabo host (production). Create the unit file, `daemon-reload`, `enable --now`. | +| G14 | **Reboot race — Docker binding to the tailnet IP can start before Tailscale is online, failing the bind and leaving the service unreachable after a reboot.** Fix: create a systemd unit on the CT that runs `docker compose up` and has `After=tailscale-online.target` + `Requires=tailscale-online.target` (or equivalent `tailscale status --wait` pre-check). Alternatively, `restart: unless-stopped` in the compose file will cause Docker to self-heal via restarts, but the service will be unreachable for the first ~10–30 s after reboot. Verify reboot survival explicitly (Phase 8). Proven required for Authentik (CT 105, 2026-06-18). | +| G15 | **Do NOT change the dnsmasq split-DNS entry during cutover.** The dnsmasq entry for `.echo6.co` points at the Caddy/TLS host (100.64.0.1 = Contabo), NOT the backend. Only the Caddy upstream changes. Repointing dnsmasq to the backend tailnet IP would break internal HTTPS (no cert, no TLS termination). The Caddy host is always the internal DNS target; the backend IP only appears in the Caddy `reverse_proxy` directive. | +| G16 | **SECRET_KEY and session-signing material must travel byte-for-byte for keystone/session-bearing services** (e.g. Authentik `AUTHENTIK_SECRET_KEY`, Vaultwarden `rsa_key.pem`). Carrying them verbatim means existing browser sessions survive the cutover — users drop straight in with no forced re-login. If the key is regenerated on the target, all active sessions are invalidated immediately. Confirm from startup logs that no new key was generated. | | N | The composed **Contabo-Caddy → edge2-LXC tailnet** path is unexercised for each new service. Keep the Phase 6 HTTP `/alive` 200 gate as a HARD pre-cutover requirement (use `curl`, not ICMP). | --- @@ -441,4 +446,4 @@ ssh root@100.64.0.1 'systemctl disable --now -ssh-dnat.service && rm /e --- -*Last updated: 2026-06-17 — Added LiveSync/CouchDB pilot (Variant D: cold named-volume tar + bind-mounted config, multi-token Caddy cutover); Phase 5 Variant D; Phase 7 multi-token example; previously added Forgejo pilot (Variants B/C, Phase 7a, Gotchas G11-G13)* +*Last updated: 2026-06-18 — Added Authentik pilot (PostgreSQL keystone, multi-block Caddy cutover across 2 site blocks, reboot tailscale-before-docker race); Phase 7 multi-block example; Gotchas G14 (reboot race), G15 (dnsmasq must not be repointed), G16 (SECRET_KEY must travel); previously added LiveSync/CouchDB pilot (Variant D, G11-G13)* diff --git a/runbooks/mailcow-create-mailbox.md b/vault/runbooks/mailcow-create-mailbox.md similarity index 100% rename from runbooks/mailcow-create-mailbox.md rename to vault/runbooks/mailcow-create-mailbox.md diff --git a/runbooks/meshmonitor-password-reset.md b/vault/runbooks/meshmonitor-password-reset.md similarity index 100% rename from runbooks/meshmonitor-password-reset.md rename to vault/runbooks/meshmonitor-password-reset.md diff --git a/runbooks/meshtastic-sidecar-node.md b/vault/runbooks/meshtastic-sidecar-node.md similarity index 100% rename from runbooks/meshtastic-sidecar-node.md rename to vault/runbooks/meshtastic-sidecar-node.md diff --git a/runbooks/meshtasticd-sim-nodes-runbook.md b/vault/runbooks/meshtasticd-sim-nodes-runbook.md similarity index 100% rename from runbooks/meshtasticd-sim-nodes-runbook.md rename to vault/runbooks/meshtasticd-sim-nodes-runbook.md diff --git a/runbooks/nordvpn-lxc.md b/vault/runbooks/nordvpn-lxc.md similarity index 100% rename from runbooks/nordvpn-lxc.md rename to vault/runbooks/nordvpn-lxc.md diff --git a/runbooks/peertube-remote-runner.md b/vault/runbooks/peertube-remote-runner.md similarity index 100% rename from runbooks/peertube-remote-runner.md rename to vault/runbooks/peertube-remote-runner.md diff --git a/runbooks/pg-backup.md b/vault/runbooks/pg-backup.md similarity index 100% rename from runbooks/pg-backup.md rename to vault/runbooks/pg-backup.md diff --git a/runbooks/pi-nas-omv-runbook.md b/vault/runbooks/pi-nas-omv-runbook.md similarity index 100% rename from runbooks/pi-nas-omv-runbook.md rename to vault/runbooks/pi-nas-omv-runbook.md diff --git a/runbooks/pipeline-patterns.md b/vault/runbooks/pipeline-patterns.md similarity index 100% rename from runbooks/pipeline-patterns.md rename to vault/runbooks/pipeline-patterns.md diff --git a/runbooks/proxmox-create-ubuntu-vm.md b/vault/runbooks/proxmox-create-ubuntu-vm.md similarity index 100% rename from runbooks/proxmox-create-ubuntu-vm.md rename to vault/runbooks/proxmox-create-ubuntu-vm.md diff --git a/runbooks/proxmox-onboard-node.md b/vault/runbooks/proxmox-onboard-node.md similarity index 100% rename from runbooks/proxmox-onboard-node.md rename to vault/runbooks/proxmox-onboard-node.md diff --git a/runbooks/recon-operations.md b/vault/runbooks/recon-operations.md similarity index 100% rename from runbooks/recon-operations.md rename to vault/runbooks/recon-operations.md diff --git a/runbooks/recon-service-integration.md b/vault/runbooks/recon-service-integration.md similarity index 100% rename from runbooks/recon-service-integration.md rename to vault/runbooks/recon-service-integration.md diff --git a/runbooks/syncthing-add-node.md b/vault/runbooks/syncthing-add-node.md similarity index 100% rename from runbooks/syncthing-add-node.md rename to vault/runbooks/syncthing-add-node.md diff --git a/session-resume/SESSION-HANDOFF-meshai-test.md b/vault/session-resume/SESSION-HANDOFF-meshai-test.md similarity index 100% rename from session-resume/SESSION-HANDOFF-meshai-test.md rename to vault/session-resume/SESSION-HANDOFF-meshai-test.md