echo6-docs/engine/lib/agent.py

#!/usr/bin/env python3
"""
agent.py — Vault Tagger + Existing-Doc Linker (v4 — simple model)

The agreed model: documentation library only.
- Never create entity/concept/node pages.
- Per-doc: (1) assign 1-3 category tags from topic_categories, (2) inline [[wikilinks]]
  to EXISTING docs only (most-specific match, first prose occurrence).
- Keep: related (bge-m3 nearest existing docs), title, updated.
- Preserve all human frontmatter keys.

Usage:
    python3 engine/lib/agent.py --dry-run <path>   # print diff, write nothing
    python3 engine/lib/agent.py <path>              # apply

stdlib only — no pip. All HTTP via urllib.
"""

from __future__ import annotations

import argparse
import difflib
import json
import math
import os
import re
import sys
import urllib.error
import urllib.request
from datetime import datetime, timezone
from pathlib import Path


# ---------------------------------------------------------------------------
# Config loader (minimal stdlib YAML parser)
# ---------------------------------------------------------------------------

def _parse_simple_yaml(text: str) -> dict:
    result: dict = {}
    current_key = None
    current_list: list | None = None
    for raw_line in text.splitlines():
        line = raw_line.rstrip()
        stripped = line.lstrip()
        if not stripped or stripped.startswith("#"):
            if current_list is not None and not line.startswith(" "):
                result[current_key] = current_list
                current_list = None
                current_key = None
            continue
        if stripped.startswith("- ") and current_list is not None:
            current_list.append(stripped[2:].strip().strip('"').strip("'"))
            continue
        if current_list is not None and not stripped.startswith("-"):
            result[current_key] = current_list
            current_list = None
            current_key = None
        if ":" in stripped:
            key, _, value = stripped.partition(":")
            key = key.strip()
            value = value.strip()
            if value.startswith("[") and value.endswith("]"):
                inner = value[1:-1]
                result[key] = [v.strip().strip('"').strip("'") for v in inner.split(",") if v.strip()]
            elif value == "":
                current_key = key
                current_list = []
            elif value.startswith("#"):
                result[key] = ""
            else:
                value = value.split(" #")[0].strip().strip('"').strip("'")
                result[key] = value
    if current_list is not None and current_key:
        result[current_key] = current_list
    return result


def load_config(config_path: str | Path | None = None) -> dict:
    """Load config.yaml; fall back to defaults."""
    defaults: dict = {
        "vault_dir": "/home/zvx/projects/.ref/vault",
        "engine_dir": "/home/zvx/projects/.ref/engine",
        "topic_categories": [
            "mesh", "matrix", "recon", "media", "auth",
            "dns", "vpn", "storage", "proxmox", "ai", "mail",
        ],
        "models": {
            "tagger": {
                "ollama_endpoint": "http://localhost:11434",
                "model": "vault-tagger",
                "temperature": 0.1,
            },
            "embeddings": {
                "tei_endpoint": "http://localhost:8090",
            },
        },
        "behavior": {
            "auto_apply": True,
            "changelog": "/home/zvx/projects/.ref/engine/changelog.md",
            "confidence_threshold": 0.6,
        },
    }

    if config_path is None:
        here = Path(__file__).parent
        for c in [here.parent / "config.yaml", Path("/home/zvx/projects/.ref/engine/config.yaml")]:
            if c.exists():
                config_path = c
                break

    if config_path is None or not Path(config_path).exists():
        return defaults

    cfg = dict(defaults)
    try:
        parsed = _parse_simple_yaml(Path(config_path).read_text(encoding="utf-8"))
        if "vault_dir" in parsed:
            cfg["vault_dir"] = parsed["vault_dir"]
        if "engine_dir" in parsed:
            cfg["engine_dir"] = parsed["engine_dir"]
        if "topic_categories" in parsed:
            cfg["topic_categories"] = parsed["topic_categories"]
        # Models
        if "models" in parsed:
            pass  # nested — handled by line-scan below
        # Line-scan for nested values
        lines = Path(config_path).read_text(encoding="utf-8").splitlines()
        in_tagger = in_embed = in_behavior = False
        for line in lines:
            if re.match(r"  tagger:", line):
                in_tagger, in_embed = True, False
            elif re.match(r"  embeddings:", line):
                in_embed, in_tagger = True, False
            elif re.match(r"^behavior:", line):
                in_tagger = in_embed = False
                in_behavior = True
            elif line and not line.startswith(" "):
                in_tagger = in_embed = in_behavior = False

            if in_tagger:
                m = re.match(r"\s+ollama_endpoint:\s*(\S+)", line)
                if m:
                    cfg["models"]["tagger"]["ollama_endpoint"] = m.group(1).strip()
                m = re.match(r"\s+model:\s*(\S+)", line)
                if m:
                    cfg["models"]["tagger"]["model"] = m.group(1).strip()
                m = re.match(r"\s+temperature:\s*([\d.]+)", line)
                if m:
                    cfg["models"]["tagger"]["temperature"] = float(m.group(1))
            if in_embed:
                m = re.match(r"\s+tei_endpoint:\s*(\S+)", line)
                if m:
                    cfg["models"]["embeddings"]["tei_endpoint"] = m.group(1).strip()
            if in_behavior:
                m = re.match(r"\s+auto_apply:\s*(\S+)", line)
                if m:
                    cfg["behavior"]["auto_apply"] = m.group(1).strip().lower() == "true"
                m = re.match(r"\s+confidence_threshold:\s*([\d.]+)", line)
                if m:
                    cfg["behavior"]["confidence_threshold"] = float(m.group(1))
                m = re.match(r"\s+changelog:\s*(.+)", line)
                if m:
                    cfg["behavior"]["changelog"] = m.group(1).strip()
    except Exception as e:
        print(f"[warn] Could not fully parse {config_path}: {e}", file=sys.stderr)
    return cfg


# ---------------------------------------------------------------------------
# Frontmatter parser
# ---------------------------------------------------------------------------

def parse_frontmatter(text: str) -> tuple[dict, str]:
    """Parse YAML frontmatter. Returns (fm_dict, body_text)."""
    fm: dict = {}
    m = re.match(r"^---\r?\n(.*?)\r?\n---\r?\n?(.*)", text, re.DOTALL)
    if not m:
        return fm, text

    fm_raw = m.group(1)
    body = m.group(2)

    lines = fm_raw.splitlines()
    i = 0
    while i < len(lines):
        line = lines[i]
        if not line.strip() or line.strip().startswith("#"):
            i += 1
            continue
        kv = re.match(r"^(\w[\w-]*):\s*(.*)", line)
        if not kv:
            i += 1
            continue
        key = kv.group(1)
        val_raw = kv.group(2).strip()

        if val_raw.startswith("["):
            inner = re.sub(r"^\[|\]$", "", val_raw).strip()
            fm[key] = [x.strip().strip("\"'") for x in inner.split(",") if x.strip()] if inner else []
            i += 1
            continue

        if not val_raw:
            lst = []
            j = i + 1
            while j < len(lines) and re.match(r"^\s+-\s+(.*)", lines[j]):
                item_m = re.match(r"^\s+-\s+(.*)", lines[j])
                lst.append(item_m.group(1).strip().strip("\"'"))
                j += 1
            fm[key] = lst
            i = j
            continue

        fm[key] = val_raw.strip("\"'")
        i += 1

    return fm, body


def render_frontmatter(fm: dict) -> str:
    """Render frontmatter dict to YAML string (between --- markers)."""
    lines = ["---"]
    key_order = ["title", "type", "tags", "aliases", "related", "updated"]
    written: set[str] = set()

    for key in key_order:
        if key not in fm:
            continue
        written.add(key)
        val = fm[key]
        if isinstance(val, list):
            if not val:
                lines.append(f"{key}: []")
            else:
                lines.append(f"{key}:")
                for item in val:
                    lines.append(f"  - {item}")
        else:
            sv = str(val)
            if any(c in sv for c in ':#{}[]|>&*!,?'):
                lines.append(f'{key}: "{sv}"')
            else:
                lines.append(f"{key}: {sv}")

    for key, val in fm.items():
        if key in written:
            continue
        if isinstance(val, list):
            if not val:
                lines.append(f"{key}: []")
            else:
                lines.append(f"{key}:")
                for item in val:
                    lines.append(f"  - {item}")
        else:
            sv = str(val)
            if any(c in sv for c in ':#{}[]|>&*!,?'):
                lines.append(f'{key}: "{sv}"')
            else:
                lines.append(f"{key}: {sv}")

    lines.append("---")
    return "\n".join(lines) + "\n"


# ---------------------------------------------------------------------------
# HTTP helpers
# ---------------------------------------------------------------------------

def _http_post(url: str, payload: dict, timeout: int = 120) -> dict | list:
    data = json.dumps(payload).encode("utf-8")
    req = urllib.request.Request(
        url, data=data,
        headers={"Content-Type": "application/json"},
        method="POST",
    )
    with urllib.request.urlopen(req, timeout=timeout) as resp:
        return json.loads(resp.read().decode("utf-8"))


# ---------------------------------------------------------------------------
# Existing-doc index (the core of v4)
# ---------------------------------------------------------------------------

def _norm(s: str) -> str:
    """Normalize: lowercase, spaces/underscores → hyphens."""
    return re.sub(r"[\s_]+", "-", s.strip().lower())


def build_doc_index(vault_dir: Path) -> dict[str, Path]:
    """
    Build a mapping of normalized name → Path for ALL vault docs.
    Keys: normalized basename, normalized frontmatter title, normalized aliases.
    Longer/more-specific keys win over shorter on conflict.
    Archive excluded.
    """
    index: dict[str, Path] = {}

    for p in sorted(vault_dir.rglob("*.md")):
        if "archive" in p.parts:
            continue
        try:
            text = p.read_text(encoding="utf-8", errors="replace")
            fm, _ = parse_frontmatter(text)
        except Exception:
            fm = {}

        keys: list[str] = [_norm(p.stem)]
        title = fm.get("title", "")
        if title:
            keys.append(_norm(str(title)))
        for alias in fm.get("aliases", []) or []:
            if alias:
                keys.append(_norm(str(alias)))

        for key in keys:
            if key and key not in index:
                index[key] = p
            elif key and key in index:
                # Keep longer stem (more specific)
                if len(p.stem) > len(index[key].stem):
                    index[key] = p

    return index


def resolve_term_to_doc(term: str, doc_index: dict[str, Path]) -> Path | None:
    """
    Given a surface term, find the most-specific existing doc.
    Returns Path or None.
    """
    norm = _norm(term)
    return doc_index.get(norm)


# ---------------------------------------------------------------------------
# Ollama tagger — assigns 1-3 category tags
# ---------------------------------------------------------------------------

_TAGGER_SYSTEM = """You are the Echo6 vault tagger. Given a markdown document and a list of allowed topic categories, assign tags that honestly describe what the doc is primarily about. Output ONLY valid JSON. No prose, no markdown fences.

Rules:
- Only use tags from the provided topic_categories list.
- Output the FEWEST tags that are accurate. DEFAULT to exactly ONE tag — the document's primary subject.
- Add a SECOND tag ONLY if the document is genuinely, substantially about two co-equal subjects.
- NEVER add a tag for something merely mentioned, proxied, or tangentially related.
- A third tag is almost never correct.
- Primary topic first (most central).
- confidence: 0.0-1.0

Negative examples (DO NOT do this):
- A Caddy reverse-proxy config doc is ["dns"], NOT ["dns", "matrix"] — even if it proxies Matrix.
- A PeerTube channel guide is ["media"], NOT ["media", "recon"] — even if recon uses a channel.
- A generic container runbook is ["proxmox"], NOT ["proxmox", "mesh"] — even if the CT joins the mesh.
- An Authentik access-groups doc is ["auth"], NOT ["auth", "mesh"] — tangential mesh mention does not earn a tag."""


def ollama_tag(doc_text: str, topic_categories: list[str], config: dict) -> dict:
    """
    Call Qwen vault-tagger to assign 1-3 category tags.
    Returns {tags: [...], confidence: float}.
    """
    tagger_cfg = config.get("models", {}).get("tagger", {})
    endpoint = tagger_cfg.get("ollama_endpoint", "http://localhost:11434")
    model = tagger_cfg.get("model", "vault-tagger")
    temperature = tagger_cfg.get("temperature", 0.1)

    cats_str = ", ".join(topic_categories)
    doc_truncated = doc_text[:5000]
    if len(doc_text) > 5000:
        doc_truncated += "\n\n[... truncated ...]"

    user_msg = (
        f"topic_categories (ONLY use these): {cats_str}\n\n"
        f"## Document\n\n{doc_truncated}\n\n"
        f"Return ONLY this JSON:\n"
        f'{{\n  "tags": ["primary_tag"],\n  "confidence": 0.85\n}}'
    )

    payload = {
        "model": model,
        "messages": [
            {"role": "system", "content": _TAGGER_SYSTEM},
            {"role": "user", "content": user_msg},
        ],
        "format": "json",
        "stream": False,
        "options": {"temperature": temperature},
    }

    try:
        resp = _http_post(f"{endpoint}/api/chat", payload, timeout=120)
        # /api/chat returns {"message": {"role": "assistant", "content": "..."}}
        if isinstance(resp, dict):
            raw = resp.get("message", {}).get("content", "") or resp.get("response", "")
        else:
            raw = ""
        result = json.loads(raw)
    except json.JSONDecodeError:
        m = re.search(r"\{.*\}", raw, re.DOTALL)
        result = json.loads(m.group(0)) if m else {}
    except Exception as e:
        print(f"[warn] Tagger call failed: {e}", file=sys.stderr)
        return {"tags": [], "confidence": 0.0}

    valid_cats = set(topic_categories)
    tags = []
    if isinstance(result.get("tags"), list):
        for t in result["tags"]:
            if isinstance(t, str) and t.strip() in valid_cats and t.strip() not in tags:
                tags.append(t.strip())
    tags = tags[:3]

    try:
        conf = float(result.get("confidence", 0.0))
        conf = max(0.0, min(1.0, conf))
    except (TypeError, ValueError):
        conf = 0.0

    return {"tags": tags, "confidence": conf}


# ---------------------------------------------------------------------------
# Body wikilinking — existing docs only
# ---------------------------------------------------------------------------

def _build_skip_spans(body: str) -> list[tuple[int, int]]:
    """
    Return spans that must NOT be modified:
    fenced code blocks, inline code, existing [[wikilinks]], headings, URLs,
    domain-like patterns.
    """
    spans: list[tuple[int, int]] = []

    for m in re.finditer(r"```.*?```", body, re.DOTALL):
        spans.append((m.start(), m.end()))
    for m in re.finditer(r"`[^`\n]+`", body):
        spans.append((m.start(), m.end()))
    for m in re.finditer(r"\[\[.*?\]\]", body):
        spans.append((m.start(), m.end()))
    for m in re.finditer(r"^#+\s+.*$", body, re.MULTILINE):
        spans.append((m.start(), m.end()))
    for m in re.finditer(r"https?://\S+", body):
        spans.append((m.start(), m.end()))
    # Domain-like: prevent "mesh" in "mesh.echo6.co"
    for m in re.finditer(r"\b\w[\w\-]*\.\w+\b", body):
        spans.append((m.start(), m.end()))

    return spans


def _in_skip_span(start: int, end: int, spans: list[tuple[int, int]]) -> bool:
    for s, e in spans:
        if start < e and end > s:
            return True
    return False


def _collect_link_candidates(
    body: str,
    doc_index: dict[str, Path],
    self_stems: set[str],
) -> list[tuple[str, str, Path]]:
    """
    Scan body for terms that match existing docs (excluding self and headings/code).
    Returns list of (surface_text, display_name, target_path) sorted longest-first.
    Only returns candidates that actually appear in the prose.
    """
    skip_spans = _build_skip_spans(body)
    body_lower = body.lower()
    candidates: list[tuple[str, str, Path]] = []
    seen_targets: set[str] = set()

    # Build list of (norm_key, display, path) sorted by key length descending
    # (longest/most-specific match first)
    sorted_keys = sorted(doc_index.keys(), key=len, reverse=True)

    for norm_key in sorted_keys:
        path = doc_index[norm_key]
        stem_norm = _norm(path.stem)

        # Skip self
        if stem_norm in self_stems:
            continue
        # Skip INDEX (being deleted)
        if path.stem.upper() == "INDEX":
            continue

        # Skip docs in archive
        if "archive" in path.parts:
            continue

        # Derive surface variations to search for
        surfaces = [norm_key, norm_key.replace("-", " ")]
        # Remove duplicates
        surfaces = list(dict.fromkeys(surfaces))

        # Get display name (frontmatter title or stem)
        try:
            text = path.read_text(encoding="utf-8", errors="replace")
            fm, _ = parse_frontmatter(text)
            display = fm.get("title", "") or path.stem
        except Exception:
            display = path.stem

        target_key = str(path)
        if target_key in seen_targets:
            continue

        for surface in surfaces:
            if len(surface) < 3:
                continue
            # Check if this surface appears in body (quick scan)
            if surface not in body_lower:
                continue

            # Build word-boundary pattern
            esc = re.escape(surface).replace(r"\ ", r"[\s\-]").replace(r"\-", r"[\s\-]")
            pattern = rf"(?<![a-zA-Z0-9\-_\[\]])({esc})(?![a-zA-Z0-9\-_\[\]\.])"
            for m in re.finditer(pattern, body, re.IGNORECASE):
                s, e = m.start(1), m.end(1)
                if not _in_skip_span(s, e, skip_spans):
                    candidates.append((m.group(1), str(display), path))
                    seen_targets.add(target_key)
                    break  # found at least one prose occurrence — add candidate
            if target_key in seen_targets:
                break

    return candidates


def apply_wikilinks(
    body: str,
    doc_index: dict[str, Path],
    self_stems: set[str],
) -> str:
    """
    Insert [[wikilinks]] for FIRST prose occurrence of each existing-doc name.
    - Longest match wins (most-specific doc).
    - Skip code blocks, inline code, headings, URLs, existing links.
    - Idempotent.
    - [[Doc|surface]] when surface differs from doc stem; [[Doc]] when same.
    """
    skip_spans = _build_skip_spans(body)
    result_chars = list(body)
    replaced_targets: set[str] = set()  # target paths already linked
    offset = 0  # cumulative character offset

    # Sort candidates: longest surface text first (most specific wins)
    sorted_keys = sorted(doc_index.keys(), key=len, reverse=True)

    replacements: list[tuple[int, int, str]] = []
    replacement_spans: list[tuple[int, int]] = []

    for norm_key in sorted_keys:
        path = doc_index[norm_key]
        stem_norm = _norm(path.stem)

        if stem_norm in self_stems:
            continue
        if path.stem.upper() == "INDEX":
            continue
        if "archive" in path.parts:
            continue

        target_key = str(path)
        if target_key in replaced_targets:
            continue

        surfaces = list(dict.fromkeys([norm_key, norm_key.replace("-", " ")]))

        try:
            text = path.read_text(encoding="utf-8", errors="replace")
            fm, _ = parse_frontmatter(text)
            display = str(fm.get("title", "") or path.stem)
        except Exception:
            display = path.stem

        found = False
        for surface in surfaces:
            if len(surface) < 3:
                continue

            esc = re.escape(surface).replace(r"\ ", r"[\s\-]").replace(r"\-", r"[\s\-]")
            pattern = rf"(?<![a-zA-Z0-9\-_\[\]])({esc})(?![a-zA-Z0-9\-_\[\]\.])"

            for m in re.finditer(pattern, body, re.IGNORECASE):
                s, e = m.start(1), m.end(1)
                matched_text = m.group(1)

                if _in_skip_span(s, e, skip_spans):
                    continue
                if _in_skip_span(s, e, replacement_spans):
                    continue

                # Build wikilink
                if matched_text.lower() == display.lower():
                    wiki = f"[[{display}]]"
                elif _norm(matched_text) == _norm(path.stem):
                    wiki = f"[[{path.stem}]]"
                else:
                    wiki = f"[[{path.stem}|{matched_text}]]"

                replacements.append((s, e, wiki))
                replacement_spans.append((s, e))
                replaced_targets.add(target_key)
                found = True
                break

            if found:
                break

    # Apply in reverse order
    replacements.sort(key=lambda x: x[0], reverse=True)
    chars = list(body)
    for s, e, wiki in replacements:
        chars[s:e] = list(wiki)

    return "".join(chars)


# ---------------------------------------------------------------------------
# Type inference
# ---------------------------------------------------------------------------

def infer_type(doc_path: Path, vault_dir: Path) -> str | None:
    try:
        rel = doc_path.relative_to(vault_dir)
    except ValueError:
        return None
    parts = rel.parts
    folder = parts[0].lower() if len(parts) > 1 else ""
    folder_map = {
        "runbooks": "runbook",
        "projects": "project",
        "docs": "reference",
        "notes": "note",
        "session-resume": "session",
        "plans": "note",
    }
    return folder_map.get(folder, "reference")


def extract_title(fm: dict, body: str, doc_path: Path) -> str:
    if fm.get("title"):
        return str(fm["title"])
    m = re.search(r"^#\s+(.+)", body, re.MULTILINE)
    if m:
        return m.group(1).strip()
    return re.sub(r"[-_]+", " ", doc_path.stem).title()


# ---------------------------------------------------------------------------
# TEI embedding + related
# ---------------------------------------------------------------------------

def _detect_tei_route(endpoint: str) -> str:
    for route in ["/embed", "/embeddings"]:
        try:
            resp = _http_post(f"{endpoint}{route}", {"inputs": "test"}, timeout=10)
            if isinstance(resp, list):
                return route
        except urllib.error.HTTPError as e:
            if e.code not in (404, 405):
                return route
        except Exception:
            continue
    return "/embed"


_TEI_ROUTE_CACHE: dict[str, str] = {}


def tei_embed(text: str, endpoint: str) -> list[float]:
    if endpoint not in _TEI_ROUTE_CACHE:
        _TEI_ROUTE_CACHE[endpoint] = _detect_tei_route(endpoint)
    route = _TEI_ROUTE_CACHE[endpoint]
    resp = _http_post(f"{endpoint}{route}", {"inputs": text}, timeout=30)
    if isinstance(resp, list):
        return resp[0] if resp and isinstance(resp[0], list) else resp
    raise RuntimeError(f"Unexpected TEI response: {type(resp)}")


def cosine_sim(a: list[float], b: list[float]) -> float:
    if len(a) != len(b):
        return 0.0
    dot = sum(x * y for x, y in zip(a, b))
    mag_a = math.sqrt(sum(x * x for x in a))
    mag_b = math.sqrt(sum(y * y for y in b))
    return dot / (mag_a * mag_b) if mag_a and mag_b else 0.0


def _cache_path(engine_dir: Path) -> Path:
    return engine_dir / ".embcache.json"


def load_embed_cache(engine_dir: Path) -> dict:
    p = _cache_path(engine_dir)
    if p.exists():
        try:
            return json.loads(p.read_text(encoding="utf-8"))
        except Exception:
            return {}
    return {}


def save_embed_cache(cache: dict, engine_dir: Path) -> None:
    _cache_path(engine_dir).write_text(json.dumps(cache), encoding="utf-8")


def get_or_embed(
    doc_path: Path, vault_dir: Path, tei_endpoint: str, cache: dict,
    doc_text: str | None = None,
) -> tuple[list[float], bool]:
    rel = str(doc_path.relative_to(vault_dir))
    mtime = doc_path.stat().st_mtime
    key = f"{rel}::{mtime:.3f}"
    if key in cache:
        return cache[key], True
    if doc_text is None:
        doc_text = doc_path.read_text(encoding="utf-8", errors="replace")
    vec = tei_embed(doc_text[:4000], tei_endpoint)
    cache[key] = vec
    return vec, False


def find_related(
    doc_path: Path, doc_vec: list[float], vault_dir: Path,
    tei_endpoint: str, cache: dict, k: int = 5,
) -> list[str]:
    sims: list[tuple[float, str]] = []
    for md in vault_dir.rglob("*.md"):
        if md == doc_path or "archive" in md.parts:
            continue
        try:
            vec, _ = get_or_embed(md, vault_dir, tei_endpoint, cache)
            sims.append((cosine_sim(doc_vec, vec), md.stem))
        except Exception:
            continue
    sims.sort(reverse=True)
    return [f"[[{stem}]]" for _, stem in sims[:k]]


# ---------------------------------------------------------------------------
# Core: process_doc
# ---------------------------------------------------------------------------

def process_doc(
    doc_path: Path | str,
    config: dict,
    dry_run: bool = True,
    cache: dict | None = None,
) -> dict:
    """
    Process a single vault document.
    Steps:
      1. Assign 1-3 category tags via Qwen tagger.
      2. Build existing-doc index; insert inline wikilinks to existing docs only.
      3. Update related: (bge-m3 nearest docs).
      4. Write updated frontmatter (title, type, tags, related, updated).
    Returns result dict. dry_run=True → nothing written.
    """
    doc_path = Path(doc_path).resolve()
    vault_dir = Path(config["vault_dir"]).resolve()
    engine_dir = Path(config["engine_dir"]).resolve()
    tei_endpoint = config["models"]["embeddings"]["tei_endpoint"]
    topic_categories = config.get("topic_categories", [])

    if cache is None:
        cache = load_embed_cache(engine_dir)

    doc_text = doc_path.read_text(encoding="utf-8", errors="replace")
    existing_fm, body = parse_frontmatter(doc_text)

    # --- Step 1: Tags via Qwen ---
    print(f"[tag] Calling tagger on {doc_path.name} ...", file=sys.stderr)
    tag_result = ollama_tag(doc_text, topic_categories, config)
    print(f"[tag] tags={tag_result['tags']} confidence={tag_result['confidence']:.2f}", file=sys.stderr)

    # --- Step 2: Build doc index + wikilinks ---
    print(f"[link] Building doc index ...", file=sys.stderr)
    doc_index = build_doc_index(vault_dir)

    # Self stems to exclude (don't self-link)
    self_stems = {_norm(doc_path.stem)}
    title_val = existing_fm.get("title", "")
    if title_val:
        self_stems.add(_norm(str(title_val)))

    print(f"[link] Applying wikilinks to {doc_path.name} ...", file=sys.stderr)
    linked_body = apply_wikilinks(body, doc_index, self_stems)

    body_diff = "".join(difflib.unified_diff(
        body.splitlines(keepends=True),
        linked_body.splitlines(keepends=True),
        fromfile=f"a/{doc_path.name}",
        tofile=f"b/{doc_path.name} (linked)",
        n=2,
    ))

    # --- Step 3: related (bge-m3) ---
    print(f"[embed] Embedding {doc_path.name} ...", file=sys.stderr)
    related_links: list[str] = []
    try:
        doc_vec, cache_hit = get_or_embed(doc_path, vault_dir, tei_endpoint, cache, doc_text=doc_text)
        if not cache_hit:
            save_embed_cache(cache, engine_dir)
        related_links = find_related(doc_path, doc_vec, vault_dir, tei_endpoint, cache, k=5)
        save_embed_cache(cache, engine_dir)
        print(f"[embed] {len(related_links)} related", file=sys.stderr)
    except Exception as e:
        print(f"[warn] Embedding failed: {e}", file=sys.stderr)

    # --- Step 4: Frontmatter ---
    today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
    new_fm = dict(existing_fm)

    if not new_fm.get("title"):
        new_fm["title"] = extract_title(existing_fm, body, doc_path)

    inferred = infer_type(doc_path, vault_dir)
    if inferred:
        new_fm["type"] = inferred

    # Tags: use tagger result; fall back to existing tags if tagger returns empty
    if tag_result["tags"]:
        new_fm["tags"] = tag_result["tags"]
    elif not new_fm.get("tags"):
        new_fm["tags"] = []

    if "aliases" not in new_fm:
        new_fm["aliases"] = []

    # related: replace with embedding results (or keep existing if embed failed)
    if related_links:
        new_fm["related"] = related_links

    new_fm["updated"] = today

    new_fm_text = render_frontmatter(new_fm)
    new_doc_text = new_fm_text + linked_body

    full_diff = "".join(difflib.unified_diff(
        doc_text.splitlines(keepends=True),
        new_doc_text.splitlines(keepends=True),
        fromfile=f"a/{doc_path.name}",
        tofile=f"b/{doc_path.name}",
        n=2,
    ))

    applied = False
    if not dry_run:
        if config["behavior"].get("auto_apply", True):
            doc_path.write_text(new_doc_text, encoding="utf-8")
            applied = True
            print(f"[apply] Wrote {doc_path}", file=sys.stderr)

    return {
        "path": str(doc_path),
        "tag_result": tag_result,
        "body_diff": body_diff,
        "full_diff": full_diff,
        "related_links": related_links,
        "proposed_frontmatter": new_fm_text,
        "applied": applied,
        "dry_run": dry_run,
        "existing_fm": existing_fm,
        "new_fm": new_fm,
    }


# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------

def main() -> None:
    parser = argparse.ArgumentParser(description="Vault tagger + existing-doc linker (v4).")
    parser.add_argument("path", help="Path to the vault markdown document")
    parser.add_argument("--dry-run", action="store_true", help="Print diff, write nothing")
    args = parser.parse_args()

    here = Path(__file__).parent
    config_path = here.parent / "config.yaml"
    if not config_path.exists():
        config_path = Path("/home/zvx/projects/.ref/engine/config.yaml")
    config = load_config(config_path)

    doc_path = Path(args.path).resolve()
    if not doc_path.exists():
        print(f"[error] File not found: {doc_path}", file=sys.stderr)
        sys.exit(1)

    # Skip symlinks — vault/CLAUDE-baseline.md and vault/rules point outside the
    # repo (to ~/.claude/). Editing them would silently modify external files.
    if os.path.islink(args.path) or doc_path != Path(args.path).resolve():
        print(f"[skip] Symlink — refusing to edit external target: {doc_path}", file=sys.stderr)
        sys.exit(0)

    result = process_doc(doc_path, config, dry_run=args.dry_run)

    print("\n" + "=" * 72)
    print(f"VAULT TAGGER v4 {'DRY-RUN ' if args.dry_run else ''}— {doc_path.name}")
    print("=" * 72)
    print(f"\n### Tags")
    print(f"  {result['tag_result']['tags']}  (confidence={result['tag_result']['confidence']:.2f})")
    print(f"\n### Body wikilink diff")
    print(result["body_diff"] or "  (no new wikilinks)")
    print(f"\n### Related (bge-m3 top-5)")
    for r in result["related_links"]:
        print(f"  {r}")
    print(f"\n### Proposed frontmatter")
    print(result["proposed_frontmatter"])
    if args.dry_run:
        print("[DRY-RUN] Nothing written.")
    else:
        print(f"[APPLY] applied={result['applied']}")


if __name__ == "__main__":
    main()