matt/echo6-docs
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

auto: docs sync 2026-06-18T18:00:10+00:00

Browse source 
Files changed: .gitignore CLAUDE.md credentials engine/.embcache.json engine/changelog.md engine/config.yaml engine/lib/__pycache__/agent.cpython-312.pyc engine/lib/agent.py engine/lib/lint.py engine/lint-report.md engine/sweep-full.log engine/sweep.sh vault/.obsidian/graph.json vault/.obsidian/workspace.json vault/INDEX.md vault/archive/projects/mmud/last-ember-chronicle.html vault/archive/projects/mmud/last-ember-howto.html vault/archive/projects/mmud/last-ember.html vault/archive/projects/mmud/mmud-phase5-prompt.md vault/archive/projects/mmud/mmud-phase6-prompt.md vault/archive/projects/mmud/mmud-prompts/mmud-prompts/01-update-planned.md vault/archive/projects/mmud/mmud-prompts/mmud-prompts/02-npc-nodes.md vault/archive/projects/mmud/mmud-prompts/mmud-prompts/03-darkcragg.md vault/archive/projects/mmud/mmud-prompts/mmud-prompts/04-dcrg-node.md vault/archive/projects/mmud/mmud-prompts/mmud-prompts/05-phase5.md vault/archive/projects/mmud/mmud-prompts/mmud-prompts/06-phase6.md vault/archive/projects/mmud/mmud-prompts/mmud-prompts/README.md vault/archive/projects/mmud/mmud-prompts/mmud-prompts/mmud-project.md vault/docs/hardware/environment.md vault/docs/hardware/ip-allocation.md vault/docs/matrix/archivist.md vault/docs/matrix/matrix_host.md vault/docs/matrix/mautrix_signal.md vault/docs/matrix/synapse.md vault/docs/matrix/synapse_retention_discovery.md vault/docs/navi/cc-rules.md vault/docs/navi/deployment.md vault/docs/navi/themes.md vault/docs/services/ots-setup.md vault/docs/services/services.md vault/docs/services/usenet.md vault/docs/software/authentik.md vault/docs/software/caddy.md vault/docs/software/dns.md vault/docs/software/geo-tools.md vault/docs/software/recon.md vault/docs/software/searxng.md vault/glossary.md vault/notes/echo6-landing-page-data-export.md vault/notes/ia-download-queue.md vault/plans/vaultwarden-plan.md vault/projects/advbbs-project.md vault/projects/argus.md vault/projects/deploy-livesync.md vault/projects/matrix-synapse-deployment.md vault/projects/meshtastic-headscale-runbook.md vault/projects/mmud-project.md vault/runbooks/add-peertube-channel.md vault/runbooks/authentik-access-groups.md vault/runbooks/authentik-create-invitation.md vault/runbooks/authentik-oidc-application.md vault/runbooks/authentik-upgrade.md vault/runbooks/ct-runbook.md vault/runbooks/edge2-access-reference.md vault/runbooks/expose-service-contabo.md vault/runbooks/expose-service-edge2.md vault/runbooks/expose-service-home.md vault/runbooks/headscale-onboard-node.md vault/runbooks/ia-cli-reference.md vault/runbooks/ia-download-mirror.md vault/runbooks/idahomesh-bridge-setup.md vault/runbooks/idahomesh-vpn-device-setup.md vault/runbooks/lxc-service-migration.md vault/runbooks/mailcow-create-mailbox.md vault/runbooks/meshmonitor-password-reset.md vault/runbooks/meshtastic-sidecar-node.md vault/runbooks/meshtasticd-sim-nodes-runbook.md vault/runbooks/nordvpn-lxc.md vault/runbooks/peertube-remote-runner.md vault/runbooks/pg-backup.md vault/runbooks/pi-nas-omv-runbook.md vault/runbooks/pipeline-patterns.md vault/runbooks/proxmox-create-ubuntu-vm.md vault/runbooks/proxmox-onboard-node.md vault/runbooks/recon-operations.md vault/runbooks/recon-service-integration.md vault/runbooks/syncthing-add-node.md vault/session-resume/SESSION-HANDOFF-meshai-test.md
This commit is contained in:
echo6-autocommit 2026-06-18 18:00:10 +00:00
commit eb7eade7fa
88 changed files with 5469 additions and 6038 deletions
Download patch file Download diff file Expand all files Collapse all files

561
engine/lib/lint.py
View file

@ -1,17 +1,19 @@
#!/usr/bin/env python3
"""
lint.py — Deterministic Vault Lint (no LLM, stdlib only)
lint.py — Deterministic Vault Lint (stdlib only)
Checks performed (all deterministic):
1. Frontmatter schema — required keys, valid type, tags is a list
2. Tag vocabulary — tags must be in topic_categories + {meta}
3. Dead wikilinks — [[target]] resolves against note basenames + entity vocab
2. Tag vocabulary — tags must be in topic_categories
3. Dead wikilinks — [[target]] must resolve against vault note basenames
4. Orphans — notes with zero incoming wikilinks (INFO, capped at 40)
Severity:
ERROR — dead wikilinks
WARN — schema violations, unknown tags
INFO — orphan notes
Gaps & suggestions (written to ## Gaps & suggestions in lint-report.md):
- Docs with NO tags
- True orphans (no inbound link AND shares no tag with any other doc)
- Dead wikilinks (must be 0 after v4 sweep)
- Tags not in topic_categories
- "Earned-a-doc?" candidates: terms mentioned in >= 4 docs with no dedicated doc (top 15)
Usage:
python3 engine/lib/lint.py # report mode, always exit 0
@ -23,23 +25,19 @@ Writes: engine/lint-report.md
from __future__ import annotations
import json
import os
import re
import sys
from collections import Counter, defaultdict
from datetime import datetime, timezone
from pathlib import Path
from typing import NamedTuple
# ---------------------------------------------------------------------------
# Config / vocab loading (stdlib-only minimal YAML parser)
# ---------------------------------------------------------------------------
def _parse_simple_yaml(text: str) -> dict:
"""
Minimal YAML parser: handles only the scalar/list constructs in config.yaml.
Supports: key: value, key: [a, b], - item under a key, # comments.
Does NOT handle nested dicts beyond top-level — enough for config.yaml.
"""
result: dict = {}
current_key = None
current_list: list | None = None
@ -48,58 +46,44 @@ def _parse_simple_yaml(text: str) -> dict:
line = raw_line.rstrip()
stripped = line.lstrip()
if not stripped or stripped.startswith("#"):
# Flush pending list if indent drops
if current_list is not None and not line.startswith(" "):
result[current_key] = current_list
current_list = None
current_key = None
continue
# List item under current key
if stripped.startswith("- ") and current_list is not None:
current_list.append(stripped[2:].strip().strip('"').strip("'"))
continue
# End of list block
if current_list is not None and not stripped.startswith("-"):
result[current_key] = current_list
current_list = None
current_key = None
if ":" in stripped:
key, _, value = stripped.partition(":")
key = key.strip()
value = value.strip()
if value.startswith("[") and value.endswith("]"):
# Inline list: [a, b, c]
inner = value[1:-1]
result[key] = [v.strip().strip('"').strip("'") for v in inner.split(",") if v.strip()]
elif value == "":
# May start a list block
current_key = key
current_list = []
elif value.startswith("#"):
result[key] = ""
else:
# Scalar — strip inline comment
value = value.split(" #")[0].strip().strip('"').strip("'")
result[key] = value
if current_list is not None and current_key:
result[current_key] = current_list
return result
def load_config(engine_dir: Path) -> dict:
"""Load config.yaml using minimal YAML parser."""
config_path = engine_dir / "config.yaml"
text = config_path.read_text(encoding="utf-8")
return _parse_simple_yaml(text)
return _parse_simple_yaml(config_path.read_text(encoding="utf-8"))
def load_vocab(engine_dir: Path) -> dict:
"""Load vocab.json."""
vocab_path = engine_dir / "vocab.json"
return json.loads(vocab_path.read_text(encoding="utf-8"))
@ -112,11 +96,6 @@ _FM_FENCE = re.compile(r"^---\s*$")
def parse_frontmatter(path: Path) -> tuple[dict | None, str]:
"""
Parse YAML frontmatter block from a markdown file.
Returns (frontmatter_dict_or_None, body_text).
body_text is the full file text if no frontmatter.
"""
text = path.read_text(encoding="utf-8", errors="replace")
lines = text.splitlines(keepends=True)
@ -133,23 +112,12 @@ def parse_frontmatter(path: Path) -> tuple[dict | None, str]:
return None, text
fm_text = "".join(lines[1:end_idx])
body = "".join(lines[end_idx + 1 :])
body = "".join(lines[end_idx + 1:])
fm = _parse_yaml_frontmatter(fm_text)
return fm, body
def _parse_yaml_frontmatter(text: str) -> dict:
"""
Parse simple flat YAML frontmatter. Handles:
key: scalar
key: [a, b, c]
key:
- a
- b
tags: [a, b]
tags:
- a
"""
result: dict = {}
current_key: str | None = None
current_list: list | None = None
@ -160,13 +128,9 @@ def _parse_yaml_frontmatter(text: str) -> dict:
if not stripped:
continue
# List item
if stripped.startswith("- ") and current_list is not None:
current_list.append(stripped[2:].strip().strip('"').strip("'"))
continue
# New key — flush pending list
if current_list is not None:
result[current_key] = current_list
current_list = None
@ -200,57 +164,45 @@ def _parse_yaml_frontmatter(text: str) -> dict:
# Check 1: Frontmatter schema
# ---------------------------------------------------------------------------
# v4: entity/concept types removed; valid types are the real doc types
VALID_TYPES = {"reference", "runbook", "project", "note", "index", "session"}
def check_frontmatter(fm: dict | None, config: dict) -> list[dict]:
"""
Returns list of findings dicts: {severity, check, message}
WARN severity for all schema violations.
"""
findings = []
required_keys = ["title", "type", "tags", "updated"]
if fm is None:
findings.append(
{
"severity": "WARN",
"check": "frontmatter",
"message": "missing frontmatter block entirely",
}
)
findings.append({
"severity": "WARN",
"check": "frontmatter",
"message": "missing frontmatter block entirely",
})
return findings
for key in required_keys:
if key not in fm or fm[key] is None or fm[key] == "":
findings.append(
{
"severity": "WARN",
"check": "frontmatter",
"message": f"missing required key: {key!r}",
}
)
findings.append({
"severity": "WARN",
"check": "frontmatter",
"message": f"missing required key: {key!r}",
})
if "type" in fm and fm["type"]:
if fm["type"] not in VALID_TYPES:
findings.append(
{
"severity": "WARN",
"check": "frontmatter",
"message": f"invalid type {fm['type']!r} — must be one of {sorted(VALID_TYPES)}",
}
)
findings.append({
"severity": "WARN",
"check": "frontmatter",
"message": f"invalid type {fm['type']!r} — must be one of {sorted(VALID_TYPES)}",
})
if "tags" in fm and fm["tags"] is not None:
if not isinstance(fm["tags"], list):
findings.append(
{
"severity": "WARN",
"check": "frontmatter",
"message": f"tags must be a list, got {type(fm['tags']).__name__}: {fm['tags']!r}",
}
)
findings.append({
"severity": "WARN",
"check": "frontmatter",
"message": f"tags must be a list, got {type(fm['tags']).__name__}: {fm['tags']!r}",
})
return findings
@ -259,27 +211,20 @@ def check_frontmatter(fm: dict | None, config: dict) -> list[dict]:
# Check 2: Tag vocabulary
# ---------------------------------------------------------------------------
ALLOWED_EXTRA_TAGS = {"meta"}
def check_tags(fm: dict | None, allowed_tags: set[str]) -> list[dict]:
"""Returns WARN findings for tags outside the allowed vocabulary."""
if fm is None:
return []
tags = fm.get("tags")
if not tags or not isinstance(tags, list):
return []
findings = []
for tag in tags:
if tag not in allowed_tags:
findings.append(
{
"severity": "WARN",
"check": "tag-vocab",
"message": f"unknown tag {tag!r} (not in topic_categories or allowed extras)",
}
)
findings.append({
"severity": "WARN",
"check": "tag-vocab",
"message": f"unknown tag {tag!r} (not in topic_categories)",
})
return findings
@ -287,21 +232,12 @@ def check_tags(fm: dict | None, allowed_tags: set[str]) -> list[dict]:
# Check 3: Dead wikilinks
# ---------------------------------------------------------------------------
# Matches [[target]], [[target|alias]], [[target#heading]]
# But NOT bash [[ ... ]] test syntax.
# Strategy: require the inner text to look like a note name:
# - no spaces at start/end of target
# - may contain word chars, hyphens, dots, spaces, #, | — but NOT operators like
# -n, -f, ==, !, &&, ||, etc.
# We detect shell syntax by checking for space-separated words starting with - or
# containing shell operators.
_WIKILINK_RE = re.compile(
r"""
\[\[ # opening [[
([^\[\]\n]+?) # capture: link target (non-greedy, no newlines)
(?:\|[^\[\]\n]*)? # optional |alias
\]\] # closing ]]
\[\[
([^\[\]\n]+?)
(?:\|[^\[\]\n]*)?
\]\]
""",
re.VERBOSE,
)
@ -311,18 +247,13 @@ _INLINE_CODE_RE = re.compile(r"`[^`\n]+`")
def _looks_like_shell(inner: str) -> bool:
"""Return True if the wikilink inner text looks like bash test syntax."""
# Shell: -n "$VAR", ! -f "$FILE", "$CODEC" == "value", -z, etc.
s = inner.strip()
# Contains bash operators or variable expansions
if re.search(r'\$[{(A-Za-z_]', s):
return True
# Starts with - (flag) or ! (negation), or contains == / != / && / ||
if re.match(r'^\s*[!-]', s):
return True
if re.search(r'==|!=|&&|\|\|', s):
return True
# Multiple space-separated tokens that look like args
tokens = s.split()
if len(tokens) > 1 and tokens[0].startswith("-"):
return True
@ -330,103 +261,62 @@ def _looks_like_shell(inner: str) -> bool:
def _is_note_like(inner: str) -> bool:
"""
Return True if inner text looks like a note name:
word chars, hyphens, dots, spaces, but no shell operators.
"""
if _looks_like_shell(inner):
return False
# Must match a reasonable note-name pattern
# Allow: letters, digits, hyphens, underscores, dots, spaces
return bool(re.match(r'^[\w\s.\-/]+$', inner.strip()))
def _strip_code_blocks(text: str) -> str:
"""Remove fenced code blocks and inline code to avoid false wikilink matches."""
text = _FENCED_CODE_RE.sub("", text)
text = _INLINE_CODE_RE.sub("", text)
return text
def _extract_wikilinks(body: str) -> list[str]:
"""Extract note targets from wikilinks in body text, ignoring code blocks and shell syntax."""
clean = _strip_code_blocks(body)
targets = []
for m in _WIKILINK_RE.finditer(clean):
raw = m.group(1)
# Strip heading: [[target#heading|alias]] -> target
# Strip alias already handled by regex (group 1 = before |)
target = raw.split("|")[0].split("#")[0].strip()
if target and _is_note_like(target):
targets.append(target)
return targets
def _note_basename(path: Path) -> str:
"""Return the basename (without extension) of a vault note, lowercased."""
return path.stem.lower()
def _normalize_name(name: str) -> str:
"""Normalize: lowercase, spaces↔hyphens."""
return name.lower().replace(" ", "-").replace("_", "-")
def build_note_index(vault_dir: Path) -> dict[str, Path]:
"""
Build a mapping of normalized name → Path for all vault notes.
One entry per file, keyed by normalized basename.
"""
index: dict[str, Path] = {}
for p in vault_dir.rglob("*.md"):
if "archive" in p.parts:
continue
key = _normalize_name(p.stem)
index[key] = p
return index
def build_entity_names(vocab: dict) -> set[str]:
"""
Build a set of normalized entity names + aliases from vocab.json.
These count as valid wikilink targets (they're in the knowledge graph).
"""
names: set[str] = set()
for entity in vocab.get("entities", []):
names.add(_normalize_name(entity["name"]))
names.add(_normalize_name(entity["display"]))
for alias in entity.get("aliases", []):
names.add(_normalize_name(alias))
for cat in vocab.get("topic_categories", []):
names.add(_normalize_name(cat))
return names
def check_dead_links(
path: Path,
body: str,
note_index: dict[str, Path],
entity_names: set[str],
) -> list[dict]:
"""
Returns ERROR findings for wikilinks that resolve to neither a vault note
nor a known entity name.
"""
findings = []
for target in _extract_wikilinks(body):
norm = _normalize_name(target)
if norm in note_index or norm in entity_names:
if norm in note_index:
continue
findings.append(
{
"severity": "ERROR",
"check": "dead-link",
"message": f"dead wikilink [[{target}]]",
}
)
findings.append({
"severity": "ERROR",
"check": "dead-link",
"message": f"dead wikilink [[{target}]]",
})
return findings
# ---------------------------------------------------------------------------
# Check 4: Orphans (incoming wikilink count)
# Check 4: Orphans
# ---------------------------------------------------------------------------
ORPHAN_CAP = 40
@ -435,15 +325,11 @@ ORPHAN_CAP = 40
def build_backlink_counts(
vault_dir: Path,
note_index: dict[str, Path],
entity_names: set[str],
) -> dict[str, int]:
"""
Count how many other notes link TO each note.
Returns {normalized_stem: count}.
"""
counts: dict[str, int] = {k: 0 for k in note_index}
for p in vault_dir.rglob("*.md"):
if "archive" in p.parts:
continue
try:
_, body = parse_frontmatter(p)
except Exception:
@ -452,7 +338,6 @@ def build_backlink_counts(
norm = _normalize_name(target)
if norm in counts:
counts[norm] += 1
return counts
@ -461,25 +346,233 @@ def find_orphans(
note_index: dict[str, Path],
backlink_counts: dict[str, int],
) -> list[dict]:
"""
Return INFO findings for notes with zero incoming wikilinks.
Capped at ORPHAN_CAP entries.
"""
findings = []
for key, path in sorted(note_index.items()):
if backlink_counts.get(key, 0) == 0:
rel = path.relative_to(vault_dir)
findings.append(
{
"severity": "INFO",
"check": "orphan",
"message": f"no incoming links: {rel}",
"_path": path,
}
)
findings.append({
"severity": "INFO",
"check": "orphan",
"message": f"no incoming links: {rel}",
"_path": path,
})
return findings[:ORPHAN_CAP]
# ---------------------------------------------------------------------------
# Gaps & suggestions
# ---------------------------------------------------------------------------
IMPORTANT_CANDIDATE_TERMS = [
"aida-nebra", "meshtastic", "headscale", "tailscale", "headplane",
"qdrant", "aurora", "open-webui", "gemini", "syncthing", "livesync",
"peertube", "mailcow", "forgejo", "jellyfin", "immich", "nextcloud",
"vaultwarden", "meshmonitor", "lora", "mt-isr", "sigil", "navi",
"searxng", "authentik", "docker", "proxmox", "caddy", "dnsmasq",
"acme-sh", "qdrant", "bge-m3", "qwen", "ollama", "meshtasticd",
"matrix-synapse", "element", "mautrix", "synapse",
]
def build_earned_a_doc_candidates(
vault_dir: Path,
note_index: dict[str, Path],
) -> list[tuple[str, int]]:
"""
Find terms mentioned in >= 4 docs that have no dedicated vault doc.
Returns [(term, mention_count)] sorted by count desc, top 15.
"""
existing_stems = set(note_index.keys())
doc_mentions: dict[str, set] = defaultdict(set)
for p in vault_dir.rglob("*.md"):
if "archive" in p.parts:
continue
try:
text = p.read_text(encoding="utf-8", errors="replace")
except Exception:
continue
# Strip frontmatter
fm_match = re.match(r"^---.*?---\n", text, re.DOTALL)
body = text[fm_match.end():] if fm_match else text
# Strip code blocks
body = re.sub(r"```.*?```", "", body, flags=re.DOTALL)
body = re.sub(r"`[^`]+`", "", body)
body_lower = body.lower()
path_str = str(p)
# Check candidate terms
for term in IMPORTANT_CANDIDATE_TERMS:
norm = _normalize_name(term)
if norm in existing_stems:
continue # already has a doc
# Search for the term (and space variant) in body
variants = [term, term.replace("-", " ")]
for v in variants:
if v in body_lower:
doc_mentions[term].add(path_str)
break
# Also scan for hyphenated identifiers in wikilinks (already-linked terms
# that point to non-existent docs would show up as dead links above;
# here we look for plain-text mentions of kebab terms)
for m in re.finditer(r"\b([a-z][a-z0-9]{2,}-[a-z0-9][a-z0-9\-]{2,})\b", body_lower):
t = m.group(1)
if t not in existing_stems and len(t) >= 6:
# Filter out version strings, IPs, etc.
if not re.match(r"^\d", t) and "--" not in t:
doc_mentions[t].add(path_str)
# Filter to >= 4 mentions, sort by count desc, top 15
candidates = [
(term, len(docs))
for term, docs in doc_mentions.items()
if len(docs) >= 4
]
candidates.sort(key=lambda x: -x[1])
return candidates[:15]
def build_tag_coverage(
vault_dir: Path,
note_index: dict[str, Path],
) -> tuple[list[str], dict[str, list[str]]]:
"""
Returns (no_tag_paths, tag_to_paths) for shared-tag orphan check.
no_tag_paths: relative paths of docs with empty tags.
tag_to_paths: {tag: [doc_stem, ...]}
"""
no_tag_paths = []
tag_to_paths: dict[str, list[str]] = defaultdict(list)
for p in vault_dir.rglob("*.md"):
if "archive" in p.parts:
continue
try:
fm, _ = parse_frontmatter(p)
except Exception:
fm = None
tags = []
if fm:
t = fm.get("tags")
if isinstance(t, list):
tags = t
stem = _normalize_name(p.stem)
if not tags:
no_tag_paths.append(str(p.relative_to(vault_dir)))
for tag in tags:
tag_to_paths[tag].append(stem)
return no_tag_paths, tag_to_paths
def build_gaps_section(
vault_dir: Path,
note_index: dict[str, Path],
backlink_counts: dict[str, int],
all_results: list,
allowed_tags: set[str],
) -> str:
"""Build the ## Gaps & suggestions section."""
lines = ["## Gaps & suggestions", ""]
# 1. Docs with NO tags
no_tag_paths, tag_to_paths = build_tag_coverage(vault_dir, note_index)
lines.append("### Docs with no tags")
lines.append("")
if no_tag_paths:
for p in sorted(no_tag_paths):
lines.append(f"- `{p}`")
else:
lines.append("_None — all docs have at least one tag._")
lines.append("")
# 2. True orphans (no inbound link AND no shared tag)
lines.append("### True orphans (no inbound link, no shared tag)")
lines.append("")
true_orphans = []
for key, path in sorted(note_index.items()):
if backlink_counts.get(key, 0) > 0:
continue
# Check if any of its tags appear in other docs
try:
fm, _ = parse_frontmatter(path)
except Exception:
fm = None
tags = []
if fm:
t = fm.get("tags")
if isinstance(t, list):
tags = t
# Does any other doc share a tag?
shares_tag = False
for tag in tags:
others = tag_to_paths.get(tag, [])
# Others = docs with this tag; if any != self, shares_tag = True
self_key = _normalize_name(path.stem)
if any(s != self_key for s in others):
shares_tag = True
break
if not shares_tag:
rel = path.relative_to(vault_dir)
true_orphans.append(str(rel))
if true_orphans:
for p in true_orphans[:20]:
lines.append(f"- `{p}`")
else:
lines.append("_None._")
lines.append("")
# 3. Dead wikilinks (already in ERROR section, just summarize)
dead_count = sum(
1 for r in all_results
for f in r.findings
if f["severity"] == "ERROR" and f["check"] == "dead-link"
)
lines.append("### Dead wikilinks")
lines.append("")
if dead_count == 0:
lines.append("_None — zero dead wikilinks. All [[links]] resolve._")
else:
lines.append(f"**{dead_count} dead wikilinks** — see ERROR section above for details.")
lines.append("")
# 4. Tags not in topic_categories
lines.append("### Unknown tags (not in topic_categories)")
lines.append("")
unknown_tags: list[str] = []
for r in all_results:
for f in r.findings:
if f["severity"] == "WARN" and f["check"] == "tag-vocab":
unknown_tags.append(f["message"])
if unknown_tags:
for msg in unknown_tags:
lines.append(f"- {msg}")
else:
lines.append("_None — all tags are in the controlled vocabulary._")
lines.append("")
# 5. "Earned-a-doc?" candidates
lines.append("### Earned-a-doc? candidates (terms in ≥4 docs, no dedicated doc)")
lines.append("")
lines.append("Terms mentioned frequently across the vault but with no dedicated doc.")
lines.append("Matt decides whether to create a real doc — when he does, future sweeps will link to it.")
lines.append("")
candidates = build_earned_a_doc_candidates(vault_dir, note_index)
if candidates:
lines.append("| Term | Docs mentioning it |")
lines.append("|------|--------------------|")
for term, count in candidates:
lines.append(f"| `{term}` | {count} |")
else:
lines.append("_No candidates found (all frequent terms already have dedicated docs)._")
lines.append("")
return "\n".join(lines)
# ---------------------------------------------------------------------------
# Reporting
# ---------------------------------------------------------------------------
@ -494,53 +587,47 @@ def format_report(
vault_dir: Path,
total_docs: int,
elapsed_s: float,
gaps_section: str,
) -> str:
"""Format a human-readable + markdown report."""
ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
errors = [f for r in results for f in r.findings if f["severity"] == "ERROR"]
warns = [f for r in results for f in r.findings if f["severity"] == "WARN"]
infos = [f for r in results for f in r.findings if f["severity"] == "INFO"]
# Count specific warn subcategories
fm_missing = sum(
1
for r in results
for f in r.findings
if f["severity"] == "WARN"
and f["check"] == "frontmatter"
1 for r in results for f in r.findings
if f["severity"] == "WARN" and f["check"] == "frontmatter"
and "missing frontmatter block" in f["message"]
)
fm_invalid = sum(
1
for r in results
for f in r.findings
if f["severity"] == "WARN"
and f["check"] == "frontmatter"
1 for r in results for f in r.findings
if f["severity"] == "WARN" and f["check"] == "frontmatter"
and "missing frontmatter block" not in f["message"]
)
tag_warns = sum(
1 for r in results for f in r.findings if f["severity"] == "WARN" and f["check"] == "tag-vocab"
1 for r in results for f in r.findings
if f["severity"] == "WARN" and f["check"] == "tag-vocab"
)
lines = [
f"# Vault Lint Report",
f"",
"# Vault Lint Report",
"",
f"Generated: {ts} | Docs scanned: {total_docs} | Elapsed: {elapsed_s:.1f}s",
f"",
f"## Summary",
f"",
f"| Severity | Count |",
f"|----------|-------|",
"",
"## Summary",
"",
"| Severity | Count |",
"|----------|-------|",
f"| ERROR (dead links) | {len(errors)} |",
f"| WARN (schema) | {len(warns)} |",
f"| INFO (orphans) | {len(infos)} |",
f"",
f"### WARN breakdown",
"",
"### WARN breakdown",
f"- Missing frontmatter block: {fm_missing}",
f"- Invalid/missing frontmatter fields: {fm_invalid}",
f"- Unknown tags: {tag_warns}",
f"",
"",
]
if errors:
@ -565,7 +652,10 @@ def format_report(
for r in results:
ws = [f for f in r.findings if f["severity"] == "WARN"]
if ws:
rel = r.path.relative_to(vault_dir)
try:
rel = r.path.relative_to(vault_dir)
except ValueError:
rel = r.path
for f in ws:
lines.append(f"- `{rel}` — {f['message']}")
lines.append("")
@ -577,32 +667,27 @@ def format_report(
lines.append(f"- {f['message']}")
lines.append("")
# Append gaps & suggestions section
lines.append(gaps_section)
return "\n".join(lines)
# ---------------------------------------------------------------------------
# Public API (importable by sweep.sh or other scripts)
# Public API
# ---------------------------------------------------------------------------
def run_lint(vault_dir: Path, engine_dir: Path) -> tuple[list[LintResult], dict]:
"""
Run all lint checks. Returns (results, stats_dict).
Importable entry point.
"""
import time
config = load_config(engine_dir)
vocab = load_vocab(engine_dir)
# Build allowed tag set
topic_cats = set(vocab.get("topic_categories", []))
topic_cats |= ALLOWED_EXTRA_TAGS
# No extra allowed tags in v4 (removed 'meta' catch-all)
note_index = build_note_index(vault_dir)
entity_names = build_entity_names(vocab)
# Build backlink counts (single pass over all docs)
backlink_counts = build_backlink_counts(vault_dir, note_index, entity_names)
backlink_counts = build_backlink_counts(vault_dir, note_index)
all_paths = sorted(vault_dir.rglob("*.md"))
results: list[LintResult] = []
@ -610,24 +695,27 @@ def run_lint(vault_dir: Path, engine_dir: Path) -> tuple[list[LintResult], dict]
t0 = time.monotonic()
for path in all_paths:
if "archive" in path.parts:
continue
findings: list[dict] = []
try:
fm, body = parse_frontmatter(path)
except Exception as exc:
findings.append(
{"severity": "WARN", "check": "parse-error", "message": f"could not parse: {exc}"}
)
findings.append({
"severity": "WARN",
"check": "parse-error",
"message": f"could not parse: {exc}",
})
results.append(LintResult(path=path, findings=findings))
continue
findings.extend(check_frontmatter(fm, config))
findings.extend(check_tags(fm, topic_cats))
findings.extend(check_dead_links(path, body, note_index, entity_names))
findings.extend(check_dead_links(path, body, note_index))
results.append(LintResult(path=path, findings=findings))
# Orphan check (uses backlink counts already computed)
# Orphan check
orphan_findings = find_orphans(vault_dir, note_index, backlink_counts)
# Attach orphan findings as a synthetic "vault" result
if orphan_findings:
results.append(LintResult(path=vault_dir / "_orphans_", findings=orphan_findings))
@ -639,6 +727,9 @@ def run_lint(vault_dir: Path, engine_dir: Path) -> tuple[list[LintResult], dict]
"warns": sum(1 for r in results for f in r.findings if f["severity"] == "WARN"),
"infos": sum(1 for r in results for f in r.findings if f["severity"] == "INFO"),
"elapsed_s": elapsed,
"note_index": note_index,
"backlink_counts": backlink_counts,
"topic_cats": topic_cats,
}
return results, stats
@ -652,17 +743,12 @@ def main() -> None:
strict = "--strict" in sys.argv
# Locate engine/vault dirs relative to this file
this_file = Path(__file__).resolve()
engine_dir = this_file.parent.parent # engine/lib/lint.py → engine/
engine_dir = this_file.parent.parent
config = load_config(engine_dir)
vault_str = config.get("vault_dir", "")
if not vault_str:
# Fallback: sibling of engine_dir named "vault"
vault_dir = engine_dir.parent / "vault"
else:
vault_dir = Path(vault_str)
vault_dir = Path(vault_str) if vault_str else engine_dir.parent / "vault"
if not vault_dir.exists():
print(f"ERROR: vault_dir not found: {vault_dir}", file=sys.stderr)
@ -670,19 +756,24 @@ def main() -> None:
results, stats = run_lint(vault_dir, engine_dir)
report_text = format_report(
results, vault_dir, stats["total_docs"], stats["elapsed_s"]
gaps_section = build_gaps_section(
vault_dir,
stats["note_index"],
stats["backlink_counts"],
results,
stats["topic_cats"],
)
report_text = format_report(
results, vault_dir, stats["total_docs"], stats["elapsed_s"],
gaps_section,
)
# Write report
report_path = engine_dir / "lint-report.md"
report_path.write_text(report_text, encoding="utf-8")
# Print to stdout
print(report_text)
print(f"--- Wrote: {report_path} ---")
# Summary line
print(
f"\nSummary: {stats['total_docs']} docs | "
f"{stats['errors']} ERRORs | {stats['warns']} WARNs | {stats['infos']} INFOs"