auto: docs sync 2026-06-18T12:00:08+00:00
Files changed: .gitignore engine/.embcache.json engine/changelog.md engine/lib/__pycache__/__init__.cpython-312.pyc engine/lib/__pycache__/agent.cpython-312.pyc engine/lib/agent.py engine/lint-report.md engine/prompts/system.md engine/sweep.sh vault/.obsidian/graph.json vault/docs/software/authentik.md vault/docs/software/caddy.md vault/docs/software/recon.md vault/notes/echo6-landing-page-data-export.md vault/projects/argus.md vault/projects/meshtastic-headscale-runbook.md vault/runbooks/add-peertube-channel.md vault/runbooks/authentik-access-groups.md vault/runbooks/ct-runbook.md vault/runbooks/meshtastic-sidecar-node.md
This commit is contained in:
echo6-autocommit
parent
1d30335963
commit
c30ce9f1e3
20 changed files with 1490 additions and 703 deletions
235
engine/sweep.sh
235
engine/sweep.sh
|
|
@ -1,66 +1,221 @@
|
|||
#!/usr/bin/env bash
|
||||
# sweep.sh — Echo6 Vault Engine daily maintenance sweep
|
||||
# sweep.sh — Echo6 Vault Engine orchestrator
|
||||
#
|
||||
# Invoked by cron (schedule: "0 9 * * *" from config.yaml).
|
||||
# Also callable manually: ./sweep.sh
|
||||
#
|
||||
# What this does (when fully implemented):
|
||||
# 1. GPU-busy guard: check VRAM usage; defer if > defer_if_gpu_busy_mib (6000 MiB default)
|
||||
# 2. Run lint (lib/lint.py) over all vault docs — fix or flag frontmatter issues
|
||||
# 3. Run agent (lib/agent.py) over changed/new docs since last run — tag + embed
|
||||
# 4. Append a summary entry to changelog.md
|
||||
#
|
||||
# Configuration is read from config.yaml (engine_dir, vault_dir, thresholds, changelog path).
|
||||
# Usage:
|
||||
# ./sweep.sh # default: process docs changed since last sweep
|
||||
# ./sweep.sh --all # process every non-archived vault doc
|
||||
# ./sweep.sh --dry-run # pass --dry-run to agent.py (no writes)
|
||||
# ./sweep.sh --all --dry-run # both
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
VAULT_DIR="${SCRIPT_DIR}/../vault"
|
||||
CONFIG="${SCRIPT_DIR}/config.yaml"
|
||||
LAST_SWEEP="${SCRIPT_DIR}/.last-sweep"
|
||||
CHANGELOG="${SCRIPT_DIR}/changelog.md"
|
||||
LINT_REPORT="${SCRIPT_DIR}/lint-report.md"
|
||||
|
||||
echo "==> Echo6 vault sweep — $(date -u '+%Y-%m-%dT%H:%M:%SZ')"
|
||||
# Parse args
|
||||
OPT_ALL=0
|
||||
OPT_DRY=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--all) OPT_ALL=1 ;;
|
||||
--dry-run) OPT_DRY=1 ;;
|
||||
*)
|
||||
echo "[error] Unknown argument: $arg" >&2
|
||||
echo "Usage: $0 [--all] [--dry-run]" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
SWEEP_START="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "==> Echo6 vault sweep — ${SWEEP_START}"
|
||||
if [ "$OPT_DRY" -eq 1 ]; then
|
||||
echo " (DRY-RUN mode — no files will be modified)"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 1 — GPU-busy guard
|
||||
# ---------------------------------------------------------------------------
|
||||
# TODO: Query nvidia-smi for used VRAM; compare to defer_if_gpu_busy_mib from config.yaml.
|
||||
# If busy, log a deferred entry to changelog and exit 0 (not an error, just deferred).
|
||||
#
|
||||
# Example skeleton:
|
||||
# USED_MIB=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -1)
|
||||
# THRESHOLD=6000 # read from config.yaml
|
||||
# if [ "$USED_MIB" -gt "$THRESHOLD" ]; then
|
||||
# echo "GPU busy (${USED_MIB} MiB > ${THRESHOLD} MiB threshold) — deferring sweep."
|
||||
# exit 0
|
||||
# fi
|
||||
echo ""
|
||||
echo "[1/4] GPU-busy guard..."
|
||||
|
||||
# Helper: run nvidia-smi, using docker exec ollama if host lacks it
|
||||
_nvidia_smi() {
|
||||
if command -v nvidia-smi &>/dev/null; then
|
||||
nvidia-smi "$@" 2>/dev/null
|
||||
elif command -v docker &>/dev/null && docker ps --format '{{.Names}}' 2>/dev/null | grep -q '^ollama$'; then
|
||||
docker exec ollama nvidia-smi "$@" 2>/dev/null
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
GPU_DEFERRED=0
|
||||
DEFER_REASON=""
|
||||
|
||||
# Guard 1 — competing GPU processes (share the A4000)
|
||||
if pgrep -fa 'peertube-runner|whisper|ffmpeg' >/dev/null 2>&1; then
|
||||
COMPETING="$(pgrep -fa 'peertube-runner|whisper|ffmpeg' | head -1)"
|
||||
DEFER_REASON="competing GPU process: ${COMPETING}"
|
||||
GPU_DEFERRED=1
|
||||
fi
|
||||
|
||||
# Guard 2 — sustained GPU utilization > 60 %
|
||||
if [ "$GPU_DEFERRED" -eq 0 ]; then
|
||||
GPU_UTIL="$(_nvidia_smi --query-gpu=utilization.gpu --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d '[:space:]' || true)"
|
||||
if [ -n "$GPU_UTIL" ] && [[ "$GPU_UTIL" =~ ^[0-9]+$ ]]; then
|
||||
echo " GPU utilization: ${GPU_UTIL}%"
|
||||
if [ "$GPU_UTIL" -gt 60 ]; then
|
||||
DEFER_REASON="GPU utilization ${GPU_UTIL}% > 60%"
|
||||
GPU_DEFERRED=1
|
||||
fi
|
||||
else
|
||||
echo " nvidia-smi not available or could not read utilization — skipping util guard."
|
||||
fi
|
||||
fi
|
||||
|
||||
# Guard 3 (safety cap) — very high VRAM usage (>14000 MiB) — not triggered by vault-tagger itself (~8-10 GB)
|
||||
if [ "$GPU_DEFERRED" -eq 0 ]; then
|
||||
GPU_USED_MIB="$(_nvidia_smi --query-gpu=memory.used --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d '[:space:]' || true)"
|
||||
if [ -n "$GPU_USED_MIB" ] && [[ "$GPU_USED_MIB" =~ ^[0-9]+$ ]]; then
|
||||
echo " GPU VRAM used: ${GPU_USED_MIB} MiB (safety cap: 14000 MiB)"
|
||||
if [ "$GPU_USED_MIB" -gt 14000 ]; then
|
||||
DEFER_REASON="VRAM ${GPU_USED_MIB} MiB > 14000 MiB safety cap"
|
||||
GPU_DEFERRED=1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$GPU_DEFERRED" -eq 1 ]; then
|
||||
echo " GPU busy — deferring sweep. Reason: ${DEFER_REASON}"
|
||||
if [ "$OPT_DRY" -eq 0 ]; then
|
||||
printf "\n## %s — sweep deferred (%s)\n" "$SWEEP_START" "$DEFER_REASON" >> "$CHANGELOG"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo " GPU OK — proceeding."
|
||||
|
||||
echo "[1/4] TODO — GPU-busy guard not yet implemented."
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 2 — Run lint
|
||||
# Step 2 — Lint
|
||||
# ---------------------------------------------------------------------------
|
||||
# TODO: Call lib/lint.py to validate/fix frontmatter across vault docs.
|
||||
# Lint should be idempotent and log all changes to changelog.
|
||||
#
|
||||
# python3 "${SCRIPT_DIR}/lib/lint.py" --config "${CONFIG}"
|
||||
echo ""
|
||||
echo "[2/4] Running lint..."
|
||||
|
||||
echo "[2/4] TODO — lint.py not yet implemented (Step 2)."
|
||||
python3 "${SCRIPT_DIR}/lib/lint.py" --report 2>&1 | tee /tmp/sweep-lint.log || {
|
||||
echo "[warn] lint.py exited non-zero — continuing sweep"
|
||||
}
|
||||
echo " Lint complete. Report: ${LINT_REPORT}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 3 — Run agent over changed/new docs
|
||||
# Step 3 — Select docs and run agent
|
||||
# ---------------------------------------------------------------------------
|
||||
# TODO: Call lib/agent.py to tag + embed documents modified since last sweep.
|
||||
# Agent tracks last-run timestamp in a state file (e.g. engine/.last_sweep).
|
||||
#
|
||||
# python3 "${SCRIPT_DIR}/lib/agent.py" --config "${CONFIG}"
|
||||
echo ""
|
||||
echo "[3/4] Selecting documents..."
|
||||
|
||||
echo "[3/4] TODO — agent.py not yet implemented (Step 5)."
|
||||
# Build file list
|
||||
DOC_LIST=()
|
||||
if [ "$OPT_ALL" -eq 1 ] || [ ! -f "$LAST_SWEEP" ]; then
|
||||
if [ "$OPT_ALL" -eq 1 ]; then
|
||||
echo " Mode: --all (full vault scan)"
|
||||
else
|
||||
echo " Mode: no .last-sweep marker found — treating as first run (full scan)"
|
||||
fi
|
||||
while IFS= read -r -d f; do
|
||||
# Skip archive
|
||||
if [[ "$f" == *"/archive/"* ]]; then
|
||||
continue
|
||||
fi
|
||||
DOC_LIST+=("$f")
|
||||
done < <(find "${VAULT_DIR}" -name "*.md" -print0 | sort -z)
|
||||
else
|
||||
LAST_TS="$(cat "$LAST_SWEEP")"
|
||||
echo " Mode: incremental — docs modified since ${LAST_TS}"
|
||||
# Compare file mtime to .last-sweep mtime (not contents)
|
||||
while IFS= read -r -d f; do
|
||||
if [[ "$f" == *"/archive/"* ]]; then
|
||||
continue
|
||||
fi
|
||||
DOC_LIST+=("$f")
|
||||
done < <(find "${VAULT_DIR}" -name "*.md" -newer "$LAST_SWEEP" -print0 | sort -z)
|
||||
fi
|
||||
|
||||
TOTAL="${#DOC_LIST[@]}"
|
||||
echo " ${TOTAL} document(s) selected."
|
||||
|
||||
if [ "$TOTAL" -eq 0 ]; then
|
||||
echo " Nothing to process — vault is up to date."
|
||||
if [ "$OPT_DRY" -eq 0 ]; then
|
||||
date -u +%Y-%m-%dT%H:%M:%SZ > "$LAST_SWEEP"
|
||||
printf "\n## %s — sweep complete (0 docs selected)\n" "$SWEEP_START" >> "$CHANGELOG"
|
||||
fi
|
||||
echo "==> Done."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo " Processing..."
|
||||
|
||||
N_PROCESSED=0
|
||||
N_WRITTEN=0
|
||||
N_FLAGGED=0
|
||||
N_ERRORS=0
|
||||
|
||||
AGENT_DRY_FLAG=""
|
||||
if [ "$OPT_DRY" -eq 1 ]; then
|
||||
AGENT_DRY_FLAG="--dry-run"
|
||||
fi
|
||||
|
||||
for doc in "${DOC_LIST[@]}"; do
|
||||
echo ""
|
||||
echo " --- $(basename "$doc") ---"
|
||||
set +e
|
||||
python3 "${SCRIPT_DIR}/lib/agent.py" $AGENT_DRY_FLAG "$doc" 2>&1
|
||||
EXIT_CODE=$?
|
||||
set -e
|
||||
N_PROCESSED=$(( N_PROCESSED + 1 ))
|
||||
if [ "$EXIT_CODE" -eq 0 ]; then
|
||||
N_WRITTEN=$(( N_WRITTEN + 1 ))
|
||||
else
|
||||
echo " [warn] agent.py exited ${EXIT_CODE} for ${doc}"
|
||||
N_ERRORS=$(( N_ERRORS + 1 ))
|
||||
fi
|
||||
done
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 4 — Append changelog summary
|
||||
# Step 4 — Update marker + changelog summary
|
||||
# ---------------------------------------------------------------------------
|
||||
# TODO: agent.py and lint.py both append to changelog.md directly.
|
||||
# This step adds a sweep-level summary entry.
|
||||
echo ""
|
||||
echo "[4/4] Finalising..."
|
||||
|
||||
echo "[4/4] TODO — changelog summary not yet implemented."
|
||||
SWEEP_END="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
|
||||
echo "==> sweep.sh done."
|
||||
if [ "$OPT_DRY" -eq 0 ]; then
|
||||
date -u +%Y-%m-%dT%H:%M:%SZ > "$LAST_SWEEP"
|
||||
{
|
||||
printf "\n## %s — sweep run\n" "$SWEEP_START"
|
||||
printf "- end: %s\n" "$SWEEP_END"
|
||||
printf "- mode: %s\n" "$([ "$OPT_ALL" -eq 1 ] && echo "--all" || echo "incremental")"
|
||||
printf "- docs selected: %d\n" "$TOTAL"
|
||||
printf "- processed: %d | written: %d | flagged: %d | errors: %d\n" \
|
||||
"$N_PROCESSED" "$N_WRITTEN" "$N_FLAGGED" "$N_ERRORS"
|
||||
} >> "$CHANGELOG"
|
||||
echo " .last-sweep updated. Changelog appended."
|
||||
else
|
||||
echo " [DRY-RUN] .last-sweep NOT updated. Changelog NOT written."
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "==> Sweep complete."
|
||||
echo " Docs selected : ${TOTAL}"
|
||||
echo " Processed : ${N_PROCESSED}"
|
||||
echo " Written/OK : ${N_WRITTEN}"
|
||||
echo " Errors : ${N_ERRORS}"
|
||||
if [ "$OPT_DRY" -eq 1 ]; then
|
||||
echo " [DRY-RUN] No vault docs were modified."
|
||||
fi
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue