226 lines
7.4 KiB
Bash
Executable file
226 lines
7.4 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# sweep.sh — Echo6 Vault Engine orchestrator
|
|
#
|
|
# Usage:
|
|
# ./sweep.sh # default: process docs changed since last sweep
|
|
# ./sweep.sh --all # process every non-archived vault doc
|
|
# ./sweep.sh --dry-run # pass --dry-run to agent.py (no writes)
|
|
# ./sweep.sh --all --dry-run # both
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
VAULT_DIR="${SCRIPT_DIR}/../vault"
|
|
CONFIG="${SCRIPT_DIR}/config.yaml"
|
|
LAST_SWEEP="${SCRIPT_DIR}/.last-sweep"
|
|
CHANGELOG="${SCRIPT_DIR}/changelog.md"
|
|
LINT_REPORT="${SCRIPT_DIR}/lint-report.md"
|
|
|
|
# Parse args
|
|
OPT_ALL=0
|
|
OPT_DRY=0
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--all) OPT_ALL=1 ;;
|
|
--dry-run) OPT_DRY=1 ;;
|
|
*)
|
|
echo "[error] Unknown argument: $arg" >&2
|
|
echo "Usage: $0 [--all] [--dry-run]" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
SWEEP_START="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
echo "==> Echo6 vault sweep — ${SWEEP_START}"
|
|
if [ "$OPT_DRY" -eq 1 ]; then
|
|
echo " (DRY-RUN mode — no files will be modified)"
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Step 1 — GPU-busy guard
|
|
# ---------------------------------------------------------------------------
|
|
echo ""
|
|
echo "[1/4] GPU-busy guard..."
|
|
|
|
# Helper: run nvidia-smi, using docker exec ollama if host lacks it
|
|
_nvidia_smi() {
|
|
if command -v nvidia-smi &>/dev/null; then
|
|
nvidia-smi "$@" 2>/dev/null
|
|
elif command -v docker &>/dev/null && docker ps --format '{{.Names}}' 2>/dev/null | grep -q '^ollama$'; then
|
|
docker exec ollama nvidia-smi "$@" 2>/dev/null
|
|
else
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
GPU_DEFERRED=0
|
|
DEFER_REASON=""
|
|
|
|
# Guard 1 — competing GPU processes (share the A4000)
|
|
# Note: peertube-runner is intentionally excluded — it's a persistent server
|
|
# daemon that's always running, so its mere presence doesn't mean the GPU is
|
|
# busy (only an active transcode job does, and that spawns ffmpeg, which is
|
|
# still caught here; actual peertube contention is caught by the util/VRAM
|
|
# guards below).
|
|
if pgrep -fa 'whisper|ffmpeg' >/dev/null 2>&1; then
|
|
COMPETING="$(pgrep -fa 'whisper|ffmpeg' | head -1)"
|
|
DEFER_REASON="competing GPU process: ${COMPETING}"
|
|
GPU_DEFERRED=1
|
|
fi
|
|
|
|
# Guard 2 — sustained GPU utilization > 60 %
|
|
if [ "$GPU_DEFERRED" -eq 0 ]; then
|
|
GPU_UTIL="$(_nvidia_smi --query-gpu=utilization.gpu --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d '[:space:]' || true)"
|
|
if [ -n "$GPU_UTIL" ] && [[ "$GPU_UTIL" =~ ^[0-9]+$ ]]; then
|
|
echo " GPU utilization: ${GPU_UTIL}%"
|
|
if [ "$GPU_UTIL" -gt 60 ]; then
|
|
DEFER_REASON="GPU utilization ${GPU_UTIL}% > 60%"
|
|
GPU_DEFERRED=1
|
|
fi
|
|
else
|
|
echo " nvidia-smi not available or could not read utilization — skipping util guard."
|
|
fi
|
|
fi
|
|
|
|
# Guard 3 (safety cap) — very high VRAM usage (>14000 MiB) — not triggered by vault-tagger itself (~8-10 GB)
|
|
if [ "$GPU_DEFERRED" -eq 0 ]; then
|
|
GPU_USED_MIB="$(_nvidia_smi --query-gpu=memory.used --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d '[:space:]' || true)"
|
|
if [ -n "$GPU_USED_MIB" ] && [[ "$GPU_USED_MIB" =~ ^[0-9]+$ ]]; then
|
|
echo " GPU VRAM used: ${GPU_USED_MIB} MiB (safety cap: 14000 MiB)"
|
|
if [ "$GPU_USED_MIB" -gt 14000 ]; then
|
|
DEFER_REASON="VRAM ${GPU_USED_MIB} MiB > 14000 MiB safety cap"
|
|
GPU_DEFERRED=1
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
if [ "$GPU_DEFERRED" -eq 1 ]; then
|
|
echo " GPU busy — deferring sweep. Reason: ${DEFER_REASON}"
|
|
if [ "$OPT_DRY" -eq 0 ]; then
|
|
printf "\n## %s — sweep deferred (%s)\n" "$SWEEP_START" "$DEFER_REASON" >> "$CHANGELOG"
|
|
fi
|
|
exit 0
|
|
fi
|
|
|
|
echo " GPU OK — proceeding."
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Step 2 — Lint
|
|
# ---------------------------------------------------------------------------
|
|
echo ""
|
|
echo "[2/4] Running lint..."
|
|
|
|
python3 "${SCRIPT_DIR}/lib/lint.py" --report 2>&1 | tee /tmp/sweep-lint.log || {
|
|
echo "[warn] lint.py exited non-zero — continuing sweep"
|
|
}
|
|
echo " Lint complete. Report: ${LINT_REPORT}"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Step 3 — Select docs and run agent
|
|
# ---------------------------------------------------------------------------
|
|
echo ""
|
|
echo "[3/4] Selecting documents..."
|
|
|
|
# Build file list
|
|
DOC_LIST=()
|
|
if [ "$OPT_ALL" -eq 1 ] || [ ! -f "$LAST_SWEEP" ]; then
|
|
if [ "$OPT_ALL" -eq 1 ]; then
|
|
echo " Mode: --all (full vault scan)"
|
|
else
|
|
echo " Mode: no .last-sweep marker found — treating as first run (full scan)"
|
|
fi
|
|
while IFS= read -r -d '' f; do
|
|
# Skip archive
|
|
if [[ "$f" == *"/archive/"* ]]; then
|
|
continue
|
|
fi
|
|
DOC_LIST+=("$f")
|
|
done < <(find "${VAULT_DIR}" -name "*.md" -type f -print0 | sort -z) # -type f excludes symlinks (e.g. CLAUDE-baseline.md, vault/rules/)
|
|
else
|
|
LAST_TS="$(cat "$LAST_SWEEP")"
|
|
echo " Mode: incremental — docs modified since ${LAST_TS}"
|
|
# Compare file mtime to .last-sweep mtime (not contents)
|
|
while IFS= read -r -d '' f; do
|
|
if [[ "$f" == *"/archive/"* ]]; then
|
|
continue
|
|
fi
|
|
DOC_LIST+=("$f")
|
|
done < <(find "${VAULT_DIR}" -name "*.md" -type f -newer "$LAST_SWEEP" -print0 | sort -z) # -type f excludes symlinks
|
|
fi
|
|
|
|
TOTAL="${#DOC_LIST[@]}"
|
|
echo " ${TOTAL} document(s) selected."
|
|
|
|
if [ "$TOTAL" -eq 0 ]; then
|
|
echo " Nothing to process — vault is up to date."
|
|
if [ "$OPT_DRY" -eq 0 ]; then
|
|
date -u +%Y-%m-%dT%H:%M:%SZ > "$LAST_SWEEP"
|
|
printf "\n## %s — sweep complete (0 docs selected)\n" "$SWEEP_START" >> "$CHANGELOG"
|
|
fi
|
|
echo "==> Done."
|
|
exit 0
|
|
fi
|
|
|
|
echo ""
|
|
echo " Processing..."
|
|
|
|
N_PROCESSED=0
|
|
N_WRITTEN=0
|
|
N_FLAGGED=0
|
|
N_ERRORS=0
|
|
|
|
AGENT_DRY_FLAG=""
|
|
if [ "$OPT_DRY" -eq 1 ]; then
|
|
AGENT_DRY_FLAG="--dry-run"
|
|
fi
|
|
|
|
for doc in "${DOC_LIST[@]}"; do
|
|
echo ""
|
|
echo " --- $(basename "$doc") ---"
|
|
set +e
|
|
python3 "${SCRIPT_DIR}/lib/agent.py" $AGENT_DRY_FLAG "$doc" 2>&1
|
|
EXIT_CODE=$?
|
|
set -e
|
|
N_PROCESSED=$(( N_PROCESSED + 1 ))
|
|
if [ "$EXIT_CODE" -eq 0 ]; then
|
|
N_WRITTEN=$(( N_WRITTEN + 1 ))
|
|
else
|
|
echo " [warn] agent.py exited ${EXIT_CODE} for ${doc}"
|
|
N_ERRORS=$(( N_ERRORS + 1 ))
|
|
fi
|
|
done
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Step 4 — Update marker + changelog summary
|
|
# ---------------------------------------------------------------------------
|
|
echo ""
|
|
echo "[4/4] Finalising..."
|
|
|
|
SWEEP_END="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
|
|
if [ "$OPT_DRY" -eq 0 ]; then
|
|
date -u +%Y-%m-%dT%H:%M:%SZ > "$LAST_SWEEP"
|
|
{
|
|
printf "\n## %s — sweep run\n" "$SWEEP_START"
|
|
printf "- end: %s\n" "$SWEEP_END"
|
|
printf "- mode: %s\n" "$([ "$OPT_ALL" -eq 1 ] && echo "--all" || echo "incremental")"
|
|
printf "- docs selected: %d\n" "$TOTAL"
|
|
printf "- processed: %d | written: %d | flagged: %d | errors: %d\n" \
|
|
"$N_PROCESSED" "$N_WRITTEN" "$N_FLAGGED" "$N_ERRORS"
|
|
} >> "$CHANGELOG"
|
|
echo " .last-sweep updated. Changelog appended."
|
|
else
|
|
echo " [DRY-RUN] .last-sweep NOT updated. Changelog NOT written."
|
|
fi
|
|
|
|
echo ""
|
|
echo "==> Sweep complete."
|
|
echo " Docs selected : ${TOTAL}"
|
|
echo " Processed : ${N_PROCESSED}"
|
|
echo " Written/OK : ${N_WRITTEN}"
|
|
echo " Errors : ${N_ERRORS}"
|
|
if [ "$OPT_DRY" -eq 1 ]; then
|
|
echo " [DRY-RUN] No vault docs were modified."
|
|
fi
|