echo6-docs/engine/sweep.sh

#!/usr/bin/env bash
# sweep.sh — Echo6 Vault Engine orchestrator
#
# Usage:
#   ./sweep.sh                  # default: process docs changed since last sweep
#   ./sweep.sh --all            # process every non-archived vault doc
#   ./sweep.sh --dry-run        # pass --dry-run to agent.py (no writes)
#   ./sweep.sh --all --dry-run  # both

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
VAULT_DIR="${SCRIPT_DIR}/../vault"
CONFIG="${SCRIPT_DIR}/config.yaml"
LAST_SWEEP="${SCRIPT_DIR}/.last-sweep"
CHANGELOG="${SCRIPT_DIR}/changelog.md"
LINT_REPORT="${SCRIPT_DIR}/lint-report.md"

# Parse args
OPT_ALL=0
OPT_DRY=0
for arg in "$@"; do
    case "$arg" in
        --all)      OPT_ALL=1 ;;
        --dry-run)  OPT_DRY=1 ;;
        *)
            echo "[error] Unknown argument: $arg" >&2
            echo "Usage: $0 [--all] [--dry-run]" >&2
            exit 1
            ;;
    esac
done

SWEEP_START="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "==> Echo6 vault sweep — ${SWEEP_START}"
if [ "$OPT_DRY" -eq 1 ]; then
    echo "    (DRY-RUN mode — no files will be modified)"
fi

# ---------------------------------------------------------------------------
# Step 1 — GPU-busy guard
# ---------------------------------------------------------------------------
echo ""
echo "[1/4] GPU-busy guard..."

# Helper: run nvidia-smi, using docker exec ollama if host lacks it
_nvidia_smi() {
    if command -v nvidia-smi &>/dev/null; then
        nvidia-smi "$@" 2>/dev/null
    elif command -v docker &>/dev/null && docker ps --format '{{.Names}}' 2>/dev/null | grep -q '^ollama$'; then
        docker exec ollama nvidia-smi "$@" 2>/dev/null
    else
        return 1
    fi
}

GPU_DEFERRED=0
DEFER_REASON=""

# Guard 1 — competing GPU processes (share the A4000)
# Note: peertube-runner is intentionally excluded — it's a persistent server
# daemon that's always running, so its mere presence doesn't mean the GPU is
# busy (only an active transcode job does, and that spawns ffmpeg, which is
# still caught here; actual peertube contention is caught by the util/VRAM
# guards below).
if pgrep -fa 'whisper|ffmpeg' >/dev/null 2>&1; then
    COMPETING="$(pgrep -fa 'whisper|ffmpeg' | head -1)"
    DEFER_REASON="competing GPU process: ${COMPETING}"
    GPU_DEFERRED=1
fi

# Guard 2 — sustained GPU utilization > 60 %
if [ "$GPU_DEFERRED" -eq 0 ]; then
    GPU_UTIL="$(_nvidia_smi --query-gpu=utilization.gpu --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d '[:space:]' || true)"
    if [ -n "$GPU_UTIL" ] && [[ "$GPU_UTIL" =~ ^[0-9]+$ ]]; then
        echo "    GPU utilization: ${GPU_UTIL}%"
        if [ "$GPU_UTIL" -gt 60 ]; then
            DEFER_REASON="GPU utilization ${GPU_UTIL}% > 60%"
            GPU_DEFERRED=1
        fi
    else
        echo "    nvidia-smi not available or could not read utilization — skipping util guard."
    fi
fi

# Guard 3 (safety cap) — very high VRAM usage (>14000 MiB) — not triggered by vault-tagger itself (~8-10 GB)
if [ "$GPU_DEFERRED" -eq 0 ]; then
    GPU_USED_MIB="$(_nvidia_smi --query-gpu=memory.used --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d '[:space:]' || true)"
    if [ -n "$GPU_USED_MIB" ] && [[ "$GPU_USED_MIB" =~ ^[0-9]+$ ]]; then
        echo "    GPU VRAM used: ${GPU_USED_MIB} MiB (safety cap: 14000 MiB)"
        if [ "$GPU_USED_MIB" -gt 14000 ]; then
            DEFER_REASON="VRAM ${GPU_USED_MIB} MiB > 14000 MiB safety cap"
            GPU_DEFERRED=1
        fi
    fi
fi

if [ "$GPU_DEFERRED" -eq 1 ]; then
    echo "    GPU busy — deferring sweep. Reason: ${DEFER_REASON}"
    if [ "$OPT_DRY" -eq 0 ]; then
        printf "\n## %s — sweep deferred (%s)\n" "$SWEEP_START" "$DEFER_REASON" >> "$CHANGELOG"
    fi
    exit 0
fi

echo "    GPU OK — proceeding."


# ---------------------------------------------------------------------------
# Step 2 — Lint
# ---------------------------------------------------------------------------
echo ""
echo "[2/4] Running lint..."

python3 "${SCRIPT_DIR}/lib/lint.py" --report 2>&1 | tee /tmp/sweep-lint.log || {
    echo "[warn] lint.py exited non-zero — continuing sweep"
}
echo "    Lint complete. Report: ${LINT_REPORT}"

# ---------------------------------------------------------------------------
# Step 3 — Select docs and run agent
# ---------------------------------------------------------------------------
echo ""
echo "[3/4] Selecting documents..."

# Build file list
DOC_LIST=()
if [ "$OPT_ALL" -eq 1 ] || [ ! -f "$LAST_SWEEP" ]; then
    if [ "$OPT_ALL" -eq 1 ]; then
        echo "    Mode: --all (full vault scan)"
    else
        echo "    Mode: no .last-sweep marker found — treating as first run (full scan)"
    fi
    while IFS= read -r -d  f; do
        # Skip archive
        if [[ "$f" == *"/archive/"* ]]; then
            continue
        fi
        DOC_LIST+=("$f")
    done < <(find "${VAULT_DIR}" -name "*.md" -type f -print0 | sort -z)  # -type f excludes symlinks (e.g. CLAUDE-baseline.md, vault/rules/)
else
    LAST_TS="$(cat "$LAST_SWEEP")"
    echo "    Mode: incremental — docs modified since ${LAST_TS}"
    # Compare file mtime to .last-sweep mtime (not contents)
    while IFS= read -r -d  f; do
        if [[ "$f" == *"/archive/"* ]]; then
            continue
        fi
        DOC_LIST+=("$f")
    done < <(find "${VAULT_DIR}" -name "*.md" -type f -newer "$LAST_SWEEP" -print0 | sort -z)  # -type f excludes symlinks
fi

TOTAL="${#DOC_LIST[@]}"
echo "    ${TOTAL} document(s) selected."

if [ "$TOTAL" -eq 0 ]; then
    echo "    Nothing to process — vault is up to date."
    if [ "$OPT_DRY" -eq 0 ]; then
        date -u +%Y-%m-%dT%H:%M:%SZ > "$LAST_SWEEP"
        printf "\n## %s — sweep complete (0 docs selected)\n" "$SWEEP_START" >> "$CHANGELOG"
    fi
    echo "==> Done."
    exit 0
fi

echo ""
echo "    Processing..."

N_PROCESSED=0
N_WRITTEN=0
N_FLAGGED=0
N_ERRORS=0

AGENT_DRY_FLAG=""
if [ "$OPT_DRY" -eq 1 ]; then
    AGENT_DRY_FLAG="--dry-run"
fi

for doc in "${DOC_LIST[@]}"; do
    echo ""
    echo "  --- $(basename "$doc") ---"
    set +e
    python3 "${SCRIPT_DIR}/lib/agent.py" $AGENT_DRY_FLAG "$doc" 2>&1
    EXIT_CODE=$?
    set -e
    N_PROCESSED=$(( N_PROCESSED + 1 ))
    if [ "$EXIT_CODE" -eq 0 ]; then
        N_WRITTEN=$(( N_WRITTEN + 1 ))
    else
        echo "  [warn] agent.py exited ${EXIT_CODE} for ${doc}"
        N_ERRORS=$(( N_ERRORS + 1 ))
    fi
done

# ---------------------------------------------------------------------------
# Step 4 — Update marker + changelog summary
# ---------------------------------------------------------------------------
echo ""
echo "[4/4] Finalising..."

SWEEP_END="$(date -u +%Y-%m-%dT%H:%M:%SZ)"

if [ "$OPT_DRY" -eq 0 ]; then
    date -u +%Y-%m-%dT%H:%M:%SZ > "$LAST_SWEEP"
    {
        printf "\n## %s — sweep run\n" "$SWEEP_START"
        printf "- end: %s\n" "$SWEEP_END"
        printf "- mode: %s\n" "$([ "$OPT_ALL" -eq 1 ] && echo "--all" || echo "incremental")"
        printf "- docs selected: %d\n" "$TOTAL"
        printf "- processed: %d | written: %d | flagged: %d | errors: %d\n" \
            "$N_PROCESSED" "$N_WRITTEN" "$N_FLAGGED" "$N_ERRORS"
    } >> "$CHANGELOG"
    echo "    .last-sweep updated. Changelog appended."
else
    echo "    [DRY-RUN] .last-sweep NOT updated. Changelog NOT written."
fi

echo ""
echo "==> Sweep complete."
echo "    Docs selected : ${TOTAL}"
echo "    Processed     : ${N_PROCESSED}"
echo "    Written/OK    : ${N_WRITTEN}"
echo "    Errors        : ${N_ERRORS}"
if [ "$OPT_DRY" -eq 1 ]; then
    echo "    [DRY-RUN] No vault docs were modified."
fi