From 45c3bb8d56d431e32fc8ecc5b57aa5cc65c488c2 Mon Sep 17 00:00:00 2001
From: Matt <matt@echo6.co>
Date: Sat, 18 Apr 2026 21:03:39 +0000
Subject: [PATCH] Add scraper job queue management (delete, clear failed)

New API endpoints: DELETE single job, clear all failed/cancelled.
Dashboard now shows Delete buttons on completed/failed jobs,
Retry+Delete on failed jobs, and a Clear Failed bulk action.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 lib/api.py                   | 30 ++++++++++++++++++++++++++++++
 static/js/scraper.js         | 29 ++++++++++++++++++++++++++---
 templates/kiwix/scraper.html |  5 ++++-
 3 files changed, 60 insertions(+), 4 deletions(-)

diff --git a/lib/api.py b/lib/api.py
index aa13a39..ce0381f 100644
--- a/lib/api.py
+++ b/lib/api.py
@@ -2373,6 +2373,36 @@ def api_scraper_retry(job_id):
     return jsonify({'ok': True})
 
 
+@app.route('/api/scraper/delete/<int:job_id>', methods=['POST'])
+def api_scraper_delete(job_id):
+    """Delete a scrape job (only if not currently running)."""
+    db = StatusDB()
+    job = db.get_scrape_job(job_id)
+    if not job:
+        return jsonify({'error': 'Job not found'}), 404
+
+    if job['status'] == 'running':
+        return jsonify({'error': 'Cannot delete a running job — cancel it first'}), 400
+
+    conn = db._get_conn()
+    conn.execute("DELETE FROM scrape_jobs WHERE id = ?", (job_id,))
+    conn.commit()
+    logger.info(f"Scraper job {job_id} deleted")
+    return jsonify({'ok': True})
+
+
+@app.route('/api/scraper/clear-failed', methods=['POST'])
+def api_scraper_clear_failed():
+    """Delete all failed and cancelled scrape jobs."""
+    db = StatusDB()
+    conn = db._get_conn()
+    result = conn.execute("DELETE FROM scrape_jobs WHERE status IN ('failed', 'cancelled')")
+    conn.commit()
+    count = result.rowcount
+    logger.info(f"Cleared {count} failed/cancelled scraper jobs")
+    return jsonify({'ok': True, 'deleted': count})
+
+
 # ── Metrics API ──
 
 @app.route('/api/metrics/history')
diff --git a/static/js/scraper.js b/static/js/scraper.js
index 6aa23d7..49ce178 100644
--- a/static/js/scraper.js
+++ b/static/js/scraper.js
@@ -11,7 +11,7 @@
             var active = 0, complete = 0, failed = 0;
             jobs.forEach(function(j) {
                 if (j.status === 'complete') complete++;
-                else if (j.status === 'failed') failed++;
+                else if (j.status === 'failed' || j.status === 'cancelled') failed++;
                 else if (j.status === 'running' || j.status === 'pending') active++;
             });
             RECON.set('sc-total', RECON.fmt(total));
@@ -19,6 +19,10 @@
             RECON.set('sc-complete', RECON.fmt(complete));
             RECON.set('sc-failed', RECON.fmt(failed));
 
+            // Show/hide Clear Failed button
+            var clearBtn = document.getElementById('sc-clear-btn');
+            if (clearBtn) clearBtn.style.display = failed > 0 ? '' : 'none';
+
             // Table
             var html = '';
             jobs.forEach(function(j) {
@@ -33,7 +37,10 @@
                 if (j.status === 'running' || j.status === 'pending') {
                     actions = '<button class="btn btn-danger" onclick="SCRAPER.cancel(' + j.id + ')">Cancel</button>';
                 } else if (j.status === 'failed' || j.status === 'cancelled') {
-                    actions = '<button class="btn" onclick="SCRAPER.retry(' + j.id + ')">Retry</button>';
+                    actions = '<button class="btn" onclick="SCRAPER.retry(' + j.id + ')">Retry</button> ' +
+                              '<button class="btn btn-danger" onclick="SCRAPER.remove(' + j.id + ')">Delete</button>';
+                } else if (j.status === 'complete') {
+                    actions = '<button class="btn btn-danger" onclick="SCRAPER.remove(' + j.id + ')">Delete</button>';
                 }
 
                 // Truncate URL for display
@@ -146,8 +153,24 @@
         });
     }
 
+    function remove(jobId) {
+        if (!confirm('Delete job #' + jobId + '? This cannot be undone.')) return;
+        RECON.postJSON('/api/scraper/delete/' + jobId).then(function(data) {
+            if (data.ok) loadJobs();
+            else alert('Error: ' + (data.error || 'Unknown'));
+        });
+    }
+
+    function clearFailed() {
+        if (!confirm('Delete all failed and cancelled jobs?')) return;
+        RECON.postJSON('/api/scraper/clear-failed').then(function(data) {
+            if (data.ok) loadJobs();
+            else alert('Error: ' + (data.error || 'Unknown'));
+        });
+    }
+
     // Expose for inline onclick
-    window.SCRAPER = { submit: submit, cancel: cancel, retry: retry };
+    window.SCRAPER = { submit: submit, cancel: cancel, retry: retry, remove: remove, clearFailed: clearFailed };
 
     document.addEventListener('DOMContentLoaded', function() {
         RECON.startRefresh(loadJobs, 10000);
diff --git a/templates/kiwix/scraper.html b/templates/kiwix/scraper.html
index 53d3e23..3c42f43 100644
--- a/templates/kiwix/scraper.html
+++ b/templates/kiwix/scraper.html
@@ -65,7 +65,10 @@
 
     <!-- Jobs Table -->
     <div class="panel">
-        <h3 class="section-title" style="margin-bottom:12px;">Scrape Jobs</h3>
+        <div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:12px;">
+            <h3 class="section-title" style="margin:0;">Scrape Jobs</h3>
+            <button class="btn btn-danger" onclick="SCRAPER.clearFailed()" id="sc-clear-btn" style="display:none;">Clear Failed</button>
+        </div>
         <table class="data-table" id="sc-table">
             <thead>
                 <tr>