matt/recon
1
0
Fork 0
mirror of https://github.com/zvx-echo6/recon.git synced 2026-05-20 06:34:40 +02:00
Code Issues Projects Releases Packages Wiki Activity

Add scraper dashboard UI under Kiwix tab

Browse source 
New /kiwix/scraper page with submit form (URL, title, language,
crawl mode), stats cards, and auto-refreshing jobs table with
cancel/retry actions. Kiwix section now has Library/Scraper subnav.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt 2026-04-18 20:47:17 +00:00
commit 1ce9a3731f
4 changed files with 257 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

11
lib/api.py
View file

@ -60,7 +60,10 @@ PEERTUBE_SUBNAV = [
]
KIWIX_SUBNAV = [] # Single-page, no subnav needed
KIWIX_SUBNAV = [
{'href': '/kiwix', 'label': 'Library'},
{'href': '/kiwix/scraper', 'label': 'Scraper'},
]
SETTINGS_SUBNAV = [
{'href': '/settings/keys', 'label': 'API Keys'},
{'href': '/settings/cookies', 'label': 'YouTube Cookies'},
@ -1956,6 +1959,12 @@ def kiwix_dashboard():
domain='kiwix', subnav=KIWIX_SUBNAV, active_page='/kiwix')
@app.route('/kiwix/scraper')
def kiwix_scraper():
return render_template('kiwix/scraper.html',
domain='kiwix', subnav=KIWIX_SUBNAV, active_page='/kiwix/scraper')
@app.route('/api/kiwix/sources')
def api_kiwix_sources():
"""Serve pre-cached Kiwix sources data (never blocks)."""

1
static/css/recon.css
View file

@ -331,3 +331,4 @@ tr:hover { background: var(--bg-secondary); }
.badge-detected { background: #333; color: #888; padding: 2px 8px; border-radius: var(--radius); font-size: 11px; }
.badge-processing { background: #4a3a1a; color: #f59e0b; padding: 2px 8px; border-radius: var(--radius); font-size: 11px; }
.badge-extracting { background: #1a3a5a; color: #0ea5e9; padding: 2px 8px; border-radius: var(--radius); font-size: 11px; }
.badge-failed { background: #4a1a1a; color: #ff4444; padding: 2px 8px; border-radius: var(--radius); font-size: 11px; }

155
static/js/scraper.js Normal file
View file

@ -0,0 +1,155 @@
/* RECON Scraper Dashboard JS */
(function() {
'use strict';
function loadJobs() {
return RECON.fetchJSON('/api/scraper/jobs').then(function(data) {
var jobs = data.jobs || [];
// Stats
var total = jobs.length;
var active = 0, complete = 0, failed = 0;
jobs.forEach(function(j) {
if (j.status === 'complete') complete++;
else if (j.status === 'failed') failed++;
else if (j.status === 'running' || j.status === 'pending') active++;
});
RECON.set('sc-total', RECON.fmt(total));
RECON.set('sc-active', RECON.fmt(active));
RECON.set('sc-complete', RECON.fmt(complete));
RECON.set('sc-failed', RECON.fmt(failed));
// Table
var html = '';
jobs.forEach(function(j) {
var badge = statusBadge(j.status);
var mode = j.crawl_mode ?
'<span class="text-small">' + j.crawl_mode + '</span>' : '<span class="text-muted">\u2014</span>';
var pages = j.page_count ? RECON.fmt(j.page_count) : '\u2014';
var zim = j.zim_filename ?
'<span class="text-small">' + j.zim_filename + '</span>' : '\u2014';
var actions = '';
if (j.status === 'running' || j.status === 'pending') {
actions = '<button class="btn btn-danger" onclick="SCRAPER.cancel(' + j.id + ')">Cancel</button>';
} else if (j.status === 'failed' || j.status === 'cancelled') {
actions = '<button class="btn" onclick="SCRAPER.retry(' + j.id + ')">Retry</button>';
}
// Truncate URL for display
var displayUrl = j.url.length > 40 ? j.url.substring(0, 40) + '\u2026' : j.url;
html += '<tr>' +
'<td>' + j.id + '</td>' +
'<td><a href="' + escHtml(j.url) + '" target="_blank" title="' + escHtml(j.url) + '">' + escHtml(displayUrl) + '</a></td>' +
'<td>' + escHtml(j.title || '\u2014') + '</td>' +
'<td>' + mode + '</td>' +
'<td>' + pages + '</td>' +
'<td>' + badge + errorTooltip(j) + '</td>' +
'<td>' + zim + '</td>' +
'<td>' + actions + '</td>' +
'</tr>';
});
if (!html) html = '<tr><td colspan="8" class="text-muted">No scrape jobs</td></tr>';
RECON.setHTML('sc-table-body', html);
}).catch(function(err) {
console.error('Scraper dashboard error:', err);
});
}
function statusBadge(status) {
var map = {
'pending': '<span class="badge-detected">PENDING</span>',
'running': '<span class="badge-processing">RUNNING</span>',
'complete': '<span class="badge-complete">COMPLETE</span>',
'failed': '<span class="badge-failed">FAILED</span>',
'cancelled': '<span class="badge-detected">CANCELLED</span>'
};
return map[status] || '<span class="badge-detected">' + (status || 'UNKNOWN').toUpperCase() + '</span>';
}
function errorTooltip(job) {
if (!job.error_message) return '';
var short = job.error_message.length > 80 ?
job.error_message.substring(0, 80) + '\u2026' : job.error_message;
return '<div class="text-small text-muted" style="max-width:200px;word-break:break-all;" title="' +
escHtml(job.error_message) + '">' + escHtml(short) + '</div>';
}
function escHtml(str) {
if (!str) return '';
return str.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;')
.replace(/"/g, '&quot;').replace(/'/g, '&#39;');
}
function submit(e) {
e.preventDefault();
var url = document.getElementById('sf-url').value.trim();
if (!url) return false;
var body = { url: url };
var title = document.getElementById('sf-title').value.trim();
var lang = document.getElementById('sf-lang').value;
var category = document.getElementById('sf-category').value.trim();
var mode = document.getElementById('sf-mode').value;
if (title) body.title = title;
if (lang) body.language = lang;
if (category) body.category = category;
if (mode) body.crawl_mode = mode;
var btn = document.getElementById('sf-submit-btn');
var feedback = document.getElementById('sf-feedback');
btn.disabled = true;
btn.textContent = 'Submitting...';
RECON.postJSON('/api/scraper/submit', body).then(function(data) {
btn.disabled = false;
btn.textContent = 'Submit';
if (data.ok) {
feedback.style.display = 'block';
feedback.style.color = '#00ff41';
feedback.textContent = 'Job #' + data.job_id + ' submitted successfully';
document.getElementById('sf-url').value = '';
document.getElementById('sf-title').value = '';
document.getElementById('sf-category').value = '';
setTimeout(function() { feedback.style.display = 'none'; }, 4000);
loadJobs();
} else {
feedback.style.display = 'block';
feedback.style.color = '#ff4444';
feedback.textContent = 'Error: ' + (data.error || 'Unknown error');
}
}).catch(function(err) {
btn.disabled = false;
btn.textContent = 'Submit';
feedback.style.display = 'block';
feedback.style.color = '#ff4444';
feedback.textContent = 'Network error: ' + err.message;
});
return false;
}
function cancel(jobId) {
if (!confirm('Cancel job #' + jobId + '?')) return;
RECON.postJSON('/api/scraper/cancel/' + jobId).then(function(data) {
if (data.ok) loadJobs();
else alert('Error: ' + (data.error || 'Unknown'));
});
}
function retry(jobId) {
RECON.postJSON('/api/scraper/retry/' + jobId).then(function(data) {
if (data.ok) loadJobs();
else alert('Error: ' + (data.error || 'Unknown'));
});
}
// Expose for inline onclick
window.SCRAPER = { submit: submit, cancel: cancel, retry: retry };
document.addEventListener('DOMContentLoaded', function() {
RECON.startRefresh(loadJobs, 10000);
});
})();

91
templates/kiwix/scraper.html Normal file
View file

@ -0,0 +1,91 @@
{% extends "base.html" %}
{% block content %}
<div id="scraper-page">
<!-- Submit Form -->
<div class="panel">
<h3 class="section-title" style="margin-bottom:12px;">Submit Scrape Job</h3>
<form id="scraper-form" onsubmit="return SCRAPER.submit(event)">
<div style="display:grid;grid-template-columns:1fr 1fr;gap:12px;margin-bottom:12px;">
<div>
<label class="text-small text-muted" style="display:block;margin-bottom:4px;">URL *</label>
<input type="url" id="sf-url" placeholder="https://example.com/" required
style="width:100%;padding:8px 12px;background:var(--bg-secondary);border:1px solid var(--border);color:var(--text-primary);border-radius:var(--radius);font-family:inherit;font-size:13px;">
</div>
<div>
<label class="text-small text-muted" style="display:block;margin-bottom:4px;">Title</label>
<input type="text" id="sf-title" placeholder="Optional display title"
style="width:100%;padding:8px 12px;background:var(--bg-secondary);border:1px solid var(--border);color:var(--text-primary);border-radius:var(--radius);font-family:inherit;font-size:13px;">
</div>
</div>
<div style="display:grid;grid-template-columns:1fr 1fr 1fr auto;gap:12px;align-items:end;">
<div>
<label class="text-small text-muted" style="display:block;margin-bottom:4px;">Language</label>
<select id="sf-lang"
style="width:100%;padding:8px 12px;background:var(--bg-secondary);border:1px solid var(--border);color:var(--text-primary);border-radius:var(--radius);font-family:inherit;font-size:13px;">
<option value="eng" selected>English</option>
<option value="spa">Spanish</option>
<option value="fra">French</option>
<option value="deu">German</option>
<option value="por">Portuguese</option>
<option value="rus">Russian</option>
<option value="jpn">Japanese</option>
<option value="zho">Chinese</option>
<option value="mul">Multilingual</option>
</select>
</div>
<div>
<label class="text-small text-muted" style="display:block;margin-bottom:4px;">Category</label>
<input type="text" id="sf-category" placeholder="Optional"
style="width:100%;padding:8px 12px;background:var(--bg-secondary);border:1px solid var(--border);color:var(--text-primary);border-radius:var(--radius);font-family:inherit;font-size:13px;">
</div>
<div>
<label class="text-small text-muted" style="display:block;margin-bottom:4px;">Crawl Mode</label>
<select id="sf-mode"
style="width:100%;padding:8px 12px;background:var(--bg-secondary);border:1px solid var(--border);color:var(--text-primary);border-radius:var(--radius);font-family:inherit;font-size:13px;">
<option value="" selected>Auto-detect</option>
<option value="static">Static (wget)</option>
<option value="browser">Browser (SingleFile)</option>
</select>
</div>
<div>
<button type="submit" class="btn" id="sf-submit-btn">Submit</button>
</div>
</div>
<div id="sf-feedback" style="margin-top:8px;font-size:12px;display:none;"></div>
</form>
</div>
<!-- Stats row -->
<div class="stat-grid" style="grid-template-columns:repeat(4, 1fr);">
<div class="stat-card"><div class="label">Total Jobs</div><div class="value" id="sc-total">&mdash;</div></div>
<div class="stat-card"><div class="label">Active</div><div class="value" id="sc-active">&mdash;</div></div>
<div class="stat-card"><div class="label">Complete</div><div class="value" id="sc-complete">&mdash;</div></div>
<div class="stat-card"><div class="label">Failed</div><div class="value" id="sc-failed">&mdash;</div></div>
</div>
<!-- Jobs Table -->
<div class="panel">
<h3 class="section-title" style="margin-bottom:12px;">Scrape Jobs</h3>
<table class="data-table" id="sc-table">
<thead>
<tr>
<th>ID</th>
<th>URL</th>
<th>Title</th>
<th>Mode</th>
<th>Pages</th>
<th>Status</th>
<th>ZIM</th>
<th></th>
</tr>
</thead>
<tbody id="sc-table-body">
<tr><td colspan="8" class="text-muted">Loading...</td></tr>
</tbody>
</table>
</div>
</div>
{% endblock %}
{% block scripts %}
<script src="/static/js/scraper.js"></script>
{% endblock %}