matt/recon
1
0
Fork 0
mirror of https://github.com/zvx-echo6/recon.git synced 2026-05-20 06:34:40 +02:00
Code Issues Projects Releases Packages Wiki Activity

Replace wget/SingleFile/Playwright backends with Zimit

Browse source 
- Zimit Docker container handles all site types (static, SPA, JS redirects)
- Removed: _detect_crawl_mode, _crawl_wget, _crawl_singlefile, preflight logic
- Added: _crawl_zimit() with Docker lifecycle management
- Simplified pipeline: submit → Zimit crawl → kiwix-manage register → done
- No more zimwriterfs step — Zimit produces ZIM directly
- Dashboard UI simplified: removed crawl mode dropdown
- Config simplified: removed reject patterns, preflight, singlefile sections

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt 2026-04-19 14:06:23 +00:00
commit 8945c82e3f
5 changed files with 212 additions and 606 deletions
Download patch file Download diff file Expand all files Collapse all files

15
static/js/scraper.js
View file

@ -12,7 +12,7 @@
jobs.forEach(function(j) {
if (j.status === 'complete') complete++;
else if (j.status === 'failed' || j.status === 'cancelled') failed++;
else if (j.status === 'running' || j.status === 'pending') active++;
else if (j.status === 'scraping' || j.status === 'registering' || j.status === 'pending') active++;
});
RECON.set('sc-total', RECON.fmt(total));
RECON.set('sc-active', RECON.fmt(active));
@ -27,14 +27,12 @@
var html = '';
jobs.forEach(function(j) {
var badge = statusBadge(j.status);
var mode = j.crawl_mode ?
'<span class="text-small">' + j.crawl_mode + '</span>' : '<span class="text-muted">\u2014</span>';
var pages = j.page_count ? RECON.fmt(j.page_count) : '\u2014';
var zim = j.zim_filename ?
'<span class="text-small">' + j.zim_filename + '</span>' : '\u2014';
var actions = '';
if (j.status === 'running' || j.status === 'pending') {
if (j.status === 'scraping' || j.status === 'registering' || j.status === 'pending') {
actions = '<button class="btn btn-danger" onclick="SCRAPER.cancel(' + j.id + ')">Cancel</button>';
} else if (j.status === 'failed' || j.status === 'cancelled') {
actions = '<button class="btn" onclick="SCRAPER.retry(' + j.id + ')">Retry</button> ' +
@ -50,14 +48,13 @@
'<td>' + j.id + '</td>' +
'<td><a href="' + escHtml(j.url) + '" target="_blank" title="' + escHtml(j.url) + '">' + escHtml(displayUrl) + '</a></td>' +
'<td>' + escHtml(j.title || '\u2014') + '</td>' +
'<td>' + mode + '</td>' +
'<td>' + pages + '</td>' +
'<td>' + badge + errorTooltip(j) + '</td>' +
'<td>' + zim + '</td>' +
'<td>' + actions + '</td>' +
'</tr>';
});
if (!html) html = '<tr><td colspan="8" class="text-muted">No scrape jobs</td></tr>';
if (!html) html = '<tr><td colspan="7" class="text-muted">No scrape jobs</td></tr>';
RECON.setHTML('sc-table-body', html);
}).catch(function(err) {
console.error('Scraper dashboard error:', err);
@ -67,7 +64,8 @@
function statusBadge(status) {
var map = {
'pending': '<span class="badge-detected">PENDING</span>',
'running': '<span class="badge-processing">RUNNING</span>',
'scraping': '<span class="badge-processing">SCRAPING</span>',
'registering': '<span class="badge-processing">REGISTERING</span>',
'complete': '<span class="badge-complete">COMPLETE</span>',
'failed': '<span class="badge-failed">FAILED</span>',
'cancelled': '<span class="badge-detected">CANCELLED</span>'
@ -98,12 +96,9 @@
var title = document.getElementById('sf-title').value.trim();
var lang = document.getElementById('sf-lang').value;
var category = document.getElementById('sf-category').value.trim();
var mode = document.getElementById('sf-mode').value;
if (title) body.title = title;
if (lang) body.language = lang;
if (category) body.category = category;
if (mode) body.crawl_mode = mode;
var btn = document.getElementById('sf-submit-btn');
var feedback = document.getElementById('sf-feedback');