matt/recon
1
0
Fork 0
mirror of https://github.com/zvx-echo6/recon.git synced 2026-05-20 14:44:54 +02:00
Code Issues Projects Releases Packages Wiki Activity

Initial commit: RECON codebase baseline

Browse source 
Current state of the pipeline code as of 2026-04-14 (Phase 1 scaffolding complete).
Config has new_pipeline.enabled=false and crawler.sites=[] per refactor plan.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt 2026-04-14 14:57:23 +00:00
commit 563c16bb71
59 changed files with 18327 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

120
static/js/channels.js Normal file
View file

@ -0,0 +1,120 @@
/* RECON PeerTube Channels page JS */
(function() {
'use strict';
async function loadChannelStats() {
try {
var resp = await fetch('/api/peertube/channels/stats');
var data = await resp.json();
if (resp.ok) {
document.getElementById('pt-total-ch').textContent = data.total_channels;
document.getElementById('pt-total-vid').textContent = data.total_videos;
var dlEl = document.getElementById('pt-dl-status');
dlEl.textContent = data.downloader_active ? 'Active' : 'Stopped';
dlEl.style.color = data.downloader_active ? '#00ff41' : '#ff4444';
}
} catch(e) {
console.error('Stats error:', e);
}
}
async function loadChannels() {
try {
var resp = await fetch('/api/peertube/channels');
var data = await resp.json();
if (!resp.ok) throw new Error(data.error || 'Failed');
var tbody = document.getElementById('pt-channel-tbody');
if (!data.length) {
tbody.innerHTML = '<tr><td colspan="6" style="text-align:center;padding:20px;color:#555;">No channels configured</td></tr>';
return;
}
var cats = [];
var catSet = {};
data.forEach(function(c) { if (c.category && !catSet[c.category]) { catSet[c.category] = true; cats.push(c.category); } });
document.getElementById('pt-cat-list').innerHTML = cats.map(function(c) { return '<option value="' + c + '">'; }).join('');
var html = '';
data.forEach(function(ch) {
var vids = ch.videos_in_peertube || 0;
var statusColor = vids > 0 ? '#00ff41' : '#ffa500';
var statusText = vids > 0 ? 'syncing' : 'new';
var ytLink = ch.youtube_url ? '<a href="' + ch.youtube_url + '" target="_blank" style="color:#00a0d0;text-decoration:none;">' + ch.channel_name + '</a>' : ch.channel_name;
html += '<tr style="border-bottom:1px solid #1a1a1a;">' +
'<td style="padding:8px 10px;">' + ytLink + '</td>' +
'<td style="padding:8px 10px;text-align:center;">' + vids + '</td>' +
'<td style="padding:8px 10px;color:#888;">' + (ch.category || '') + '</td>' +
'<td style="padding:8px 10px;text-align:center;">' + (ch.priority || 'M') + '</td>' +
'<td style="padding:8px 10px;text-align:center;"><span style="color:' + statusColor + ';">' + statusText + '</span></td>' +
'<td style="padding:8px 10px;text-align:center;"><button onclick="removeChannel(\'' + ch.actor_name + '\')" style="background:none;border:1px solid #333;color:#ff4444;cursor:pointer;padding:2px 8px;font-size:11px;font-family:inherit;">x</button></td>' +
'</tr>';
});
tbody.innerHTML = html;
} catch(e) {
document.getElementById('pt-channel-tbody').innerHTML = '<tr><td colspan="6" style="text-align:center;padding:20px;color:#ff4444;">Error: ' + e.message + '</td></tr>';
}
}
window.addChannel = async function() {
var fb = document.getElementById('pt-feedback');
var url = document.getElementById('pt-yt-url').value.trim();
if (!url) {
fb.style.color = '#ff4444';
fb.textContent = 'Enter a YouTube channel URL';
return;
}
var category = document.getElementById('pt-category').value.trim();
var priority = document.getElementById('pt-priority').value;
var btn = document.getElementById('pt-add-btn');
btn.disabled = true;
fb.style.color = '#ffa500';
fb.textContent = 'Resolving channel...';
try {
var resp = await fetch('/api/peertube/channels/add', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({youtube_url: url, category: category, priority: priority})
});
var data = await resp.json();
if (resp.ok) {
fb.style.color = '#00ff41';
fb.textContent = 'Added: ' + (data.channel_name || 'channel');
document.getElementById('pt-yt-url').value = '';
loadChannels();
loadChannelStats();
} else {
fb.style.color = '#ff4444';
fb.textContent = data.error || 'Failed to add channel';
}
} catch(e) {
fb.style.color = '#ff4444';
fb.textContent = 'Error: ' + e.message;
}
btn.disabled = false;
};
window.removeChannel = async function(actorName) {
if (!confirm('Remove channel ' + actorName + '?')) return;
var fb = document.getElementById('pt-feedback');
fb.style.color = '#ffa500';
fb.textContent = 'Removing...';
try {
var resp = await fetch('/api/peertube/channels/' + encodeURIComponent(actorName), {method: 'DELETE'});
var data = await resp.json();
if (resp.ok) {
fb.style.color = '#00ff41';
fb.textContent = data.message || 'Removed';
loadChannels();
loadChannelStats();
} else {
fb.style.color = '#ff4444';
fb.textContent = data.error || 'Failed';
}
} catch(e) {
fb.style.color = '#ff4444';
fb.textContent = 'Error: ' + e.message;
}
};
loadChannelStats();
loadChannels();
})();

186
static/js/charts.js Normal file
View file

@ -0,0 +1,186 @@
/* RECON Lightweight Canvas Line Chart
* No dependencies. drawLineChart(canvasId, datasets, opts)
* DPI-aware rendering for sharp lines on all displays.
*/
var ReconChart = (function() {
'use strict';
var COLORS = ['#00ff41', '#0ea5e9', '#ffa500', '#ff4444', '#7c3aed', '#fbbf24'];
function drawLineChart(canvasId, datasets, opts) {
opts = opts || {};
var canvas = document.getElementById(canvasId);
if (!canvas) return;
// DPI-aware sizing — match canvas bitmap to actual CSS pixels
var dpr = window.devicePixelRatio || 1;
var rect = canvas.getBoundingClientRect();
var cssW = rect.width || 800;
var cssH = rect.height || 200;
canvas.width = cssW * dpr;
canvas.height = cssH * dpr;
var ctx = canvas.getContext('2d');
ctx.scale(dpr, dpr);
var W = cssW;
var H = cssH;
var pad = {top: 20, right: 20, bottom: 30, left: 60};
var plotW = W - pad.left - pad.right;
var plotH = H - pad.top - pad.bottom;
// Clear
ctx.fillStyle = '#111';
ctx.fillRect(0, 0, W, H);
if (!datasets || datasets.length === 0) {
ctx.fillStyle = '#666';
ctx.font = '12px Courier New';
ctx.textAlign = 'center';
ctx.fillText('No data', W/2, H/2);
return;
}
// Find global min/max Y
var allY = [];
var allX = [];
datasets.forEach(function(ds) {
ds.points.forEach(function(p) {
allY.push(p.y);
allX.push(p.x);
});
});
if (allY.length === 0) return;
var minY = Math.min.apply(null, allY);
var maxY = Math.max.apply(null, allY);
var minX = Math.min.apply(null, allX);
var maxX = Math.max.apply(null, allX);
// Add 10% padding to Y
var yRange = maxY - minY || 1;
minY = Math.max(0, minY - yRange * 0.05);
maxY = maxY + yRange * 0.1;
var xRange = maxX - minX || 1;
function xToCanvas(x) { return pad.left + ((x - minX) / xRange) * plotW; }
function yToCanvas(y) { return pad.top + plotH - ((y - minY) / (maxY - minY)) * plotH; }
// Grid lines
ctx.strokeStyle = '#222';
ctx.lineWidth = 1;
var ySteps = 5;
for (var i = 0; i <= ySteps; i++) {
var yVal = minY + (maxY - minY) * (i / ySteps);
var cy = yToCanvas(yVal);
ctx.beginPath();
ctx.moveTo(pad.left, cy);
ctx.lineTo(W - pad.right, cy);
ctx.stroke();
// Y labels
ctx.fillStyle = '#666';
ctx.font = '10px Courier New';
ctx.textAlign = 'right';
ctx.fillText(Math.round(yVal).toLocaleString(), pad.left - 6, cy + 3);
}
// X labels (time)
ctx.textAlign = 'center';
ctx.fillStyle = '#666';
var xSteps = Math.min(6, allX.length);
for (var j = 0; j < xSteps; j++) {
var xVal = minX + xRange * (j / (xSteps - 1 || 1));
var cx = xToCanvas(xVal);
var d = new Date(xVal);
var label = d.getHours().toString().padStart(2, '0') + ':' + d.getMinutes().toString().padStart(2, '0');
ctx.fillText(label, cx, H - 8);
}
// Draw lines + dots at each data point
datasets.forEach(function(ds, idx) {
var color = ds.color || COLORS[idx % COLORS.length];
ctx.strokeStyle = color;
ctx.lineWidth = 2;
ctx.beginPath();
var pts = ds.points.sort(function(a, b) { return a.x - b.x; });
pts.forEach(function(p, i) {
var x = xToCanvas(p.x);
var y = yToCanvas(p.y);
if (i === 0) ctx.moveTo(x, y);
else ctx.lineTo(x, y);
});
ctx.stroke();
// Draw dots at each point for visibility with sparse data
ctx.fillStyle = color;
pts.forEach(function(p) {
var x = xToCanvas(p.x);
var y = yToCanvas(p.y);
ctx.beginPath();
ctx.arc(x, y, 3, 0, Math.PI * 2);
ctx.fill();
});
// Legend label
if (ds.label) {
ctx.fillStyle = color;
ctx.font = '10px Courier New';
ctx.textAlign = 'left';
ctx.fillText(ds.label, pad.left + idx * 100, 12);
}
});
}
function loadAndDraw(canvasId, metricType, keys, labels, hours) {
hours = hours || 24;
RECON.fetchJSON('/api/metrics/history?type=' + metricType + '&hours=' + hours).then(function(data) {
if (!data.points || data.points.length < 2) {
// Show "collecting data" message instead of hiding
var canvas = document.getElementById(canvasId);
if (!canvas) return;
var container = canvas.parentElement;
if (container) container.style.display = 'block';
var dpr = window.devicePixelRatio || 1;
var rect = canvas.getBoundingClientRect();
canvas.width = (rect.width || 800) * dpr;
canvas.height = (rect.height || 200) * dpr;
var ctx = canvas.getContext('2d');
ctx.scale(dpr, dpr);
ctx.fillStyle = '#111';
ctx.fillRect(0, 0, rect.width, rect.height);
ctx.fillStyle = '#555';
ctx.font = '12px Courier New';
ctx.textAlign = 'center';
var msg = data.points && data.points.length === 1
? 'Collecting data... (1 snapshot, need 2+)'
: 'Collecting data... (snapshots every 2 min)';
ctx.fillText(msg, (rect.width || 800) / 2, (rect.height || 200) / 2);
return;
}
var container = document.getElementById(canvasId).parentElement;
if (container) container.style.display = 'block';
var datasets = keys.map(function(key, i) {
return {
label: labels[i] || key,
color: COLORS[i % COLORS.length],
points: data.points.map(function(p) {
return {
x: new Date(p.timestamp).getTime(),
y: p.data[key] || 0
};
})
};
});
drawLineChart(canvasId, datasets);
}).catch(function() {});
}
return {
drawLineChart: drawLineChart,
loadAndDraw: loadAndDraw
};
})();

163
static/js/common.js Normal file
View file

@ -0,0 +1,163 @@
/* RECON Common Utilities
* Shared fetch helpers, formatters, auto-refresh
*/
var RECON = (function() {
'use strict';
// Pipeline color/label maps
var pipeColors = {
queued: '#555', extracting: '#b45309', extracted: '#d97706',
enriching: '#0284c7', enriched: '#0ea5e9', embedding: '#7c3aed',
complete: '#16a34a', failed: '#dc2626'
};
var pipeLabels = {
queued: 'Queued', extracting: 'Extracting', extracted: 'Extracted',
enriching: 'Enriching', enriched: 'Enriched', embedding: 'Embedding',
complete: 'Complete', failed: 'Failed'
};
var _refreshTimers = [];
var _heartbeatEl = null;
function fetchJSON(url) {
return fetch(url).then(function(r) {
if (!r.ok) throw new Error('HTTP ' + r.status);
return r.json();
});
}
function postJSON(url, body) {
return fetch(url, {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify(body || {})
}).then(function(r) { return r.json(); });
}
function set(id, text) {
var el = document.getElementById(id);
if (el) el.textContent = text;
}
function setHTML(id, html) {
var el = document.getElementById(id);
if (el) el.innerHTML = html;
}
function fmt(n) {
if (typeof n !== 'number' || isNaN(n)) return '—';
return n.toLocaleString();
}
function fmtBytes(bytes) {
if (!bytes || bytes === 0) return '0 B';
var units = ['B', 'KB', 'MB', 'GB', 'TB'];
var i = Math.floor(Math.log(bytes) / Math.log(1024));
return (bytes / Math.pow(1024, i)).toFixed(1) + ' ' + units[i];
}
function pct(n, total) {
if (!total || total === 0) return '0';
return (n / total * 100).toFixed(1);
}
// Trend indicator: compare current to previous
function trend(current, previous) {
if (previous === undefined || previous === null) return '';
var diff = current - previous;
if (diff > 0) return '<span class="trend trend-up">+' + fmt(diff) + ' &#9650;</span>';
if (diff < 0) return '<span class="trend trend-down">' + fmt(diff) + ' &#9660;</span>';
return '<span class="trend trend-flat">&mdash; &#9654;</span>';
}
// Build a segmented pipeline progress bar
function progressBar(segments, total) {
var html = '';
segments.forEach(function(seg) {
var w = total > 0 ? (seg.count / total * 100) : 0;
if (w > 0) {
html += '<div class="segment" style="width:' + w + '%;background:' +
(seg.color || pipeColors[seg.status] || '#555') + ';" title="' +
(seg.label || pipeLabels[seg.status] || seg.status) + ': ' + fmt(seg.count) + '"></div>';
}
});
return html;
}
// Build legend for pipeline bar
function progressLegend(segments) {
var html = '';
segments.forEach(function(seg) {
if (seg.count > 0) {
html += '<span><span class="legend-dot" style="background:' +
(seg.color || pipeColors[seg.status] || '#555') + ';"></span>' +
(seg.label || pipeLabels[seg.status] || seg.status) + ': ' + fmt(seg.count) + '</span>';
}
});
return html;
}
// Auto-refresh with heartbeat
function startRefresh(callback, intervalMs) {
_heartbeatEl = document.getElementById('heartbeat');
function tick() {
try {
var result = callback();
if (result && typeof result.then === 'function') {
result.then(function() {
if (_heartbeatEl) {
_heartbeatEl.classList.remove('dead');
}
}).catch(function() {
if (_heartbeatEl) {
_heartbeatEl.classList.add('dead');
}
});
} else {
if (_heartbeatEl) _heartbeatEl.classList.remove('dead');
}
} catch(e) {
if (_heartbeatEl) _heartbeatEl.classList.add('dead');
}
}
// Initial load
tick();
var timer = setInterval(tick, intervalMs || 30000);
_refreshTimers.push(timer);
return timer;
}
function stopRefresh(timer) {
if (timer) clearInterval(timer);
}
// Quick-stats loader for header
function loadQuickStats() {
fetchJSON('/api/quick-stats').then(function(data) {
setHTML('qs-docs', fmt(data.catalogued));
setHTML('qs-vectors', fmt(data.vectors));
setHTML('qs-pipeline', fmt(data.in_pipeline));
}).catch(function() {});
}
return {
fetchJSON: fetchJSON,
postJSON: postJSON,
set: set,
setHTML: setHTML,
fmt: fmt,
fmtBytes: fmtBytes,
pct: pct,
trend: trend,
progressBar: progressBar,
progressLegend: progressLegend,
startRefresh: startRefresh,
stopRefresh: stopRefresh,
loadQuickStats: loadQuickStats,
pipeColors: pipeColors,
pipeLabels: pipeLabels
};
})();

232
static/js/dashboard.js Normal file
View file

@ -0,0 +1,232 @@
/* RECON Knowledge Dashboard */
(function() {
'use strict';
var pipeColors = RECON.pipeColors;
var pipeLabels = RECON.pipeLabels;
function loadDashboard() {
return RECON.fetchJSON('/api/knowledge-stats').then(function(data) {
var t = data.totals;
// Top cards
RECON.set('kv-catalogued', RECON.fmt(t.catalogued || 0));
RECON.set('kv-pipeline', RECON.fmt(t.in_pipeline || 0));
var pipeSub = document.getElementById('kv-pipeline-sub');
if (t.in_pipeline > 0) {
var active = data.pipeline.filter(function(p) { return ['extracting','enriching','embedding'].indexOf(p.status) >= 0; });
var activeText = active.map(function(p) { return p.count + ' ' + p.status; }).join(', ');
pipeSub.textContent = activeText || 'processing';
} else { pipeSub.textContent = 'idle'; }
RECON.set('kv-complete', RECON.fmt(t.complete || 0));
var failEl = document.getElementById('kv-failed');
failEl.textContent = RECON.fmt(t.failed || 0);
failEl.style.color = t.failed > 0 ? '#ff4444' : '#00ff41';
RECON.set('kv-concepts', RECON.fmt(t.concepts || 0));
RECON.set('kv-vectors', RECON.fmt(t.vectors || 0));
RECON.set('kv-pages', RECON.fmt(t.pages_processed || 0));
// Progress bar
var total = t.catalogued || 1;
var notYetQueued = total - (t.documents || 0);
var segments = [];
if (notYetQueued > 0) {
segments.push({status: 'unqueued', count: notYetQueued, color: '#1a1a1a', label: 'Not queued'});
}
data.pipeline.forEach(function(p) {
if (p.count > 0) segments.push(p);
});
RECON.setHTML('progress-bar', RECON.progressBar(segments, total));
var completePct = total > 0 ? (t.complete / total * 100).toFixed(1) : 0;
RECON.set('progress-pct', completePct + '% complete (' + RECON.fmt(t.complete || 0) + ' / ' + RECON.fmt(total) + ')');
// Legend
var legendSegments = [];
if (notYetQueued > 0) legendSegments.push({status: 'unqueued', count: notYetQueued, color: '#1a1a1a', label: 'Not queued'});
data.pipeline.forEach(function(p) { if (p.count > 0) legendSegments.push(p); });
RECON.setHTML('progress-legend', RECON.progressLegend(legendSegments));
// Pipeline activity
var activeStatuses = data.pipeline.filter(function(p) { return ['extracting','enriching','embedding'].indexOf(p.status) >= 0 && p.count > 0; });
var actDiv = document.getElementById('pipeline-activity');
if (activeStatuses.length > 0) {
actDiv.style.display = 'block';
var actHtml = '';
activeStatuses.forEach(function(p) {
actHtml += '<div style="margin:4px 0;"><span style="color:' + (pipeColors[p.status]||'#ffa500') + ';">&#9679; ' + (pipeLabels[p.status]||p.status) + ':</span> ' + p.count + ' documents</div>';
});
if (data.active_titles) {
Object.keys(data.active_titles).forEach(function(st) {
var titles = data.active_titles[st];
if (titles.length > 0) actHtml += '<div style="color:#666;font-size:11px;margin-left:16px;">' + titles.slice(0,3).join(', ') + (titles.length > 3 ? ', ...' : '') + '</div>';
});
}
RECON.setHTML('activity-content', actHtml);
} else { actDiv.style.display = 'none'; }
// Qdrant health
var q = data.qdrant;
var qEl = document.getElementById('qdrant-status');
if (q.error) {
qEl.innerHTML = '<span style="color:#ff4444;">&#9679; Offline</span> &mdash; ' + q.error;
} else {
var idxType = q.index_type || (q.vectors >= 20000 ? 'HNSW' : 'brute-force');
var idxColor = idxType === 'HNSW' ? '#00ff41' : '#ffa500';
qEl.innerHTML = '<span style="color:#00ff41;">&#9679; Online</span> | ' +
RECON.fmt(q.vectors) + ' vectors | ' +
'<span style="color:' + idxColor + ';">' + idxType + '</span>' +
(idxType === 'HNSW' ? ' (' + RECON.fmt(q.indexed||0) + ' indexed)' : ' (HNSW auto-builds at 20K)') +
' | <span style="color:#555;">recon_knowledge</span>';
}
// Sources table
var tbody = document.getElementById('sources-tbody');
var totalCat = 0, totalComp = 0, totalPipe = 0, totalConcepts = 0, totalVectors = 0;
tbody.innerHTML = data.sources.map(function(s) {
var catCount = s.catalogued || 0;
var compCount = s.complete || 0;
var pipeCount = s.in_pipeline || 0;
totalCat += catCount; totalComp += compCount; totalPipe += pipeCount;
totalConcepts += s.concepts; totalVectors += s.vectors;
var badge = s.type === 'web' ? '<span class="badge-web">WEB</span>' : '<span class="badge-pdf">PDF</span>';
var compPct = catCount > 0 ? (compCount / catCount * 100) : 0;
var pipePct = catCount > 0 ? (pipeCount / catCount * 100) : 0;
var compColor = compPct >= 100 ? '#00ff41' : compPct > 0 ? '#ffa500' : '#666';
var pipeColor = pipeCount > 0 ? '#0ea5e9' : '#555';
var barW = 80;
var compW = (compPct / 100 * barW).toFixed(1);
var pipeW = (pipePct / 100 * barW).toFixed(1);
var miniBar = '<div style="display:flex;align-items:center;gap:6px;">' +
'<div style="width:' + barW + 'px;height:10px;background:#1a1a1a;border-radius:3px;overflow:hidden;display:flex;">' +
'<div style="width:' + compW + 'px;background:#16a34a;height:100%;"></div>' +
'<div style="width:' + pipeW + 'px;background:#0284c7;height:100%;"></div>' +
'</div><span style="color:#888;font-size:10px;">' + compPct.toFixed(0) + '%</span></div>';
return '<tr><td>' + s.name + '</td><td>' + badge + '</td><td>' +
RECON.fmt(catCount) + '</td><td><span style="color:' + compColor + ';">' +
RECON.fmt(compCount) + '</span></td><td><span style="color:' + pipeColor + ';">' +
RECON.fmt(pipeCount) + '</span></td><td>' + miniBar + '</td><td>' +
RECON.fmt(s.concepts) + '</td><td>' + RECON.fmt(s.vectors) + '</td></tr>';
}).join('');
RECON.setHTML('sources-tfoot',
'<tr style="border-top:1px solid #333;font-weight:bold;"><td>TOTAL</td><td></td><td>' +
RECON.fmt(totalCat) + '</td><td>' + RECON.fmt(totalComp) + '</td><td>' +
RECON.fmt(totalPipe) + '</td><td></td><td>' +
RECON.fmt(totalConcepts) + '</td><td>' + RECON.fmt(totalVectors) + '</td></tr>');
// Domain bars
var dc = document.getElementById('domain-bars');
var domEntries = Object.entries(data.domains);
if (domEntries.length === 0) {
dc.innerHTML = '<span class="text-dim">No domain data</span>';
} else {
var maxD = Math.max.apply(null, domEntries.map(function(e) { return e[1]; }));
dc.innerHTML = domEntries.map(function(entry) {
var name = entry[0], count = entry[1];
var pct = (count / maxD * 100).toFixed(1);
return '<div style="display:flex;align-items:center;gap:10px;margin:5px 0;">' +
'<span style="width:160px;text-align:right;color:#aaa;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">' + name + '</span>' +
'<div style="flex:1;height:18px;background:#1a1a1a;border-radius:3px;overflow:hidden;">' +
'<div style="height:100%;background:#00cc66;border-radius:3px;width:' + pct + '%;"></div></div>' +
'<span style="width:50px;color:#ccc;text-align:right;">' + RECON.fmt(count) + '</span></div>';
}).join('');
}
// Knowledge Type bars
var ktEl = document.getElementById('knowledge-type-bars');
var ktEntries = Object.entries(data.knowledge_types || {});
var totalKt = ktEntries.reduce(function(a, e) { return a + e[1]; }, 0);
if (ktEntries.length === 0) {
ktEl.innerHTML = '<span class="text-dim">No data yet (migration in progress)</span>';
} else {
var ktColors = {foundational: '#60a5fa', procedural: '#4ade80', operational: '#fbbf24'};
var maxKt = Math.max.apply(null, ktEntries.map(function(e) { return e[1]; }));
ktEl.innerHTML = ktEntries.map(function(entry) {
var name = entry[0], count = entry[1];
var pctVal = totalKt > 0 ? (count / totalKt * 100).toFixed(0) : 0;
var barPct = (count / maxKt * 100).toFixed(1);
var color = ktColors[name] || '#888';
return '<div style="display:flex;align-items:center;gap:10px;margin:5px 0;">' +
'<span style="width:100px;text-align:right;color:' + color + ';">' + name + '</span>' +
'<div style="flex:1;height:18px;background:#1a1a1a;border-radius:3px;overflow:hidden;">' +
'<div style="height:100%;background:' + color + ';opacity:0.6;border-radius:3px;width:' + barPct + '%;"></div></div>' +
'<span style="width:80px;color:#ccc;text-align:right;">' + RECON.fmt(count) + ' (' + pctVal + '%)</span></div>';
}).join('');
}
var ktMig = document.getElementById('knowledge-type-migration');
ktMig.textContent = RECON.fmt(totalKt) + ' / ' + RECON.fmt(data.sample_size) + ' migrated';
// Complexity bars
var cxEl = document.getElementById('complexity-bars');
var cxEntries = Object.entries(data.complexities || {});
var totalCx = cxEntries.reduce(function(a, e) { return a + e[1]; }, 0);
if (cxEntries.length === 0) {
cxEl.innerHTML = '<span class="text-dim">No data yet (migration in progress)</span>';
} else {
var cxColors = {basic: '#4ade80', intermediate: '#fbbf24', advanced: '#f87171'};
var maxCx = Math.max.apply(null, cxEntries.map(function(e) { return e[1]; }));
cxEl.innerHTML = cxEntries.map(function(entry) {
var name = entry[0], count = entry[1];
var pctVal = totalCx > 0 ? (count / totalCx * 100).toFixed(0) : 0;
var barPct = (count / maxCx * 100).toFixed(1);
var color = cxColors[name] || '#888';
return '<div style="display:flex;align-items:center;gap:10px;margin:5px 0;">' +
'<span style="width:100px;text-align:right;color:' + color + ';">' + name + '</span>' +
'<div style="flex:1;height:18px;background:#1a1a1a;border-radius:3px;overflow:hidden;">' +
'<div style="height:100%;background:' + color + ';opacity:0.6;border-radius:3px;width:' + barPct + '%;"></div></div>' +
'<span style="width:80px;color:#ccc;text-align:right;">' + RECON.fmt(count) + ' (' + pctVal + '%)</span></div>';
}).join('');
}
var cxMig = document.getElementById('complexity-migration');
cxMig.textContent = RECON.fmt(totalCx) + ' / ' + RECON.fmt(data.sample_size) + ' migrated';
// Recent completions
var rtb = document.getElementById('recent-tbody');
if (data.recent_complete.length === 0) {
rtb.innerHTML = '<tr><td colspan="4" class="text-dim">None yet</td></tr>';
} else {
rtb.innerHTML = data.recent_complete.map(function(r) {
var badge = r.type === 'web' ? '<span class="badge-web">WEB</span>' : '<span class="badge-pdf">PDF</span>';
return '<tr><td>' + r.title + '</td><td>' + badge + '</td><td>' +
r.concepts + '</td><td>' + r.vectors + '</td></tr>';
}).join('');
}
});
}
function loadCharts() {
if (typeof ReconChart !== 'undefined') {
ReconChart.loadAndDraw('kb-chart', 'knowledge',
['complete', 'concepts'], ['Completed', 'Concepts'], 24);
}
}
function initSourcesToggle() {
var toggle = document.getElementById('sources-toggle');
var arrow = document.getElementById('sources-arrow');
var thead = document.getElementById('sources-thead');
var tbody = document.getElementById('sources-tbody');
var expanded = localStorage.getItem('recon-sources-expanded') === 'true';
function apply() {
var show = expanded ? '' : 'none';
thead.style.display = show;
tbody.style.display = show;
arrow.innerHTML = expanded ? '&#9660;' : '&#9654;';
}
toggle.addEventListener('click', function() {
expanded = !expanded;
localStorage.setItem('recon-sources-expanded', expanded);
apply();
});
apply();
}
document.addEventListener('DOMContentLoaded', function() {
initSourcesToggle();
RECON.startRefresh(loadDashboard, 30000);
loadCharts();
setInterval(loadCharts, 300000); // refresh charts every 5 min
});
})();

106
static/js/peertube.js Normal file
View file

@ -0,0 +1,106 @@
/* RECON PeerTube Dashboard JS */
(function() {
'use strict';
function loadPTDashboard() {
return RECON.fetchJSON('/api/peertube/dashboard').then(function(data) {
// Video states
var vs = data.video_states || {};
// PeerTube state codes: 1=published, 2=to_transcode, 3=to_import, 4=waiting_for_live, 5=live_ended, 6=to_move_to_external_storage, 7=transcoding_failed, 8=to_edit, 9=waiting_for_live_to_end
var published = vs['1'] || 0;
var inPipeline = (vs['2'] || 0) + (vs['3'] || 0) + (vs['6'] || 0) + (vs['8'] || 0);
var failed = vs['7'] || 0;
RECON.set('pt-published', RECON.fmt(published));
RECON.set('pt-in-pipeline', RECON.fmt(inPipeline));
var failEl = document.getElementById('pt-failed');
failEl.textContent = RECON.fmt(failed);
failEl.style.color = failed > 0 ? '#ff4444' : '#00ff41';
// Import rate from downloader state
var ds = data.downloader_state || {};
var rate = ds.imports_last_hour || 0;
RECON.set('pt-import-rate', RECON.fmt(rate));
// GPU
var gpu = data.gpu || {};
if (gpu.name) {
RECON.set('pt-gpu-util', gpu.utilization_gpu || '—');
RECON.set('pt-gpu-temp', gpu.temperature_gpu || '—');
var gpuPanel = document.getElementById('pt-gpu-panel');
gpuPanel.style.display = 'block';
document.getElementById('pt-gpu-detail').innerHTML =
'<strong>' + gpu.name + '</strong> | VRAM: ' +
RECON.fmt(parseInt(gpu.memory_used || 0)) + ' / ' + RECON.fmt(parseInt(gpu.memory_total || 0)) + ' MiB | ' +
'Util: ' + (gpu.utilization_gpu || '?') + '% | ' +
'Temp: ' + (gpu.temperature_gpu || '?') + '&deg;C';
} else {
RECON.set('pt-gpu-util', '—');
RECON.set('pt-gpu-temp', '—');
document.getElementById('pt-gpu-panel').style.display = 'none';
}
// Services
var svcs = data.services || {};
['downloader', 'importer', 'transcoder', 'runner'].forEach(function(s) {
var el = document.getElementById('svc-' + s);
el.className = 'svc-dot ' + (svcs[s] === 'active' ? 'active' : svcs[s] === 'inactive' ? 'inactive' : 'unknown');
});
// Pipeline dirs
var dirs = data.pipeline_dirs || {};
var storageHtml = '';
var dirOrder = ['staging', 'completed', 'transcoded', 'failed'];
var dirLabels = {staging: 'Downloaded', completed: 'Awaiting Transcode', transcoded: 'Ready to Import', failed: 'Failed'};
var dirColors = {staging: '#b45309', completed: '#0284c7', transcoded: '#7c3aed', failed: '#dc2626'};
var totalVideos = 0;
dirOrder.forEach(function(d) {
var info = dirs[d] || {};
var videos = info.videos || 0;
var bytes = info.bytes || 0;
totalVideos += videos;
storageHtml += '<div class="flex-between" style="margin:4px 0;">' +
'<span><span class="legend-dot" style="background:' + (dirColors[d] || '#555') + ';"></span>' + (dirLabels[d] || d) + '</span>' +
'<span>' + videos + ' videos / ' + RECON.fmtBytes(bytes) + '</span></div>';
});
RECON.setHTML('pt-storage-content', storageHtml);
// Pipeline bar (using video counts)
var segments = dirOrder.map(function(d) {
return {status: d, count: (dirs[d] || {}).videos || 0, color: dirColors[d], label: dirLabels[d] || d};
});
RECON.setHTML('pt-pipeline-bar', RECON.progressBar(segments, totalVideos || 1));
RECON.setHTML('pt-pipeline-legend', RECON.progressLegend(segments));
RECON.set('pt-pipeline-summary', totalVideos + ' videos in pipeline');
// Errors
var errors = data.recent_errors || [];
var errPanel = document.getElementById('pt-errors-panel');
RECON.set('pt-error-count', errors.length);
if (errors.length > 0) {
errPanel.classList.add('has-errors');
var errHtml = '';
errors.forEach(function(e) {
errHtml += '<div class="error-line">' + e + '</div>';
});
RECON.setHTML('pt-errors-content', errHtml);
} else {
errPanel.classList.remove('has-errors');
}
}).catch(function(err) {
console.error('PT dashboard error:', err);
});
}
function loadCharts() {
if (typeof ReconChart !== 'undefined') {
ReconChart.loadAndDraw('pt-chart', 'peertube',
['published', 'backlog'], ['Published', 'Backlog'], 24);
}
}
document.addEventListener('DOMContentLoaded', function() {
RECON.startRefresh(loadPTDashboard, 30000);
loadCharts();
setInterval(loadCharts, 300000);
});
})();

193
static/js/web-ingest.js Normal file
View file

@ -0,0 +1,193 @@
/* RECON Web Ingest page JS */
(function() {
'use strict';
window.showSection = function(name) {
document.getElementById('section-single').style.display = name === 'single' ? '' : 'none';
document.getElementById('section-crawl').style.display = name === 'crawl' ? '' : 'none';
document.getElementById('tab-single').className = 'btn' + (name === 'single' ? ' active' : '');
document.getElementById('tab-crawl').className = 'btn' + (name === 'crawl' ? ' active' : '');
};
window.doWebIngest = async function() {
var btn = document.getElementById('wi-btn');
var status = document.getElementById('wi-status');
var resultsDiv = document.getElementById('wi-results');
var urlText = document.getElementById('wi-urls').value.trim();
var category = document.getElementById('wi-category').value.trim() || 'Web';
if (!urlText) {
status.style.color = '#ff4444';
status.textContent = 'Enter at least one URL';
return;
}
var urls = urlText.split('\n').map(function(u) { return u.trim(); }).filter(function(u) { return u && !u.startsWith('#'); });
if (urls.length === 0) {
status.style.color = '#ff4444';
status.textContent = 'No valid URLs';
return;
}
btn.disabled = true;
status.style.color = '#ffa500';
resultsDiv.style.display = 'none';
if (urls.length === 1) {
status.textContent = 'Fetching and extracting...';
try {
var resp = await fetch('/api/ingest-url', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({ url: urls[0], category: category, process: true })
});
var data = await resp.json();
if (resp.ok || resp.status === 409) {
var color = data.status === 'duplicate' ? '#888' : '#00ff41';
status.style.color = color;
status.textContent = data.status.toUpperCase() + ': ' + (data.title || urls[0]);
resultsDiv.style.display = 'block';
resultsDiv.innerHTML = '<span style="color:' + color + ';">' + data.status.toUpperCase() + '</span><br>' +
'<span class="text-dim">Hash: ' + data.hash + '</span><br>' +
(data.page_count ? '<span class="text-dim">Pages: ' + data.page_count + '</span><br>' : '') +
(data.title ? '<span class="text-dim">Title: ' + data.title + '</span><br>' : '') +
(data.pipeline ? '<span style="color:#00ff41;">Pipeline: enriched ' + (data.pipeline.enriched || 0) + ', embedded ' + (data.pipeline.embedded || 0) + '</span>' : '');
} else {
status.style.color = '#ff4444';
status.textContent = data.error || 'Ingestion failed';
}
} catch (err) {
status.style.color = '#ff4444';
status.textContent = 'Network error: ' + err.message;
}
} else {
status.textContent = 'Processing ' + urls.length + ' URLs...';
try {
var resp = await fetch('/api/ingest-urls', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({ urls: urls, category: category, process: true })
});
var data = await resp.json();
if (resp.ok) {
var s = data.summary;
status.style.color = '#00ff41';
var batchPipe = data.pipeline && data.pipeline.enriched ? ' | enriched: ' + data.pipeline.enriched + ', embedded: ' + data.pipeline.embedded : '';
status.textContent = s.succeeded + ' new, ' + s.duplicates + ' dupes, ' + s.failed + ' failed' + batchPipe;
resultsDiv.style.display = 'block';
var html = '';
for (var i = 0; i < data.results.length; i++) {
var r = data.results[i];
var c = r.status === 'failed' ? '#ff4444' : r.status === 'duplicate' ? '#888' : '#00ff41';
html += '<div style="margin-bottom:4px;"><span style="color:' + c + ';">' +
r.status.toUpperCase() + '</span> ' + (r.title || r.url) + '</div>';
}
resultsDiv.innerHTML = html;
} else {
status.style.color = '#ff4444';
status.textContent = data.error || 'Batch ingestion failed';
}
} catch (err) {
status.style.color = '#ff4444';
status.textContent = 'Network error: ' + err.message;
}
}
btn.disabled = false;
};
window.doCrawl = async function(dryRun) {
var status = document.getElementById('crawl-status');
var resultsDiv = document.getElementById('crawl-results');
var url = document.getElementById('crawl-url').value.trim();
var category = document.getElementById('crawl-category').value.trim() || 'Web';
var maxPages = parseInt(document.getElementById('crawl-max-pages').value) || 500;
var includeRaw = document.getElementById('crawl-include').value.trim();
var excludeRaw = document.getElementById('crawl-exclude').value.trim();
if (!url) {
status.style.color = '#ff4444';
status.textContent = 'Enter a site URL';
return;
}
var include = includeRaw ? includeRaw.split(',').map(function(s) { return s.trim(); }).filter(Boolean) : null;
var exclude = excludeRaw ? excludeRaw.split(',').map(function(s) { return s.trim(); }).filter(Boolean) : null;
var btnP = document.getElementById('crawl-preview-btn');
var btnC = document.getElementById('crawl-btn');
btnP.disabled = true;
btnC.disabled = true;
status.style.color = '#ffa500';
status.textContent = dryRun ? 'Discovering URLs...' : 'Starting crawl...';
resultsDiv.style.display = 'none';
try {
var body = { url: url, category: category, max_pages: maxPages, dry_run: dryRun };
if (include) body.include = include;
if (exclude) body.exclude = exclude;
var resp = await fetch('/api/crawl', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify(body)
});
var data = await resp.json();
if (dryRun) {
var urls = data.urls || [];
status.style.color = '#00ff41';
status.textContent = urls.length + ' URLs found (' + (data.discovery_method || 'unknown') + ')';
resultsDiv.style.display = 'block';
var html = '<div style="color:#00ff41;margin-bottom:8px;">Discovery: ' + (data.discovery_method || 'unknown') + ' — ' + urls.length + ' URLs</div>';
urls.forEach(function(u, i) {
html += '<div class="text-muted">' + (i+1) + '. ' + u + '</div>';
});
resultsDiv.innerHTML = html;
} else if (data.crawl_id) {
status.style.color = '#00ff41';
status.textContent = 'Crawl started — ID: ' + data.crawl_id;
resultsDiv.style.display = 'block';
resultsDiv.innerHTML = '<div style="color:#ffa500;">Crawl running in background...</div>' +
'<div class="text-dim" style="margin-top:4px;">ID: ' + data.crawl_id + '</div>';
pollCrawl(data.crawl_id, resultsDiv);
} else {
status.style.color = '#ff4444';
status.textContent = data.error || 'Crawl failed';
}
} catch (err) {
status.style.color = '#ff4444';
status.textContent = 'Network error: ' + err.message;
}
btnP.disabled = false;
btnC.disabled = false;
};
function pollCrawl(crawlId, resultsDiv) {
var check = async function() {
try {
var resp = await fetch('/api/crawl/' + crawlId + '/status');
var data = await resp.json();
if (data.status === 'running') {
var stageText = data.stage ? ' (' + data.stage + ')' : '';
resultsDiv.innerHTML = '<div style="color:#ffa500;">Pipeline running' + stageText + '...</div>' +
'<div class="text-dim">Site: ' + (data.site || '') + '</div>';
setTimeout(check, 5000);
} else if (data.summary) {
var s = data.summary;
var pipeInfo = data.pipeline ? ' | Enriched: ' + (data.pipeline.enriched || 0) + ' | Embedded: ' + (data.pipeline.embedded || 0) : '';
resultsDiv.innerHTML = '<div style="color:#00ff41;">Pipeline complete!</div>' +
'<div class="text-dim" style="margin-top:4px;">New: ' + s.succeeded + ' | Duplicates: ' + s.duplicates + ' | Failed: ' + s.failed + ' | Total: ' + s.total + pipeInfo + '</div>';
document.getElementById('crawl-status').style.color = '#00ff41';
document.getElementById('crawl-status').textContent = 'Complete: ' + s.succeeded + ' new' + pipeInfo;
} else if (data.error) {
resultsDiv.innerHTML = '<div style="color:#ff4444;">Crawl failed: ' + data.error + '</div>';
}
} catch (err) {
resultsDiv.innerHTML += '<div style="color:#ff4444;">Poll error: ' + err.message + '</div>';
}
};
setTimeout(check, 5000);
}
showSection('single');
})();