matt/recon
1
0
Fork 0
mirror of https://github.com/zvx-echo6/recon.git synced 2026-05-20 06:34:40 +02:00
Code Issues Projects Releases Packages Wiki Activity

Initial commit: RECON codebase baseline

Browse source 
Current state of the pipeline code as of 2026-04-14 (Phase 1 scaffolding complete).
Config has new_pipeline.enabled=false and crawler.sites=[] per refactor plan.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt 2026-04-14 14:57:23 +00:00
commit 563c16bb71
59 changed files with 18327 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

94
templates/settings/cookies.html Normal file
View file

@ -0,0 +1,94 @@
{% extends "base.html" %}
{% block content %}
<h3 class="section-title mb-16">YouTube Cookies</h3>
<div class="panel">
<div id="cookie-status" style="margin-bottom:16px;font-size:12px;color:#666;">Loading cookie status...</div>
<div class="mb-16">
<label class="text-dim text-xs" style="text-transform:uppercase;display:block;margin-bottom:4px;">Cookies.txt File (Netscape format)</label>
<input type="file" id="cookie-file" accept=".txt"
style="background:#0a0a0a;border:1px solid #333;color:#c0c0c0;padding:8px;width:100%;font-family:inherit;">
</div>
<button class="btn" id="cookie-btn" onclick="uploadCookies()">Upload Cookies</button>
<span id="cookie-upload-status" style="margin-left:12px;font-size:12px;"></span>
<div id="cookie-result" style="display:none;background:#0a0a0a;border:1px solid #222;padding:12px;margin-top:16px;font-size:11px;white-space:pre-wrap;color:#888;max-height:200px;overflow-y:auto;"></div>
</div>
{% endblock %}
{% block scripts %}
<script>
async function loadCookieStatus() {
try {
var resp = await fetch('/api/cookies/status');
var data = await resp.json();
if (resp.ok) {
var age = data.age_hours;
var ageStr, ageColor;
if (age < 24) {
ageStr = Math.round(age) + ' hours ago';
ageColor = '#00ff41';
} else {
var days = Math.round(age / 24);
ageStr = days + ' days ago';
ageColor = days > 14 ? '#ff4444' : days > 7 ? '#ffa500' : '#00ff41';
}
var html = '<span style="color:' + ageColor + ';">Last updated: ' + ageStr + '</span>';
if (data.is_stale) {
html += ' <span style="color:#ff4444;font-weight:bold;">[STALE - cookies likely expired]</span>';
}
if (data.recent_rate_limits > 0) {
html += '<br><span style="color:#ffa500;">YouTube rate limits in last 30min: ' + data.recent_rate_limits + '</span>';
}
html += '<br><span class="text-faint">Downloader: ' + (data.downloader_active ? 'active' : 'stopped') + '</span>';
document.getElementById('cookie-status').innerHTML = html;
} else {
document.getElementById('cookie-status').innerHTML = '<span class="text-red">Could not check cookie status</span>';
}
} catch(e) {
document.getElementById('cookie-status').innerHTML = '<span class="text-red">Error: ' + e.message + '</span>';
}
}
async function uploadCookies() {
var fileInput = document.getElementById('cookie-file');
var btn = document.getElementById('cookie-btn');
var status = document.getElementById('cookie-upload-status');
var result = document.getElementById('cookie-result');
if (!fileInput.files.length) {
status.style.color = '#ff4444';
status.textContent = 'No file selected';
return;
}
btn.disabled = true;
status.style.color = '#ffa500';
status.textContent = 'Uploading and testing cookies...';
result.style.display = 'none';
var formData = new FormData();
formData.append('file', fileInput.files[0]);
try {
var resp = await fetch('/api/cookies/upload', { method: 'POST', body: formData });
var data = await resp.json();
if (data.ok) {
status.style.color = '#00ff41';
status.textContent = 'Cookies updated and verified';
result.style.display = 'block';
result.style.borderColor = '#00ff41';
result.innerHTML = '<span style="color:#00ff41;">SUCCESS</span><br>' + (data.test_output || '') + '<br>Data lines: ' + data.data_lines;
loadCookieStatus();
} else {
status.style.color = data.error ? '#ff4444' : '#ffa500';
status.textContent = data.error || data.message || 'Upload issue';
if (data.test_output) {
result.style.display = 'block';
result.style.borderColor = '#ff4444';
result.textContent = data.test_output;
}
}
} catch(e) {
status.style.color = '#ff4444';
status.textContent = 'Network error: ' + e.message;
}
btn.disabled = false;
}
loadCookieStatus();
</script>
{% endblock %}

68
templates/settings/health.html Normal file
View file

@ -0,0 +1,68 @@
{% extends "base.html" %}
{% block content %}
<h3 class="section-title mb-16">Service Health</h3>
<div id="health-grid" class="stat-grid" style="grid-template-columns:repeat(auto-fit, minmax(250px, 1fr));">
<div class="stat-card">
<div class="label">Qdrant</div>
<div class="value text-small" id="h-qdrant"><span class="svc-dot unknown"></span>Checking...</div>
</div>
<div class="stat-card">
<div class="label">TEI Embeddings</div>
<div class="value text-small" id="h-tei"><span class="svc-dot unknown"></span>Checking...</div>
</div>
<div class="stat-card">
<div class="label">NFS Mount</div>
<div class="value text-small" id="h-nfs"><span class="svc-dot unknown"></span>Checking...</div>
</div>
<div class="stat-card">
<div class="label">Gemini API</div>
<div class="value text-small" id="h-gemini"><span class="svc-dot unknown"></span>Checking...</div>
</div>
</div>
<h3 class="section-title mt-24">Pipeline Status</h3>
<div id="h-pipeline" class="panel text-small text-dim">Loading...</div>
{% endblock %}
{% block scripts %}
<script>
async function loadHealth() {
try {
var resp = await fetch('/api/health');
var data = await resp.json();
var c = data.components || {};
function dot(status) {
var cls = status === 'up' ? 'active' : (status === 'configured' ? 'active' : 'inactive');
return '<span class="svc-dot ' + cls + '"></span>';
}
var q = c.qdrant || {};
document.getElementById('h-qdrant').innerHTML = dot(q.status) + (q.status === 'up' ? 'Online — ' + RECON.fmt(q.vectors) + ' vectors' : 'Offline' + (q.error ? ' — ' + q.error : ''));
var t = c.tei || {};
document.getElementById('h-tei').innerHTML = dot(t.status) + (t.status === 'up' ? 'Online' : 'Offline' + (t.error ? ' — ' + t.error : ''));
var n = c.nfs || {};
document.getElementById('h-nfs').innerHTML = dot(n.status) + (n.status === 'up' ? 'Mounted' : 'Not mounted');
var g = c.gemini || {};
document.getElementById('h-gemini').innerHTML = dot(g.status === 'configured' ? 'up' : 'down') + (g.status === 'configured' ? g.keys + ' keys configured' : 'No keys');
// Pipeline
var p = data.pipeline || {};
var html = '';
Object.keys(p).forEach(function(k) {
html += '<div style="margin:4px 0;"><span class="status status-' + k + '">' + k + '</span>: ' + p[k] + '</div>';
});
document.getElementById('h-pipeline').innerHTML = html || '<span class="text-dim">No pipeline data</span>';
} catch(e) {
document.getElementById('h-qdrant').innerHTML = '<span class="svc-dot inactive"></span>Error: ' + e.message;
}
}
document.addEventListener('DOMContentLoaded', function() {
RECON.startRefresh(loadHealth, 30000);
});
</script>
{% endblock %}

137
templates/settings/keys.html Normal file
View file

@ -0,0 +1,137 @@
{% extends "base.html" %}
{% block content %}
<h3 class="section-title mb-16">API Keys</h3>
<div style="margin-bottom:20px;">
<button class="btn" onclick="validateAll()" id="btn-validate">Validate All</button>
<button class="btn" onclick="reloadKeys()" style="margin-left:8px;">Reload from .env</button>
<button class="btn btn-warn" onclick="restartService()" style="margin-left:8px;">Restart Service</button>
<span id="validate-status" style="margin-left:12px;color:#666;font-size:12px;"></span>
</div>
<table id="keys-table">
<tr><th>#</th><th>Key</th><th>Status</th><th>Calls</th><th>Errors</th><th>Last Used</th><th>Actions</th></tr>
{% for k in keys_data %}
<tr id="key-row-{{ k.index }}">
<td>{{ k.index + 1 }}</td>
<td class="mono text-small">{{ k.masked }}</td>
<td>
{% if k.valid is true %}
<span class="text-green">Valid</span>
{% elif k.valid is false %}
<span class="text-red">Invalid</span>
{% else %}
<span class="text-dim">&mdash;</span>
{% endif %}
</td>
<td>{{ k.calls }}</td>
<td class="{% if k.errors %}text-red{% else %}text-muted{% endif %}">{{ k.errors }}</td>
<td class="text-dim text-xs">{{ k.last_used or '&mdash;' }}</td>
<td>
<button class="btn text-xs" onclick="validateKey({{ k.index }})">Test</button>
<button class="btn btn-danger text-xs" onclick="removeKey({{ k.index }})">Remove</button>
</td>
</tr>
{% endfor %}
</table>
<div style="margin-top:24px;border-top:1px solid #222;padding-top:16px;">
<h4 class="text-muted" style="margin-bottom:12px;">Add Key</h4>
<div class="flex gap-8" style="align-items:center;">
<input type="text" id="new-key" placeholder="Paste Gemini API key..."
style="flex:1;background:#1a1a1a;border:1px solid #333;color:#ccc;padding:8px 12px;border-radius:4px;font-family:monospace;font-size:13px;">
<button class="btn" onclick="addKey()">Add</button>
</div>
<div id="add-result" style="margin-top:8px;font-size:12px;"></div>
</div>
<div style="margin-top:24px;border-top:1px solid #222;padding-top:16px;">
<h4 class="text-muted" style="margin-bottom:12px;">Replace Key</h4>
<div class="flex gap-8" style="align-items:center;">
<input type="number" id="replace-index" placeholder="#" min="0" max="9"
style="width:50px;background:#1a1a1a;border:1px solid #333;color:#ccc;padding:8px;border-radius:4px;text-align:center;">
<input type="text" id="replace-key" placeholder="New Gemini API key..."
style="flex:1;background:#1a1a1a;border:1px solid #333;color:#ccc;padding:8px 12px;border-radius:4px;font-family:monospace;font-size:13px;">
<button class="btn" onclick="replaceKey()">Replace</button>
</div>
<div id="replace-result" style="margin-top:8px;font-size:12px;"></div>
</div>
{% endblock %}
{% block scripts %}
<script>
async function validateAll() {
document.getElementById('btn-validate').disabled = true;
document.getElementById('validate-status').textContent = 'Validating...';
try {
var r = await fetch('/api/keys/validate', {method:'POST'});
var data = await r.json();
document.getElementById('validate-status').textContent = 'Done — ' + data.results.filter(function(r){return r.valid;}).length + '/' + data.results.length + ' valid';
setTimeout(function() { location.reload(); }, 1000);
} catch(e) {
document.getElementById('validate-status').textContent = 'Error: ' + e;
}
document.getElementById('btn-validate').disabled = false;
}
async function validateKey(idx) {
try {
var r = await fetch('/api/keys/' + idx + '/validate', {method:'POST'});
var data = await r.json();
alert('Key ' + (idx+1) + ': ' + data.message);
location.reload();
} catch(e) { alert('Error: ' + e); }
}
async function removeKey(idx) {
if (!confirm('Remove key ' + (idx+1) + '? Pipeline needs at least 1 key.')) return;
try {
var r = await fetch('/api/keys/' + idx, {method:'DELETE'});
var data = await r.json();
if (data.error) { alert(data.error); return; }
location.reload();
} catch(e) { alert('Error: ' + e); }
}
async function addKey() {
var key = document.getElementById('new-key').value.trim();
if (!key) return;
try {
var r = await fetch('/api/keys', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify({key:key})});
var data = await r.json();
if (data.error) { document.getElementById('add-result').innerHTML = '<span class="text-red">' + data.error + '</span>'; return; }
document.getElementById('add-result').innerHTML = '<span class="text-green">Added at position ' + (data.index+1) + '</span>';
setTimeout(function() { location.reload(); }, 1000);
} catch(e) { document.getElementById('add-result').innerHTML = '<span class="text-red">' + e + '</span>'; }
}
async function replaceKey() {
var idx = parseInt(document.getElementById('replace-index').value) - 1;
var key = document.getElementById('replace-key').value.trim();
if (isNaN(idx) || !key) return;
try {
var r = await fetch('/api/keys/' + idx, {method:'PUT', headers:{'Content-Type':'application/json'}, body:JSON.stringify({key:key})});
var data = await r.json();
if (data.error) { document.getElementById('replace-result').innerHTML = '<span class="text-red">' + data.error + '</span>'; return; }
document.getElementById('replace-result').innerHTML = '<span class="text-green">Replaced key ' + (idx+1) + '</span>';
setTimeout(function() { location.reload(); }, 1000);
} catch(e) { document.getElementById('replace-result').innerHTML = '<span class="text-red">' + e + '</span>'; }
}
async function restartService() {
if (!confirm('Restart RECON service? Pipeline will pause for ~10 seconds.')) return;
document.getElementById('validate-status').textContent = 'Restarting...';
try {
await fetch('/api/service/restart', {method:'POST'});
} catch(e) {}
document.getElementById('validate-status').innerHTML = '<span style="color:#ff8800;">Restarting... page will reload in 10s</span>';
setTimeout(function() { location.reload(); }, 30000);
}
async function reloadKeys() {
try {
var r = await fetch('/api/keys/reload', {method:'POST'});
var data = await r.json();
alert('Reloaded ' + data.count + ' key(s) from .env');
location.reload();
} catch(e) { alert('Error: ' + e); }
}
</script>
{% endblock %}

97
templates/settings/vpn.html Normal file
View file

@ -0,0 +1,97 @@
{% extends "base.html" %}
{% block content %}
<h3 class="section-title mb-16">NordVPN</h3>
<div class="panel">
<div id="vpn-status" style="margin-bottom:16px;font-size:12px;color:#666;">Loading VPN status...</div>
<div class="flex gap-8" style="flex-wrap:wrap;margin-bottom:12px;">
<button class="btn" onclick="vpnRotate()" id="vpn-rotate-btn">Rotate</button>
<button class="btn" onclick="vpnDisconnect()" id="vpn-disconnect-btn">Disconnect</button>
<select id="vpn-country" style="background:#0a0a0a;border:1px solid #333;color:#c0c0c0;padding:6px;font-family:inherit;font-size:12px;">
<option value="United_States">United States</option>
<option value="Canada">Canada</option>
<option value="United_Kingdom">United Kingdom</option>
<option value="Germany">Germany</option>
<option value="Netherlands">Netherlands</option>
<option value="Sweden">Sweden</option>
</select>
<button class="btn" onclick="vpnConnect()" id="vpn-connect-btn">Connect</button>
</div>
<span id="vpn-action-status" style="font-size:12px;"></span>
<details style="margin-top:16px;">
<summary class="text-faint" style="cursor:pointer;font-size:11px;">Setup (one-time)</summary>
<div style="margin-top:8px;">
<input type="password" id="vpn-token" placeholder="NordVPN token"
style="background:#0a0a0a;border:1px solid #333;color:#c0c0c0;padding:6px;width:300px;font-family:inherit;font-size:12px;">
<button class="btn" onclick="vpnLogin()">Login</button>
<span id="vpn-login-status" style="font-size:11px;margin-left:8px;"></span>
</div>
</details>
</div>
{% endblock %}
{% block scripts %}
<script>
async function loadVpnStatus() {
try {
var resp = await fetch('/api/vpn/status');
var data = await resp.json();
if (resp.ok) {
var dot = data.connected ? '<span style="color:#00ff41;">&#9679;</span>' : '<span style="color:#ff4444;">&#9679;</span>';
var html = dot + ' ' + (data.connected ? 'Connected' : 'Disconnected');
if (data.connected) {
html += ' &mdash; <span style="color:#00ff41;">' + data.country + '</span>';
html += ' <span class="text-faint">(' + data.ip + ')</span>';
}
if (data.rotations_today > 0) {
html += '<br><span class="text-faint">Rotations today: ' + data.rotations_today + '</span>';
}
document.getElementById('vpn-status').innerHTML = html;
}
} catch(e) {
document.getElementById('vpn-status').innerHTML = '<span class="text-red">Error: ' + e.message + '</span>';
}
}
async function vpnAction(url, opts, statusEl) {
var el = document.getElementById(statusEl || 'vpn-action-status');
el.style.color = '#ffa500';
el.textContent = 'Working...';
try {
var resp = await fetch(url, opts);
var data = await resp.json();
if (data.ok) {
el.style.color = '#00ff41';
el.textContent = data.country ? (data.country + ' (' + data.ip + ')') : (data.message || 'Done');
} else {
el.style.color = '#ff4444';
el.textContent = data.error || data.message || 'Failed';
}
loadVpnStatus();
} catch(e) {
el.style.color = '#ff4444';
el.textContent = 'Error: ' + e.message;
}
}
function vpnRotate() { vpnAction('/api/vpn/rotate', {method:'POST'}); }
function vpnDisconnect() { vpnAction('/api/vpn/disconnect', {method:'POST'}); }
function vpnConnect() {
var country = document.getElementById('vpn-country').value;
vpnAction('/api/vpn/connect', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({country: country})
});
}
function vpnLogin() {
var token = document.getElementById('vpn-token').value;
if (!token) return;
vpnAction('/api/vpn/login', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({token: token})
}, 'vpn-login-status');
}
loadVpnStatus();
</script>
{% endblock %}