/* RECON Web Ingest page JS */ (function() { 'use strict'; window.showSection = function(name) { document.getElementById('section-single').style.display = name === 'single' ? '' : 'none'; document.getElementById('section-crawl').style.display = name === 'crawl' ? '' : 'none'; document.getElementById('tab-single').className = 'btn' + (name === 'single' ? ' active' : ''); document.getElementById('tab-crawl').className = 'btn' + (name === 'crawl' ? ' active' : ''); }; window.doWebIngest = async function() { var btn = document.getElementById('wi-btn'); var status = document.getElementById('wi-status'); var resultsDiv = document.getElementById('wi-results'); var urlText = document.getElementById('wi-urls').value.trim(); var category = document.getElementById('wi-category').value.trim() || 'Web'; if (!urlText) { status.style.color = '#ff4444'; status.textContent = 'Enter at least one URL'; return; } var urls = urlText.split('\n').map(function(u) { return u.trim(); }).filter(function(u) { return u && !u.startsWith('#'); }); if (urls.length === 0) { status.style.color = '#ff4444'; status.textContent = 'No valid URLs'; return; } btn.disabled = true; status.style.color = '#ffa500'; resultsDiv.style.display = 'none'; if (urls.length === 1) { status.textContent = 'Fetching and extracting...'; try { var resp = await fetch('/api/ingest-url', { method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({ url: urls[0], category: category, process: true }) }); var data = await resp.json(); if (resp.ok || resp.status === 409) { var color = data.status === 'duplicate' ? '#888' : '#00ff41'; status.style.color = color; status.textContent = data.status.toUpperCase() + ': ' + (data.title || urls[0]); resultsDiv.style.display = 'block'; resultsDiv.innerHTML = '' + data.status.toUpperCase() + '
' + 'Hash: ' + data.hash + '
' + (data.page_count ? 'Pages: ' + data.page_count + '
' : '') + (data.title ? 'Title: ' + data.title + '
' : '') + (data.pipeline ? 'Pipeline: enriched ' + (data.pipeline.enriched || 0) + ', embedded ' + (data.pipeline.embedded || 0) + '' : ''); } else { status.style.color = '#ff4444'; status.textContent = data.error || 'Ingestion failed'; } } catch (err) { status.style.color = '#ff4444'; status.textContent = 'Network error: ' + err.message; } } else { status.textContent = 'Processing ' + urls.length + ' URLs...'; try { var resp = await fetch('/api/ingest-urls', { method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({ urls: urls, category: category, process: true }) }); var data = await resp.json(); if (resp.ok) { var s = data.summary; status.style.color = '#00ff41'; var batchPipe = data.pipeline && data.pipeline.enriched ? ' | enriched: ' + data.pipeline.enriched + ', embedded: ' + data.pipeline.embedded : ''; status.textContent = s.succeeded + ' new, ' + s.duplicates + ' dupes, ' + s.failed + ' failed' + batchPipe; resultsDiv.style.display = 'block'; var html = ''; for (var i = 0; i < data.results.length; i++) { var r = data.results[i]; var c = r.status === 'failed' ? '#ff4444' : r.status === 'duplicate' ? '#888' : '#00ff41'; html += '
' + r.status.toUpperCase() + ' ' + (r.title || r.url) + '
'; } resultsDiv.innerHTML = html; } else { status.style.color = '#ff4444'; status.textContent = data.error || 'Batch ingestion failed'; } } catch (err) { status.style.color = '#ff4444'; status.textContent = 'Network error: ' + err.message; } } btn.disabled = false; }; window.doCrawl = async function(dryRun) { var status = document.getElementById('crawl-status'); var resultsDiv = document.getElementById('crawl-results'); var url = document.getElementById('crawl-url').value.trim(); var category = document.getElementById('crawl-category').value.trim() || 'Web'; var maxPages = parseInt(document.getElementById('crawl-max-pages').value) || 500; var includeRaw = document.getElementById('crawl-include').value.trim(); var excludeRaw = document.getElementById('crawl-exclude').value.trim(); if (!url) { status.style.color = '#ff4444'; status.textContent = 'Enter a site URL'; return; } var include = includeRaw ? includeRaw.split(',').map(function(s) { return s.trim(); }).filter(Boolean) : null; var exclude = excludeRaw ? excludeRaw.split(',').map(function(s) { return s.trim(); }).filter(Boolean) : null; var btnP = document.getElementById('crawl-preview-btn'); var btnC = document.getElementById('crawl-btn'); btnP.disabled = true; btnC.disabled = true; status.style.color = '#ffa500'; status.textContent = dryRun ? 'Discovering URLs...' : 'Starting crawl...'; resultsDiv.style.display = 'none'; try { var body = { url: url, category: category, max_pages: maxPages, dry_run: dryRun }; if (include) body.include = include; if (exclude) body.exclude = exclude; var resp = await fetch('/api/crawl', { method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify(body) }); var data = await resp.json(); if (dryRun) { var urls = data.urls || []; status.style.color = '#00ff41'; status.textContent = urls.length + ' URLs found (' + (data.discovery_method || 'unknown') + ')'; resultsDiv.style.display = 'block'; var html = '
Discovery: ' + (data.discovery_method || 'unknown') + ' — ' + urls.length + ' URLs
'; urls.forEach(function(u, i) { html += '
' + (i+1) + '. ' + u + '
'; }); resultsDiv.innerHTML = html; } else if (data.crawl_id) { status.style.color = '#00ff41'; status.textContent = 'Crawl started — ID: ' + data.crawl_id; resultsDiv.style.display = 'block'; resultsDiv.innerHTML = '
Crawl running in background...
' + '
ID: ' + data.crawl_id + '
'; pollCrawl(data.crawl_id, resultsDiv); } else { status.style.color = '#ff4444'; status.textContent = data.error || 'Crawl failed'; } } catch (err) { status.style.color = '#ff4444'; status.textContent = 'Network error: ' + err.message; } btnP.disabled = false; btnC.disabled = false; }; function pollCrawl(crawlId, resultsDiv) { var check = async function() { try { var resp = await fetch('/api/crawl/' + crawlId + '/status'); var data = await resp.json(); if (data.status === 'running') { var stageText = data.stage ? ' (' + data.stage + ')' : ''; resultsDiv.innerHTML = '
Pipeline running' + stageText + '...
' + '
Site: ' + (data.site || '') + '
'; setTimeout(check, 5000); } else if (data.summary) { var s = data.summary; var pipeInfo = data.pipeline ? ' | Enriched: ' + (data.pipeline.enriched || 0) + ' | Embedded: ' + (data.pipeline.embedded || 0) : ''; resultsDiv.innerHTML = '
Pipeline complete!
' + '
New: ' + s.succeeded + ' | Duplicates: ' + s.duplicates + ' | Failed: ' + s.failed + ' | Total: ' + s.total + pipeInfo + '
'; document.getElementById('crawl-status').style.color = '#00ff41'; document.getElementById('crawl-status').textContent = 'Complete: ' + s.succeeded + ' new' + pipeInfo; } else if (data.error) { resultsDiv.innerHTML = '
Crawl failed: ' + data.error + '
'; } } catch (err) { resultsDiv.innerHTML += '
Poll error: ' + err.message + '
'; } }; setTimeout(check, 5000); } showSection('single'); })();