From 2986dd3fd09b741588f3d2768497a504a0beab9b Mon Sep 17 00:00:00 2001 From: malice Date: Tue, 7 Jul 2026 00:49:42 -0600 Subject: [PATCH] feat(gui): no-code editor for generic data sources (+ URL preview) (#79) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Whitelist generic_sources as a config section and add a Data Sources page: add/edit/delete sources with the full field-mapping UI (items/id/lat/lon/ geometry/title paths + field_mappings list + summary template) and a server-side URL Preview that shows the endpoint's JSON so operators can map fields without knowing the structure ahead of time. Makes the generic adapter truly no-code — point it at any public REST/GeoJSON feed from the GUI. Co-authored-by: Matt Johnson Co-authored-by: Claude Opus 4.8 (1M context) --- work/dashboard-frontend/src/App.tsx | 2 + .../src/components/Layout.tsx | 1 + work/dashboard-frontend/src/lib/api.ts | 68 +++ .../src/pages/GenericSources.tsx | 562 ++++++++++++++++++ work/meshai/dashboard/api/config_routes.py | 2 + .../dashboard/api/generic_sources_routes.py | 118 ++++ work/meshai/dashboard/server.py | 2 + 7 files changed, 755 insertions(+) create mode 100644 work/dashboard-frontend/src/pages/GenericSources.tsx create mode 100644 work/meshai/dashboard/api/generic_sources_routes.py diff --git a/work/dashboard-frontend/src/App.tsx b/work/dashboard-frontend/src/App.tsx index a8fdd75..6de0848 100644 --- a/work/dashboard-frontend/src/App.tsx +++ b/work/dashboard-frontend/src/App.tsx @@ -22,6 +22,7 @@ import ScheduledBroadcasts from './pages/ScheduledBroadcasts' import MeshtasticDangerZones from './pages/MeshtasticDangerZones' import MeshCoreDangerZones from './pages/MeshCoreDangerZones' import Coverage from './pages/Coverage' +import GenericSources from './pages/GenericSources' import { ToastProvider } from './components/ToastProvider' import { DirtyProvider } from './context/DirtyContext' @@ -44,6 +45,7 @@ function App() { {/* New aggregated pages */} } /> } /> + } /> {/* De-navved routes still work */} } /> diff --git a/work/dashboard-frontend/src/components/Layout.tsx b/work/dashboard-frontend/src/components/Layout.tsx index cdd9885..40740ff 100644 --- a/work/dashboard-frontend/src/components/Layout.tsx +++ b/work/dashboard-frontend/src/components/Layout.tsx @@ -50,6 +50,7 @@ const navGroups: NavGroup[] = [ { path: '/', label: 'Dashboard', icon: LayoutDashboard }, { path: '/config', label: 'Settings', icon: Settings }, { path: '/environment', label: 'Data Feeds', icon: Cloud }, + { path: '/data-sources', label: 'Data Sources', icon: Layers }, { path: '/activity', label: 'Activity Log', icon: Activity }, { path: '/places', label: 'Places', icon: MapPin }, { path: '/coverage', label: 'Coverage', icon: Map }, diff --git a/work/dashboard-frontend/src/lib/api.ts b/work/dashboard-frontend/src/lib/api.ts index ce57195..f9fb38b 100644 --- a/work/dashboard-frontend/src/lib/api.ts +++ b/work/dashboard-frontend/src/lib/api.ts @@ -302,6 +302,74 @@ export async function updateConfig( return response.json() } +// ---- Generic (no-code) data sources ------------------------------------ + +export interface FieldMapping { + source_path: string + dest_key: string +} + +export interface GenericSource { + name: string + enabled: boolean + url: string + items_path: string + id_path: string + lat_path?: string + lon_path?: string + geometry_path?: string + title_path?: string + time_path?: string + category: string + poll_seconds: number + severity: string + field_mappings: FieldMapping[] + summary_template?: string + emoji?: string +} + +export interface GenericSourcePreview { + ok: boolean + status?: number + error?: string + sample?: string + item_count?: number | null + first_item?: string + items_path_note?: string +} + +export async function fetchGenericSources(): Promise { + return fetchJson('/api/config/generic_sources') +} + +export async function saveGenericSources( + sources: GenericSource[] +): Promise<{ saved: boolean; restart_required: boolean; changed_keys?: string[] }> { + const response = await fetch('/api/config/generic_sources', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(sources), + }) + const result = await response.json() + if (!response.ok) { + throw new Error((result as { detail?: string }).detail || `Save failed (${response.status})`) + } + return result +} + +export async function previewGenericSource( + url: string, + items_path?: string +): Promise { + const response = await fetch('/api/generic-sources/preview', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ url, items_path }), + }) + // The endpoint never raises; it returns {ok:false,error} on failure too. + return response.json() +} + export async function fetchAlerts(): Promise { return fetchJson('/api/alerts/active') } diff --git a/work/dashboard-frontend/src/pages/GenericSources.tsx b/work/dashboard-frontend/src/pages/GenericSources.tsx new file mode 100644 index 0000000..1a4298d --- /dev/null +++ b/work/dashboard-frontend/src/pages/GenericSources.tsx @@ -0,0 +1,562 @@ +import { useEffect, useState } from 'react' +import { Save, RotateCcw, Plus, Trash2, Eye, Loader2 } from 'lucide-react' +import { notifyRestartRequired } from '@/components/RestartBanner' +import { useDirty } from '@/context/DirtyContext' +import { + fetchGenericSources, + saveGenericSources, + previewGenericSource, + type GenericSource, + type FieldMapping, + type GenericSourcePreview, +} from '@/lib/api' + +const SEVERITIES = ['routine', 'priority', 'immediate'] as const + +function blankSource(n: number): GenericSource { + return { + name: `source_${n}`, + enabled: true, + url: '', + items_path: 'features', + id_path: '', + lat_path: '', + lon_path: '', + geometry_path: 'geometry', + title_path: '', + time_path: '', + category: 'generic_alert', + poll_seconds: 300, + severity: 'routine', + field_mappings: [], + summary_template: '', + emoji: '', + } +} + +// Small labelled text input used throughout the cards. +function Field({ + label, + value, + onChange, + placeholder, + type = 'text', + mono = false, +}: { + label: string + value: string | number + onChange: (v: string) => void + placeholder?: string + type?: string + mono?: boolean +}) { + return ( +
+ + onChange(e.target.value)} + className={`w-full bg-[#0d0d0d] border border-border px-2 py-1.5 text-xs ${ + mono ? 'font-mono' : '' + } text-[#e0e0e0] placeholder:text-[#555]`} + /> +
+ ) +} + +function SectionLabel({ children }: { children: React.ReactNode }) { + return ( +
+ {children} +
+ ) +} + +export default function GenericSources() { + const [sources, setSources] = useState(null) + const [original, setOriginal] = useState('') + const [loading, setLoading] = useState(true) + const [saving, setSaving] = useState(false) + const [error, setError] = useState(null) + const [success, setSuccess] = useState(null) + const { setDirty } = useDirty() + + // Per-card preview state, keyed by source index. + const [previews, setPreviews] = useState>({}) + const [previewing, setPreviewing] = useState>({}) + + useEffect(() => { + document.title = 'Data Sources — MeshAI' + fetchGenericSources() + .then((data) => { + const list = Array.isArray(data) ? data : [] + // Normalize so every field the form binds to is defined (avoids + // controlled/uncontrolled input churn). + const norm = list.map((s, i) => ({ ...blankSource(i + 1), ...s })) + setSources(norm) + setOriginal(JSON.stringify(norm)) + }) + .catch((e) => setError(e instanceof Error ? e.message : String(e))) + .finally(() => setLoading(false)) + }, []) + + const hasChanges = sources !== null && JSON.stringify(sources) !== original + + useEffect(() => { + setDirty(hasChanges) + return () => setDirty(false) + }, [hasChanges, setDirty]) + + const update = (idx: number, patch: Partial) => { + setSources((prev) => + prev ? prev.map((s, i) => (i === idx ? { ...s, ...patch } : s)) : prev, + ) + } + + const addSource = () => { + setSources((prev) => [...(prev ?? []), blankSource((prev?.length ?? 0) + 1)]) + } + + const deleteSource = (idx: number) => { + setSources((prev) => (prev ? prev.filter((_, i) => i !== idx) : prev)) + setPreviews((prev) => { + const next = { ...prev } + delete next[idx] + return next + }) + } + + const addMapping = (idx: number) => { + setSources((prev) => + prev + ? prev.map((s, i) => + i === idx + ? { ...s, field_mappings: [...s.field_mappings, { source_path: '', dest_key: '' }] } + : s, + ) + : prev, + ) + } + + const updateMapping = (idx: number, mIdx: number, patch: Partial) => { + setSources((prev) => + prev + ? prev.map((s, i) => + i === idx + ? { + ...s, + field_mappings: s.field_mappings.map((m, j) => + j === mIdx ? { ...m, ...patch } : m, + ), + } + : s, + ) + : prev, + ) + } + + const removeMapping = (idx: number, mIdx: number) => { + setSources((prev) => + prev + ? prev.map((s, i) => + i === idx + ? { ...s, field_mappings: s.field_mappings.filter((_, j) => j !== mIdx) } + : s, + ) + : prev, + ) + } + + const runPreview = async (idx: number) => { + if (!sources) return + const src = sources[idx] + setPreviewing((p) => ({ ...p, [idx]: true })) + try { + const res = await previewGenericSource(src.url, src.items_path) + setPreviews((p) => ({ ...p, [idx]: res })) + } catch (e) { + setPreviews((p) => ({ + ...p, + [idx]: { ok: false, error: e instanceof Error ? e.message : String(e) }, + })) + } finally { + setPreviewing((p) => ({ ...p, [idx]: false })) + } + } + + const discard = () => { + if (original) { + setSources(JSON.parse(original) as GenericSource[]) + setPreviews({}) + } + } + + const save = async () => { + if (!sources) return + setSaving(true) + setError(null) + setSuccess(null) + try { + const result = await saveGenericSources(sources) + setOriginal(JSON.stringify(sources)) + setSuccess('Data sources saved') + setTimeout(() => setSuccess(null), 3000) + if (result.restart_required) { + notifyRestartRequired(Array.isArray(result.changed_keys) ? result.changed_keys : []) + } + } catch (e) { + setError(e instanceof Error ? e.message : 'Save failed') + } finally { + setSaving(false) + } + } + + if (loading) { + return ( +
+ Loading data sources… +
+ ) + } + if (!sources) { + return ( +
+ {error || 'No config'} +
+ ) + } + + const actionButtons = ( +
+ + +
+ ) + + return ( +
+ {/* Page description + action bar */} +
+

+ Point MeshAI at any public REST or GeoJSON feed — no code. Each source polls a URL, + maps its JSON fields to an event via dotted paths, and broadcasts through the normal + coverage-gated pipeline. Use Preview to fetch a + URL and read its structure before filling in the paths. +

+ {hasChanges && actionButtons} +
+ + {error &&
{error}
} + {success &&
{success}
} + + {sources.length === 0 && ( +
+ No data sources yet. Click “Add source” to wire up a public feed. +
+ )} + +
+ {sources.map((src, i) => { + const preview = previews[i] + const isPreviewing = previewing[i] + return ( +
+ {/* Header: name + enabled + delete */} +
+ update(i, { name: e.target.value })} + placeholder="source name (unique id)" + className="flex-1 bg-[#0d0d0d] border border-border px-2 py-1.5 text-sm font-medium text-[#e0e0e0]" + /> + + +
+ + {/* Basics */} + Basics +
+
+ update(i, { url: v })} + placeholder="https://example.com/api/feed.geojson" + mono + /> +
+
+ update(i, { poll_seconds: parseInt(v, 10) || 0 })} + /> +
+
+ update(i, { category: v })} + placeholder="generic_alert" + /> +
+
+ + +
+
+ update(i, { emoji: v })} + placeholder="⚡" + /> +
+
+ + {/* Extraction */} + Extraction +
+ update(i, { items_path: v })} + placeholder="features · object.outages" + mono + /> + update(i, { id_path: v })} + placeholder="id · omsOutageId · properties.id" + mono + /> +
+ + {/* Location */} + Location — GeoJSON geometry OR lat + lon paths +
+ update(i, { geometry_path: v })} + placeholder="geometry" + mono + /> + update(i, { lat_path: v })} + placeholder="properties.lat" + mono + /> + update(i, { lon_path: v })} + placeholder="properties.lon" + mono + /> +
+ + {/* Display */} + Display +
+ update(i, { title_path: v })} + placeholder="properties.headline" + mono + /> + update(i, { time_path: v })} + placeholder="properties.updated" + mono + /> +
+ update(i, { summary_template: v })} + placeholder="⚡ Power out — {customers} affected, ETA {eta}" + mono + /> +
+
+ + {/* Field mappings */} +
+
+
+ Field Mappings +
+ +
+ {src.field_mappings.length === 0 ? ( +

+ No mappings. Each mapping pulls a dotted source_path from an item + into a dest_key you can reference in the summary template. +

+ ) : ( +
+ {src.field_mappings.map((m, mIdx) => ( +
+ updateMapping(i, mIdx, { source_path: e.target.value })} + placeholder="source_path (e.g. properties.customers)" + className="flex-1 bg-[#0d0d0d] border border-border px-2 py-1.5 text-xs font-mono text-[#e0e0e0] placeholder:text-[#555]" + /> + → + updateMapping(i, mIdx, { dest_key: e.target.value })} + placeholder="dest_key (e.g. customers)" + className="flex-1 bg-[#0d0d0d] border border-border px-2 py-1.5 text-xs font-mono text-[#e0e0e0] placeholder:text-[#555]" + /> + +
+ ))} +
+ )} +
+ + {/* Preview */} +
+
+
+ Preview +
+ +
+ {preview && ( +
+ {preview.ok ? ( +
+ HTTP {preview.status ?? '200'} + {typeof preview.item_count === 'number' && ( + + {' '} + — items_path resolved {preview.item_count} item + {preview.item_count === 1 ? '' : 's'} + + )} +
+ ) : ( +
{preview.error}
+ )} + {preview.items_path_note && ( +
+ {preview.items_path_note} +
+ )} + {preview.first_item && ( +
+
+ First item +
+ 
+                          {preview.first_item}
+
+
+ )} + {preview.sample && ( +
+ + Raw response + + 
+                          {preview.sample}
+
+
+ )} +
+ )} +
+
+ ) + })} +
+ + {/* Add + bottom save bar */} +
+ + {hasChanges && actionButtons} +
+
+ ) +} diff --git a/work/meshai/dashboard/api/config_routes.py b/work/meshai/dashboard/api/config_routes.py index cd485e0..1682b92 100644 --- a/work/meshai/dashboard/api/config_routes.py +++ b/work/meshai/dashboard/api/config_routes.py @@ -34,6 +34,7 @@ RESTART_REQUIRED_SECTIONS = { "dashboard", "environmental", "coverage", + "generic_sources", } # Valid config section names @@ -58,6 +59,7 @@ VALID_SECTIONS = { "dashboard", "danger_zones", "coverage", + "generic_sources", } diff --git a/work/meshai/dashboard/api/generic_sources_routes.py b/work/meshai/dashboard/api/generic_sources_routes.py new file mode 100644 index 0000000..af069f7 --- /dev/null +++ b/work/meshai/dashboard/api/generic_sources_routes.py @@ -0,0 +1,118 @@ +"""Generic-source URL preview endpoint. + +Powers the no-code Data Sources editor: the operator pastes a public REST / +GeoJSON URL, hits Preview, and the SERVER (not the browser — avoids CORS) fetches +it, so they can SEE the JSON structure and figure out the dotted paths +(``items_path`` / ``id_path`` / ``lat_path`` / ``field_mappings`` …) without +knowing the schema ahead of time. + +The single route never raises: on any error it returns +``{"ok": false, "error": ...}`` so the UI can show the problem inline. +""" + +import json +import logging +from urllib.error import HTTPError, URLError +from urllib.request import Request as UrlRequest, urlopen + +from fastapi import APIRouter, Request +from starlette.concurrency import run_in_threadpool + +# Reuse the adapter's dotted-path walker so Preview resolves items_path exactly +# the way the running GenericHttpAdapter will. +from meshai.env.generic_http import _dig + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["generic-sources"]) + +# Cap the pretty-printed JSON we ship back so a huge feed can't bloat the +# response / freeze the browser. ~8 KB is plenty to read the structure. +_SAMPLE_MAX = 8192 +_ITEM_MAX = 2048 +_FETCH_TIMEOUT = 30 +_USER_AGENT = "MeshAI/1.0" + + +def _truncate(text: str, limit: int) -> str: + if len(text) <= limit: + return text + return text[:limit] + f"\n… (truncated, {len(text)} chars total)" + + +def _fetch_preview(url: str, items_path: str) -> dict: + """Blocking fetch + parse. NEVER raises — always returns a result dict.""" + if not url or not isinstance(url, str): + return {"ok": False, "error": "A url is required."} + + try: + req = UrlRequest(url, headers={"User-Agent": _USER_AGENT}) + with urlopen(req, timeout=_FETCH_TIMEOUT) as resp: + status = getattr(resp, "status", None) or resp.getcode() + raw = resp.read() + except HTTPError as e: + return {"ok": False, "status": e.code, "error": f"HTTP {e.code}: {e.reason}"} + except URLError as e: + return {"ok": False, "error": f"Could not reach URL: {e.reason}"} + except Exception as e: # timeout, DNS, connection reset, … + return {"ok": False, "error": f"Fetch failed: {e}"} + + try: + text = raw.decode("utf-8", errors="replace") + except Exception as e: + return {"ok": False, "status": status, "error": f"Decode failed: {e}"} + + try: + data = json.loads(text) + except Exception as e: + # Not JSON — still show a raw sample so the operator sees what came back. + return { + "ok": False, + "status": status, + "error": f"Response is not valid JSON: {e}", + "sample": _truncate(text, _SAMPLE_MAX), + } + + result: dict = { + "ok": True, + "status": status, + "sample": _truncate(json.dumps(data, indent=2, ensure_ascii=False), _SAMPLE_MAX), + } + + # If the operator gave an items_path, resolve it so they can confirm the + # array + see a single item's shape (the fields they'll map). + if items_path: + items = _dig(data, items_path) + if isinstance(items, list): + result["item_count"] = len(items) + if items: + result["first_item"] = _truncate( + json.dumps(items[0], indent=2, ensure_ascii=False), _ITEM_MAX + ) + else: + result["item_count"] = None + result["items_path_note"] = ( + f"items_path '{items_path}' did not resolve to a list " + f"(got {type(items).__name__})." + ) + + return result + + +@router.post("/generic-sources/preview") +async def preview_generic_source(request: Request): + """Server-side fetch of a candidate feed URL for the no-code editor. + + Body: ``{"url": str, "items_path"?: str}``. + Returns ``{ok, status?, error?, sample?, item_count?, first_item?}`` — never + raises; failures come back as ``{ok: false, error: ...}``. + """ + try: + body = await request.json() + except Exception: + body = {} + + url = (body or {}).get("url") + items_path = (body or {}).get("items_path") or "" + + return await run_in_threadpool(_fetch_preview, url, items_path) diff --git a/work/meshai/dashboard/server.py b/work/meshai/dashboard/server.py index d07441b..e79558c 100644 --- a/work/meshai/dashboard/server.py +++ b/work/meshai/dashboard/server.py @@ -52,6 +52,7 @@ def create_app() -> FastAPI: # Import and include API routers from .api.system_routes import router as system_router from .api.config_routes import router as config_router + from .api.generic_sources_routes import router as generic_sources_router from .api.mesh_routes import router as mesh_router from .api.mesh_send_routes import router as mesh_send_router from .api.env_routes import router as env_router @@ -65,6 +66,7 @@ def create_app() -> FastAPI: app.include_router(adapter_config_router, prefix="/api") app.include_router(curation_router, prefix="/api") app.include_router(config_router, prefix="/api") + app.include_router(generic_sources_router, prefix="/api") app.include_router(mesh_router, prefix="/api") app.include_router(mesh_send_router, prefix="/api") app.include_router(env_router, prefix="/api")