diff --git a/src/central/gui/routes.py b/src/central/gui/routes.py index e6bb580..74a6d6b 100644 --- a/src/central/gui/routes.py +++ b/src/central/gui/routes.py @@ -1,10 +1,11 @@ """Route handlers for Central GUI.""" import base64 +import html import json import logging import re -from datetime import datetime +from datetime import datetime, timezone from typing import Any logger = logging.getLogger("central.gui.routes") @@ -2727,6 +2728,24 @@ def _parse_events_params(params) -> tuple[dict | None, str | None]: }, None +def _derive_subject(event: dict) -> str | None: + """Derive an event's plain-text subject for the JSON API. + + Renders the same per-adapter ``_event_summaries/{adapter}.html`` partial + the /events table uses (falling back to ``_default.html``), so the JSON + subject carries the same human text as the GUI's Subject cell with no + duplicated derivation logic. The partials are HTML-autoescaped for the + table (e.g. ``>`` -> ``>``); we ``html.unescape`` so JSON consumers get + plain text. Returns ``None`` when the partial yields no text -- an unknown + adapter, or an event whose source fields don't support a subject (e.g. a + wfigs row with neither county nor state). + """ + template = _get_templates().env.select_template( + [f"_event_summaries/{event.get('adapter')}.html", "_event_summaries/_default.html"] + ) + return html.unescape(template.render(event=event)).strip() or None + + async def _fetch_events(parsed_params: dict) -> EventsQueryResult: """ Fetch events from database using parsed parameters. @@ -2794,7 +2813,6 @@ async def _fetch_events(parsed_params: dict) -> EventsQueryResult: received, adapter, category, - payload->>'subject' as subject, ST_AsGeoJSON(geom) as geometry, payload as data, regions @@ -2824,17 +2842,23 @@ async def _fetch_events(parsed_params: dict) -> EventsQueryResult: if row["geometry"]: geometry = json.loads(row["geometry"]) - events.append({ + event = { "id": row["id"], "time": row["time"].isoformat(), "received": row["received"].isoformat(), "adapter": row["adapter"], "category": row["category"], - "subject": row["subject"], "geometry": geometry, "data": dict(row["data"]) if row["data"] else {}, "regions": list(row["regions"]) if row["regions"] else [], - }) + } + # Subject is derived from the inner adapter payload by rendering the + # same _event_summaries partial the /events table uses, so the JSON + # `subject` matches the GUI's Subject cell. (The CloudEvents envelope + # has no top-level `subject`; the old `payload->>'subject'` was always + # null for every consumer.) + event["subject"] = _derive_subject(event) + events.append(event) # Build next_cursor if there are more results next_cursor = None @@ -2870,6 +2894,31 @@ def _geometry_summary(geometry: dict | None) -> str: return geom_type +def _format_event_time(iso: str | None) -> str: + """Format an ISO-8601 timestamp as 'MM-DD-YYYY HH:MM UTC' (24h, no seconds).""" + if not iso: + return "" + try: + dt = datetime.fromisoformat(iso).astimezone(timezone.utc) + except (ValueError, TypeError): + return iso + return dt.strftime("%m-%d-%Y %H:%M") + " UTC" + + +def _decorate_table_events(events: list[dict]) -> None: + """Add display-only fields used by the HTML events table (in place). + + These are for the table chrome only and are deliberately NOT added in + _fetch_events, so the /events.json payload is unchanged. adapter_display + is sourced from the registry (display_name), with the bare name as fallback. + """ + display = {cls.name: cls.display_name for cls in discover_adapters().values()} + for event in events: + event["geometry_summary"] = _geometry_summary(event.get("geometry")) + event["time_human"] = _format_event_time(event.get("time")) + event["adapter_display"] = display.get(event.get("adapter"), event.get("adapter")) + + @router.get("/events.json") async def events_json(request: Request): @@ -2958,9 +3007,15 @@ async def events_list(request: Request) -> HTMLResponse: events = result.events next_cursor = result.next_cursor - # Add geometry summary to each event - for event in events: - event["geometry_summary"] = _geometry_summary(event.get("geometry")) + # Add table-only display fields (time_human, adapter_display, geometry_summary) + _decorate_table_events(events) + + # Registry-derived adapter list for the filter - - - + {% for a in adapters %} + + {% endfor %}
@@ -163,18 +170,12 @@
+ {% for a in adapters %}
-
- NWS (Weather) -
-
-
- FIRMS (Fire) -
-
-
- USGS (Quake) +
+ {{ a.display_name }}
+ {% endfor %}
@@ -189,17 +190,57 @@ var tileUrl = {{ tile_url | tojson }}; var tileAttr = {{ tile_attribution | tojson }}; - // Adapter color mapping + // Adapter color mapping — built from the registry-derived adapter list and + // the same palette the legend uses, keyed by sorted index. No adapter name + // or color is hardcoded here. + var PALETTE = {{ palette | tojson }}; var ADAPTER_COLORS = { - "nws": "#f59e0b", - "firms": "#dc2626", - "usgs_quake": "#7c3aed" + {% for a in adapters %}{{ a.name | tojson }}: PALETTE[{{ loop.index0 }} % PALETTE.length]{{ "," if not loop.last }} + {% endfor %} }; function getAdapterColor(adapter) { return ADAPTER_COLORS[adapter] || "#3388ff"; } + // Flatten arbitrarily-nested GeoJSON coordinates into a flat [lng, lat] list. + function flattenCoords(coords, out) { + if (coords.length && typeof coords[0] === "number") { + out.push(coords); + return; + } + for (var i = 0; i < coords.length; i++) { + flattenCoords(coords[i], out); + } + } + + // A geometry whose vertices all collapse to a single point (zero extent in + // both dimensions). Enrichment stores some point-like sources as degenerate + // polygons; Leaflet would draw these invisibly, so we plot a marker instead. + function isDegenerate(geom) { + if (!geom || !geom.coordinates) return false; + var pts = []; + flattenCoords(geom.coordinates, pts); + if (pts.length === 0) return false; + var minX = pts[0][0], maxX = pts[0][0], minY = pts[0][1], maxY = pts[0][1]; + for (var i = 1; i < pts.length; i++) { + if (pts[i][0] < minX) minX = pts[i][0]; + if (pts[i][0] > maxX) maxX = pts[i][0]; + if (pts[i][1] < minY) minY = pts[i][1]; + if (pts[i][1] > maxY) maxY = pts[i][1]; + } + return (maxX - minX) < 1e-9 && (maxY - minY) < 1e-9; + } + + // Mean of all vertices, returned as Leaflet [lat, lng]. + function centroidLatLng(geom) { + var pts = []; + flattenCoords(geom.coordinates, pts); + var sx = 0, sy = 0; + for (var i = 0; i < pts.length; i++) { sx += pts[i][0]; sy += pts[i][1]; } + return [sy / pts.length, sx / pts.length]; + } + // Initialize map var map = L.map("events-map").setView([39, -98], 4); @@ -275,23 +316,33 @@ var adapter = row.dataset.adapter || ""; var color = getAdapterColor(adapter); - var layer = L.geoJSON(geom, { - style: { - color: color, - weight: 2, - fillColor: color, - fillOpacity: 0.25 - }, - pointToLayer: function(feature, latlng) { - return L.circleMarker(latlng, { - radius: 8, + var markerStyle = { + radius: 8, + color: color, + weight: 2, + fillColor: color, + fillOpacity: 0.25 + }; + + // Real polygons/lines render as geometry; zero-extent geometries + // (degenerate polygons from enrichment) render as a point marker + // so every non-null geometry is actually visible on the map. + var layer; + if (isDegenerate(geom)) { + layer = L.circleMarker(centroidLatLng(geom), markerStyle); + } else { + layer = L.geoJSON(geom, { + style: { color: color, weight: 2, fillColor: color, fillOpacity: 0.25 - }); - } - }); + }, + pointToLayer: function(feature, latlng) { + return L.circleMarker(latlng, markerStyle); + } + }); + } layer.bindPopup(buildPopup(row)); layer.on("click", function() { @@ -414,17 +465,80 @@ // Fit to results button document.getElementById("fit-to-results").addEventListener("click", fitToAllLayers); + // Client-side sort of the displayed rows; state persists across HTMX swaps. + var sortState = { col: null, dir: 1 }; // dir: 1 asc, -1 desc + + function sortKey(row, col) { + // Time sorts on the ISO timestamp for true chronological order. + if (col === 1) return row.dataset.time || ""; + var cell = row.children[col]; + return cell ? cell.textContent.trim().toLowerCase() : ""; + } + + function applySort() { + if (sortState.col === null) return; + var tbody = document.querySelector("#events-rows table.events-table tbody"); + if (!tbody) return; + var pairs = []; + tbody.querySelectorAll("tr.event-row").forEach(function(r) { + pairs.push([r, r.nextElementSibling]); // main row + its detail row + }); + // Array.prototype.sort is stable (ES2019+), so equal keys keep order. + pairs.sort(function(a, b) { + var ka = sortKey(a[0], sortState.col), kb = sortKey(b[0], sortState.col); + if (ka < kb) return -sortState.dir; + if (ka > kb) return sortState.dir; + return 0; + }); + pairs.forEach(function(p) { + tbody.appendChild(p[0]); + if (p[1]) tbody.appendChild(p[1]); + }); + } + + function updateSortIndicators(ths) { + ths.forEach(function(th, idx) { + th.querySelectorAll(".sort-ind").forEach(function(s) { s.remove(); }); + if (idx === sortState.col) { + var s = document.createElement("span"); + s.className = "sort-ind"; + s.textContent = sortState.dir === 1 ? " ▲" : " ▼"; + th.appendChild(s); + } + }); + } + + function bindSortHandlers() { + var ths = document.querySelectorAll("#events-rows table.events-table thead th"); + ths.forEach(function(th, idx) { + if (idx === 0) return; // expand column is not sortable + th.style.cursor = "pointer"; + th.onclick = function() { + if (sortState.col === idx) { sortState.dir *= -1; } + else { sortState.col = idx; sortState.dir = 1; } + updateSortIndicators(ths); + applySort(); + }; + }); + updateSortIndicators(ths); + } + // Initial load - bind layers and fit bounds rebindEventLayers(); // Initial load only + bindSortHandlers(); if (false) { // DISABLED: map never auto-fits fitToAllLayers(); isInitialLoad = false; } - // Re-bind layers after HTMX swap (but do NOT fit bounds) + // Re-bind layers after HTMX swap so the map tracks the current (filtered / + // paginated) result set. Viewport is preserved — we never auto-fit here. document.body.addEventListener("htmx:afterSwap", function(evt) { if (evt.detail.target.id === "events-rows") { - // rebindEventLayers(); // DISABLED: map shows all events, only table filters + rebindEventLayers(); + // Re-bind sort handlers to the new rows and re-apply the active sort. + bindSortHandlers(); + applySort(); // Do NOT call fitToAllLayers - preserve user viewport } }); diff --git a/src/central/supervisor.py b/src/central/supervisor.py index 90913a7..26db1da 100644 --- a/src/central/supervisor.py +++ b/src/central/supervisor.py @@ -76,9 +76,12 @@ async def apply_enrichment( No-op when the adapter declares no enrichment_locations or no enrichers are registered. Uses the first (lat_path, lon_path) tuple that resolves to - a non-null coordinate pair in event.data. Each enricher's result is keyed - by enricher.name. Mutates the data dict in place (Event is frozen, but its - data dict is not — this avoids a model_copy on every published event). + a non-null coordinate pair in event.data. If no declared pair resolves to + coordinates, still attaches an all-null bundle so that every event from an + enriched adapter carries _enriched (consumers get a stable field set). + Each enricher's result is keyed by enricher.name. Mutates the data dict in + place (Event is frozen, but its data dict is not — this avoids a + model_copy on every published event). """ if not enrichment_locations or not enrichers: return @@ -93,6 +96,15 @@ async def apply_enrichment( enriched[enricher.name] = await enricher.enrich(location) event.data["_enriched"] = enriched return + # No declared pair resolved to coordinates. Still attach _enriched: each + # enricher resolves the null location to its own all-null bundle (per the + # never-raise contract), so coordless events (e.g. removal tombstones) + # carry the same shape as enriched ones. + null_location = {"lat": None, "lon": None} + enriched = {} + for enricher in enrichers: + enriched[enricher.name] = await enricher.enrich(null_location) + event.data["_enriched"] = enriched # Stream subject mappings -- derived from the registry; every stream is included # (META too: supervisor must create it in JetStream even though archive skips it). diff --git a/tests/conftest.py b/tests/conftest.py index 97a4f5d..fca89f0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,6 +13,25 @@ from unittest.mock import AsyncMock, MagicMock, patch from central.bootstrap_config import Settings +@pytest.fixture(autouse=True) +def isolate_enrichment_cache(tmp_path, monkeypatch): + """Redirect the supervisor's enrichment cache off the production path. + + `central.supervisor.ENRICHMENT_CACHE_DB_PATH` defaults to + /var/lib/central/enrichment_cache.db. Constructing a Supervisor opens it, + so without this fixture the suite writes to (or, for any user without write + access to /var/lib/central, fails on) the live cache. Point it at a + per-test temp dir so no test ever touches the production path. + """ + import central.supervisor as supervisor_mod + + monkeypatch.setattr( + supervisor_mod, + "ENRICHMENT_CACHE_DB_PATH", + tmp_path / "enrichment_cache.db", + ) + + @pytest.fixture(scope="session") def event_loop(): """Create an event loop for the test session.""" diff --git a/tests/test_apply_enrichment_coordless.py b/tests/test_apply_enrichment_coordless.py new file mode 100644 index 0000000..870485d --- /dev/null +++ b/tests/test_apply_enrichment_coordless.py @@ -0,0 +1,78 @@ +"""Regression tests for apply_enrichment's coordless path. + +Design principle: every event from an adapter that declares enrichment_locations +must carry data["_enriched"] — populated when coordinates resolve, an all-null +bundle when they don't (e.g. removal tombstones with no lat/lon). Adapters that +declare no enrichment_locations are still skipped entirely. +""" + +from datetime import datetime, timezone +from typing import Any + +import pytest + +from central.config_models import EnrichmentConfig +from central.enrichment.cache import EnrichmentCache +from central.enrichment.geocoder import GeocoderEnricher, all_null_bundle +from central.models import Event, Geo +from central.supervisor import apply_enrichment, build_enrichers + + +def _make_event(data: dict[str, Any]) -> Event: + return Event( + id="evt-1", + adapter="usgs_quake", + category="quake.event.test", + time=datetime(2026, 1, 1, tzinfo=timezone.utc), + geo=Geo(), + data=data, + ) + + +class _PopulatingBackend: + """Deterministic backend that resolves any real coords to a fixed place.""" + + async def reverse(self, lat: float, lon: float) -> dict[str, Any]: + return {**all_null_bundle(), "city": "Boise", "state": "ID"} + + +@pytest.mark.asyncio +async def test_coordless_event_with_declared_locations_gets_null_bundle(tmp_path): + """An event whose declared coord paths are all None still gets _enriched.""" + cache = EnrichmentCache(tmp_path / "enrichment_cache.db") + enrichers = build_enrichers(EnrichmentConfig(), cache) + event = _make_event( + {"latitude": None, "longitude": None, "reason": "fallen_off_current_service"} + ) + assert "_enriched" not in event.data + + await apply_enrichment(event, [("latitude", "longitude")], enrichers) + + assert event.data["_enriched"]["geocoder"] == all_null_bundle() + + +@pytest.mark.asyncio +async def test_event_with_coords_still_enriches_normally(tmp_path): + """The coord-bearing path is unchanged: the backend is consulted and its + resolved fields land in the bundle.""" + cache = EnrichmentCache(tmp_path / "enrichment_cache.db") + enricher = GeocoderEnricher(_PopulatingBackend(), cache=cache) + event = _make_event({"latitude": 43.0, "longitude": -116.0}) + + await apply_enrichment(event, [("latitude", "longitude")], [enricher]) + + bundle = event.data["_enriched"]["geocoder"] + assert bundle["state"] == "ID" + assert bundle["city"] == "Boise" + + +@pytest.mark.asyncio +async def test_adapter_with_no_enrichment_locations_still_skipped(tmp_path): + """Adapters declaring no enrichment_locations are skipped — no _enriched.""" + cache = EnrichmentCache(tmp_path / "enrichment_cache.db") + enrichers = build_enrichers(EnrichmentConfig(), cache) + event = _make_event({"latitude": 43.0, "longitude": -116.0}) + + await apply_enrichment(event, [], enrichers) + + assert "_enriched" not in event.data diff --git a/tests/test_config_source.py b/tests/test_config_source.py index bc944c1..a26a12a 100644 --- a/tests/test_config_source.py +++ b/tests/test_config_source.py @@ -12,6 +12,7 @@ from central.config_source import ( ConfigSource, DbConfigSource, ) +from central.bootstrap_config import get_settings from central.crypto import KEY_SIZE, clear_key_cache # Test database DSN @@ -31,11 +32,20 @@ def master_key_path(tmp_path_factory: pytest.TempPathFactory) -> Path: @pytest.fixture(autouse=True) -def setup_master_key(master_key_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Configure master key path for all tests.""" - clear_key_cache() +def setup_master_key(master_key_path: Path, monkeypatch: pytest.MonkeyPatch): + """Configure master key path for all tests. + + Clear get_settings (and the crypto key cache) AFTER setting the env so + crypto rebuilds from the test key regardless of suite order, and again on + teardown so the test key never leaks into a later test. See PR M-b. + """ monkeypatch.setenv("CENTRAL_DB_DSN", TEST_DB_DSN) monkeypatch.setenv("CENTRAL_MASTER_KEY_PATH", str(master_key_path)) + clear_key_cache() + get_settings.cache_clear() + yield + clear_key_cache() + get_settings.cache_clear() @pytest_asyncio.fixture diff --git a/tests/test_config_store.py b/tests/test_config_store.py index 4653e32..e80d515 100644 --- a/tests/test_config_store.py +++ b/tests/test_config_store.py @@ -13,6 +13,7 @@ import asyncpg import pytest import pytest_asyncio +from central.bootstrap_config import get_settings from central.config_store import ConfigStore from central.crypto import KEY_SIZE, clear_key_cache @@ -34,12 +35,24 @@ def master_key_path(tmp_path_factory: pytest.TempPathFactory) -> Path: @pytest.fixture(autouse=True) -def setup_master_key(master_key_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Configure master key path for all tests.""" - clear_key_cache() +def setup_master_key(master_key_path: Path, monkeypatch: pytest.MonkeyPatch): + """Configure master key path for all tests. + + CENTRAL_MASTER_KEY_PATH feeds Settings, which get_settings() lru-caches. An + earlier test can warm that cache with the default /etc/central/master.key + before this fixture runs, so the env change alone is not enough — clear + get_settings (and the crypto key cache) AFTER setting the env so crypto + rebuilds from the test key regardless of suite order, and again on teardown + so the test key never leaks into a later test. + """ monkeypatch.setenv("CENTRAL_DB_DSN", TEST_DB_DSN) monkeypatch.setenv("CENTRAL_MASTER_KEY_PATH", str(master_key_path)) monkeypatch.setenv("CENTRAL_CSRF_SECRET", "test-csrf-secret-for-testing-only-32chars") + clear_key_cache() + get_settings.cache_clear() + yield + clear_key_cache() + get_settings.cache_clear() @pytest_asyncio.fixture @@ -338,3 +351,13 @@ class TestListenerReconnect: pytest.fail("Listener did not stop after cancellation") assert listen_task.cancelled() or listen_task.done() + + +def test_master_key_path_is_isolated(master_key_path: Path) -> None: + """Contract: after setup_master_key runs, get_settings() resolves the master + key to the per-session test key — never the production /etc/central path — + regardless of suite order. Fails on the pre-fix code in a full-suite run + where get_settings was warmed with the default path by an earlier test. + """ + assert get_settings().master_key_path == master_key_path + assert get_settings().master_key_path != Path("/etc/central/master.key") diff --git a/tests/test_events_feed_frontend.py b/tests/test_events_feed_frontend.py index 2ec1700..f1ab1e7 100644 --- a/tests/test_events_feed_frontend.py +++ b/tests/test_events_feed_frontend.py @@ -1,12 +1,15 @@ """Tests for events feed frontend routes.""" +import html import json from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock, MagicMock, patch import pytest -from central.gui.routes import events_list, events_rows, events_json +from central.adapter_discovery import discover_adapters +from central.gui import templates as _gui_templates +from central.gui.routes import events_list, events_rows, events_json, _derive_subject class TestEventsFeedFrontendAuthenticated: @@ -683,4 +686,512 @@ class TestEventRowDataAttributes: assert len(context["events"]) == 1 assert context["events"][0]["adapter"] == "usgs_quake" assert context["events"][0]["category"] == "quake.event" - assert context["events"][0]["subject"] == "M4.2 Earthquake" + # `subject` is now derived from the inner payload (rendered partial), + # not a DB pass-through, so the mock's input value is no longer echoed; + # just confirm the field is present. See TestEventsJsonSubject for the + # derivation contract. + assert "subject" in context["events"][0] + + +# --- PR L-b: operator /events tab polish --------------------------------- + + +def _events_context(events): + """Minimal context for rendering _events_rows.html as a standalone fragment.""" + return { + "events": events, + "next_cursor": None, + "filter_error": None, + "filter_values": { + "adapter": "", "category": "", "since": "", "until": "", + "region_north": "", "region_south": "", "region_east": "", + "region_west": "", "limit": "50", + }, + } + + +def _event(adapter, inner=None, geometry=None): + """Build an event dict matching _fetch_events output shape. + + `inner` populates payload->data->data (the adapter-specific payload) at + event["data"]["data"]["data"], which the per-adapter partials read. + """ + return { + "id": "evt-" + adapter, + "time": "2026-05-17T12:00:00+00:00", + "received": "2026-05-17T12:00:00+00:00", + "adapter": adapter, + "category": adapter + ".test", + "subject": "subject", + "geometry": geometry, + "geometry_summary": "", + "data": {"data": {"data": inner or {}}}, + "regions": [], + } + + +def _render_rows(events): + """Render _events_rows.html through the real Jinja environment.""" + from central.gui import templates as gui_templates + return gui_templates.env.get_template("_events_rows.html").render( + **_events_context(events) + ) + + +class TestRegistryDrivenAdapterFilter: + """(A) Adapter filter