From c1c3d2576dcfb9a1e51eb0cc5ebe83ca9414d5f3 Mon Sep 17 00:00:00 2001 From: malice Date: Sun, 14 Jun 2026 13:16:48 -0600 Subject: [PATCH 01/17] v0.14.4: propagate WFIGS perimeter updates (dedup key includes attr_ModifiedOnDateTime_dt) (#110) Sibling fix to #109/v0.14.3 for wfigs_perimeters. Dedup key was bare attr_IrwinID, silencing every geometry refinement after first publish. Fix: id = f"{irwin_id}:{attr_ModifiedOnDateTime_dt}" (prefixed field; no recency filter so None path needs no monkeypatch). No new field/event-type/subject/migration. Closes the v0.14.3 deferred follow-up. --- src/central/adapters/wfigs_perimeters.py | 10 +- tests/test_wfigs.py | 116 ++++++++++++++++++++++- 2 files changed, 123 insertions(+), 3 deletions(-) diff --git a/src/central/adapters/wfigs_perimeters.py b/src/central/adapters/wfigs_perimeters.py index 541ed17..fd1f944 100644 --- a/src/central/adapters/wfigs_perimeters.py +++ b/src/central/adapters/wfigs_perimeters.py @@ -287,9 +287,15 @@ class WFIGSPerimetersAdapter(SourceAdapter): ) # Build event with geometry in data - # Use normalized field names in event data for consistency + # Use normalized field names in event data for consistency. + # v0.14.4: the dedup key includes attr_ModifiedOnDateTime_dt so each + # genuine upstream perimeter refinement mints a new id and + # republishes, while identical re-polls (same modified time) still + # dedup. The bare IrwinID published a perimeter once and silenced + # every later geometry refinement (same fix as v0.14.3 for the + # sibling wfigs_incidents adapter). event = Event( - id=irwin_id, + id=f"{irwin_id}:{props.get('attr_ModifiedOnDateTime_dt')}", adapter=self.name, category=f"fire.perimeter.{incident_type}", time=discovery_time or datetime.now(timezone.utc), diff --git a/tests/test_wfigs.py b/tests/test_wfigs.py index 6d537dc..d9f910e 100644 --- a/tests/test_wfigs.py +++ b/tests/test_wfigs.py @@ -690,7 +690,9 @@ class TestWFIGSPerimetersAdapter: assert len(events) == 1 event = events[0] - assert event.id == "GUID-001-GLACIER" + # v0.14.4: dedup key is IrwinID + attr_ModifiedOnDateTime_dt so perimeter + # refinements republish (see TestWFIGSPerimetersUpdatePropagation below). + assert event.id == "GUID-001-GLACIER:1716100000000" assert event.adapter == "wfigs_perimeters" # Category uses normalized incident type assert event.category == "fire.perimeter.wildfire" # NOT fire.perimeter.wf @@ -864,6 +866,33 @@ def _incident_response(irwin: str, mod_dt, county: str = "Ada", name: str = "Tes } +def _perimeter_response(irwin: str, mod_dt, county: str = "Ada", name: str = "Test Fire"): + """Single perimeter feature (attr_*/poly_* prefixed) with a controllable + attr_ModifiedOnDateTime_dt and a Polygon intersecting the test region bbox.""" + return { + "type": "FeatureCollection", + "features": [{ + "type": "Feature", + "geometry": {"type": "Polygon", "coordinates": [[ + [-116.6, 43.4], [-116.4, 43.4], [-116.4, 43.6], + [-116.6, 43.6], [-116.6, 43.4], + ]]}, # Idaho + "properties": { + "attr_IrwinID": irwin, + "attr_IncidentName": name, + "attr_IncidentTypeCategory": "WF", + "attr_IncidentSize": 100, + "poly_GISAcres": 98.5, + "attr_PercentContained": 0, + "attr_FireDiscoveryDateTime": 1716000000000, + "attr_ModifiedOnDateTime_dt": mod_dt, + "attr_POOState": "US-ID", + "attr_POOCounty": county, + }, + }], + } + + async def _poll_once(adapter, response): mr = AsyncMock() mr.raise_for_status = MagicMock() @@ -968,3 +997,88 @@ class TestWFIGSIncidentsUpdatePropagation: assert len(pub1) == 1 assert pub1[0].id == "GUID-A:None" assert len(pub2) == 0 # still dedups consistently on the ':None' key + + +# --- v0.14.4: perimeter UPDATE propagation ------------------------------------ +# Same fix as v0.14.3 applied to the sibling wfigs_perimeters adapter: the dedup +# key is now attr_IrwinID + attr_ModifiedOnDateTime_dt (was bare attr_IrwinID, +# which published a perimeter once and silenced every later geometry refinement). +# Note: wfigs_perimeters has NO client-side recency filter, so the None path +# needs no monkeypatch (unlike incidents). + +class TestWFIGSPerimetersUpdatePropagation: + """v0.14.4: dedup key includes attr_ModifiedOnDateTime_dt.""" + + def _adapter(self, tmp_path: Path): + from central.adapters.wfigs_perimeters import WFIGSPerimetersAdapter + config = AdapterConfig( + name="wfigs_perimeters", enabled=True, cadence_s=300, + settings={"region": {"north": 49.0, "south": 31.0, "east": -102.0, "west": -124.0}}, + updated_at=datetime.now(timezone.utc), + ) + return WFIGSPerimetersAdapter(config, MagicMock(), tmp_path / "cursors.db") + + @pytest.mark.asyncio + async def test_unchanged_perimeter_dedups_across_polls(self, tmp_path: Path): + """Same IrwinID + same attr_ModifiedOnDateTime_dt: first publishes, second deduped.""" + adapter = self._adapter(tmp_path) + await adapter.startup() + resp = _perimeter_response("GUID-P", 1_716_100_000_000) + + pub1 = _dedup_publish(adapter, await _poll_once(adapter, resp)) + pub2 = _dedup_publish(adapter, await _poll_once(adapter, resp)) + + await adapter.shutdown() + assert len(pub1) == 1 + assert len(pub2) == 0 # collapsed via published_ids on the identical key + assert pub1[0].id == "GUID-P:1716100000000" + + @pytest.mark.asyncio + async def test_modified_perimeter_republishes_with_distinct_id(self, tmp_path: Path): + """Same IrwinID, DIFFERENT attr_ModifiedOnDateTime_dt: BOTH publish, distinct ids. + + This is the load-bearing test: it proves the perimeter silencing is fixed.""" + adapter = self._adapter(tmp_path) + await adapter.startup() + resp_t1 = _perimeter_response("GUID-P", 1_716_100_000_000) + resp_t2 = _perimeter_response("GUID-P", 1_716_100_300_000) # perimeter refined upstream + + pub1 = _dedup_publish(adapter, await _poll_once(adapter, resp_t1)) + pub2 = _dedup_publish(adapter, await _poll_once(adapter, resp_t2)) + + await adapter.shutdown() + assert len(pub1) == 1 + assert len(pub2) == 1 # the refinement propagated, not silenced + assert pub1[0].id == "GUID-P:1716100000000" + assert pub2[0].id == "GUID-P:1716100300000" + assert pub1[0].id != pub2[0].id + + @pytest.mark.asyncio + async def test_subject_derivation_unchanged(self, tmp_path: Path): + """The new id does not affect subject_for: still central.fire.perimeter...""" + adapter = self._adapter(tmp_path) + await adapter.startup() + events = await _poll_once( + adapter, _perimeter_response("GUID-P", 1_716_100_300_000, county="Ada") + ) + await adapter.shutdown() + assert len(events) == 1 + assert events[0].id == "GUID-P:1716100300000" + assert adapter.subject_for(events[0]) == "central.fire.perimeter.id.ada" + + @pytest.mark.asyncio + async def test_none_modified_time_is_deterministic(self, tmp_path: Path): + """Defensive: a None attr_ModifiedOnDateTime_dt yields a deterministic + ':None' id without raising. No recency filter on perimeters, so this + reaches the construction path directly (no monkeypatch needed).""" + adapter = self._adapter(tmp_path) + await adapter.startup() + resp = _perimeter_response("GUID-P", None) + + pub1 = _dedup_publish(adapter, await _poll_once(adapter, resp)) + pub2 = _dedup_publish(adapter, await _poll_once(adapter, resp)) + + await adapter.shutdown() + assert len(pub1) == 1 + assert pub1[0].id == "GUID-P:None" + assert len(pub2) == 0 # still dedups consistently on the ':None' key From 3361abaa923084f3821bbf14bea14228059088a6 Mon Sep 17 00:00:00 2001 From: malice Date: Mon, 15 Jun 2026 22:02:02 -0600 Subject: [PATCH 02/17] v0.14.5: migration 042 re-asserts central ownership of config.monitoring_areas (#111) Append idempotent ALTER ... OWNER TO central for the table + SERIAL sequence so fresh installs are self-healing; prod already patched inline during v0.14.0. Adds whitespace-insensitive static-check assertions. Co-Authored-By: Claude Opus 4.8 (1M context) --- sql/migrations/042_monitoring_area_to_multi_areas.sql | 10 ++++++++++ tests/test_migration_042.py | 7 +++++++ 2 files changed, 17 insertions(+) diff --git a/sql/migrations/042_monitoring_area_to_multi_areas.sql b/sql/migrations/042_monitoring_area_to_multi_areas.sql index 77d2754..1fd109f 100644 --- a/sql/migrations/042_monitoring_area_to_multi_areas.sql +++ b/sql/migrations/042_monitoring_area_to_multi_areas.sql @@ -46,3 +46,13 @@ WHERE id = true AND monitor_east IS NOT NULL AND monitor_west IS NOT NULL ON CONFLICT (name) DO NOTHING; + +-- Ownership fix (v0.14.5). During the v0.14.0 prod deploy (2026-06-12) this +-- migration was applied as `sudo -u postgres`, so the table + its SERIAL +-- sequence ended up owned by postgres while the `central` app role expects +-- ownership-based access (it could read but not manage the new config table). +-- We patched prod inline with these same ALTERs; making them part of the file +-- keeps fresh installs self-healing. Idempotent: a no-op when already owned by +-- central. (See central-manual-migration-owner-role.) +ALTER TABLE config.monitoring_areas OWNER TO central; +ALTER SEQUENCE config.monitoring_areas_id_seq OWNER TO central; diff --git a/tests/test_migration_042.py b/tests/test_migration_042.py index 3a1902f..182cf67 100644 --- a/tests/test_migration_042.py +++ b/tests/test_migration_042.py @@ -38,3 +38,10 @@ def test_does_not_drop_old_columns_in_v0_14_0(): upper = _NORM.upper() assert "DROP COLUMN" not in upper assert "DROP TABLE" not in upper + + +def test_grants_table_and_sequence_ownership_to_central(): + # v0.14.5: applied-as-postgres left the table/sequence postgres-owned; the + # file now re-asserts central ownership so fresh installs are self-healing. + assert "ALTER TABLE config.monitoring_areas OWNER TO central" in _NORM + assert "ALTER SEQUENCE config.monitoring_areas_id_seq OWNER TO central" in _NORM From 39161cc99368dfc24be88bb6c582c5064e0fd485 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 18 Jun 2026 21:09:53 +0000 Subject: [PATCH 03/17] =?UTF-8?q?docs:=20correct=20stale=20README=20?= =?UTF-8?q?=E2=80=94=20central=20is=20live=20in=20production=20(v0.14.5),?= =?UTF-8?q?=20not=20scaffold?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f057003..c0e4bc8 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,17 @@ # Central -Central is the data hub spine for the infrastructure. Adapters normalize upstream sources into a canonical event shape, publish CloudEvents to NATS/JetStream, and archive to TimescaleDB for historical query. Single-LXC deployment. +Central is the data hub spine for the Echo6 infrastructure. Adapters normalize upstream sources into a canonical event shape, publish CloudEvents to NATS/JetStream, and archive to TimescaleDB for historical query. Single-LXC deployment. ## Status -Phase 0 — scaffold. Not yet operational. +**Live in production** (v0.14.5) on utility CT 104 (`central.echo6.mesh`). + +~25 adapters across domains: traffic, wildfire, weather, space-weather, hydrology, earthquakes, avalanche, disasters, and satellite. Events flow CloudEvents -> NATS/JetStream -> TimescaleDB/PostGIS. FastAPI + HTMX GUI/API on :8000. + +Three systemd services manage the deployment: +- `central-supervisor` — adapter lifecycle manager +- `central-archive` — NATS consumer persisting events to TimescaleDB +- `central-gui` — FastAPI + HTMX web interface / API (:8000) ## Architecture From d3a1be82feca7875856a0bcb42c06d74e6eb46a9 Mon Sep 17 00:00:00 2001 From: malice Date: Sun, 28 Jun 2026 14:13:40 -0600 Subject: [PATCH 04/17] v0.14.6: commit orphaned migration 036 + fix stale README/version metadata (#112) Migration 036 (avalanche_org adapter row) was code-shipped in v0.10.10 (PR #98) but the SQL file was never committed to the repo. It has been applied in production since 2026-06-18; this commit adds the file so clean rebuilds can reach the same schema state. Also bumps pyproject.toml version from the stale 0.3.0 to 0.14.6 to match the deployed tag, and updates the README Status section to reflect current reality (~22 adapters, three systemd units, operational). Co-authored-by: Matt Johnson Co-authored-by: Claude Opus 4.8 (1M context) --- README.md | 6 ++-- pyproject.toml | 2 +- .../036_add_avalanche_org_adapter.sql | 33 +++++++++++++++++++ 3 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 sql/migrations/036_add_avalanche_org_adapter.sql diff --git a/README.md b/README.md index c0e4bc8..dc88dca 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,11 @@ Central is the data hub spine for the Echo6 infrastructure. Adapters normalize u ## Status -**Live in production** (v0.14.5) on utility CT 104 (`central.echo6.mesh`). +**Operational** (v0.14.6) — deployed on utility CT 104 (`central.echo6.mesh`). -~25 adapters across domains: traffic, wildfire, weather, space-weather, hydrology, earthquakes, avalanche, disasters, and satellite. Events flow CloudEvents -> NATS/JetStream -> TimescaleDB/PostGIS. FastAPI + HTMX GUI/API on :8000. +~22 adapters live across traffic, wildfire, weather, space-weather, hydrology, earthquakes, avalanche, disasters, and satellite. CloudEvents published to NATS/JetStream, archived to TimescaleDB/PostGIS. FastAPI + HTMX GUI/API on :8000. -Three systemd services manage the deployment: +Three systemd units on a single LXC: - `central-supervisor` — adapter lifecycle manager - `central-archive` — NATS consumer persisting events to TimescaleDB - `central-gui` — FastAPI + HTMX web interface / API (:8000) diff --git a/pyproject.toml b/pyproject.toml index a9d4cad..cbe0d0f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "central" -version = "0.3.0" +version = "0.14.6" requires-python = ">=3.12,<3.13" description = "Data hub spine — adapters, bus, archive." readme = "README.md" diff --git a/sql/migrations/036_add_avalanche_org_adapter.sql b/sql/migrations/036_add_avalanche_org_adapter.sql new file mode 100644 index 0000000..3e8db54 --- /dev/null +++ b/sql/migrations/036_add_avalanche_org_adapter.sql @@ -0,0 +1,33 @@ +-- Migration 036: register avalanche_org adapter row in config.adapters (v0.10.10.1) +-- +-- v0.10.10 (PR #98) shipped the avalanche_org adapter code, GUI templates, +-- stream registry entry, and migration 035 (CENTRAL_AVY config.streams seed). +-- It also (incorrectly) assumed the supervisor would schedule a new adapter +-- the moment its code landed. It won't: supervisor.list_enabled_adapters() +-- reads from config.adapters and skips any adapter without a row, so the +-- v0.10.10 deploy left avalanche_org code-shipped-but-inactive. +-- +-- This migration closes that gap. Mirrors the pattern of migration 031 +-- (itd_511) -- INSERT a row with default settings, idempotent via +-- ON CONFLICT (name) DO NOTHING. +-- +-- Unlike itd_511 (which shipped disabled because it needed an API key +-- staged first), avalanche_org has no auth requirement and is enabled +-- immediately: the upstream API is public and the off-season severity +-- gate means the adapter yields zero events during summer regardless, +-- so 'enabled at install' is the right default. +-- +-- Settings JSON matches AvalancheOrgSettings: center_ids defaults to +-- SNFAC + PAC (Idaho-region coverage). Operator can extend via GUI to +-- any avalanche.org-recognised center (NWAC, CAIC, etc.). +-- +-- Idempotent: re-running is a no-op for existing rows. + +INSERT INTO config.adapters (name, enabled, cadence_s, settings) +VALUES ( + 'avalanche_org', + true, + 1800, + '{"center_ids": ["SNFAC", "PAC"]}'::jsonb +) +ON CONFLICT (name) DO NOTHING; From 2d4bf022dbcb360c0d8668bc6dd48cc076f47861 Mon Sep 17 00:00:00 2001 From: malice Date: Sun, 28 Jun 2026 14:13:45 -0600 Subject: [PATCH 05/17] eonet: bypass bbox filter for global disaster feed (#113) EONET events are global-by-design and were being dropped 100% by the supervisor's monitoring_areas bbox filter. Set bypass_bbox_filter = True on EONETAdapter (mirrors the v0.14.2/#108 satellite-telemetry exemption). Also adds "eonet" to archive._BYPASS_BBOX_ADAPTERS to satisfy the consumer-side mirror enforced by tests/test_bypass_bbox_consistency.py. Co-authored-by: Ubuntu Co-authored-by: Claude Opus 4.8 (1M context) --- src/central/adapters/eonet.py | 6 ++++++ src/central/archive.py | 17 +++++++++-------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/central/adapters/eonet.py b/src/central/adapters/eonet.py index 36d9226..6b3d955 100644 --- a/src/central/adapters/eonet.py +++ b/src/central/adapters/eonet.py @@ -152,6 +152,12 @@ class EONETAdapter(SourceAdapter): # Event lat/lon mirrored from Geo.centroid into event.data (see poll()). enrichment_locations = [("latitude", "longitude")] + # Global-by-design: EONET disaster events (earthquakes, floods, wildfires, + # etc.) span the entire globe. A geographic monitoring area (e.g. Idaho) + # drops everything outside the bbox, yielding zero events. Skip the + # publish-time/archive bbox filter (mirrors v0.14.2/#108 satellite exemption). + # Keep in sync with archive._BYPASS_BBOX_ADAPTERS. + bypass_bbox_filter = True def __init__( self, diff --git a/src/central/archive.py b/src/central/archive.py index 4380fee..269de27 100644 --- a/src/central/archive.py +++ b/src/central/archive.py @@ -36,14 +36,15 @@ BATCH_SIZE = 100 FETCH_TIMEOUT = 5.0 ACK_WAIT = 30 -# v0.14.2: adapters whose events are global-by-design (satellite telemetry) and -# must bypass the geographic monitoring-area bbox filter. The archive consumer -# only has the adapter NAME at runtime (it reads off the wire, not the adapter -# class), so it can't reach SourceAdapter.bypass_bbox_filter directly -- this -# static set is the consumer-side mirror. It MUST stay in sync with the adapter -# classes that set bypass_bbox_filter = True; tests/test_bypass_bbox_consistency.py -# fails CI if the two drift. -_BYPASS_BBOX_ADAPTERS = {"sat_positions", "sat_orbits"} +# v0.14.2: adapters whose events are global-by-design (satellite telemetry, +# global disaster feeds) and must bypass the geographic monitoring-area bbox +# filter. The archive consumer only has the adapter NAME at runtime (it reads +# off the wire, not the adapter class), so it can't reach +# SourceAdapter.bypass_bbox_filter directly -- this static set is the +# consumer-side mirror. It MUST stay in sync with the adapter classes that set +# bypass_bbox_filter = True; tests/test_bypass_bbox_consistency.py fails CI if +# the two drift. +_BYPASS_BBOX_ADAPTERS = {"sat_positions", "sat_orbits", "eonet"} def consumer_name_for(stream: str) -> str: From 1ae3f3e7fda55728a2b4c2e5e9ba854460084d86 Mon Sep 17 00:00:00 2001 From: malice Date: Sun, 28 Jun 2026 14:13:50 -0600 Subject: [PATCH 06/17] supervisor: shrink sat dedup window + WAL cursors.db to cut CPU/IO (#114) sat_positions and sat_orbits produce dedup IDs that are unique per second (:), so the inherited 14-day sweep window accumulates ~3.5M rows in cursors.db with no benefit. Shrink to 1 day (positions) and 2 days (orbits, to tolerate stable TLE re-emission). Open cursors.db in WAL + NORMAL sync mode in both adapters' startup(). WAL mode is file-level persistent, so this covers all adapters sharing the same cursors.db. Previously journal_mode=DELETE + synchronous=FULL forced an fsync on every mark_published() call (~190 calls/60s for sat_positions alone), sustaining ~17% CPU on the supervisor process. A redeploy is needed; the box is currently detached at v0.14.5. Co-authored-by: Ubuntu Co-authored-by: Claude Opus 4.8 (1M context) --- src/central/adapters/sat_orbits.py | 8 ++++++++ src/central/adapters/sat_positions.py | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/src/central/adapters/sat_orbits.py b/src/central/adapters/sat_orbits.py index 8b74721..a2855d5 100644 --- a/src/central/adapters/sat_orbits.py +++ b/src/central/adapters/sat_orbits.py @@ -126,6 +126,10 @@ class SatOrbitsAdapter(SourceAdapter): # Skip the publish-time/archive bbox filter (v0.14.2). Keep in sync with # archive._BYPASS_BBOX_ADAPTERS. bypass_bbox_filter = True + # Dedup IDs are ":" -- unique per second by design, + # so a 14-day window accumulates rows unnecessarily. TLEs are stable for days, + # so 2 days is sufficient to catch re-emitted orbits without bloating the table. + dedup_sweep_days = 2 def __init__( self, @@ -147,6 +151,10 @@ class SatOrbitsAdapter(SourceAdapter): async def startup(self) -> None: self._db = sqlite3.connect(self._cursor_db_path) + # WAL + NORMAL sync: eliminates per-commit fsync overhead on cursors.db. + # WAL mode is file-level persistent; applies to all adapters on this DB. + self._db.execute("PRAGMA journal_mode=WAL") + self._db.execute("PRAGMA synchronous=NORMAL") self._db.execute(_DEDUP_DDL) self._db.execute( "CREATE INDEX IF NOT EXISTS published_ids_last_seen ON published_ids (last_seen)" diff --git a/src/central/adapters/sat_positions.py b/src/central/adapters/sat_positions.py index 91d8bdb..11c2a74 100644 --- a/src/central/adapters/sat_positions.py +++ b/src/central/adapters/sat_positions.py @@ -136,6 +136,9 @@ class SatPositionsAdapter(SourceAdapter): # queries. Skip the publish-time/archive bbox filter (v0.14.2). Keep in sync # with archive._BYPASS_BBOX_ADAPTERS. bypass_bbox_filter = True + # Dedup IDs are ":" -- unique per second by design, + # so a 14-day window accumulates ~3.5M rows unnecessarily. 1 day is enough. + dedup_sweep_days = 1 def __init__( self, @@ -155,6 +158,10 @@ class SatPositionsAdapter(SourceAdapter): async def startup(self) -> None: self._db = sqlite3.connect(self._cursor_db_path) + # WAL + NORMAL sync: eliminates per-commit fsync overhead on cursors.db. + # WAL mode is file-level persistent; applies to all adapters on this DB. + self._db.execute("PRAGMA journal_mode=WAL") + self._db.execute("PRAGMA synchronous=NORMAL") self._db.execute(_DEDUP_DDL) self._db.execute( "CREATE INDEX IF NOT EXISTS published_ids_last_seen ON published_ids (last_seen)" From 9b506dc9d37af9a1ec133c2a1c2ed87180b54d72 Mon Sep 17 00:00:00 2001 From: malice Date: Sun, 28 Jun 2026 14:13:55 -0600 Subject: [PATCH 07/17] scripts/deploy.sh: one-command tag-based deploy with pre-flight backup + verify (#115) Codifies the previously-manual tag-based deploy of central on CT 104. Performs pre-flight drift check, mandatory pg_dump, detached-HEAD checkout + uv sync, interactive confirm, migration apply, service restart, and health-check verify. ERR trap prints rollback instructions. Co-authored-by: Ubuntu Co-authored-by: Claude Opus 4.8 (1M context) --- scripts/README.md | 65 ++++++++++ scripts/deploy.sh | 304 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 369 insertions(+) create mode 100644 scripts/README.md create mode 100755 scripts/deploy.sh diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..3356e96 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,65 @@ +# scripts/ + +## deploy.sh — tag-based deploy for central on CT 104 + +Codifies the manual deploy procedure for the `central` service running on +Proxmox CT 104 (`utility`, Tailscale `100.64.0.12`). + +### What it does + +1. **Preflight** — checks that `/opt/central` is a git repo, captures the + currently-deployed ref for rollback messaging, reports unit status (warns but + does not abort on inactive units), runs `central-migrate --check` to gate + on migration drift, and takes a pre-deploy `pg_dump` backup. +2. **Deploy** — fetches from `origin`, verifies the requested ref exists, + checks it out as a detached HEAD, and runs `uv sync` to update the venv + against the checked-out `uv.lock`. +3. **Confirm** — shows a `--dry-run` migration preview and prompts for + confirmation before applying any changes to the running system (skip with + `-y`). +4. **Apply** — runs `central-migrate` to apply pending SQL migrations, then + restarts all three systemd units (`central-supervisor`, `central-archive`, + `central-gui`). +5. **Verify** — confirms all units are active, re-runs `central-migrate + --check` for a clean post-deploy state, and polls `http://localhost:8000/health` + (up to 5 retries, 2 s apart) for an HTTP 200. +6. **ERR trap** — on any unexpected failure, prints a ROLLBACK block with + the exact commands to re-checkout the previous ref, re-sync the venv, + restart services, and (if needed) restore from the pre-deploy dump. + +### Usage + +Run on CT 104 as a user with passwordless `sudo` (e.g. `zvx`): + +``` +/opt/central/scripts/deploy.sh [-y|--yes] +``` + +- `` — any Git tag, branch, or commit SHA (tags are the standard + deploy unit; e.g. `v0.14.5`). +- `-y` / `--yes` — skip the interactive confirmation prompt (safe for + automation once you have reviewed the dry-run output manually). + +### Pre-flight backup + +Before applying any changes, the script takes a `pg_dump -Fc` of the +`central` database and writes it to `/var/backups/central/`. The 10 newest +dumps are retained; older ones are pruned automatically. + +**Migrations are forward-only.** There are no down-scripts. The `pg_dump` is +the only automated mechanism for rolling back the database. If you need to +revert after migrations have run, restore from the dump printed in the SUCCESS +(or ERR-trap) output. + +### One-time cutover steps + +Some releases require manual cutover steps that cannot be automated (e.g. +removing a deprecated EONET region key from `config.adapters`). These are +intentionally out of scope for this script. See the vault runbook +`central-deploy-cutover.md` for guidance on release-specific procedures. + +### Bootstrap caveat + +This script is version-controlled inside the `central` repository. The very +first deploy that introduces it must still be performed manually (the script +ships in the repo it deploys and cannot deploy itself from scratch). diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100755 index 0000000..b8ae5f6 --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,304 @@ +#!/usr/bin/env bash +# deploy.sh — tag-based deploy for the central service on CT 104 +# +# Usage: deploy.sh [-y|--yes] +# +# Must run on CT 104 as a sudo-capable user (e.g. zvx). +# Performs a detached-HEAD checkout of , syncs the uv venv, +# runs a pre-flight migration drift check, takes a pg_dump backup, runs +# migrations, restarts all three systemd units, and verifies health. +# +# One-time cutover steps (e.g. EONET region-key removal) are NOT handled +# here — see the vault runbook central-deploy-cutover.md. +# +# Bootstrap caveat: the very first deploy that introduces this script is +# still manual (the script ships inside the repo it deploys). + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- +DEPLOY_DIR=/opt/central +VENV="$DEPLOY_DIR/.venv" +CENTRAL_USER=central +UV=/usr/local/bin/uv +BACKUP_DIR=/var/backups/central +UNITS=(central-supervisor central-archive central-gui) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +log() { + echo "" + echo "==> $*" +} + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +# Run a command as $CENTRAL_USER via sudo. +cen() { + sudo -u "$CENTRAL_USER" "$@" +} + +# --------------------------------------------------------------------------- +# ERR trap — fired on any unhandled non-zero exit inside set -e +# --------------------------------------------------------------------------- +on_error() { + local exit_code=$? + echo "" >&2 + echo "================================================================" >&2 + echo " DEPLOY FAILED (exit $exit_code)" >&2 + echo "================================================================" >&2 + echo "" >&2 + echo " ROLLBACK GUIDE:" >&2 + echo "" >&2 + echo " 1. Checkout the previously-deployed ref:" >&2 + echo " sudo -u $CENTRAL_USER git -C $DEPLOY_DIR checkout ${PREV_REF:-}" >&2 + echo " sudo -u $CENTRAL_USER bash -c \"cd $DEPLOY_DIR && $UV sync\"" >&2 + echo " sudo systemctl restart ${UNITS[*]}" >&2 + echo "" >&2 + echo " 2. DB rollback (migrations are forward-only, no down-scripts):" >&2 + echo " The ONLY automated DB rollback is the pg_dump taken before deploy." >&2 + if [[ -n "${DUMP:-}" ]]; then + echo " Dump path: $DUMP" >&2 + echo " Restore: sudo -u $CENTRAL_USER pg_restore -d central -Fc --clean '$DUMP'" >&2 + else + echo " (Dump was not yet created — no DB changes were made.)" >&2 + fi + echo "" >&2 + echo "================================================================" >&2 +} + +trap on_error ERR + +# --------------------------------------------------------------------------- +# Parse arguments +# --------------------------------------------------------------------------- +REF="" +SKIP_CONFIRM=0 + +for arg in "$@"; do + case "$arg" in + -y|--yes) + SKIP_CONFIRM=1 + ;; + -h|--help) + echo "Usage: $0 [-y|--yes]" + echo " Git tag, branch, or commit to deploy." + echo " -y / --yes Skip interactive confirmation prompt." + exit 0 + ;; + -*) + die "Unknown option: $arg" + ;; + *) + if [[ -z "$REF" ]]; then + REF="$arg" + else + die "Unexpected argument: $arg" + fi + ;; + esac +done + +if [[ -z "$REF" ]]; then + echo "Usage: $0 [-y|--yes]" >&2 + exit 2 +fi + +# --------------------------------------------------------------------------- +# PREFLIGHT +# --------------------------------------------------------------------------- + +log "PREFLIGHT" + +# 1. Verify DEPLOY_DIR is a git repo. +[[ -d "$DEPLOY_DIR/.git" ]] \ + || die "$DEPLOY_DIR/.git not found — is this the right deploy directory?" + +# 2. Capture current ref for rollback messaging. +PREV_REF="$(cen git -C "$DEPLOY_DIR" describe --tags --always 2>/dev/null || echo unknown)" +log "Currently deployed: $PREV_REF → deploying: $REF" + +# 3. Report unit status (warn, don't die — a redeploy may be fixing unhealthy units). +log "Current service status" +for unit in "${UNITS[@]}"; do + status="$(systemctl is-active "$unit" 2>/dev/null || true)" + if [[ "$status" != "active" ]]; then + echo " WARNING: $unit is $status (will attempt restart anyway)" + else + echo " $unit: $status" + fi +done + +# 4. Migration drift gate — refuse to deploy onto a drifted migration state. +log "Migration drift check (--check)" +cen "$VENV/bin/central-migrate" --check \ + || die "Migration drift detected (central-migrate --check exited non-zero). Resolve drift before deploying." + +# 5. Ensure backup directory exists and is owned by $CENTRAL_USER. +log "Ensuring backup directory: $BACKUP_DIR" +sudo mkdir -p "$BACKUP_DIR" +sudo chown "$CENTRAL_USER:$CENTRAL_USER" "$BACKUP_DIR" + +# 6. Take a pre-deploy pg_dump backup. +TS="$(date -u +%Y%m%dT%H%M%SZ)" +# Sanitise REF for use in a filename (replace / and : with _). +REF_SAFE="${REF//\//_}" +REF_SAFE="${REF_SAFE//:/_}" +DUMP="$BACKUP_DIR/central-pre-${REF_SAFE}-${TS}.pgdump" + +log "Taking pre-deploy backup: $DUMP" +# Run as $CENTRAL_USER so the file is owned by that user; redirect inside sudo. +cen bash -c "pg_dump -Fc central > '$DUMP'" + +# Verify the dump is non-empty (>1 KB sanity check). +if [[ ! -f "$DUMP" ]]; then + die "Dump file was not created: $DUMP" +fi +dump_size="$(stat -c%s "$DUMP" 2>/dev/null || stat -f%z "$DUMP" 2>/dev/null || echo 0)" +if [[ "$dump_size" -lt 1024 ]]; then + die "Dump file is suspiciously small (${dump_size} bytes): $DUMP — aborting." +fi +echo " Backup written: $DUMP (${dump_size} bytes)" + +# 7. Prune old backups — keep the 10 newest central-pre-*.pgdump files. +log "Pruning old backups (keep 10 newest)" +# ls -t lists newest first; tail -n +11 skips the 10 newest → these are the old ones. +old_backups="$(ls -t "$BACKUP_DIR"/central-pre-*.pgdump 2>/dev/null | tail -n +11 || true)" +if [[ -n "$old_backups" ]]; then + echo "$old_backups" | while IFS= read -r f; do + echo " Removing old backup: $f" + rm -f "$f" + done +else + echo " No old backups to prune." +fi + +# --------------------------------------------------------------------------- +# DEPLOY +# --------------------------------------------------------------------------- + +log "DEPLOY" + +# 8. Fetch latest tags and refs from origin. +log "Fetching from origin (tags)" +cen git -C "$DEPLOY_DIR" fetch origin --tags + +# 9. Verify the requested ref resolves to a commit (fail fast with a clear message). +log "Resolving ref: $REF" +cen git -C "$DEPLOY_DIR" rev-parse --verify "${REF}^{commit}" > /dev/null \ + || die "Ref '$REF' does not resolve to a commit. Check the tag/branch name and try again." + +# 10. Checkout the ref as a detached HEAD (standard deploy mode). +log "Checking out: $REF" +cen git -C "$DEPLOY_DIR" checkout "$REF" + +# 11. Sync the venv against the checked-out lockfile (uv respects uv.lock). +log "Syncing venv (uv sync)" +cen bash -c "cd '$DEPLOY_DIR' && '$UV' sync" + +# 12. Migration preview — show what would run without applying. +log "Migration dry-run (preview)" +cen "$VENV/bin/central-migrate" --dry-run + +# 13. CONFIRM gate — unless -y was passed. +# +# NOTE: At this point the code and venv are already updated to the new ref, +# but services have NOT been restarted and migrations have NOT been applied. +# If the operator aborts here, the new code is staged on disk but production +# is still running the old code. To return to a clean state manually: +# sudo -u central git -C /opt/central checkout +# sudo -u central bash -c "cd /opt/central && /usr/local/bin/uv sync" +if [[ "$SKIP_CONFIRM" -eq 0 ]]; then + echo "" + read -r -p "Apply migrations and restart services? [y/N] " confirm < /dev/tty + if [[ "$confirm" != "y" && "$confirm" != "Y" ]]; then + echo "" + echo "Aborted by operator." + echo "" + echo " Code and venv are now at: $REF" + echo " Services are still running: $PREV_REF" + echo " Migrations have NOT been applied." + echo "" + echo " To stage-abort cleanly (revert code on disk):" + echo " sudo -u $CENTRAL_USER git -C $DEPLOY_DIR checkout $PREV_REF" + echo " sudo -u $CENTRAL_USER bash -c \"cd $DEPLOY_DIR && $UV sync\"" + exit 0 + fi +fi + +# 14. Apply migrations. +log "Applying migrations" +cen "$VENV/bin/central-migrate" + +# 15. Restart all three systemd units. +log "Restarting services" +sudo systemctl restart "${UNITS[@]}" + +# --------------------------------------------------------------------------- +# VERIFY +# --------------------------------------------------------------------------- + +log "VERIFY" + +# 16. Confirm all units are active after restart. +log "Checking unit status" +failed_units=() +for unit in "${UNITS[@]}"; do + status="$(systemctl is-active "$unit" 2>/dev/null || true)" + echo " $unit: $status" + if [[ "$status" != "active" ]]; then + failed_units+=("$unit") + fi +done +if [[ "${#failed_units[@]}" -gt 0 ]]; then + die "Unit(s) not active after restart: ${failed_units[*]}" +fi + +# 17. Post-deploy migration check — should be clean (0 pending). +log "Post-deploy migration check (--check)" +cen "$VENV/bin/central-migrate" --check \ + || die "Post-deploy migration check failed — schema may be inconsistent." + +# 18. Health check — up to 5 retries with 2-second sleep between attempts. +log "Health check: http://localhost:8000/health" +HEALTH_OK=0 +for attempt in 1 2 3 4 5; do + http_status="$(curl -sf -o /dev/null -w "%{http_code}" http://localhost:8000/health 2>/dev/null || true)" + if [[ "$http_status" == "200" ]]; then + echo " Attempt $attempt: HTTP $http_status — OK" + HEALTH_OK=1 + break + else + echo " Attempt $attempt: HTTP ${http_status:-no-response} — retrying in 2s..." + sleep 2 + fi +done + +if [[ "$HEALTH_OK" -eq 0 ]]; then + die "Health check failed after 5 attempts (http://localhost:8000/health did not return 200)." +fi + +# --------------------------------------------------------------------------- +# SUCCESS +# --------------------------------------------------------------------------- +DEPLOYED_DESC="$(cen git -C "$DEPLOY_DIR" describe --tags --always 2>/dev/null || echo "$REF")" + +echo "" +echo "================================================================" +echo " SUCCESS" +echo "================================================================" +echo " Deployed: $DEPLOYED_DESC" +echo " Previous: $PREV_REF" +echo " Pre-deploy backup: $DUMP" +echo "" +echo " REMINDER: one-time cutover steps (e.g. EONET region-key removal)" +echo " are NOT performed by this script. See vault runbook:" +echo " central-deploy-cutover.md" +echo "================================================================" From acef894b219296f0a108babc7586bd243425a6ba Mon Sep 17 00:00:00 2001 From: malice Date: Sun, 28 Jun 2026 16:40:34 -0600 Subject: [PATCH 08/17] deploy.sh: fix central-migrate env (cd + EnvironmentFile) so pre-flight check works (#116) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit central-migrate invocations were run via bare `cen "$VENV/bin/central-migrate"`, which does not set the working directory or source the EnvironmentFile that the systemd unit provides. This caused the pre-flight --check to fail during the v0.14.6 deploy with `PermissionError: [Errno 13] Permission error: '.env'` because central-migrate tried to read .env from the caller's cwd instead of /opt/central, and the DB DSN was missing. The daemons themselves are unaffected — they always get WorkingDirectory= /opt/central and EnvironmentFile=/etc/central/central.env from systemd. The deploy script is the only place that invoked central-migrate outside that context. Fix: add ENV_FILE=/etc/central/central.env to the config block and a cen_migrate() helper that wraps every central-migrate call with: sudo -u central bash -c "cd /opt/central && set -a && . /etc/central/central.env && set +a && /opt/central/.venv/bin/central-migrate " Replace all four bare invocations (--check preflight, --dry-run, apply, --check post-deploy) with cen_migrate. Exit-code propagation is unchanged — bash -c forwards the inner exit status, so set -e / the ERR trap still fires on non-zero exits. Co-authored-by: Ubuntu Co-authored-by: Claude Opus 4.8 (1M context) --- scripts/deploy.sh | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/scripts/deploy.sh b/scripts/deploy.sh index b8ae5f6..286ecda 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -25,6 +25,7 @@ CENTRAL_USER=central UV=/usr/local/bin/uv BACKUP_DIR=/var/backups/central UNITS=(central-supervisor central-archive central-gui) +ENV_FILE=/etc/central/central.env # --------------------------------------------------------------------------- # Helpers @@ -45,6 +46,14 @@ cen() { sudo -u "$CENTRAL_USER" "$@" } +# Run central-migrate as $CENTRAL_USER with the service's working directory and +# environment file, matching what the systemd unit provides via WorkingDirectory= +# and EnvironmentFile=. Without this, central-migrate tries to read .env from +# the caller's cwd and fails with PermissionError. +cen_migrate() { + sudo -u "$CENTRAL_USER" bash -c "cd '$DEPLOY_DIR' && set -a && . '$ENV_FILE' && set +a && '$VENV/bin/central-migrate' $*" +} + # --------------------------------------------------------------------------- # ERR trap — fired on any unhandled non-zero exit inside set -e # --------------------------------------------------------------------------- @@ -138,7 +147,7 @@ done # 4. Migration drift gate — refuse to deploy onto a drifted migration state. log "Migration drift check (--check)" -cen "$VENV/bin/central-migrate" --check \ +cen_migrate --check \ || die "Migration drift detected (central-migrate --check exited non-zero). Resolve drift before deploying." # 5. Ensure backup directory exists and is owned by $CENTRAL_USER. @@ -205,7 +214,7 @@ cen bash -c "cd '$DEPLOY_DIR' && '$UV' sync" # 12. Migration preview — show what would run without applying. log "Migration dry-run (preview)" -cen "$VENV/bin/central-migrate" --dry-run +cen_migrate --dry-run # 13. CONFIRM gate — unless -y was passed. # @@ -235,7 +244,7 @@ fi # 14. Apply migrations. log "Applying migrations" -cen "$VENV/bin/central-migrate" +cen_migrate # 15. Restart all three systemd units. log "Restarting services" @@ -263,7 +272,7 @@ fi # 17. Post-deploy migration check — should be clean (0 pending). log "Post-deploy migration check (--check)" -cen "$VENV/bin/central-migrate" --check \ +cen_migrate --check \ || die "Post-deploy migration check failed — schema may be inconsistent." # 18. Health check — up to 5 retries with 2-second sleep between attempts. From a38eaaa4fd3b2b490deada774a0f27cbd854f39e Mon Sep 17 00:00:00 2001 From: malice Date: Sun, 28 Jun 2026 19:41:39 -0600 Subject: [PATCH 09/17] gui: add consumers admin page (view + delete JetStream consumers) (#117) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * gui: add consumers admin page (view + delete JetStream consumers) Adds GET /consumers (list all stream consumers grouped by stream) and POST /consumers/{stream}/{consumer}/delete. Archive-* consumers are protected in both the template (no delete button rendered) and the POST handler (hard refuse before touching NATS). CSRF validated, audit logged via CONSUMER_DELETE action, DB conn acquired same pattern as api_keys_delete. Co-Authored-By: Claude Opus 4.8 (1M context) * gui: fix consumers_info coroutine usage + list-returning test mock + None-guard counts - routes.py: change `async for ci in js.consumers_info(stream_name)` to `for ci in await js.consumers_info(stream_name)` — nats-py 2.14.0 consumers_info() is a plain coroutine returning list[ConsumerInfo], not an async iterable; the old form threw TypeError silently (swallowed by except), causing every stream to show "unavailable" and zero consumers. - test_consumers.py: replace async-generator mock with AsyncMock returning a list, matching the real API; also fix inline consumers_info_raising in the error test (remove dead yield); add explicit regression guard asserting consumer names appear in the template context. - consumers_list.html: guard num_pending/num_ack_pending/num_redelivered/ num_waiting with `… if … is not none else '—'` to prevent "None" in cells. Co-Authored-By: Claude Opus 4.8 (1M context) * gui: make consumers_delete guards DB-independent + add rendered-HTML tests Builds on the consumers_info coroutine fix: - consumers_delete: acquire the DB pool only when actually writing the audit (after the CSRF / archive-guard / NATS-unavailable early exits) and use a local `get_js` import. Previously `pool = get_pool()` ran at the top, so the CSRF-reject, archive-refuse and NATS-down paths all needed an initialized DB pool, and the module-level get_js bound at import time ignored test patches of central.gui.nats.get_js. Mirrors the local-import pattern the streams routes already use. - tests: add TestConsumersListHtmlRender — renders consumers_list.html through the real Jinja2 environment and asserts the consumer NAME reaches the HTML body, the central-owned label gates on `protected`, and None counts render an em dash rather than the literal "None". Stronger than the context-dict checks; the coroutine/None regressions cannot return. All 4 delete-route tests now pass (were failing on an uninitialized-pool RuntimeError). Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Ubuntu Co-authored-by: Claude Opus 4.8 (1M context) --- src/central/gui/audit.py | 1 + src/central/gui/routes.py | 117 ++++++ src/central/gui/templates/base.html | 1 + src/central/gui/templates/consumers_list.html | 70 ++++ tests/test_consumers.py | 379 ++++++++++++++++++ 5 files changed, 568 insertions(+) create mode 100644 src/central/gui/templates/consumers_list.html create mode 100644 tests/test_consumers.py diff --git a/src/central/gui/audit.py b/src/central/gui/audit.py index 520fffc..ada29df 100644 --- a/src/central/gui/audit.py +++ b/src/central/gui/audit.py @@ -14,6 +14,7 @@ STREAM_UPDATE = "stream.update" API_KEY_CREATE = "api_key.create" API_KEY_ROTATE = "api_key.rotate" API_KEY_DELETE = "api_key.delete" +CONSUMER_DELETE = "consumer.delete" SYSTEM_UPDATE = "system.update" MONITORING_AREA_CREATE = "monitoring_area.create" MONITORING_AREA_UPDATE = "monitoring_area.update" diff --git a/src/central/gui/routes.py b/src/central/gui/routes.py index 235f80b..8dff9c5 100644 --- a/src/central/gui/routes.py +++ b/src/central/gui/routes.py @@ -44,6 +44,7 @@ from central.gui.audit import ( AUTH_LOGIN_FAILED, AUTH_LOGOUT, AUTH_PASSWORD_CHANGE, + CONSUMER_DELETE, MONITORING_AREA_CREATE, MONITORING_AREA_DELETE, MONITORING_AREA_UPDATE, @@ -2194,6 +2195,122 @@ async def streams_update( return RedirectResponse(url="/streams", status_code=302) +# ============================================================================= +# Consumers routes +# ============================================================================= + + +@router.get("/consumers", response_class=HTMLResponse) +async def consumers_list(request: Request) -> HTMLResponse: + """List all JetStream consumers across all registered streams.""" + from central.gui.nats import get_js + + templates = _get_templates() + operator = request.state.operator + js = get_js() + + streams_data = [] + for stream_entry in STREAM_REGISTRY: + stream_name = stream_entry.name + consumers = [] + stream_error = None + + if js is not None: + try: + for ci in await js.consumers_info(stream_name): + consumers.append({ + "name": ci.name, + "num_pending": ci.num_pending, + "num_ack_pending": ci.num_ack_pending, + "num_redelivered": ci.num_redelivered, + "num_waiting": ci.num_waiting, + "created": ci.created, + "protected": ci.name.startswith("archive-"), + }) + except Exception as e: + logger.warning( + "consumers_info failed", + extra={"stream": stream_name, "err": type(e).__name__}, + ) + stream_error = f"unavailable: {type(e).__name__}" + else: + stream_error = "NATS unavailable" + + streams_data.append({ + "stream": stream_name, + "consumers": consumers, + "error": stream_error, + }) + + csrf_token = request.state.csrf_token + response = templates.TemplateResponse( + request=request, + name="consumers_list.html", + context={ + "operator": operator, + "csrf_token": csrf_token, + "streams": streams_data, + }, + ) + return response + + +@router.post("/consumers/{stream}/{consumer}/delete", response_class=HTMLResponse) +async def consumers_delete(request: Request, stream: str, consumer: str) -> Response: + """Delete a JetStream consumer.""" + from central.gui.nats import get_js + + operator = request.state.operator + + form = await request.form() + form_csrf = form.get("csrf_token", "") + if not form_csrf or form_csrf != request.state.csrf_token: + raise CsrfValidationError("Invalid CSRF token") + + # Hard guard: never delete archive-* consumers even if path is forged + if consumer.startswith("archive-"): + return RedirectResponse("/consumers", status_code=302) + + js = get_js() + if js is None: + return RedirectResponse("/consumers", status_code=302) + + # Capture before state from NATS for audit log + try: + before_info = await js.consumer_info(stream, consumer) + before = { + "name": before_info.name, + "stream": stream, + "num_pending": before_info.num_pending, + } + except Exception: + before = {"name": consumer, "stream": stream} + + # Delete the consumer + try: + await js.delete_consumer(stream, consumer) + except Exception: + logger.exception( + "delete_consumer failed", + extra={"stream": stream, "consumer": consumer}, + ) + return RedirectResponse("/consumers", status_code=302) + + # Write audit log + pool = get_pool() + async with pool.acquire() as conn: + await write_audit( + conn, + CONSUMER_DELETE, + operator_id=operator.id, + target=f"{stream}/{consumer}", + before=before, + after=None, + ) + + return RedirectResponse("/consumers", status_code=302) + + # ============================================================================= # Enrichment config route # ============================================================================= diff --git a/src/central/gui/templates/base.html b/src/central/gui/templates/base.html index 5732a68..3f2e990 100644 --- a/src/central/gui/templates/base.html +++ b/src/central/gui/templates/base.html @@ -18,6 +18,7 @@ Events Telemetry Streams + Consumers Enrichment Monitoring Area API Keys diff --git a/src/central/gui/templates/consumers_list.html b/src/central/gui/templates/consumers_list.html new file mode 100644 index 0000000..252c1c4 --- /dev/null +++ b/src/central/gui/templates/consumers_list.html @@ -0,0 +1,70 @@ +{% extends "base.html" %} + +{% block title %}Central — Consumers{% endblock %} + +{% block content %} +

Consumers

+

JetStream consumers across all registered streams. A consumer with high +Pending and zero Waiting has accumulated unacknowledged messages and +has no active subscriber — it is safe to delete if it is not a central-owned consumer.

+ +
+{% for stream in streams %} +
+
{{ stream.stream }}
+ + {% if stream.error %} +

({{ stream.error }})

+ {% elif stream.consumers %} + + + + + + + + + + + + + + {% for c in stream.consumers %} + + + + + + + + + + {% endfor %} + +
NamePendingAck PendingRedeliveredWaitingCreatedAction
{{ c.name }}{{ c.num_pending if c.num_pending is not none else '—' }}{{ c.num_ack_pending if c.num_ack_pending is not none else '—' }}{{ c.num_redelivered if c.num_redelivered is not none else '—' }}{{ c.num_waiting if c.num_waiting is not none else '—' }}{{ c.created.isoformat() if c.created else '—' }} + {% if c.protected %} + central-owned + {% else %} +
+ + +
+ {% endif %} +
+ {% else %} +

(no consumers)

+ {% endif %} +
+{% endfor %} +
+ +

+ Legend: Pending = messages not yet delivered to this consumer; + Ack Pending = delivered but not yet acknowledged; + Waiting = active pull requests from a live subscriber. + A consumer with high Pending and zero Waiting is abandoned — no subscriber + is pulling from it and messages are piling up. + Consumers marked central-owned (archive-*) are managed by central and cannot be deleted here. +

+{% endblock %} diff --git a/tests/test_consumers.py b/tests/test_consumers.py new file mode 100644 index 0000000..743dcb8 --- /dev/null +++ b/tests/test_consumers.py @@ -0,0 +1,379 @@ +"""Tests for consumers admin routes (GET /consumers, POST /consumers/{s}/{c}/delete).""" + +import os +from datetime import datetime, timezone +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +# Set required env vars before importing central modules +os.environ.setdefault("CENTRAL_DB_DSN", "postgresql://test:test@localhost/test") +os.environ.setdefault("CENTRAL_CSRF_SECRET", "testsecret12345678901234567890ab") +os.environ.setdefault("CENTRAL_NATS_URL", "nats://localhost:4222") + + +def _make_consumer_info(name: str, num_pending: int = 0, num_ack_pending: int = 0, + num_redelivered: int = 0, num_waiting: int = 0): + ci = MagicMock() + ci.name = name + ci.num_pending = num_pending + ci.num_ack_pending = num_ack_pending + ci.num_redelivered = num_redelivered + ci.num_waiting = num_waiting + ci.created = datetime(2026, 5, 17, 12, 0, 0, tzinfo=timezone.utc) + return ci + + +def _make_js_with_consumers(consumers_by_stream: dict): + """Build a mock JetStreamContext whose consumers_info is a coroutine returning a list.""" + mock_js = MagicMock() + mock_js.consumers_info = AsyncMock( + side_effect=lambda stream, **kw: consumers_by_stream.get(stream, []) + ) + mock_js.consumer_info = AsyncMock() + mock_js.delete_consumer = AsyncMock() + return mock_js + + +class TestConsumersListNatsUnavailable: + """GET /consumers when NATS is down shows per-stream error.""" + + @pytest.mark.asyncio + async def test_nats_unavailable_shows_error_per_stream(self): + from central.gui.routes import consumers_list + + mock_request = MagicMock() + mock_request.state.operator = MagicMock(id=1, username="testop") + mock_request.state.csrf_token = "test_csrf" + + mock_templates = MagicMock() + mock_templates.TemplateResponse.return_value = MagicMock() + + with patch("central.gui.routes._get_templates", return_value=mock_templates): + with patch("central.gui.nats.get_js", return_value=None): + await consumers_list(mock_request) + + call_args = mock_templates.TemplateResponse.call_args + context = call_args.kwargs.get("context", call_args[1].get("context")) + streams = context["streams"] + # All streams should show the NATS unavailable error + assert all(s["error"] == "NATS unavailable" for s in streams) + # And no consumers listed + assert all(s["consumers"] == [] for s in streams) + + +class TestConsumersListWithConsumers: + """GET /consumers with live NATS returns consumers per stream.""" + + @pytest.mark.asyncio + async def test_consumers_listed_with_protected_flag(self): + from central.gui.routes import consumers_list + + mock_request = MagicMock() + mock_request.state.operator = MagicMock(id=1, username="testop") + mock_request.state.csrf_token = "test_csrf" + + mock_templates = MagicMock() + mock_templates.TemplateResponse.return_value = MagicMock() + + consumers_by_stream = { + "CENTRAL_WX": [ + _make_consumer_info("archive-CENTRAL_WX", num_pending=5, num_waiting=1), + _make_consumer_info("meshai-wx", num_pending=1000, num_waiting=0), + ], + } + mock_js = _make_js_with_consumers(consumers_by_stream) + + with patch("central.gui.routes._get_templates", return_value=mock_templates): + with patch("central.gui.nats.get_js", return_value=mock_js): + await consumers_list(mock_request) + + call_args = mock_templates.TemplateResponse.call_args + context = call_args.kwargs.get("context", call_args[1].get("context")) + streams = context["streams"] + + wx = next(s for s in streams if s["stream"] == "CENTRAL_WX") + assert wx["error"] is None + assert len(wx["consumers"]) == 2 + + archive_c = next(c for c in wx["consumers"] if c["name"] == "archive-CENTRAL_WX") + assert archive_c["protected"] is True + assert archive_c["num_pending"] == 5 + + meshai_c = next(c for c in wx["consumers"] if c["name"] == "meshai-wx") + assert meshai_c["protected"] is False + assert meshai_c["num_pending"] == 1000 + assert meshai_c["num_waiting"] == 0 + + # Regression guard: consumer names must appear in the template context so + # they are rendered into the HTML body (guards against the coroutine/iterator + # bug where consumers_info was consumed as an async-iterable instead of awaited). + consumer_names_in_context = {c["name"] for c in wx["consumers"]} + assert "archive-CENTRAL_WX" in consumer_names_in_context + assert "meshai-wx" in consumer_names_in_context + + @pytest.mark.asyncio + async def test_stream_with_no_consumers_shows_empty(self): + from central.gui.routes import consumers_list + + mock_request = MagicMock() + mock_request.state.operator = MagicMock(id=1, username="testop") + mock_request.state.csrf_token = "test_csrf" + + mock_templates = MagicMock() + mock_templates.TemplateResponse.return_value = MagicMock() + + mock_js = _make_js_with_consumers({}) # No consumers on any stream + + with patch("central.gui.routes._get_templates", return_value=mock_templates): + with patch("central.gui.nats.get_js", return_value=mock_js): + await consumers_list(mock_request) + + call_args = mock_templates.TemplateResponse.call_args + context = call_args.kwargs.get("context", call_args[1].get("context")) + streams = context["streams"] + assert all(s["consumers"] == [] for s in streams) + assert all(s["error"] is None for s in streams) + + @pytest.mark.asyncio + async def test_one_stream_error_does_not_break_page(self): + from central.gui.routes import consumers_list + + mock_request = MagicMock() + mock_request.state.operator = MagicMock(id=1, username="testop") + mock_request.state.csrf_token = "test_csrf" + + mock_templates = MagicMock() + mock_templates.TemplateResponse.return_value = MagicMock() + + mock_js = MagicMock() + + async def consumers_info_raising(stream_name): + if stream_name == "CENTRAL_FIRE": + raise RuntimeError("stream not found") + # other streams: empty list (coroutine returning a list, not an async generator) + return [] + + mock_js.consumers_info = consumers_info_raising + + with patch("central.gui.routes._get_templates", return_value=mock_templates): + with patch("central.gui.nats.get_js", return_value=mock_js): + await consumers_list(mock_request) + + call_args = mock_templates.TemplateResponse.call_args + context = call_args.kwargs.get("context", call_args[1].get("context")) + streams = context["streams"] + + fire = next(s for s in streams if s["stream"] == "CENTRAL_FIRE") + assert "unavailable" in fire["error"] + assert fire["consumers"] == [] + + +class TestConsumersListHtmlRender: + """Render consumers_list.html through the real Jinja2 environment. + + Stronger than the context-dict checks above: these prove the values + actually reach the rendered HTML body. Guards two regressions: + - the consumer NAME must appear in the rendered HTML (proves the + ``await js.consumers_info(...)`` list reaches the template, not the + coroutine/async-iterator bug) + - Optional[int] count fields that are None must not render the literal + string ``None`` (they are guarded to an em dash). + """ + + PROTECTED_LABEL = 'central-owned' + + def _render(self, streams): + from central.gui import templates as templates_mod + template = templates_mod.env.get_template("consumers_list.html") + return template.render( + operator=MagicMock(username="testop"), + csrf_token="test_csrf", + streams=streams, + ) + + def test_consumer_name_appears_in_html(self): + streams = [ + { + "stream": "CENTRAL_WX", + "error": None, + "consumers": [ + { + "name": "meshai-wx", + "num_pending": 1000, + "num_ack_pending": 0, + "num_redelivered": 0, + "num_waiting": 0, + "created": datetime(2026, 5, 17, 12, 0, 0, tzinfo=timezone.utc), + "protected": False, + }, + ], + }, + ] + html = self._render(streams) + assert "meshai-wx" in html + # Non-protected consumer renders a delete form + assert "/consumers/CENTRAL_WX/meshai-wx/delete" in html + # ...and not the central-owned label span (which only the legend prose + # mentions, so we match the exact span markup, not the bare phrase) + assert self.PROTECTED_LABEL not in html + + def test_protected_consumer_renders_label_not_button(self): + streams = [ + { + "stream": "CENTRAL_WX", + "error": None, + "consumers": [ + { + "name": "archive-CENTRAL_WX", + "num_pending": 5, + "num_ack_pending": 0, + "num_redelivered": 0, + "num_waiting": 1, + "created": datetime(2026, 5, 17, 12, 0, 0, tzinfo=timezone.utc), + "protected": True, + }, + ], + }, + ] + html = self._render(streams) + assert "archive-CENTRAL_WX" in html + assert self.PROTECTED_LABEL in html + # No delete form for the protected consumer + assert "/consumers/CENTRAL_WX/archive-CENTRAL_WX/delete" not in html + + def test_none_counts_render_dash_not_literal_none(self): + streams = [ + { + "stream": "CENTRAL_WX", + "error": None, + "consumers": [ + { + "name": "meshai-wx", + "num_pending": None, + "num_ack_pending": None, + "num_redelivered": None, + "num_waiting": None, + "created": None, + "protected": False, + }, + ], + }, + ] + html = self._render(streams) + assert "meshai-wx" in html + # The literal "None" must never leak into a rendered table cell + assert ">None<" not in html + # The guarded fallback em dash is rendered instead + assert "—" in html + + +class TestConsumersDeleteArchiveGuard: + """POST /consumers/{stream}/archive-*/delete must be refused.""" + + @pytest.mark.asyncio + async def test_archive_consumer_refused_redirects(self): + from central.gui.routes import consumers_delete + + mock_request = MagicMock() + mock_request.state.operator = MagicMock(id=1) + mock_request.state.csrf_token = "tok" + form_data = MagicMock() + form_data.get.side_effect = lambda k, d="": {"csrf_token": "tok"}.get(k, d) + mock_request.form = AsyncMock(return_value=form_data) + + with patch("central.gui.nats.get_js", return_value=MagicMock()): + result = await consumers_delete(mock_request, "CENTRAL_WX", "archive-CENTRAL_WX") + + assert result.status_code == 302 + assert result.headers["location"] == "/consumers" + + +class TestConsumersDeleteSuccess: + """POST /consumers/{stream}/{consumer}/delete happy path.""" + + @pytest.mark.asyncio + async def test_delete_non_protected_consumer_audits_and_redirects(self): + from central.gui.routes import consumers_delete + + mock_request = MagicMock() + mock_request.state.operator = MagicMock(id=1) + mock_request.state.csrf_token = "tok" + form_data = MagicMock() + form_data.get.side_effect = lambda k, d="": {"csrf_token": "tok"}.get(k, d) + mock_request.form = AsyncMock(return_value=form_data) + + before_ci = _make_consumer_info("meshai-wx", num_pending=500) + mock_js = MagicMock() + mock_js.consumer_info = AsyncMock(return_value=before_ci) + mock_js.delete_consumer = AsyncMock() + + mock_conn = AsyncMock() + mock_pool = MagicMock() + mock_pool.acquire.return_value.__aenter__ = AsyncMock(return_value=mock_conn) + mock_pool.acquire.return_value.__aexit__ = AsyncMock(return_value=None) + + captured_audit = {} + + async def capture_audit(conn, action, operator_id=None, target=None, before=None, after=None): + captured_audit["action"] = action + captured_audit["operator_id"] = operator_id + captured_audit["target"] = target + captured_audit["before"] = before + captured_audit["after"] = after + + with patch("central.gui.nats.get_js", return_value=mock_js): + with patch("central.gui.routes.get_pool", return_value=mock_pool): + with patch("central.gui.routes.write_audit", side_effect=capture_audit): + result = await consumers_delete(mock_request, "CENTRAL_WX", "meshai-wx") + + assert result.status_code == 302 + assert result.headers["location"] == "/consumers" + + mock_js.delete_consumer.assert_awaited_once_with("CENTRAL_WX", "meshai-wx") + + assert captured_audit["action"] == "consumer.delete" + assert captured_audit["operator_id"] == 1 + assert captured_audit["target"] == "CENTRAL_WX/meshai-wx" + assert captured_audit["before"]["name"] == "meshai-wx" + assert captured_audit["after"] is None + + +class TestConsumersDeleteCsrfGuard: + """POST /consumers/{stream}/{consumer}/delete CSRF mismatch raises.""" + + @pytest.mark.asyncio + async def test_csrf_mismatch_raises(self): + from central.gui.routes import consumers_delete + from central.gui.auth import CsrfValidationError + + mock_request = MagicMock() + mock_request.state.operator = MagicMock(id=1) + mock_request.state.csrf_token = "real_token" + form_data = MagicMock() + form_data.get.side_effect = lambda k, d="": {"csrf_token": "wrong_token"}.get(k, d) + mock_request.form = AsyncMock(return_value=form_data) + + with pytest.raises(CsrfValidationError): + await consumers_delete(mock_request, "CENTRAL_WX", "meshai-wx") + + +class TestConsumersDeleteNatsUnavailable: + """POST /consumers/{stream}/{consumer}/delete when NATS is down redirects.""" + + @pytest.mark.asyncio + async def test_nats_unavailable_redirects(self): + from central.gui.routes import consumers_delete + + mock_request = MagicMock() + mock_request.state.operator = MagicMock(id=1) + mock_request.state.csrf_token = "tok" + form_data = MagicMock() + form_data.get.side_effect = lambda k, d="": {"csrf_token": "tok"}.get(k, d) + mock_request.form = AsyncMock(return_value=form_data) + + with patch("central.gui.nats.get_js", return_value=None): + result = await consumers_delete(mock_request, "CENTRAL_WX", "meshai-wx") + + assert result.status_code == 302 + assert result.headers["location"] == "/consumers" From 832804817ef6b58308c0f8c065efcd531f263b0e Mon Sep 17 00:00:00 2001 From: Matt Johnson Date: Mon, 29 Jun 2026 01:42:13 +0000 Subject: [PATCH 10/17] chore: bump version to 0.14.7 (consumers admin page release) Co-Authored-By: Claude Opus 4.8 (1M context) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cbe0d0f..77c5b8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "central" -version = "0.14.6" +version = "0.14.7" requires-python = ">=3.12,<3.13" description = "Data hub spine — adapters, bus, archive." readme = "README.md" From a84ac0f911a4da2762ed9b0233a03071c9f8fdaf Mon Sep 17 00:00:00 2001 From: malice Date: Sun, 28 Jun 2026 21:38:13 -0600 Subject: [PATCH 11/17] gui: stack consumers page stream blocks full-width (fix table overflow) (#118) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 7-column consumer tables were rendering inside the shared `.cols` multi-column grid (minmax 240px), causing every table to overflow its card horizontally — right-hand columns and Delete buttons appeared outside the card border, visually detached from their rows. Fix: replace `.cols` with a new `.consumer-streams` flex-column container (consumers_list.html only) so each stream block is full-width and stacked vertically. Added a `.consumer-table-wrap` div with `overflow-x: auto` as a safety net for narrow viewports. The shared `.cols` class and the streams page are untouched. All existing content is preserved: 7 columns, None-guards (em dash), the `central-owned` label for archive-* consumers (no Delete button), the red Delete form + CSRF for deletable consumers, the legend, and the "(no consumers)" empty state. Version bump 0.14.7 → 0.14.8. Co-authored-by: Ubuntu Co-authored-by: Claude Opus 4.8 (1M context) --- pyproject.toml | 2 +- src/central/gui/static/css/central.css | 9 +++++++++ src/central/gui/templates/consumers_list.html | 4 +++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 77c5b8d..ffb56d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "central" -version = "0.14.7" +version = "0.14.8" requires-python = ">=3.12,<3.13" description = "Data hub spine — adapters, bus, archive." readme = "README.md" diff --git a/src/central/gui/static/css/central.css b/src/central/gui/static/css/central.css index 735ec39..2387a5c 100644 --- a/src/central/gui/static/css/central.css +++ b/src/central/gui/static/css/central.css @@ -194,6 +194,15 @@ button:disabled, .btn:disabled { opacity: 0.5; cursor: not-allowed; } .cols { display: grid; gap: 16px; grid-template-columns: repeat(auto-fit, minmax(240px, 1fr)); } +/* ─── consumers page: full-width stacked stream blocks ──────────────────── */ +/* Unlike .cols (multi-column grid used by streams/dashboard), these stream + blocks need the full page width because each table has 7 columns. */ +.consumer-streams { display: flex; flex-direction: column; gap: 16px; } +.consumer-streams > article { margin-bottom: 0; width: 100%; } +/* Safety net: scroll horizontally on very narrow viewports instead of + overflowing the card border. Default desktop view shows the full table. */ +.consumer-table-wrap { overflow-x: auto; } + /* cards (formerly the framework's
) */ article, .card { border: 1px solid var(--rule); diff --git a/src/central/gui/templates/consumers_list.html b/src/central/gui/templates/consumers_list.html index 252c1c4..11cfac8 100644 --- a/src/central/gui/templates/consumers_list.html +++ b/src/central/gui/templates/consumers_list.html @@ -8,7 +8,7 @@ Pending and zero Waiting has accumulated unacknowledged messages and has no active subscriber — it is safe to delete if it is not a central-owned consumer.

-
+
{% for stream in streams %}
{{ stream.stream }}
@@ -16,6 +16,7 @@ has no active subscriber — it is safe to delete if it is not a central-owned c {% if stream.error %}

({{ stream.error }})

{% elif stream.consumers %} +
@@ -52,6 +53,7 @@ has no active subscriber — it is safe to delete if it is not a central-owned c {% endfor %}
+
{% else %}

(no consumers)

{% endif %} From 9450696fe9f8e6e76bd61a045043766959e82c25 Mon Sep 17 00:00:00 2001 From: malice Date: Sun, 28 Jun 2026 22:32:50 -0600 Subject: [PATCH 12/17] gui: add Delivered + Confirmed (acked) columns to consumers page (#119) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Surfaces ConsumerInfo.delivered.consumer_seq (total delivered) and ConsumerInfo.ack_floor.consumer_seq (total acknowledged) as new columns on the /consumers page between WAITING and CREATED. Reuses the existing ConsumerInfo objects already fetched — no new NATS calls. Both fields use safe getattr access and render '—' when None. Legend updated to clarify these are consumer-level counters, not end-to-end mesh delivery confirmation. Bump version 0.14.8 → 0.14.9. Co-authored-by: Ubuntu Co-authored-by: Claude Opus 4.8 (1M context) --- pyproject.toml | 2 +- src/central/gui/routes.py | 2 + src/central/gui/templates/consumers_list.html | 9 ++++- tests/test_consumers.py | 40 ++++++++++++++++++- 4 files changed, 50 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ffb56d9..bcc35c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "central" -version = "0.14.8" +version = "0.14.9" requires-python = ">=3.12,<3.13" description = "Data hub spine — adapters, bus, archive." readme = "README.md" diff --git a/src/central/gui/routes.py b/src/central/gui/routes.py index 8dff9c5..7258fb4 100644 --- a/src/central/gui/routes.py +++ b/src/central/gui/routes.py @@ -2224,6 +2224,8 @@ async def consumers_list(request: Request) -> HTMLResponse: "num_ack_pending": ci.num_ack_pending, "num_redelivered": ci.num_redelivered, "num_waiting": ci.num_waiting, + "delivered": getattr(getattr(ci, "delivered", None), "consumer_seq", None), + "acked": getattr(getattr(ci, "ack_floor", None), "consumer_seq", None), "created": ci.created, "protected": ci.name.startswith("archive-"), }) diff --git a/src/central/gui/templates/consumers_list.html b/src/central/gui/templates/consumers_list.html index 11cfac8..07693a5 100644 --- a/src/central/gui/templates/consumers_list.html +++ b/src/central/gui/templates/consumers_list.html @@ -25,6 +25,8 @@ has no active subscriber — it is safe to delete if it is not a central-owned c Ack Pending Redelivered Waiting + Delivered + Confirmed Created Action @@ -37,6 +39,8 @@ has no active subscriber — it is safe to delete if it is not a central-owned c {{ c.num_ack_pending if c.num_ack_pending is not none else '—' }} {{ c.num_redelivered if c.num_redelivered is not none else '—' }} {{ c.num_waiting if c.num_waiting is not none else '—' }} + {{ c.delivered if c.delivered is not none else '—' }} + {{ c.acked if c.acked is not none else '—' }} {{ c.created.isoformat() if c.created else '—' }} {% if c.protected %} @@ -64,7 +68,10 @@ has no active subscriber — it is safe to delete if it is not a central-owned c

Legend: Pending = messages not yet delivered to this consumer; Ack Pending = delivered but not yet acknowledged; - Waiting = active pull requests from a live subscriber. + Waiting = active pull requests from a live subscriber; + Delivered = total messages this consumer has received from the stream (lifetime counter); + Confirmed = total messages this consumer has acknowledged (processed) — these are consumer-level + counters (the subscriber got/acked it) and are NOT confirmation that the message reached a mesh device. A consumer with high Pending and zero Waiting is abandoned — no subscriber is pulling from it and messages are piling up. Consumers marked central-owned (archive-*) are managed by central and cannot be deleted here. diff --git a/tests/test_consumers.py b/tests/test_consumers.py index 743dcb8..95aa39c 100644 --- a/tests/test_consumers.py +++ b/tests/test_consumers.py @@ -2,6 +2,7 @@ import os from datetime import datetime, timezone +from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -13,13 +14,16 @@ os.environ.setdefault("CENTRAL_NATS_URL", "nats://localhost:4222") def _make_consumer_info(name: str, num_pending: int = 0, num_ack_pending: int = 0, - num_redelivered: int = 0, num_waiting: int = 0): + num_redelivered: int = 0, num_waiting: int = 0, + delivered_seq: int = 0, ack_floor_seq: int = 0): ci = MagicMock() ci.name = name ci.num_pending = num_pending ci.num_ack_pending = num_ack_pending ci.num_redelivered = num_redelivered ci.num_waiting = num_waiting + ci.delivered = SimpleNamespace(consumer_seq=delivered_seq) + ci.ack_floor = SimpleNamespace(consumer_seq=ack_floor_seq) ci.created = datetime(2026, 5, 17, 12, 0, 0, tzinfo=timezone.utc) return ci @@ -204,6 +208,8 @@ class TestConsumersListHtmlRender: "num_ack_pending": 0, "num_redelivered": 0, "num_waiting": 0, + "delivered": 5000, + "acked": 4000, "created": datetime(2026, 5, 17, 12, 0, 0, tzinfo=timezone.utc), "protected": False, }, @@ -230,6 +236,8 @@ class TestConsumersListHtmlRender: "num_ack_pending": 0, "num_redelivered": 0, "num_waiting": 1, + "delivered": 100, + "acked": 95, "created": datetime(2026, 5, 17, 12, 0, 0, tzinfo=timezone.utc), "protected": True, }, @@ -254,6 +262,8 @@ class TestConsumersListHtmlRender: "num_ack_pending": None, "num_redelivered": None, "num_waiting": None, + "delivered": None, + "acked": None, "created": None, "protected": False, }, @@ -267,6 +277,34 @@ class TestConsumersListHtmlRender: # The guarded fallback em dash is rendered instead assert "—" in html + def test_delivered_and_confirmed_columns_render(self): + streams = [ + { + "stream": "CENTRAL_WX", + "error": None, + "consumers": [ + { + "name": "meshai-wx", + "num_pending": 10, + "num_ack_pending": 2, + "num_redelivered": 0, + "num_waiting": 1, + "delivered": 7777, + "acked": 6543, + "created": datetime(2026, 5, 17, 12, 0, 0, tzinfo=timezone.utc), + "protected": False, + }, + ], + }, + ] + html = self._render(streams) + # Delivered and Confirmed column headers are present + assert "Delivered" in html + assert "Confirmed" in html + # The actual counter values appear in the rendered HTML + assert "7777" in html + assert "6543" in html + class TestConsumersDeleteArchiveGuard: """POST /consumers/{stream}/archive-*/delete must be refused.""" From 13722d07dd3e7a4d3b6cfeae57c291886b3a7326 Mon Sep 17 00:00:00 2001 From: malice Date: Tue, 30 Jun 2026 00:53:34 -0600 Subject: [PATCH 13/17] refactor: decouple adapter class-identity (kind) from instance-identity (name) (#120) Adds a `kind` TEXT column to `config.adapters` (migration 043) so one adapter class can later back many operator-created instances. Every built-in adapter row is back-filled with `kind = name`, keeping all 23 adapters working unchanged. - Migration 043: ADD COLUMN IF NOT EXISTS kind, back-fill, SET NOT NULL. - AdapterConfig gains `kind: str | None` with a model_validator that falls back to `name` when the column is NULL/absent (pre-043 rows). - config_store.get_adapter / list_adapters: add `kind` to SELECT. - supervisor._create_adapter AND _start_adapter (api-key precondition): resolve the class by config.kind (class key) not config.name (instance key); runtime state keying stays on config.name. - routes.py GET/POST /adapters/{name} and adapters_list: add `kind` to SELECT, resolve the adapter class by row["kind"]. - adapter_discovery.py: comment clarifying .name is the kind (class identity). - Out of scope, left as-is (built-ins only, name==kind): setup wizard paths in routes.py (~750/~891/~1109) and gui/__init__.py. - Tests: 17 new pure-unit tests (migration SQL shape, AdapterConfig back-compat, _create_adapter + _start_adapter kind dispatch). Existing test mock rows updated to include the `kind` field. Co-authored-by: Ubuntu Co-authored-by: Claude Sonnet 4.6 --- .../043_add_adapters_kind_column.sql | 25 ++ src/central/adapter_discovery.py | 2 + src/central/config_models.py | 17 +- src/central/config_store.py | 4 +- src/central/gui/routes.py | 26 +- src/central/supervisor.py | 19 +- tests/test_adapter_config_kind.py | 230 ++++++++++++++++++ tests/test_adapters.py | 14 +- tests/test_gui_adapter_edit.py | 2 +- tests/test_migration_043.py | 48 ++++ tests/test_region_picker.py | 6 + tests/test_requires_api_key.py | 3 +- 12 files changed, 370 insertions(+), 26 deletions(-) create mode 100644 sql/migrations/043_add_adapters_kind_column.sql create mode 100644 tests/test_adapter_config_kind.py create mode 100644 tests/test_migration_043.py diff --git a/sql/migrations/043_add_adapters_kind_column.sql b/sql/migrations/043_add_adapters_kind_column.sql new file mode 100644 index 0000000..d6083c8 --- /dev/null +++ b/sql/migrations/043_add_adapters_kind_column.sql @@ -0,0 +1,25 @@ +-- Migration 043: decouple adapter class-identity (kind) from instance-identity (name) +-- +-- Until now config.adapters.name served two roles: +-- 1. Instance primary key — the unique name used in runtime state, logs, dedup +-- tables, NATS subjects, etc. +-- 2. Registry key — the key used to look up the adapter class in the +-- supervisor's in-memory registry (discover_adapters returns {class.name: cls}). +-- +-- These roles must be split so that one adapter class can later back many +-- operator-created instances (e.g. a generic HTTP adapter). The new `kind` +-- column holds the class identity (registry key), while `name` remains the +-- unique instance identifier. All built-in adapters have name == kind, so +-- every existing row is back-filled with kind = name and behaviour is unchanged. +-- +-- Idempotent: ADD COLUMN IF NOT EXISTS is safe on re-run. +-- No DEFAULT is set — future INSERT paths must always supply kind explicitly. + +ALTER TABLE config.adapters ADD COLUMN IF NOT EXISTS kind TEXT; + +-- Back-fill: existing rows get kind = name (class identity == instance identity +-- for all 23 built-in adapters shipped before v0.15.0). +UPDATE config.adapters SET kind = name WHERE kind IS NULL; + +-- Enforce non-null going forward. +ALTER TABLE config.adapters ALTER COLUMN kind SET NOT NULL; diff --git a/src/central/adapter_discovery.py b/src/central/adapter_discovery.py index e26729f..5be71f9 100644 --- a/src/central/adapter_discovery.py +++ b/src/central/adapter_discovery.py @@ -30,5 +30,7 @@ def discover_adapters() -> dict[str, type[SourceAdapter]]: and attr is not SourceAdapter and hasattr(attr, "name") ): + # attr.name is the *kind* (class identity); AdapterConfig.name + # is the instance identity. For all built-ins kind == name. registry[attr.name] = attr return registry diff --git a/src/central/config_models.py b/src/central/config_models.py index 557423a..8533238 100644 --- a/src/central/config_models.py +++ b/src/central/config_models.py @@ -30,7 +30,15 @@ class RegionConfig(BaseModel): class AdapterConfig(BaseModel): """Configuration for a single adapter.""" - name: str = Field(description="Unique adapter identifier") + name: str = Field(description="Unique adapter identifier (instance identity)") + kind: str | None = Field( + default=None, + description=( + "Adapter class identifier (registry key / class identity). " + "Matches SourceAdapter.name on the class. Defaults to `name` " + "for back-compat when the DB column is absent or NULL (pre-043)." + ), + ) enabled: bool = Field(default=True, description="Whether adapter is active") cadence_s: int = Field(ge=10, description="Poll interval in seconds") settings: dict[str, Any] = Field( @@ -41,6 +49,13 @@ class AdapterConfig(BaseModel): ) updated_at: datetime = Field(description="Last configuration update time") + @model_validator(mode="after") + def _backfill_kind(self) -> "AdapterConfig": + """Default kind to name when the DB column is NULL or absent (pre-043).""" + if self.kind is None: + self.kind = self.name + return self + @property def is_paused(self) -> bool: """Check if adapter is currently paused.""" diff --git a/src/central/config_store.py b/src/central/config_store.py index 8fe35fc..be3d8ce 100644 --- a/src/central/config_store.py +++ b/src/central/config_store.py @@ -95,7 +95,7 @@ class ConfigStore: async with self._pool.acquire() as conn: row = await conn.fetchrow( """ - SELECT name, enabled, cadence_s, settings, paused_at, updated_at + SELECT name, kind, enabled, cadence_s, settings, paused_at, updated_at FROM config.adapters WHERE name = $1 """, @@ -110,7 +110,7 @@ class ConfigStore: async with self._pool.acquire() as conn: rows = await conn.fetch( """ - SELECT name, enabled, cadence_s, settings, paused_at, updated_at + SELECT name, kind, enabled, cadence_s, settings, paused_at, updated_at FROM config.adapters ORDER BY name """ diff --git a/src/central/gui/routes.py b/src/central/gui/routes.py index 7258fb4..32d078e 100644 --- a/src/central/gui/routes.py +++ b/src/central/gui/routes.py @@ -1451,7 +1451,7 @@ async def adapters_list( async with pool.acquire() as conn: rows = await conn.fetch( """ - SELECT name, enabled, cadence_s, settings, paused_at, updated_at, last_error + SELECT name, kind, enabled, cadence_s, settings, paused_at, updated_at, last_error FROM config.adapters ORDER BY name """ @@ -1460,7 +1460,8 @@ async def adapters_list( adapters = [] for row in rows: settings = row["settings"] or {} - adapter_cls = adapter_classes.get(row["name"]) + # Resolve the class by kind (class identity), not instance name. + adapter_cls = adapter_classes.get(row["kind"]) # Check if required API key is missing — resolve via the per-row # settings[api_key_field] (operator-selected alias), falling back @@ -1540,14 +1541,10 @@ async def adapters_edit_form( pool = get_pool() operator = request.state.operator - # Look up the adapter class - adapter_classes = _adapter_classes() - adapter_cls = adapter_classes.get(name) - async with pool.acquire() as conn: row = await conn.fetchrow( """ - SELECT name, enabled, cadence_s, settings, paused_at, updated_at, last_error + SELECT name, kind, enabled, cadence_s, settings, paused_at, updated_at, last_error FROM config.adapters WHERE name = $1 """, @@ -1564,6 +1561,10 @@ async def adapters_edit_form( tile_url = sys_row["map_tile_url"] if sys_row else "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png" tile_attribution = sys_row["map_attribution"] if sys_row else "© OpenStreetMap contributors" + # Look up the adapter class by kind (class identity), not instance name. + adapter_classes = _adapter_classes() + adapter_cls = adapter_classes.get(row["kind"]) + settings = row["settings"] or {} # Build adapter dict with class metadata @@ -1609,6 +1610,7 @@ async def adapters_edit_form( settings_obj = adapter_cls.settings_schema(**settings) preview_cfg = AdapterConfig( name=row["name"], + kind=row["kind"], enabled=row["enabled"], cadence_s=row["cadence_s"], settings=settings, @@ -1672,10 +1674,6 @@ async def adapters_edit_submit( if not form_csrf or form_csrf != request.state.csrf_token: raise CsrfValidationError("Invalid CSRF token") - # Look up the adapter class - adapter_classes = _adapter_classes() - adapter_cls = adapter_classes.get(name) - # Parse common form fields enabled = "enabled" in form cadence_s_str = form.get("cadence_s", "") @@ -1701,7 +1699,7 @@ async def adapters_edit_submit( # Get current adapter state row = await conn.fetchrow( """ - SELECT name, enabled, cadence_s, settings, paused_at, updated_at, last_error + SELECT name, kind, enabled, cadence_s, settings, paused_at, updated_at, last_error FROM config.adapters WHERE name = $1 """, @@ -1711,6 +1709,10 @@ async def adapters_edit_submit( if row is None: return Response(status_code=404, content="Adapter not found") + # Look up the adapter class by kind (class identity), not instance name. + adapter_classes = _adapter_classes() + adapter_cls = adapter_classes.get(row["kind"]) + current_settings = row["settings"] or {} # Parse and validate settings via Pydantic if we have the adapter class diff --git a/src/central/supervisor.py b/src/central/supervisor.py index 7b269c6..7e79a50 100644 --- a/src/central/supervisor.py +++ b/src/central/supervisor.py @@ -355,10 +355,18 @@ class Supervisor: ) def _create_adapter(self, config: AdapterConfig) -> SourceAdapter: - """Create an adapter instance based on config name.""" - cls = self._adapters.get(config.name) + """Create an adapter instance based on config kind (class identity). + + config.kind names the adapter *class* in the registry (discover_adapters + keys by SourceAdapter.name on the class). config.name is the unique + *instance* identifier and is unchanged here — runtime state, logs, and + dedup tables continue to key on config.name. + """ + cls = self._adapters.get(config.kind) if cls is None: - raise ValueError(f"Unknown adapter type: {config.name}") + raise ValueError( + f"Unknown adapter kind: {config.kind!r} (instance: {config.name!r})" + ) return cls( config=config, config_store=self._config_store, @@ -524,8 +532,9 @@ class Supervisor: # API key precondition — resolve via per-row settings[api_key_field] # (operator-selected alias), falling back to the class-attribute # default when settings hasn't been set. Returns None when no key - # is required. - adapter_cls = self._adapters.get(config.name) + # is required. Resolve the class by config.kind (class identity), not + # config.name (instance identity) — mirrors _create_adapter. + adapter_cls = self._adapters.get(config.kind) alias = resolve_api_key_alias(adapter_cls, config.settings) if alias is not None: key_value = await self._config_store.get_api_key(alias) diff --git a/tests/test_adapter_config_kind.py b/tests/test_adapter_config_kind.py new file mode 100644 index 0000000..81b60d0 --- /dev/null +++ b/tests/test_adapter_config_kind.py @@ -0,0 +1,230 @@ +"""Tests for AdapterConfig.kind and supervisor._create_adapter kind-based dispatch. + +All tests are pure-unit (no DB, no NATS) so they run anywhere. +""" + +import os +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock + +import pytest + +# Provide required env vars before importing central modules. +os.environ.setdefault("CENTRAL_DB_DSN", "postgresql://test:test@localhost/test") +os.environ.setdefault("CENTRAL_CSRF_SECRET", "testsecret12345678901234567890ab") +os.environ.setdefault("CENTRAL_NATS_URL", "nats://localhost:4222") + +from central.config_models import AdapterConfig + + +_NOW = datetime(2026, 1, 1, tzinfo=timezone.utc) + + +def _cfg(**kwargs) -> AdapterConfig: + """Build a minimal AdapterConfig with sensible defaults.""" + defaults = { + "name": "usgs_quake", + "cadence_s": 120, + "updated_at": _NOW, + } + defaults.update(kwargs) + return AdapterConfig(**defaults) + + +# --------------------------------------------------------------------------- +# AdapterConfig.kind back-compat +# --------------------------------------------------------------------------- + + +class TestAdapterConfigKind: + def test_kind_defaults_to_name_when_absent(self): + """When kind is not supplied, it falls back to name (pre-043 rows).""" + cfg = _cfg(name="usgs_quake") + assert cfg.kind == "usgs_quake" + + def test_kind_defaults_to_name_when_none(self): + """When kind is explicitly None (NULL from DB), it falls back to name.""" + cfg = _cfg(name="nws", kind=None) + assert cfg.kind == "nws" + + def test_kind_preserved_when_set(self): + """When kind is explicitly supplied, it is kept as-is.""" + cfg = _cfg(name="my_quake_instance", kind="usgs_quake") + assert cfg.kind == "usgs_quake" + assert cfg.name == "my_quake_instance" + + def test_name_unchanged_when_kind_differs(self): + """Instance identity (name) is never overwritten by the kind back-fill.""" + cfg = _cfg(name="my_custom_quake", kind="usgs_quake") + assert cfg.name == "my_custom_quake" + + def test_kind_equals_name_for_builtin_pattern(self): + """Built-in adapters: name == kind both before and after the migration.""" + cfg = _cfg(name="nws", kind="nws") + assert cfg.kind == cfg.name == "nws" + + +# --------------------------------------------------------------------------- +# supervisor._create_adapter resolves class by kind +# --------------------------------------------------------------------------- + + +class TestCreateAdapterResolvesbyKind: + """Unit-test _create_adapter without a live Supervisor (no NATS, no DB).""" + + def _make_supervisor_with_registry(self, registry: dict): + """Return a minimal Supervisor-like object with _adapters = registry.""" + # Import lazily to avoid triggering the ENRICHMENT_CACHE_DB_PATH side + # effect before conftest patches it (conftest autouse fixture handles it + # in the full test run; here we patch manually). + from unittest.mock import patch + import central.supervisor as sup_mod + + with tempfile.TemporaryDirectory() as td: + with patch.object(sup_mod, "ENRICHMENT_CACHE_DB_PATH", Path(td) / "ec.db"): + with patch.object(sup_mod, "CURSOR_DB_PATH", Path(td) / "cursor.db"): + supervisor = MagicMock() + supervisor._adapters = registry + supervisor._config_store = MagicMock() + # Bind the real _create_adapter method to our mock object. + supervisor._create_adapter = ( + sup_mod.Supervisor._create_adapter.__get__(supervisor) + ) + return supervisor + + def _make_mock_adapter_cls(self, class_name: str): + """Return a mock adapter class whose constructor returns a mock instance.""" + instance = MagicMock() + instance.name = class_name # class-level .name attribute + + cls = MagicMock() + cls.return_value = instance + return cls + + def test_resolves_class_by_kind_when_kind_differs_from_name(self): + """_create_adapter uses config.kind (class key), not config.name.""" + usgs_cls = self._make_mock_adapter_cls("usgs_quake") + registry = {"usgs_quake": usgs_cls} + sup = self._make_supervisor_with_registry(registry) + + cfg = _cfg(name="my_quake_instance", kind="usgs_quake") + result = sup._create_adapter(cfg) + + # Class was looked up by kind and instantiated. + usgs_cls.assert_called_once() + assert result is usgs_cls.return_value + + def test_builtin_pattern_kind_equals_name_still_works(self): + """Regression: existing adapters with name == kind construct correctly.""" + usgs_cls = self._make_mock_adapter_cls("usgs_quake") + registry = {"usgs_quake": usgs_cls} + sup = self._make_supervisor_with_registry(registry) + + cfg = _cfg(name="usgs_quake", kind="usgs_quake") + result = sup._create_adapter(cfg) + + usgs_cls.assert_called_once() + assert result is usgs_cls.return_value + + def test_raises_on_unknown_kind(self): + """_create_adapter raises ValueError mentioning both kind and instance name.""" + registry = {} + sup = self._make_supervisor_with_registry(registry) + + cfg = _cfg(name="my_instance", kind="nonexistent_adapter") + with pytest.raises(ValueError) as exc_info: + sup._create_adapter(cfg) + + msg = str(exc_info.value) + assert "nonexistent_adapter" in msg + assert "my_instance" in msg + + def test_instance_identity_passed_through_to_constructor(self): + """The adapter constructor receives the full config (name = instance id).""" + some_cls = self._make_mock_adapter_cls("some_adapter") + registry = {"some_adapter": some_cls} + sup = self._make_supervisor_with_registry(registry) + + cfg = _cfg(name="prod_instance_1", kind="some_adapter") + sup._create_adapter(cfg) + + # Check the config passed to constructor has the instance name. + call_kwargs = some_cls.call_args + passed_config = call_kwargs[1].get("config") or call_kwargs[0][0] + assert passed_config.name == "prod_instance_1" + + +class TestStartAdapterResolvesByKind: + """The api-key precondition in _start_adapter also resolves the class by kind. + + _start_adapter is callable directly with mocks (no live DB), mirroring + tests/test_requires_api_key.py::TestSupervisorApiKeyPrecondition. A generic + instance where name != kind must still find its class via the registry's + kind key — otherwise the lookup returns None, resolve_api_key_alias treats + it as "no key required", and the adapter silently skips its api-key check + and starts. Resolving by kind keeps the precondition enforced. + """ + + @pytest.mark.asyncio + async def test_precondition_refuses_when_key_missing_for_instance_named_differently( + self, tmp_path: Path + ): + from central.supervisor import Supervisor + from central.adapters.firms import FIRMSAdapter + + mock_config_store = MagicMock() + mock_config_store.get_api_key = AsyncMock(return_value=None) # key missing + mock_config_store.set_adapter_last_error = AsyncMock() + + mock_nats = MagicMock() + mock_nats.publish = AsyncMock() + + supervisor = Supervisor.__new__(Supervisor) + supervisor._config_store = mock_config_store + # Registry is keyed by class identity (kind == FIRMSAdapter.name == "firms"). + supervisor._adapters = {"firms": FIRMSAdapter} + supervisor._adapter_states = {} + supervisor._nats = mock_nats + supervisor._cursor_db_path = tmp_path / "cursors.db" + supervisor._log = MagicMock() + + # Instance name differs from kind — the generic-instance case. + config = _cfg( + name="my_firms_instance", + kind="firms", + enabled=True, + cadence_s=300, + settings={"api_key_alias": "firms", "satellites": ["VIIRS_SNPP_NRT"]}, + ) + + await supervisor._start_adapter(config) + + # The class WAS resolved by kind: requires_api_key fired, key was found + # missing, and the adapter refused to start. Had the lookup used + # config.name ("my_firms_instance", absent from the registry), the + # precondition would have been skipped and the adapter would have started. + mock_config_store.get_api_key.assert_called_once_with("firms") + mock_config_store.set_adapter_last_error.assert_called_once() + err_args = mock_config_store.set_adapter_last_error.call_args[0] + assert err_args[0] == "my_firms_instance" # error keyed by instance id + assert "missing api key" in err_args[1].lower() + # Did not start. + assert "my_firms_instance" not in supervisor._adapter_states + mock_nats.publish.assert_not_called() + + def test_start_path_lookup_expression_uses_kind(self): + """Belt-and-suspenders: the start-path lookup keys on config.kind. + + Direct assertion of the resolution expression (supervisor._adapters.get( + config.kind)) independent of the heavier _start_adapter flow above. + """ + from central.adapters.firms import FIRMSAdapter + + registry = {"firms": FIRMSAdapter} + config = _cfg(name="my_firms_instance", kind="firms") + + # Resolving by kind finds the class; resolving by name would miss. + assert registry.get(config.kind) is FIRMSAdapter + assert registry.get(config.name) is None diff --git a/tests/test_adapters.py b/tests/test_adapters.py index 8ed0259..bf2fb9d 100644 --- a/tests/test_adapters.py +++ b/tests/test_adapters.py @@ -42,9 +42,9 @@ class TestAdaptersListAuthenticated: mock_conn = AsyncMock() mock_conn.fetch.return_value = [ - {"name": "firms", "enabled": True, "cadence_s": 300, "settings": {"api_key_alias": "firms"}, "paused_at": None, "updated_at": None, "last_error": None}, - {"name": "nws", "enabled": True, "cadence_s": 60, "settings": {"contact_email": "test@test.com"}, "paused_at": None, "updated_at": None, "last_error": None}, - {"name": "usgs_quake", "enabled": True, "cadence_s": 120, "settings": {"feed": "all_hour"}, "paused_at": None, "updated_at": None, "last_error": None}, + {"name": "firms", "kind": "firms", "enabled": True, "cadence_s": 300, "settings": {"api_key_alias": "firms"}, "paused_at": None, "updated_at": None, "last_error": None}, + {"name": "nws", "kind": "nws", "enabled": True, "cadence_s": 60, "settings": {"contact_email": "test@test.com"}, "paused_at": None, "updated_at": None, "last_error": None}, + {"name": "usgs_quake", "kind": "usgs_quake", "enabled": True, "cadence_s": 120, "settings": {"feed": "all_hour"}, "paused_at": None, "updated_at": None, "last_error": None}, ] mock_pool = MagicMock() @@ -97,6 +97,7 @@ class TestAdaptersEditForm: mock_conn.fetchrow.side_effect = [ { "name": "nws", + "kind": "nws", "enabled": True, "cadence_s": 60, "settings": {"contact_email": "test@example.com", "region": {"north": 49, "south": 24, "east": -66, "west": -125}}, @@ -178,6 +179,7 @@ class TestAdaptersEditSubmit: mock_conn = AsyncMock() mock_conn.fetchrow.return_value = { "name": "nws", + "kind": "nws", "enabled": True, "cadence_s": 60, "settings": {"contact_email": "old@example.com", "region": {"north": 49, "south": 24, "east": -66, "west": -125}}, @@ -227,6 +229,7 @@ class TestAdaptersEditSubmit: mock_conn.fetchrow.side_effect = [ { "name": "nws", + "kind": "nws", "enabled": True, "cadence_s": 60, "settings": {"contact_email": "test@example.com", "region": {"north": 49, "south": 24, "east": -66, "west": -125}}, @@ -286,6 +289,7 @@ class TestAdaptersAudit: mock_conn = AsyncMock() mock_conn.fetchrow.return_value = { "name": "nws", + "kind": "nws", "enabled": True, "cadence_s": 60, "settings": {"contact_email": "old@example.com", "region": {"north": 49, "south": 24, "east": -66, "west": -125}}, @@ -350,6 +354,7 @@ class TestAdaptersJsonbRegression: mock_conn = AsyncMock() mock_conn.fetchrow.return_value = { "name": "nws", + "kind": "nws", "enabled": True, "cadence_s": 60, "settings": {"contact_email": "old@example.com", "region": {"north": 49, "south": 24, "east": -66, "west": -125}}, # dict, as asyncpg returns @@ -402,6 +407,7 @@ class TestAdaptersJsonbRegression: mock_conn = AsyncMock() mock_conn.fetchrow.return_value = { "name": "nws", + "kind": "nws", "enabled": True, "cadence_s": 60, "settings": {"contact_email": "old@example.com", "region": {"north": 49, "south": 24, "east": -66, "west": -125}}, # dict @@ -443,7 +449,7 @@ class TestAdaptersJsonbRegression: mock_conn = MagicMock() mock_conn.fetchrow = AsyncMock(side_effect=[ # Adapter row - {"name": "firms", "enabled": True, "cadence_s": 300, "settings": {}, + {"name": "firms", "kind": "firms", "enabled": True, "cadence_s": 300, "settings": {}, "paused_at": None, "updated_at": None, "last_error": None}, # System row {"map_tile_url": "https://tile.example.com", "map_attribution": "Test"}, diff --git a/tests/test_gui_adapter_edit.py b/tests/test_gui_adapter_edit.py index d0ee06c..27d9c76 100644 --- a/tests/test_gui_adapter_edit.py +++ b/tests/test_gui_adapter_edit.py @@ -116,7 +116,7 @@ def _post_req(pairs, cadence_s="1800"): def _pool(settings=_SETTINGS): conn = AsyncMock() conn.fetchrow.side_effect = [ - {"name": "tomtom_incidents", "enabled": True, "cadence_s": 1800, "settings": settings, + {"name": "tomtom_incidents", "kind": "tomtom_incidents", "enabled": True, "cadence_s": 1800, "settings": settings, "paused_at": None, "updated_at": None, "last_error": None}, {"map_tile_url": None, "map_attribution": None}, ] diff --git a/tests/test_migration_043.py b/tests/test_migration_043.py new file mode 100644 index 0000000..b01c94b --- /dev/null +++ b/tests/test_migration_043.py @@ -0,0 +1,48 @@ +"""v0.15.0 migration 043: add kind column to config.adapters. + +No live Postgres required — asserts the migration SQL is structurally correct: +adds the column idempotently, back-fills kind = name for pre-existing rows, and +enforces NOT NULL going forward. +""" + +from pathlib import Path + +_SQL = Path("sql/migrations/043_add_adapters_kind_column.sql").read_text() +_NORM = " ".join(_SQL.split()) # whitespace-insensitive matching + + +def test_uses_add_column_if_not_exists(): + """Must be idempotent — safe to re-run after partial application.""" + assert "ADD COLUMN IF NOT EXISTS kind TEXT" in _NORM + + +def test_backfills_kind_equals_name_for_null_rows(): + """Existing rows must get kind = name so built-in adapters keep working.""" + assert "UPDATE config.adapters SET kind = name WHERE kind IS NULL" in _NORM + + +def test_enforces_not_null_after_backfill(): + """kind must be NOT NULL once every row is back-filled.""" + assert "ALTER COLUMN kind SET NOT NULL" in _NORM + + +def test_does_not_add_default_clause(): + """No DEFAULT — future INSERT paths must always supply kind explicitly.""" + # The ADD COLUMN line should not contain DEFAULT + add_line = next( + (line for line in _SQL.splitlines() if "ADD COLUMN" in line), "" + ) + assert "DEFAULT" not in add_line.upper() + + +def test_does_not_drop_any_columns(): + """Pure additive migration — must not drop anything.""" + upper = _NORM.upper() + assert "DROP COLUMN" not in upper + assert "DROP TABLE" not in upper + + +def test_explains_name_vs_kind_split_in_comment(): + """Header comment must explain the instance-vs-class identity split.""" + assert "instance" in _SQL.lower() + assert "class" in _SQL.lower() diff --git a/tests/test_region_picker.py b/tests/test_region_picker.py index 41fcbce..9a29762 100644 --- a/tests/test_region_picker.py +++ b/tests/test_region_picker.py @@ -27,6 +27,7 @@ class TestRegionPickerInTemplate: mock_conn.fetchrow.side_effect = [ { # Adapter row "name": "firms", + "kind": "firms", "enabled": True, "cadence_s": 300, "settings": { @@ -94,6 +95,7 @@ class TestRegionValidation: mock_conn = AsyncMock() mock_conn.fetchrow.return_value = { "name": "firms", + "kind": "firms", "enabled": True, "cadence_s": 300, "settings": {"api_key_alias": "firms", "satellites": ["VIIRS_SNPP_NRT"], "region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}}, @@ -153,6 +155,7 @@ class TestRegionValidation: mock_conn.fetchrow.side_effect = [ { "name": "firms", + "kind": "firms", "enabled": True, "cadence_s": 300, "settings": {"api_key_alias": "firms", "satellites": ["VIIRS_SNPP_NRT"], "region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}}, @@ -209,6 +212,7 @@ class TestRegionValidation: mock_conn.fetchrow.side_effect = [ { "name": "firms", + "kind": "firms", "enabled": True, "cadence_s": 300, "settings": {"api_key_alias": "firms", "satellites": ["VIIRS_SNPP_NRT"], "region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}}, @@ -265,6 +269,7 @@ class TestRegionValidation: mock_conn.fetchrow.side_effect = [ { "name": "firms", + "kind": "firms", "enabled": True, "cadence_s": 300, "settings": {"api_key_alias": "firms", "satellites": ["VIIRS_SNPP_NRT"], "region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}}, @@ -323,6 +328,7 @@ class TestRegionAuditLog: mock_conn = AsyncMock() mock_conn.fetchrow.return_value = { "name": "firms", + "kind": "firms", "enabled": True, "cadence_s": 300, "settings": { diff --git a/tests/test_requires_api_key.py b/tests/test_requires_api_key.py index 50f6118..a727211 100644 --- a/tests/test_requires_api_key.py +++ b/tests/test_requires_api_key.py @@ -74,7 +74,7 @@ class TestRoutesApiKeyMissing: mock_pool = MagicMock() mock_conn = MagicMock() mock_conn.fetch = AsyncMock(return_value=[ - {"name": "firms", "enabled": False, "cadence_s": 300, "settings": {}, "paused_at": None, "updated_at": None, "last_error": None}, + {"name": "firms", "kind": "firms", "enabled": False, "cadence_s": 300, "settings": {}, "paused_at": None, "updated_at": None, "last_error": None}, ]) mock_conn.fetchval = AsyncMock(return_value=None) # No API key exists mock_conn.__aenter__ = AsyncMock(return_value=mock_conn) @@ -307,6 +307,7 @@ class TestAdaptersEditSubmitErrorRerender: # First call: adapter row { "name": "firms", + "kind": "firms", "enabled": False, "cadence_s": 300, "settings": {"api_key_alias": "firms", "satellites": ["VIIRS_SNPP_NRT"]}, From 3effd677a541813ce8cbec25affbf6c2fd6323b3 Mon Sep 17 00:00:00 2001 From: malice Date: Tue, 30 Jun 2026 11:13:06 -0600 Subject: [PATCH 14/17] feat: add GenericHttpAdapter (kind=generic_http, v0.15.0 PR2) (#121) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One config-driven adapter class that operators can instantiate many times via distinct config.adapters rows (unique name, kind="generic_http"). Ships class + Pydantic settings schema + unit tests only; no GUI create route, no migration, no seeded row (PR3). Class is dormant until an operator row exists. Key design decisions: - self.name = config.name in __init__ scopes dedup to each instance - domain validated against STREAMS registry at import time - subject_for derives region from event.data["_enriched"]["geocoder"] identically to usgs_quake (no static subject_region setting) - _dig() helper supports dotted paths into nested dicts/lists Also adds: GUI partials (_event_summaries, _event_rows), CONSUMER- INTEGRATION.md §6 subsection, and test_events_feed_frontend.py sample entry — all required by the existing adapter-coverage consistency tests. Co-authored-by: Ubuntu Co-authored-by: Claude Sonnet 4.6 --- docs/CONSUMER-INTEGRATION.md | 49 ++ src/central/adapters/generic_http.py | 442 ++++++++++++ .../templates/_event_rows/generic_http.html | 3 + .../_event_summaries/generic_http.html | 2 + tests/test_events_feed_frontend.py | 1 + tests/test_generic_http.py | 666 ++++++++++++++++++ 6 files changed, 1163 insertions(+) create mode 100644 src/central/adapters/generic_http.py create mode 100644 src/central/gui/templates/_event_rows/generic_http.html create mode 100644 src/central/gui/templates/_event_summaries/generic_http.html create mode 100644 tests/test_generic_http.py diff --git a/docs/CONSUMER-INTEGRATION.md b/docs/CONSUMER-INTEGRATION.md index a7e342a..0d75ad7 100644 --- a/docs/CONSUMER-INTEGRATION.md +++ b/docs/CONSUMER-INTEGRATION.md @@ -2023,6 +2023,55 @@ at parameter `00060`, gage height (ft) at `00065`, water temperature (°C) at \ --- +### generic_http — Generic HTTP Source (v0.15.0) + +- **Source:** Any public REST / GeoJSON endpoint; configured per-instance + via `config.adapters.settings.url`. v1 supports public (unauthenticated) + feeds only; auth is a planned follow-up. +- **Stream:** Determined by the `domain` setting, which must match an + existing Central stream domain (e.g. `wx`, `fire`, `quake`). Subject + filter: `central..>`. +- **Subject:** `central..` — `` is derived from + `event.data["_enriched"]["geocoder"]` by the Photon enrichment pipeline + (identical convention to `usgs_quake`). Returns `unknown` when enrichment + has not yet run or found no geo signal. +- **Category:** `.` (default suffix: `alert`, so + e.g. `wx.alert`). +- **Dedup key:** the value resolved by `id_path` within each source item + (required field; coerced to string). +- **Geo:** GeoJSON geometry from `geometry_path` (default `"geometry"`) + when it resolves to a dict; `geo.centroid` is also set for Point + geometries. Falls back to `lat_path` / `lon_path` for non-GeoJSON + responses. +- **Event.data fields:** operator-configured. `title_path` → `data["title"]` + when set; additional fields via `field_mappings` (`source_path → dest_key` + pairs). No guaranteed fixed schema — varies per operator configuration. +- **Settings summary:** + + | Setting | Required | Default | Notes | + |---|---|---|---| + | `url` | yes | — | Polled endpoint (GET, no auth in v1) | + | `domain` | yes | — | Must be an existing stream domain | + | `id_path` | yes | — | Dotted path to stable unique item id | + | `items_path` | no | `features` | Dotted path to the items array | + | `format` | no | `geojson` | `geojson` or `json` (informational) | + | `time_path` | no | `null` | ISO-8601 timestamp path; uses poll time if absent | + | `title_path` | no | `null` | Path → `data["title"]` | + | `geometry_path` | no | `geometry` | GeoJSON geometry path | + | `lat_path` / `lon_path` | no | `null` | Numeric lat/lon paths (non-GeoJSON fallback) | + | `severity_path` | no | `null` | Int severity (0–4) path | + | `category_suffix` | no | `alert` | Appended to domain for Event.category | + | `field_mappings` | no | `[]` | List of `{source_path, dest_key}` | + +- **Cadence:** 300s (5 min) default. +- **Multiple instances:** one `generic_http` class supports many + `config.adapters` rows (each with a distinct instance `name`); dedup is + scoped per-instance. The class is dormant until an operator row is + created — no built-in seeded instance. + +\ +--- + ## 7. Fall-off / removal semantics Central adapters fall into three buckets for handling upstream events that diff --git a/src/central/adapters/generic_http.py b/src/central/adapters/generic_http.py new file mode 100644 index 0000000..98312c5 --- /dev/null +++ b/src/central/adapters/generic_http.py @@ -0,0 +1,442 @@ +"""Generic HTTP / GeoJSON adapter — operator-instantiable, no Python required. + +One class (kind="generic_http") supports many config.adapters rows. Each +row carries a distinct ``name`` (instance identity) and a ``settings`` dict +that drives a separate poll loop. PR3 ships the GUI create route; this PR +ships only the class. + +v1: public feeds only; auth is a follow-up. +""" + +import logging +import sqlite3 +from collections.abc import AsyncIterator +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Literal + +import aiohttp +from pydantic import BaseModel, field_validator +from tenacity import ( + retry, + retry_if_exception_type, + stop_after_attempt, + wait_exponential_jitter, +) + +from central.adapter import SourceAdapter +from central.adapters._subject_helpers import subject_for_region +from central.config_models import AdapterConfig +from central.config_store import ConfigStore +from central.models import Event, Geo +from central.streams import STREAMS + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Derive the valid domain set from the stream registry at import time. +# Subject filters are always "central..>" so we split on "." and +# take index 1. +# --------------------------------------------------------------------------- +_VALID_DOMAINS: frozenset[str] = frozenset( + s.subject_filter.split(".")[1] for s in STREAMS +) + +_DEDUP_DDL = ( + "CREATE TABLE IF NOT EXISTS published_ids (" + "adapter TEXT NOT NULL, event_id TEXT NOT NULL, " + "first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, " + "last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, " + "PRIMARY KEY (adapter, event_id))" +) +_DEDUP_IDX = ( + "CREATE INDEX IF NOT EXISTS published_ids_last_seen " + "ON published_ids (last_seen)" +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _dig(obj: Any, path: str) -> Any: + """Walk a nested dict/list by a dotted path string. + + Each segment is tried as a dict key first; if the current node is a + list/tuple the segment is coerced to an integer index. Returns None on + any miss (missing key, out-of-range index, wrong node type). + + Examples:: + + _dig({"a": {"b": 1}}, "a.b") # 1 + _dig({"a": [10, 20]}, "a.1") # 20 + _dig({"a": 1}, "a.b") # None + _dig(None, "anything") # None + """ + parts = path.split(".") + cur = obj + for part in parts: + if cur is None: + return None + if isinstance(cur, dict): + cur = cur.get(part) + elif isinstance(cur, (list, tuple)): + try: + cur = cur[int(part)] + except (ValueError, IndexError): + return None + else: + return None + return cur + + +# --------------------------------------------------------------------------- +# Settings schema +# --------------------------------------------------------------------------- + +class FieldMapping(BaseModel): + """Map one path inside a source item to a key in Event.data. + + Mirrors TomTomFlowSettings / TileCoord so the GUI model_list widget + renders it identically. + """ + source_path: str # dotted path into the source item + dest_key: str # key written into Event.data + + +class GenericHttpSettings(BaseModel): + """Settings schema for GenericHttpAdapter instances.""" + + url: str + """Endpoint to poll (GET, no auth in v1).""" + + format: Literal["geojson", "json"] = "geojson" + """Response format hint (currently informational; both paths use the same + JSON fetch; the geometry_path / lat_path controls shape extraction).""" + + items_path: str = "features" + """Dotted path to the array of items inside the response object.""" + + domain: str + """Event domain; must match an existing NATS stream (e.g. ``wx``, ``fire``, + ``quake``). Determines which JetStream stream the event lands in.""" + + @field_validator("domain") + @classmethod + def _validate_domain(cls, v: str) -> str: + if v not in _VALID_DOMAINS: + valid = ", ".join(sorted(_VALID_DOMAINS)) + raise ValueError( + f"unknown domain '{v}'; must be one of: {valid}" + ) + return v + + category_suffix: str = "alert" + """Appended to domain to form Event.category: ``{domain}.{category_suffix}``.""" + + id_path: str + """Dotted path to a stable, unique identifier within each item (required + for deduplication).""" + + time_path: str | None = None + """Dotted path to an ISO-8601 timestamp string. When None, poll time + (UTC now) is used.""" + + title_path: str | None = None + """Dotted path to a human-readable title → written to ``data['title']``.""" + + geometry_path: str = "geometry" + """(GeoJSON) dotted path to a GeoJSON geometry object within each item. + Ignored when the resolved value is not a dict.""" + + lat_path: str | None = None + """(JSON) dotted path to a numeric latitude. Used only when + geometry_path does not resolve to a geometry dict.""" + + lon_path: str | None = None + """(JSON) dotted path to a numeric longitude. See lat_path.""" + + severity_path: str | None = None + """Dotted path to an integer severity (0-4). When None, severity is + omitted from the event.""" + + field_mappings: list[FieldMapping] = [] + """Extra source-path → dest-key pairs written into Event.data.""" + + +# --------------------------------------------------------------------------- +# Adapter class +# --------------------------------------------------------------------------- + +class GenericHttpAdapter(SourceAdapter): + """Config-driven REST/GeoJSON source adapter. + + A single class that operators can instantiate many times via distinct + ``config.adapters`` rows (each row has a unique ``name`` / instance + identity and ``kind="generic_http"``). No Python required to add a + new source. + """ + + # Class-level identity (kind) — used by discover_adapters() as the + # registry key. Each *instance* overrides self.name = config.name in + # __init__ so dedup and logs are scoped to the instance. + name = "generic_http" + display_name = "Generic HTTP Source" + description = ( + "Config-driven adapter that polls any public REST or GeoJSON endpoint " + "and maps response fields to Central events. Create one row per " + "source; no Python required." + ) + settings_schema = GenericHttpSettings + default_cadence_s = 300 + data_class = "event" + wizard_order = None # not in the setup wizard; created via operator GUI + enrichment_locations = [] + bypass_bbox_filter = False + + def __init__( + self, + config: AdapterConfig, + config_store: ConfigStore, # unused; accepted for signature uniformity + cursor_db_path: Path, + ) -> None: + # Override class-level name with the *instance* name so that the + # inherited dedup helpers (is_published / mark_published / sweep_old_ids) + # scope their SQL queries to this instance, not the generic_http class. + self.name = config.name + self._cursor_db_path = cursor_db_path + self._session: aiohttp.ClientSession | None = None + self._db: sqlite3.Connection | None = None + self._settings = GenericHttpSettings(**config.settings) + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + async def startup(self) -> None: + """Initialize HTTP session and dedup tracker.""" + self._session = aiohttp.ClientSession( + headers={"User-Agent": "Central/1.0 (generic_http adapter)"}, + timeout=aiohttp.ClientTimeout(total=30), + ) + self._db = sqlite3.connect(str(self._cursor_db_path)) + self._db.execute(_DEDUP_DDL) + self._db.execute(_DEDUP_IDX) + self._db.commit() + self.sweep_old_ids() + logger.info( + "generic_http adapter started", + extra={"adapter": self.name, "url": self._settings.url}, + ) + + async def shutdown(self) -> None: + """Close HTTP session and database.""" + if self._session: + await self._session.close() + self._session = None + if self._db: + self._db.close() + self._db = None + logger.info("generic_http adapter shut down", extra={"adapter": self.name}) + + # ------------------------------------------------------------------ + # Configuration hot-reload + # ------------------------------------------------------------------ + + async def apply_config(self, new_config: AdapterConfig) -> None: + """Re-parse settings in place (no reconstruction needed).""" + self._settings = GenericHttpSettings(**new_config.settings) + logger.info( + "generic_http config applied", + extra={"adapter": self.name, "url": self._settings.url}, + ) + + # ------------------------------------------------------------------ + # Fetch + # ------------------------------------------------------------------ + + @retry( + stop=stop_after_attempt(3), + wait=wait_exponential_jitter(initial=1, max=15), + retry=retry_if_exception_type((aiohttp.ClientError,)), + reraise=True, + ) + async def _fetch(self, url: str) -> dict[str, Any]: + """GET ``url`` and return parsed JSON. Retried up to 3 times.""" + if not self._session: + raise RuntimeError("Session not initialized") + async with self._session.get(url) as resp: + resp.raise_for_status() + return await resp.json(content_type=None) + + # ------------------------------------------------------------------ + # Poll + # ------------------------------------------------------------------ + + async def poll(self) -> AsyncIterator[Event]: + """Fetch the configured URL and yield new Events.""" + self.sweep_old_ids() + + s = self._settings + try: + raw = await self._fetch(s.url) + except Exception as exc: + logger.error( + "generic_http fetch failed", + extra={"adapter": self.name, "url": s.url, "error": str(exc)}, + ) + raise + + items = _dig(raw, s.items_path) + if not isinstance(items, list): + logger.warning( + "generic_http items_path did not resolve to a list", + extra={ + "adapter": self.name, + "items_path": s.items_path, + "got": type(items).__name__, + }, + ) + return + + new_count = 0 + for item in items: + event = self._item_to_event(item) + if event is None: + continue + if self.is_published(event.id): + continue + yield event + self.mark_published(event.id) + new_count += 1 + + logger.info( + "generic_http poll completed", + extra={"adapter": self.name, "count": new_count}, + ) + + # ------------------------------------------------------------------ + # Item → Event + # ------------------------------------------------------------------ + + def _item_to_event(self, item: Any) -> Event | None: + """Convert a single source item to an Event, or None to skip.""" + s = self._settings + + # --- id (required) --- + raw_id = _dig(item, s.id_path) + if raw_id is None: + logger.warning( + "generic_http item missing id", + extra={"adapter": self.name, "id_path": s.id_path}, + ) + return None + event_id = str(raw_id) + + # --- category --- + category = f"{s.domain}.{s.category_suffix}" + + # --- time --- + if s.time_path: + raw_time = _dig(item, s.time_path) + if raw_time is not None: + try: + # Python 3.11+ fromisoformat handles "Z"; older needs replace. + event_time = datetime.fromisoformat( + str(raw_time).replace("Z", "+00:00") + ) + if event_time.tzinfo is None: + event_time = event_time.replace(tzinfo=timezone.utc) + except (ValueError, TypeError): + logger.warning( + "generic_http could not parse time; using now", + extra={ + "adapter": self.name, + "time_path": s.time_path, + "raw": raw_time, + }, + ) + event_time = datetime.now(timezone.utc) + else: + event_time = datetime.now(timezone.utc) + else: + event_time = datetime.now(timezone.utc) + + # --- geo --- + geo_kwargs: dict[str, Any] = {} + + # Try GeoJSON geometry first. + if s.geometry_path: + geom = _dig(item, s.geometry_path) + if isinstance(geom, dict): + geo_kwargs["geometry"] = geom + # Also set centroid when geometry is a simple Point. + if geom.get("type") == "Point": + coords = geom.get("coordinates") or [] + if len(coords) >= 2: + try: + geo_kwargs["centroid"] = ( + float(coords[0]), + float(coords[1]), + ) + except (TypeError, ValueError): + pass + + # Fall back to explicit lat/lon paths. + if "geometry" not in geo_kwargs and s.lat_path and s.lon_path: + raw_lat = _dig(item, s.lat_path) + raw_lon = _dig(item, s.lon_path) + if raw_lat is not None and raw_lon is not None: + try: + geo_kwargs["centroid"] = (float(raw_lon), float(raw_lat)) + except (TypeError, ValueError): + pass + + geo = Geo(**geo_kwargs) + + # --- data --- + data: dict[str, Any] = {} + if s.title_path: + title = _dig(item, s.title_path) + if title is not None: + data["title"] = title + + for fm in s.field_mappings: + data[fm.dest_key] = _dig(item, fm.source_path) + + # --- severity --- + severity: int | None = None + if s.severity_path: + raw_sev = _dig(item, s.severity_path) + if raw_sev is not None: + try: + severity = int(raw_sev) + except (TypeError, ValueError): + pass + + return Event( + id=event_id, + adapter=self.name, + category=category, + time=event_time, + geo=geo, + data=data, + severity=severity, + ) + + # ------------------------------------------------------------------ + # Subject routing + # ------------------------------------------------------------------ + + def subject_for(self, event: Event) -> str: + """Return the NATS subject for a generic-http event. + + Pattern: ``central..`` + + Region is derived from ``event.data["_enriched"]["geocoder"]`` + identically to usgs_quake — the Photon enrichment pipeline populates + that key at publish time. Returns "unknown" when enrichment hasn't + run yet or found no geo signal. + """ + region = subject_for_region(event.data) + return f"central.{self._settings.domain}.{region}" diff --git a/src/central/gui/templates/_event_rows/generic_http.html b/src/central/gui/templates/_event_rows/generic_http.html new file mode 100644 index 0000000..5d60642 --- /dev/null +++ b/src/central/gui/templates/_event_rows/generic_http.html @@ -0,0 +1,3 @@ +{# Generic HTTP source. Fields from payload->data->data. #} +{% set d = (event.data.get('data') or {}).get('data') or {} %} +{% if d.get('title') is not none %}

Title
{{ d.title }}
{% endif %} diff --git a/src/central/gui/templates/_event_summaries/generic_http.html b/src/central/gui/templates/_event_summaries/generic_http.html new file mode 100644 index 0000000..a11dff4 --- /dev/null +++ b/src/central/gui/templates/_event_summaries/generic_http.html @@ -0,0 +1,2 @@ +{% set d = (event.data.get('data') or {}).get('data') or {} %} +{%- if d.get('title') %}{{ d.title }}{% endif -%} diff --git a/tests/test_events_feed_frontend.py b/tests/test_events_feed_frontend.py index 00c42de..2dfc5ea 100644 --- a/tests/test_events_feed_frontend.py +++ b/tests/test_events_feed_frontend.py @@ -1189,6 +1189,7 @@ _SAMPLE_INNER = { "current_lat_deg": -17.1487, "current_alt_km": 417.4, }, + "generic_http": {"title": "Generic source alert"}, } # Exact expected subjects for the deterministic adapters. swpc_alerts is omitted diff --git a/tests/test_generic_http.py b/tests/test_generic_http.py new file mode 100644 index 0000000..ad32b49 --- /dev/null +++ b/tests/test_generic_http.py @@ -0,0 +1,666 @@ +"""Unit tests for GenericHttpAdapter. + +Pure unit tests: no database mocking required (we use a real sqlite tmp file +for dedup tests, same as test_usgs_quake.py), and HTTP is patched. +""" + +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from pydantic import ValidationError + +from central.adapters.generic_http import ( + FieldMapping, + GenericHttpAdapter, + GenericHttpSettings, + _dig, +) +from central.config_models import AdapterConfig +from central.models import Event, Geo +from central.streams import STREAMS + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _valid_domain() -> str: + """Return a known valid domain (first event-bearing stream).""" + for s in STREAMS: + domain = s.subject_filter.split(".")[1] + if domain != "meta": + return domain + raise RuntimeError("No non-meta stream found in STREAMS") + + +def make_config( + name: str = "test_generic", + domain: str | None = None, + extra_settings: dict | None = None, +) -> AdapterConfig: + domain = domain or _valid_domain() + settings: dict = { + "url": "https://example.com/feed.json", + "domain": domain, + "id_path": "id", + "items_path": "features", + "geometry_path": "geometry", + } + if extra_settings: + settings.update(extra_settings) + return AdapterConfig( + name=name, + kind="generic_http", + enabled=True, + cadence_s=300, + settings=settings, + updated_at=datetime.now(timezone.utc), + ) + + +@pytest.fixture +def temp_db_path(): + with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as f: + yield Path(f.name) + + +@pytest.fixture +def mock_config_store(): + return MagicMock() + + +# --------------------------------------------------------------------------- +# _dig +# --------------------------------------------------------------------------- + +class TestDig: + def test_simple_key(self): + assert _dig({"a": 1}, "a") == 1 + + def test_nested_keys(self): + assert _dig({"a": {"b": {"c": 42}}}, "a.b.c") == 42 + + def test_list_index(self): + assert _dig({"a": [10, 20, 30]}, "a.1") == 20 + + def test_list_index_zero(self): + assert _dig({"items": [{"x": 99}]}, "items.0.x") == 99 + + def test_missing_key_returns_none(self): + assert _dig({"a": 1}, "b") is None + + def test_missing_nested_returns_none(self): + assert _dig({"a": {"b": 1}}, "a.c") is None + + def test_out_of_range_index_returns_none(self): + assert _dig({"a": [1, 2]}, "a.5") is None + + def test_none_root_returns_none(self): + assert _dig(None, "a.b") is None + + def test_non_dict_mid_path_returns_none(self): + assert _dig({"a": 42}, "a.b") is None + + def test_empty_list(self): + assert _dig([], "0") is None + + +# --------------------------------------------------------------------------- +# Domain validation +# --------------------------------------------------------------------------- + +class TestDomainValidation: + def test_unknown_domain_raises(self): + with pytest.raises(ValidationError) as exc_info: + GenericHttpSettings( + url="https://example.com/feed", + domain="totally_unknown_domain_xyz", + id_path="id", + ) + err_str = str(exc_info.value) + assert "unknown domain" in err_str + # Should list valid options in the error message + assert "wx" in err_str or "fire" in err_str + + def test_known_domain_ok(self): + s = GenericHttpSettings( + url="https://example.com/feed", + domain=_valid_domain(), + id_path="id", + ) + assert s.domain == _valid_domain() + + def test_wx_domain_valid(self): + s = GenericHttpSettings( + url="https://example.com/feed", + domain="wx", + id_path="id", + ) + assert s.domain == "wx" + + def test_fire_domain_valid(self): + s = GenericHttpSettings( + url="https://example.com/feed", + domain="fire", + id_path="id", + ) + assert s.domain == "fire" + + +# --------------------------------------------------------------------------- +# Category composition +# --------------------------------------------------------------------------- + +class TestCategoryComposition: + @pytest.mark.asyncio + async def test_category_default_suffix(self, temp_db_path, mock_config_store): + config = make_config(domain="wx") + adapter = GenericHttpAdapter(config, mock_config_store, temp_db_path) + await adapter.startup() + + geojson = { + "features": [ + { + "id": "item-1", + "geometry": {"type": "Point", "coordinates": [-116.0, 43.0]}, + } + ] + } + with patch.object(adapter, "_fetch", new_callable=AsyncMock) as mock_fetch: + mock_fetch.return_value = geojson + events = [e async for e in adapter.poll()] + + assert len(events) == 1 + assert events[0].category == "wx.alert" + + await adapter.shutdown() + + @pytest.mark.asyncio + async def test_category_custom_suffix(self, temp_db_path, mock_config_store): + config = make_config(domain="wx", extra_settings={"category_suffix": "warning"}) + adapter = GenericHttpAdapter(config, mock_config_store, temp_db_path) + await adapter.startup() + + geojson = { + "features": [ + { + "id": "item-wx-warn", + "geometry": {"type": "Point", "coordinates": [-116.0, 43.0]}, + } + ] + } + with patch.object(adapter, "_fetch", new_callable=AsyncMock) as mf: + mf.return_value = geojson + events = [e async for e in adapter.poll()] + + assert events[0].category == "wx.warning" + await adapter.shutdown() + + +# --------------------------------------------------------------------------- +# GeoJSON path — geometry + centroid +# --------------------------------------------------------------------------- + +GEOJSON_FIXTURE = { + "type": "FeatureCollection", + "features": [ + { + "id": "feat-001", + "type": "Feature", + "properties": { + "title": "Test Alert", + "severity": 2, + "updated": "2025-01-15T12:00:00Z", + }, + "geometry": { + "type": "Point", + "coordinates": [-116.2, 43.7], + }, + }, + { + "id": "feat-002", + "type": "Feature", + "properties": { + "title": "Polygon Alert", + "severity": 3, + "updated": "2025-01-15T13:00:00Z", + }, + "geometry": { + "type": "Polygon", + "coordinates": [[[-116, 43], [-115, 43], [-115, 44], [-116, 44], [-116, 43]]], + }, + }, + ], +} + + +class TestGeoJsonPath: + @pytest.mark.asyncio + async def test_point_geometry_and_centroid(self, temp_db_path, mock_config_store): + config = make_config( + domain="wx", + extra_settings={ + "items_path": "features", + "id_path": "id", + "geometry_path": "geometry", + "title_path": "properties.title", + "severity_path": "properties.severity", + "field_mappings": [ + {"source_path": "properties.title", "dest_key": "title"}, + ], + }, + ) + adapter = GenericHttpAdapter(config, mock_config_store, temp_db_path) + await adapter.startup() + + with patch.object(adapter, "_fetch", new_callable=AsyncMock) as mf: + mf.return_value = GEOJSON_FIXTURE + events = [e async for e in adapter.poll()] + + assert len(events) == 2 + + point_event = next(e for e in events if e.id == "feat-001") + # geo.geometry set + assert point_event.geo.geometry == { + "type": "Point", + "coordinates": [-116.2, 43.7], + } + # centroid extracted from Point + assert point_event.geo.centroid == (-116.2, 43.7) + + await adapter.shutdown() + + @pytest.mark.asyncio + async def test_polygon_geometry_no_centroid(self, temp_db_path, mock_config_store): + config = make_config(domain="wx") + adapter = GenericHttpAdapter(config, mock_config_store, temp_db_path) + await adapter.startup() + + with patch.object(adapter, "_fetch", new_callable=AsyncMock) as mf: + mf.return_value = GEOJSON_FIXTURE + events = [e async for e in adapter.poll()] + + poly_event = next(e for e in events if e.id == "feat-002") + # geometry is set + assert poly_event.geo.geometry is not None + assert poly_event.geo.geometry["type"] == "Polygon" + # centroid is NOT set (non-Point geometry) + assert poly_event.geo.centroid is None + + await adapter.shutdown() + + @pytest.mark.asyncio + async def test_field_mappings_populate_data(self, temp_db_path, mock_config_store): + config = make_config( + domain="wx", + extra_settings={ + "id_path": "id", + "title_path": "properties.title", + "field_mappings": [ + {"source_path": "properties.severity", "dest_key": "level"}, + {"source_path": "properties.updated", "dest_key": "updated_at"}, + ], + }, + ) + adapter = GenericHttpAdapter(config, mock_config_store, temp_db_path) + await adapter.startup() + + with patch.object(adapter, "_fetch", new_callable=AsyncMock) as mf: + mf.return_value = GEOJSON_FIXTURE + events = [e async for e in adapter.poll()] + + e = next(ev for ev in events if ev.id == "feat-001") + assert e.data["title"] == "Test Alert" + assert e.data["level"] == 2 + assert e.data["updated_at"] == "2025-01-15T12:00:00Z" + + await adapter.shutdown() + + +# --------------------------------------------------------------------------- +# JSON (non-GeoJSON) path — lat_path / lon_path +# --------------------------------------------------------------------------- + +JSON_FIXTURE = { + "alerts": [ + {"uid": "a1", "name": "Alert One", "lat": 43.5, "lon": -116.1, "sev": 1}, + {"uid": "a2", "name": "Alert Two", "lat": 44.0, "lon": -115.5, "sev": 3}, + ] +} + + +class TestJsonLatLonPath: + @pytest.mark.asyncio + async def test_lat_lon_centroid(self, temp_db_path, mock_config_store): + config = make_config( + domain="fire", + extra_settings={ + "items_path": "alerts", + "id_path": "uid", + "geometry_path": "", # disable geometry_path extraction + "lat_path": "lat", + "lon_path": "lon", + "title_path": "name", + "severity_path": "sev", + }, + ) + adapter = GenericHttpAdapter(config, mock_config_store, temp_db_path) + await adapter.startup() + + with patch.object(adapter, "_fetch", new_callable=AsyncMock) as mf: + mf.return_value = JSON_FIXTURE + events = [e async for e in adapter.poll()] + + assert len(events) == 2 + e1 = next(e for e in events if e.id == "a1") + # centroid is (lon, lat) per GeoJSON convention + assert e1.geo.centroid == (-116.1, 43.5) + assert e1.data["title"] == "Alert One" + assert e1.severity == 1 + + await adapter.shutdown() + + +# --------------------------------------------------------------------------- +# Dedup +# --------------------------------------------------------------------------- + +class TestDedup: + @pytest.mark.asyncio + async def test_second_poll_yields_nothing(self, temp_db_path, mock_config_store): + config = make_config(domain="wx") + adapter = GenericHttpAdapter(config, mock_config_store, temp_db_path) + await adapter.startup() + + geojson = { + "features": [ + {"id": "dedup-1", "geometry": None}, + {"id": "dedup-2", "geometry": None}, + ] + } + with patch.object(adapter, "_fetch", new_callable=AsyncMock) as mf: + mf.return_value = geojson + + first = [e async for e in adapter.poll()] + second = [e async for e in adapter.poll()] + + assert len(first) == 2 + assert len(second) == 0 + + await adapter.shutdown() + + @pytest.mark.asyncio + async def test_new_item_in_second_poll_is_yielded( + self, temp_db_path, mock_config_store + ): + config = make_config(domain="wx") + adapter = GenericHttpAdapter(config, mock_config_store, temp_db_path) + await adapter.startup() + + first_batch = {"features": [{"id": "old-1", "geometry": None}]} + second_batch = { + "features": [ + {"id": "old-1", "geometry": None}, + {"id": "new-2", "geometry": None}, + ] + } + + with patch.object(adapter, "_fetch", new_callable=AsyncMock) as mf: + mf.return_value = first_batch + [e async for e in adapter.poll()] # consume first + + mf.return_value = second_batch + second = [e async for e in adapter.poll()] + + assert len(second) == 1 + assert second[0].id == "new-2" + + await adapter.shutdown() + + @pytest.mark.asyncio + async def test_instance_name_used_for_dedup( + self, temp_db_path, mock_config_store + ): + """Two adapter instances share the same db but scope by instance name.""" + config_a = make_config(name="instance_a", domain="wx") + config_b = make_config(name="instance_b", domain="fire") + + adapter_a = GenericHttpAdapter(config_a, mock_config_store, temp_db_path) + adapter_b = GenericHttpAdapter(config_b, mock_config_store, temp_db_path) + await adapter_a.startup() + await adapter_b.startup() + + # Publish "shared-id" under instance_a + adapter_a.mark_published("shared-id") + + # same id is NOT published under instance_b + assert not adapter_b.is_published("shared-id") + # and IS published under instance_a + assert adapter_a.is_published("shared-id") + + await adapter_a.shutdown() + await adapter_b.shutdown() + + +# --------------------------------------------------------------------------- +# subject_for +# --------------------------------------------------------------------------- + +class TestSubjectFor: + @pytest.mark.asyncio + async def test_subject_no_enrichment(self, temp_db_path, mock_config_store): + """Without enrichment data, subject should end in 'unknown'.""" + config = make_config(domain="wx") + adapter = GenericHttpAdapter(config, mock_config_store, temp_db_path) + await adapter.startup() + + event = Event( + id="subj-1", + adapter="test_generic", + category="wx.alert", + time=datetime.now(timezone.utc), + geo=Geo(centroid=(-116.0, 43.0)), + data={}, + ) + subject = adapter.subject_for(event) + assert subject == "central.wx.unknown" + + await adapter.shutdown() + + @pytest.mark.asyncio + async def test_subject_with_us_enrichment(self, temp_db_path, mock_config_store): + """With US geocoder enrichment, subject should be central..us..""" + config = make_config(domain="fire") + adapter = GenericHttpAdapter(config, mock_config_store, temp_db_path) + await adapter.startup() + + event = Event( + id="subj-2", + adapter="test_generic", + category="fire.alert", + time=datetime.now(timezone.utc), + geo=Geo(centroid=(-116.0, 43.0)), + data={ + "_enriched": { + "geocoder": { + "country": "United States", + "state": "Idaho", + } + } + }, + ) + subject = adapter.subject_for(event) + assert subject == "central.fire.us.id" + + await adapter.shutdown() + + @pytest.mark.asyncio + async def test_subject_format_domain_region(self, temp_db_path, mock_config_store): + """Subject always matches central.. — NOT central...""" + config = make_config(domain="quake", extra_settings={"category_suffix": "event.minor"}) + adapter = GenericHttpAdapter(config, mock_config_store, temp_db_path) + await adapter.startup() + + event = Event( + id="subj-3", + adapter="test_generic", + category="quake.event.minor", + time=datetime.now(timezone.utc), + geo=Geo(), + data={}, + ) + subject = adapter.subject_for(event) + # Should be central.quake.unknown, NOT central.quake.event.minor.unknown + assert subject == "central.quake.unknown" + assert subject.startswith("central.quake.") + + await adapter.shutdown() + + +# --------------------------------------------------------------------------- +# time_path parsing +# --------------------------------------------------------------------------- + +class TestTimePath: + @pytest.mark.asyncio + async def test_time_path_none_uses_now(self, temp_db_path, mock_config_store): + before = datetime.now(timezone.utc) + config = make_config(domain="wx", extra_settings={"time_path": None}) + adapter = GenericHttpAdapter(config, mock_config_store, temp_db_path) + await adapter.startup() + + geojson = {"features": [{"id": "t1", "geometry": None}]} + with patch.object(adapter, "_fetch", new_callable=AsyncMock) as mf: + mf.return_value = geojson + events = [e async for e in adapter.poll()] + + after = datetime.now(timezone.utc) + assert len(events) == 1 + assert before <= events[0].time <= after + + await adapter.shutdown() + + @pytest.mark.asyncio + async def test_time_path_parsed_iso(self, temp_db_path, mock_config_store): + config = make_config( + domain="wx", + extra_settings={ + "time_path": "properties.updated", + "id_path": "id", + "items_path": "features", + }, + ) + adapter = GenericHttpAdapter(config, mock_config_store, temp_db_path) + await adapter.startup() + + geojson = { + "features": [ + { + "id": "t2", + "geometry": None, + "properties": {"updated": "2025-06-15T08:30:00Z"}, + } + ] + } + with patch.object(adapter, "_fetch", new_callable=AsyncMock) as mf: + mf.return_value = geojson + events = [e async for e in adapter.poll()] + + assert len(events) == 1 + assert events[0].time == datetime(2025, 6, 15, 8, 30, 0, tzinfo=timezone.utc) + + await adapter.shutdown() + + @pytest.mark.asyncio + async def test_time_path_missing_value_uses_now( + self, temp_db_path, mock_config_store + ): + """When time_path is set but the value is absent, fall back to now.""" + before = datetime.now(timezone.utc) + config = make_config( + domain="wx", + extra_settings={"time_path": "properties.ts"}, + ) + adapter = GenericHttpAdapter(config, mock_config_store, temp_db_path) + await adapter.startup() + + geojson = {"features": [{"id": "t3", "geometry": None, "properties": {}}]} + with patch.object(adapter, "_fetch", new_callable=AsyncMock) as mf: + mf.return_value = geojson + events = [e async for e in adapter.poll()] + + after = datetime.now(timezone.utc) + assert len(events) == 1 + assert before <= events[0].time <= after + + await adapter.shutdown() + + +# --------------------------------------------------------------------------- +# apply_config +# --------------------------------------------------------------------------- + +class TestApplyConfig: + @pytest.mark.asyncio + async def test_apply_config_updates_url(self, temp_db_path, mock_config_store): + config = make_config(domain="wx") + adapter = GenericHttpAdapter(config, mock_config_store, temp_db_path) + await adapter.startup() + + assert adapter._settings.url == "https://example.com/feed.json" + + new_config = make_config( + domain="wx", extra_settings={"url": "https://other.example.com/data.json"} + ) + await adapter.apply_config(new_config) + + assert adapter._settings.url == "https://other.example.com/data.json" + + await adapter.shutdown() + + @pytest.mark.asyncio + async def test_apply_config_updates_domain(self, temp_db_path, mock_config_store): + config = make_config(domain="wx") + adapter = GenericHttpAdapter(config, mock_config_store, temp_db_path) + await adapter.startup() + + new_config = make_config(domain="fire") + await adapter.apply_config(new_config) + + assert adapter._settings.domain == "fire" + + await adapter.shutdown() + + +# --------------------------------------------------------------------------- +# Instance name scoping +# --------------------------------------------------------------------------- + +class TestInstanceName: + def test_instance_name_is_config_name(self, temp_db_path, mock_config_store): + """adapter.name must be the instance name, not the class name.""" + config = make_config(name="my_noaa_alerts", domain="wx") + adapter = GenericHttpAdapter(config, mock_config_store, temp_db_path) + assert adapter.name == "my_noaa_alerts" + assert GenericHttpAdapter.name == "generic_http" + + @pytest.mark.asyncio + async def test_event_adapter_field_is_instance_name( + self, temp_db_path, mock_config_store + ): + config = make_config(name="my_source", domain="wx") + adapter = GenericHttpAdapter(config, mock_config_store, temp_db_path) + await adapter.startup() + + geojson = {"features": [{"id": "inst-1", "geometry": None}]} + with patch.object(adapter, "_fetch", new_callable=AsyncMock) as mf: + mf.return_value = geojson + events = [e async for e in adapter.poll()] + + assert len(events) == 1 + assert events[0].adapter == "my_source" + + await adapter.shutdown() From 7688d268a33a2901562def1003499044ff2e4e15 Mon Sep 17 00:00:00 2001 From: malice Date: Tue, 30 Jun 2026 13:03:14 -0600 Subject: [PATCH 15/17] feat: GUI create/delete for adapter instances (v0.15.0 PR3) (#122) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the CRUD loop for operator-creatable adapters. Operators can now create generic_http instances (GET+POST /adapters/new) and delete operator instances (POST /adapters/{name}/delete) without editing Python. Key changes: - SourceAdapter.operator_creatable class attr (default False); set True on GenericHttpAdapter so it appears in the kind select - ADAPTER_CREATE / ADAPTER_DELETE audit constants - _parse_adapter_settings() shared helper extracted from adapters_edit_submit (single place for widget parsing, region handling, Pydantic validation, quota blocking) — edit_submit refactored to call it - GET/POST /adapters/new: kind select → name/cadence/enabled → settings fields; INSERT with kind supplied explicitly; 302 → edit page on success - POST /adapters/{name}/delete: primary guard (name in adapter_classes → 403); second guard (kind not operator_creatable → 403); DELETE + audit; 302 → list - adapters_list.html: "New adapter" button; per-row Delete form (operator instances only; built-ins have no button); deletable flag passed from route - adapters_new.html: new template reusing same field-rendering blocks as adapters_edit.html; enabled checkbox unchecked by default - 46 new tests (all pass; mock-DB pattern mirrors test_gui_adapter_edit.py) Co-authored-by: Ubuntu Co-authored-by: Claude Sonnet 4.6 --- src/central/adapter.py | 6 + src/central/adapters/generic_http.py | 1 + src/central/gui/audit.py | 2 + src/central/gui/routes.py | 587 ++++++++++++++----- src/central/gui/templates/adapters_list.html | 11 + src/central/gui/templates/adapters_new.html | 192 ++++++ tests/test_gui_adapter_create_delete.py | 585 ++++++++++++++++++ 7 files changed, 1248 insertions(+), 136 deletions(-) create mode 100644 src/central/gui/templates/adapters_new.html create mode 100644 tests/test_gui_adapter_create_delete.py diff --git a/src/central/adapter.py b/src/central/adapter.py index 8a82c94..7f4cd02 100644 --- a/src/central/adapter.py +++ b/src/central/adapter.py @@ -66,6 +66,12 @@ class SourceAdapter(ABC): set in ``central.archive`` -- the two MUST stay in sync (enforced by ``tests/test_bypass_bbox_consistency.py``).""" + operator_creatable: bool = False + """True for adapters that operators may instantiate many times from the + GUI (one config.adapters row per instance, each with a unique name and + kind=). False (default) for singleton built-ins where the + name equals the kind and the row is seeded by migrations.""" + @abstractmethod async def poll(self) -> AsyncIterator[Event]: """ diff --git a/src/central/adapters/generic_http.py b/src/central/adapters/generic_http.py index 98312c5..70b8dba 100644 --- a/src/central/adapters/generic_http.py +++ b/src/central/adapters/generic_http.py @@ -193,6 +193,7 @@ class GenericHttpAdapter(SourceAdapter): wizard_order = None # not in the setup wizard; created via operator GUI enrichment_locations = [] bypass_bbox_filter = False + operator_creatable = True # GUI allows operators to create multiple instances def __init__( self, diff --git a/src/central/gui/audit.py b/src/central/gui/audit.py index ada29df..ef5179b 100644 --- a/src/central/gui/audit.py +++ b/src/central/gui/audit.py @@ -9,7 +9,9 @@ AUTH_LOGIN_FAILED = "auth.login_failed" AUTH_LOGOUT = "auth.logout" AUTH_PASSWORD_CHANGE = "auth.password_change" OPERATOR_CREATE = "operator.create" +ADAPTER_CREATE = "adapter.create" ADAPTER_UPDATE = "adapter.update" +ADAPTER_DELETE = "adapter.delete" STREAM_UPDATE = "stream.update" API_KEY_CREATE = "api_key.create" API_KEY_ROTATE = "api_key.rotate" diff --git a/src/central/gui/routes.py b/src/central/gui/routes.py index 32d078e..43a4281 100644 --- a/src/central/gui/routes.py +++ b/src/central/gui/routes.py @@ -36,6 +36,8 @@ from central.gui.auth import ( verify_password, ) from central.gui.audit import ( + ADAPTER_CREATE, + ADAPTER_DELETE, ADAPTER_UPDATE, API_KEY_CREATE, API_KEY_DELETE, @@ -100,6 +102,10 @@ ALIAS_REGEX = re.compile(r"^[a-zA-Z0-9_]+$") # Email validation regex (simple but effective) EMAIL_REGEX = re.compile(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$") +# Adapter instance-name regex: must start with a lowercase letter, followed by +# 1–63 lowercase letters, digits, or underscores (total 2–64 chars). +ADAPTER_NAME_REGEX = re.compile(r"^[a-z][a-z0-9_]{1,63}$") + def _get_templates(): """Get templates instance (deferred import to avoid circular).""" @@ -1437,6 +1443,164 @@ async def change_password_submit( # ============================================================================= +def _parse_adapter_settings( + form, + adapter_cls, + current_settings: dict, + cadence_s: int, +) -> tuple[dict, dict[str, str]]: + """Parse and validate adapter settings from a form submission. + + Shared by ``adapters_edit_submit`` and ``adapters_create_submit`` so that + field parsing, region handling, Pydantic validation, and quota-blocking live + in exactly one place. + + Args: + form: Starlette ``FormData`` from ``await request.form()``. + adapter_cls: The resolved ``SourceAdapter`` subclass, or ``None``. + current_settings: Existing settings dict (``{}`` for a new adapter). + Used by ``describe_fields`` to populate ``field.current_value``. + cadence_s: Validated cadence for quota estimation. + + Returns: + ``(new_settings, errors)`` — on success ``errors`` is empty and + ``new_settings`` is the Pydantic-validated dict; on failure + ``new_settings`` is ``{}`` and ``errors`` maps field names to messages. + """ + errors: dict[str, str] = {} + + if not (adapter_cls and hasattr(adapter_cls, "settings_schema")): + # No schema — preserve existing settings unchanged. + return dict(current_settings), errors + + schema = adapter_cls.settings_schema + fields = describe_fields(schema, current_settings) + + parsed_values: dict = {} + + for field in fields: + raw = form.get(field.name, "") + + if field.widget == "text": + parsed_values[field.name] = raw.strip() if raw else None + elif field.widget == "number": + try: + parsed_values[field.name] = int(raw) if raw else None + except ValueError: + errors[field.name] = f"{field.label} must be a number" + elif field.widget == "checkbox": + parsed_values[field.name] = field.name in form + elif field.widget == "csv": + if raw.strip(): + parsed_values[field.name] = [v.strip() for v in raw.split(",") if v.strip()] + else: + parsed_values[field.name] = [] + elif field.widget == "csv_int": + parsed_ints: list[int] = [] + if raw.strip(): + for tok in raw.split(","): + tok = tok.strip() + if not tok: + continue + try: + parsed_ints.append(int(tok)) + except ValueError: + logger.warning( + "csv_int: dropped non-numeric token", + extra={"field": field.name, "token": tok}, + ) + parsed_values[field.name] = parsed_ints + elif field.widget == "select": + value = raw.strip() if raw else None + if value and field.options and value not in field.options: + errors[field.name] = f"Invalid {field.label.lower()}" + else: + parsed_values[field.name] = value + elif field.widget == "checkboxes": + values = form.getlist(field.name) + if field.options: + invalid = [v for v in values if v not in field.options] + if invalid: + errors[field.name] = f"Invalid values: {', '.join(invalid)}" + else: + parsed_values[field.name] = values + else: + parsed_values[field.name] = values + elif field.widget == "api_key_select": + value = raw.strip() if raw else None + parsed_values[field.name] = value + elif field.widget == "model_list": + rows = _parse_model_list(form, field) + parsed_values[field.name] = rows + elif field.widget == "region": + pass # handled in the region block below + + # Region fields (common to adapters that expose a bounding-box region). + region_north_str = form.get("region_north", "").strip() + region_south_str = form.get("region_south", "").strip() + region_east_str = form.get("region_east", "").strip() + region_west_str = form.get("region_west", "").strip() + has_region = any([region_north_str, region_south_str, region_east_str, region_west_str]) + + if has_region: + try: + region_north = float(region_north_str) + region_south = float(region_south_str) + region_east = float(region_east_str) + region_west = float(region_west_str) + if not (-90 <= region_south < region_north <= 90): + errors["region"] = ( + "Invalid latitude: south must be less than north, " + "both between -90 and 90" + ) + elif not (-180 <= region_west < region_east <= 180): + errors["region"] = ( + "Invalid longitude: west must be less than east, " + "both between -180 and 180" + ) + else: + parsed_values["region"] = { + "north": region_north, + "south": region_south, + "east": region_east, + "west": region_west, + } + except ValueError: + errors["region"] = "Region coordinates must be valid numbers" + else: + parsed_values["region"] = None + + if errors: + return {}, errors + + # Pydantic validation + quota check. + try: + validated_data = {k: v for k, v in parsed_values.items() if v is not None} + validated = schema(**validated_data) + new_settings = validated.model_dump(mode="json") + + q = adapter_cls.quota_estimate(validated, cadence_s) + if q and q.get("blocked"): + ml = next((f.name for f in fields if f.widget == "model_list"), "quota") + errors[ml] = ( + f"Estimated {q['calls_per_month']:,} calls/month exceeds the " + f"{q['cap']:,}/month free-tier cap — raise cadence or remove rows." + ) + return {}, errors + except ValidationError as e: + ml_name = next((f.name for f in fields if f.widget == "model_list"), None) + for err in e.errors(): + loc = err["loc"] + key = str(loc[0]) if loc else (ml_name or "unknown") + if len(loc) >= 2 and isinstance(loc[1], int): + errors[key] = f"Row {loc[1] + 1}: {err['msg']}" + else: + errors[key] = err["msg"] + return {}, errors + + return new_settings, errors + + @router.get("/adapters", response_class=HTMLResponse) async def adapters_list( request: Request, @@ -1471,6 +1635,10 @@ async def adapters_list( ) api_key_missing = not has_key + # Operator instances have a name that is NOT a registered kind key. + # Built-ins always have name == kind which IS in the registry. + deletable = row["name"] not in adapter_classes + adapters.append({ "name": row["name"], "display_name": getattr(adapter_cls, "display_name", row["name"]) if adapter_cls else row["name"], @@ -1482,6 +1650,7 @@ async def adapters_list( "last_error": row["last_error"], "api_key_missing": api_key_missing, "requires_api_key_alias": requires_api_key_alias, + "deletable": deletable, }) csrf_token = request.state.csrf_token @@ -1530,6 +1699,202 @@ def _parse_model_list(form, field) -> list[dict]: return out +@router.get("/adapters/new", response_class=HTMLResponse) +async def adapters_create_form(request: Request) -> Response: + """Render the create-adapter form. + + Lists only adapter kinds where ``operator_creatable is True`` so operators + can instantiate them freely without touching Python code. + """ + templates = _get_templates() + pool = get_pool() + operator = request.state.operator + csrf_token = request.state.csrf_token + + adapter_classes = _adapter_classes() + creatable_kinds = { + kind: cls + for kind, cls in adapter_classes.items() + if getattr(cls, "operator_creatable", False) + } + + if not creatable_kinds: + return Response(status_code=404, content="No operator-creatable adapter kinds are registered.") + + # NOTE: single creatable kind today; multi-kind HTMX field-swap is a future enhancement. + first_kind, first_cls = next(iter(creatable_kinds.items())) + + fields = [] + if hasattr(first_cls, "settings_schema"): + fields = describe_fields(first_cls.settings_schema, {}) + if first_cls.api_key_field is not None: + for f in fields: + if f.name == first_cls.api_key_field: + f.widget = "api_key_select" + + async with pool.acquire() as conn: + api_key_rows = await conn.fetch("SELECT alias FROM config.api_keys ORDER BY alias") + api_keys = [{"alias": r["alias"]} for r in api_key_rows] + + return templates.TemplateResponse( + request=request, + name="adapters_new.html", + context={ + "operator": operator, + "csrf_token": csrf_token, + "creatable_kinds": [ + {"kind": kind, "display_name": getattr(cls, "display_name", kind)} + for kind, cls in creatable_kinds.items() + ], + "selected_kind": first_kind, + "default_cadence_s": first_cls.default_cadence_s, + "fields": fields, + "api_keys": api_keys, + "errors": None, + "form_data": None, + }, + ) + + +@router.post("/adapters/new") +async def adapters_create_submit(request: Request) -> Response: + """Process the create-adapter form (first INSERT in the codebase).""" + templates = _get_templates() + pool = get_pool() + operator = request.state.operator + + form = await request.form() + form_csrf = form.get("csrf_token", "") + if not form_csrf or form_csrf != request.state.csrf_token: + raise CsrfValidationError("Invalid CSRF token") + + adapter_classes = _adapter_classes() + creatable_kinds = { + kind: cls + for kind, cls in adapter_classes.items() + if getattr(cls, "operator_creatable", False) + } + + kind = (form.get("kind") or "").strip() + name = (form.get("name") or "").strip() + enabled = "enabled" in form + cadence_s_str = form.get("cadence_s", "") + + errors: dict[str, str] = {} + form_data: dict[str, Any] = { + "kind": kind, + "name": name, + "enabled": enabled, + "cadence_s": cadence_s_str, + } + + # Validate kind — must be operator-creatable. + kind_cls = creatable_kinds.get(kind) + if kind not in creatable_kinds: + errors["kind"] = f"'{kind}' is not a valid operator-creatable adapter kind." + + # Validate instance name. + if "kind" not in errors: + if not ADAPTER_NAME_REGEX.match(name): + errors["name"] = ( + "Name must start with a lowercase letter followed by 1–63 " + "lowercase letters, digits, or underscores." + ) + elif name in adapter_classes: + errors["name"] = ( + f"'{name}' is a reserved kind name and cannot be used as an " + "instance name." + ) + + # Validate cadence_s. + cadence_s = 0 + try: + cadence_s = int(cadence_s_str) + if cadence_s < 10: + errors["cadence_s"] = "Input should be greater than or equal to 10" + except ValueError: + errors["cadence_s"] = "Cadence must be a valid integer" + + # Check for duplicate name in DB (only when name passed format + kind checks). + if "name" not in errors and "kind" not in errors: + async with pool.acquire() as conn: + existing = await conn.fetchval( + "SELECT 1 FROM config.adapters WHERE name = $1", name + ) + if existing: + return Response( + status_code=409, + content=f"An adapter named '{name}' already exists.", + ) + + # Parse + validate settings via the shared helper. + new_settings: dict = {} + if not errors and kind_cls: + new_settings, settings_errors = _parse_adapter_settings( + form, kind_cls, {}, cadence_s + ) + errors.update(settings_errors) + + # Re-render on error. + if errors: + fields = [] + if kind_cls and hasattr(kind_cls, "settings_schema"): + fields = describe_fields(kind_cls.settings_schema, {}) + if kind_cls.api_key_field is not None: + for f in fields: + if f.name == kind_cls.api_key_field: + f.widget = "api_key_select" + # Populate form_data for settings fields so inputs restore values. + for field in fields: + form_data.setdefault(field.name, form.get(field.name, "")) + async with pool.acquire() as conn: + api_key_rows = await conn.fetch("SELECT alias FROM config.api_keys ORDER BY alias") + api_keys = [{"alias": r["alias"]} for r in api_key_rows] + selected_kind = kind if kind in creatable_kinds else (next(iter(creatable_kinds)) if creatable_kinds else "") + return templates.TemplateResponse( + request=request, + name="adapters_new.html", + context={ + "operator": operator, + "csrf_token": request.state.csrf_token, + "creatable_kinds": [ + {"kind": k, "display_name": getattr(c, "display_name", k)} + for k, c in creatable_kinds.items() + ], + "selected_kind": selected_kind, + "default_cadence_s": getattr(kind_cls, "default_cadence_s", 300) if kind_cls else 300, + "fields": fields, + "api_keys": api_keys, + "errors": errors, + "form_data": form_data, + }, + status_code=422, + ) + + # INSERT INTO config.adapters — kind supplied explicitly (migration 043 has no DEFAULT). + async with pool.acquire() as conn: + await conn.execute( + """ + INSERT INTO config.adapters (name, kind, enabled, cadence_s, settings, updated_at) + VALUES ($1, $2, $3, $4, $5, now()) + """, + name, + kind, + enabled, + cadence_s, + new_settings, + ) + await write_audit( + conn, + ADAPTER_CREATE, + operator_id=operator.id, + target=name, + after={"kind": kind, "enabled": enabled, "cadence_s": cadence_s, "settings": new_settings}, + ) + + return RedirectResponse(url=f"/adapters/{name}", status_code=302) + + @router.get("/adapters/{name}", response_class=HTMLResponse) async def adapters_edit_form( request: Request, @@ -1715,144 +2080,29 @@ async def adapters_edit_submit( current_settings = row["settings"] or {} - # Parse and validate settings via Pydantic if we have the adapter class - new_settings = {} + # Collect raw form values into form_data for error re-renders. if adapter_cls and hasattr(adapter_cls, "settings_schema"): - schema = adapter_cls.settings_schema - fields = describe_fields(schema, current_settings) + for _f in describe_fields(adapter_cls.settings_schema, current_settings): + if _f.widget == "checkboxes": + form_data[_f.name] = form.getlist(_f.name) + elif _f.widget == "model_list": + form_data[_f.name] = _parse_model_list(form, _f) + else: + form_data[_f.name] = form.get(_f.name, "") + form_data["region_north"] = form.get("region_north", "").strip() + form_data["region_south"] = form.get("region_south", "").strip() + form_data["region_east"] = form.get("region_east", "").strip() + form_data["region_west"] = form.get("region_west", "").strip() - # Parse form values based on widget type - parsed_values = {} - for field in fields: - raw = form.get(field.name, "") - form_data[field.name] = raw - - if field.widget == "text": - parsed_values[field.name] = raw.strip() if raw else None - elif field.widget == "number": - try: - parsed_values[field.name] = int(raw) if raw else None - except ValueError: - errors[field.name] = f"{field.label} must be a number" - elif field.widget == "checkbox": - parsed_values[field.name] = field.name in form - elif field.widget == "csv": - if raw.strip(): - parsed_values[field.name] = [v.strip() for v in raw.split(",") if v.strip()] - else: - parsed_values[field.name] = [] - elif field.widget == "csv_int": - # v0.11.3: parallel to "csv" but coerces each token through - # int(), dropping non-numeric entries with a warning. - parsed_ints: list[int] = [] - if raw.strip(): - for tok in raw.split(","): - tok = tok.strip() - if not tok: - continue - try: - parsed_ints.append(int(tok)) - except ValueError: - logger.warning( - "csv_int: dropped non-numeric token", - extra={"field": field.name, "token": tok}, - ) - parsed_values[field.name] = parsed_ints - elif field.widget == "select": - value = raw.strip() if raw else None - if value and field.options and value not in field.options: - errors[field.name] = f"Invalid {field.label.lower()}" - else: - parsed_values[field.name] = value - elif field.widget == "checkboxes": - # Use getlist for checkbox groups - values = form.getlist(field.name) - form_data[field.name] = values # Override raw value - if field.options: - invalid = [v for v in values if v not in field.options] - if invalid: - errors[field.name] = f"Invalid values: {', '.join(invalid)}" - else: - parsed_values[field.name] = values - else: - parsed_values[field.name] = values - elif field.widget == "api_key_select": - # API key select - validate against existing keys - value = raw.strip() if raw else None - parsed_values[field.name] = value - elif field.widget == "model_list": - rows = _parse_model_list(form, field) - form_data[field.name] = rows - parsed_values[field.name] = rows - elif field.widget == "region": - # Region handled separately below - pass - - # Handle region fields (common pattern) - region_north_str = form.get("region_north", "").strip() - region_south_str = form.get("region_south", "").strip() - region_east_str = form.get("region_east", "").strip() - region_west_str = form.get("region_west", "").strip() - - form_data["region_north"] = region_north_str - form_data["region_south"] = region_south_str - form_data["region_east"] = region_east_str - form_data["region_west"] = region_west_str - - # Check if any region field has a value - has_region = any([region_north_str, region_south_str, region_east_str, region_west_str]) - - if has_region: - try: - region_north = float(region_north_str) - region_south = float(region_south_str) - region_east = float(region_east_str) - region_west = float(region_west_str) - - if not (-90 <= region_south < region_north <= 90): - errors["region"] = "Invalid latitude: south must be less than north, both between -90 and 90" - elif not (-180 <= region_west < region_east <= 180): - errors["region"] = "Invalid longitude: west must be less than east, both between -180 and 180" - else: - parsed_values["region"] = { - "north": region_north, - "south": region_south, - "east": region_east, - "west": region_west, - } - except ValueError: - errors["region"] = "Region coordinates must be valid numbers" - else: - parsed_values["region"] = None - - # Only validate with Pydantic if no parse errors - if not errors: - try: - # Filter out None values for optional fields without defaults - validated_data = {k: v for k, v in parsed_values.items() if v is not None} - validated = schema(**validated_data) - new_settings = validated.model_dump(mode="json") - - # Hard-block a save that would blow the provider free tier. - q = adapter_cls.quota_estimate(validated, cadence_s) - if q and q.get("blocked"): - ml = next((f.name for f in fields if f.widget == "model_list"), "quota") - errors[ml] = ( - f"Estimated {q['calls_per_month']:,} calls/month exceeds the " - f"{q['cap']:,}/month free-tier cap — raise cadence or remove rows." - ) - except ValidationError as e: - ml_name = next((f.name for f in fields if f.widget == "model_list"), None) - for err in e.errors(): - loc = err["loc"] - key = str(loc[0]) if loc else (ml_name or "unknown") - if len(loc) >= 2 and isinstance(loc[1], int): - errors[key] = f"Row {loc[1] + 1}: {err['msg']}" - else: - errors[key] = err["msg"] - else: - # No schema - just preserve existing settings - new_settings = dict(current_settings) + # Parse + validate settings via the shared helper. + # Mirror the old behavior: skip Pydantic validation when upstream + # checks (e.g. cadence) already failed, same as the old + # "if not errors: try: validated = schema(...)" guard. + if not errors: + new_settings, settings_errors = _parse_adapter_settings( + form, adapter_cls, current_settings, cadence_s + ) + errors.update(settings_errors) # If there are errors, re-render the form if errors: @@ -1968,6 +2218,71 @@ async def adapters_edit_submit( return RedirectResponse(url="/adapters", status_code=302) +@router.post("/adapters/{name}/delete") +async def adapters_delete(request: Request, name: str) -> Response: + """Delete an operator-created adapter instance. + + Safety rule: a row is deletable iff its ``name`` is NOT a key in the + adapter class registry. Built-in adapters always have ``name == kind`` + which IS a registry key; operator instances have a unique name that is NOT. + + NOTE: orphaned ``published_ids`` rows in cursors.db (a separate SQLite + database) are left to age out via ``dedup_sweep_days``; they are not + cleaned here because the two stores are decoupled by design. + """ + pool = get_pool() + operator = request.state.operator + + form = await request.form() + form_csrf = form.get("csrf_token", "") + if not form_csrf or form_csrf != request.state.csrf_token: + raise CsrfValidationError("Invalid CSRF token") + + adapter_classes = _adapter_classes() + + async with pool.acquire() as conn: + row = await conn.fetchrow( + "SELECT name, kind FROM config.adapters WHERE name = $1", name + ) + + if row is None: + return Response(status_code=404, content=f"Adapter '{name}' not found.") + + # Primary guard: built-ins have name == kind (a registered class key). + if name in adapter_classes: + return Response( + status_code=403, + content=( + f"'{name}' is a built-in adapter and cannot be deleted; " + "disable it instead." + ), + ) + + # Second guard: the row's kind must be operator_creatable. + kind_cls = adapter_classes.get(row["kind"]) + if kind_cls is not None and not getattr(kind_cls, "operator_creatable", False): + return Response( + status_code=403, + content=( + f"Adapter kind '{row['kind']}' is not operator-creatable; " + "cannot delete." + ), + ) + + await conn.execute( + "DELETE FROM config.adapters WHERE name = $1", name + ) + await write_audit( + conn, + ADAPTER_DELETE, + operator_id=operator.id, + target=name, + before={"kind": row["kind"], "name": name}, + ) + + return RedirectResponse(url="/adapters", status_code=302) + + # ============================================================================= # Streams routes # ============================================================================= diff --git a/src/central/gui/templates/adapters_list.html b/src/central/gui/templates/adapters_list.html index 350852f..19aa398 100644 --- a/src/central/gui/templates/adapters_list.html +++ b/src/central/gui/templates/adapters_list.html @@ -4,6 +4,7 @@ {% block content %}

Adapters

+

+ New adapter

@@ -12,6 +13,7 @@ + @@ -27,6 +29,15 @@ + {% endfor %} diff --git a/src/central/gui/templates/adapters_new.html b/src/central/gui/templates/adapters_new.html new file mode 100644 index 0000000..634f43a --- /dev/null +++ b/src/central/gui/templates/adapters_new.html @@ -0,0 +1,192 @@ +{% extends "base.html" %} + +{% block title %}Central — New Adapter{% endblock %} + +{% block content %} +

New Adapter

+

Create a new adapter instance from an operator-creatable kind.

+ + + + +
+ Adapter Kind + + {# NOTE: single creatable kind today; multi-kind HTMX field-swap is a future enhancement. #} + + + {% if errors and errors.kind %} + {{ errors.kind }} + {% endif %} +
+ +
+ Instance Identity + + + + Lowercase letters, digits, and underscores; starts with a letter; 2–64 characters. + Must not match a built-in kind name. + {% if errors and errors.name %} + {{ errors.name }} + {% endif %} +
+ +
+ Core Settings + + + + + + {% if errors and errors.cadence_s %} + {{ errors.cadence_s }} + {% endif %} +
+ + {% if fields %} +
+ Adapter Settings + + {% for field in fields %} + {% if field.widget == "region" %} + {# Region is rendered in a separate fieldset below #} + {% elif field.widget == "text" %} + + + {% if field.description %} + {{ field.description }} + {% endif %} + {% if errors and errors[field.name] %} + {{ errors[field.name] }} + {% endif %} + + {% elif field.widget == "number" %} + + + {% if field.description %} + {{ field.description }} + {% endif %} + {% if errors and errors[field.name] %} + {{ errors[field.name] }} + {% endif %} + + {% elif field.widget == "checkbox" %} + + {% if field.description %} + {{ field.description }} + {% endif %} + {% if errors and errors[field.name] %} + {{ errors[field.name] }} + {% endif %} + + {% elif field.widget == "csv" %} + + + Comma-separated values{% if field.description %} — {{ field.description }}{% endif %} + {% if errors and errors[field.name] %} + {{ errors[field.name] }} + {% endif %} + + {% elif field.widget == "csv_int" %} + + + Comma-separated integers{% if field.description %} — {{ field.description }}{% endif %} + {% if errors and errors[field.name] %} + {{ errors[field.name] }} + {% endif %} + + {% elif field.widget == "select" %} + + + {% if field.description %} + {{ field.description }} + {% endif %} + {% if errors and errors[field.name] %} + {{ errors[field.name] }} + {% endif %} + + {% elif field.widget == "checkboxes" %} + + {% set current_values = form_data.getlist(field.name) if form_data and form_data.getlist else (field.current_value or []) %} + {% for opt in field.options %} + + {% endfor %} + {% if field.description %} + {{ field.description }} + {% endif %} + {% if errors and errors[field.name] %} + {{ errors[field.name] }} + {% endif %} + + {% elif field.widget == "api_key_select" %} + + + {% if field.description %} + {{ field.description }} + {% endif %} + {% if errors and errors[field.name] %} + {{ errors[field.name] }} + {% endif %} + + {% elif field.widget == "model_list" %} + {% include "_partials/model_list.html" %} + {% endif %} + {% endfor %} +
+ {% endif %} + + + Cancel + +{% endblock %} diff --git a/tests/test_gui_adapter_create_delete.py b/tests/test_gui_adapter_create_delete.py new file mode 100644 index 0000000..a05faba --- /dev/null +++ b/tests/test_gui_adapter_create_delete.py @@ -0,0 +1,585 @@ +"""v0.15.0 PR3 — GUI create + delete for adapter instances. + +Test strategy +───────────── +* Pure-unit (always run, no DB): + - ADAPTER_NAME_REGEX validation + - Deletability rule (name in registry → not deletable) + +* Mock-DB (always run; mirrors test_gui_adapter_edit.py pattern): + - GET /adapters/new renders correctly + - POST create: valid → INSERT, audit, 302 + - POST create: duplicate name → 409 + - POST create: bad name format → 422 + - POST create: non-creatable kind → 422 + - POST create: invalid settings (missing required field) → 422 + - POST delete: operator instance → DELETE, audit, 302 + - POST delete: built-in adapter → 403, no DELETE + +DB-backed INSERT/DELETE tests (test_db_* below) use the central_test +Postgres fixture. They will raise ConnectionRefusedError when the test DB +is absent — the same behaviour as other DB-backed tests in this suite (e.g. +test_config_store.py, test_supervisor_hotreload.py). +""" + +import re +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from starlette.datastructures import FormData +from starlette.requests import Request + +from central.gui import templates as gui_templates +from central.gui.routes import ( + ADAPTER_NAME_REGEX, + adapters_create_form, + adapters_create_submit, + adapters_delete, + adapters_list, +) +from central.adapters.generic_http import GenericHttpAdapter + + +# --------------------------------------------------------------------------- +# Helpers shared across test classes +# --------------------------------------------------------------------------- + +def _make_request(method="GET", form_pairs=None, csrf="x"): + """Build a mock Request with CSRF + optional form data.""" + req = MagicMock() + req.state.operator = SimpleNamespace(id=1, username="admin") + req.state.csrf_token = csrf + if form_pairs is not None: + pairs = [("csrf_token", csrf)] + list(form_pairs) + req.form = AsyncMock(return_value=FormData(pairs)) + else: + req.form = AsyncMock(return_value=FormData([("csrf_token", csrf)])) + return req + + +def _make_pool(fetchrow_returns=None, fetchval_returns=None, fetch_returns=None): + """Build a mock asyncpg pool. + + Values are set unconditionally so that None (e.g. "row not found") is + returned faithfully instead of the default truthy AsyncMock sentinel. + Pass a list for fetchrow_returns to use side_effect for sequential calls. + """ + conn = AsyncMock() + if isinstance(fetchrow_returns, list): + conn.fetchrow.side_effect = fetchrow_returns + else: + conn.fetchrow.return_value = fetchrow_returns # None = not found + conn.fetchval.return_value = fetchval_returns # None = not found + conn.fetch.return_value = fetch_returns if fetch_returns is not None else [] + pool = MagicMock() + pool.acquire.return_value.__aenter__ = AsyncMock(return_value=conn) + pool.acquire.return_value.__aexit__ = AsyncMock(return_value=None) + return pool, conn + + +# --------------------------------------------------------------------------- +# UNIT: name-regex validation +# --------------------------------------------------------------------------- + +class TestAdapterNameRegex: + """Pure-unit — no I/O, always run.""" + + VALID = [ + "my_source", + "mysource2", + "aa", # minimum length (2 chars) + "a" + "b" * 63, # maximum length (64 chars) + "a1_b2_c3", + ] + INVALID = [ + "", # empty + "a", # too short (only 1 char) + "A_source", # uppercase + "1source", # starts with digit + "_source", # starts with underscore + "my-source", # hyphen not allowed + "my source", # space not allowed + "a" + "b" * 64, # 65 chars — too long + ] + + @pytest.mark.parametrize("name", VALID) + def test_valid(self, name): + assert ADAPTER_NAME_REGEX.match(name), f"Expected {name!r} to match" + + @pytest.mark.parametrize("name", INVALID) + def test_invalid(self, name): + assert not ADAPTER_NAME_REGEX.match(name), f"Expected {name!r} not to match" + + +# --------------------------------------------------------------------------- +# UNIT: deletability rule +# --------------------------------------------------------------------------- + +class TestDeletabilityRule: + """Pure-unit — the rule is: name NOT IN adapter_classes → deletable. + + Built-ins have name == kind (the class's .name attribute) which IS a key in + the adapter class registry. Operator instances have a unique name that is + NOT a registry key. + """ + + def test_builtin_not_deletable(self): + from central.adapter_discovery import discover_adapters + classes = discover_adapters() + # Every registered kind key should be considered a built-in. + for kind in classes: + assert kind in classes, "sanity" + # The deletability check: name in adapter_classes → NOT deletable + assert kind in classes # confirms the guard fires + + def test_operator_instance_is_deletable(self): + from central.adapter_discovery import discover_adapters + classes = discover_adapters() + operator_name = "my_custom_source_42" + assert operator_name not in classes, ( + "Test assumes operator_name is not a registered kind; " + "update the name if a new kind was added with this identifier." + ) + + def test_generic_http_kind_is_not_deletable_by_name(self): + """The KIND 'generic_http' itself should not be deletable (it's a built-in key).""" + from central.adapter_discovery import discover_adapters + classes = discover_adapters() + assert "generic_http" in classes + + def test_generic_http_instance_is_deletable(self): + """An operator instance named 'my_feed' (not a kind key) should be deletable.""" + from central.adapter_discovery import discover_adapters + classes = discover_adapters() + assert "my_feed" not in classes + + +# --------------------------------------------------------------------------- +# UNIT: GenericHttpAdapter.operator_creatable +# --------------------------------------------------------------------------- + +def test_generic_http_is_operator_creatable(): + assert GenericHttpAdapter.operator_creatable is True + + +def test_base_class_default_not_creatable(): + from central.adapter import SourceAdapter + assert SourceAdapter.operator_creatable is False + + +# --------------------------------------------------------------------------- +# Mock-DB: GET /adapters/new +# --------------------------------------------------------------------------- + +class TestGetAdaptersNew: + @pytest.mark.asyncio + async def test_renders_200_with_generic_http_in_kind_select(self): + pool, conn = _make_pool(fetch_returns=[]) + tmpl = MagicMock() + tmpl.TemplateResponse.return_value = MagicMock(status_code=200) + req = _make_request() + + with patch("central.gui.routes._get_templates", return_value=tmpl), \ + patch("central.gui.routes.get_pool", return_value=pool): + await adapters_create_form(req) + + ctx = tmpl.TemplateResponse.call_args.kwargs["context"] + kind_names = [ck["kind"] for ck in ctx["creatable_kinds"]] + assert "generic_http" in kind_names + + @pytest.mark.asyncio + async def test_fields_present_for_generic_http(self): + pool, conn = _make_pool(fetch_returns=[]) + tmpl = MagicMock() + tmpl.TemplateResponse.return_value = MagicMock(status_code=200) + req = _make_request() + + with patch("central.gui.routes._get_templates", return_value=tmpl), \ + patch("central.gui.routes.get_pool", return_value=pool): + await adapters_create_form(req) + + ctx = tmpl.TemplateResponse.call_args.kwargs["context"] + field_names = [f.name for f in ctx["fields"]] + # GenericHttpSettings requires url, domain, id_path at minimum + assert "url" in field_names + assert "domain" in field_names + assert "id_path" in field_names + + @pytest.mark.asyncio + async def test_template_renders_without_errors(self): + """Smoke test: the template itself renders without crashing.""" + pool, conn = _make_pool(fetch_returns=[]) + tmpl = MagicMock() + tmpl.TemplateResponse.return_value = MagicMock(status_code=200) + req = _make_request() + + with patch("central.gui.routes._get_templates", return_value=tmpl), \ + patch("central.gui.routes.get_pool", return_value=pool): + resp = await adapters_create_form(req) + + # Template was called — no exception raised + assert tmpl.TemplateResponse.called + + +# --------------------------------------------------------------------------- +# Mock-DB: POST /adapters/new — happy path +# --------------------------------------------------------------------------- + +def _valid_generic_http_pairs(name="my_feed"): + """Minimal valid form pairs for a generic_http instance.""" + return [ + ("kind", "generic_http"), + ("name", name), + ("cadence_s", "300"), + # enabled intentionally absent → ships disabled + ("url", "https://example.com/feed.geojson"), + ("domain", "fire"), + ("id_path", "properties.id"), + ] + + +class TestPostAdaptersNewHappyPath: + @pytest.mark.asyncio + async def test_valid_creates_and_redirects(self): + pool, conn = _make_pool( + fetchval_returns=None, # name does not exist yet + fetch_returns=[], # no api keys + ) + inserted: list = [] + + async def cap_execute(q, *args): + if "INSERT INTO config.adapters" in q: + inserted.append(args) + + conn.execute.side_effect = cap_execute + + req = _make_request(form_pairs=_valid_generic_http_pairs()) + + with patch("central.gui.routes.get_pool", return_value=pool), \ + patch("central.gui.routes.write_audit", new=AsyncMock()): + resp = await adapters_create_submit(req) + + assert resp.status_code == 302 + assert "/adapters/my_feed" in resp.headers["location"] + assert len(inserted) == 1 + # args: name, kind, enabled, cadence_s, settings + _name, _kind, _enabled, _cadence, _settings = inserted[0] + assert _name == "my_feed" + assert _kind == "generic_http" + assert _enabled is False # no 'enabled' in form → ships disabled + assert _cadence == 300 + assert _settings["url"] == "https://example.com/feed.geojson" + + @pytest.mark.asyncio + async def test_enabled_flag_set_when_checked(self): + pool, conn = _make_pool(fetchval_returns=None, fetch_returns=[]) + inserted: list = [] + + async def cap(q, *args): + if "INSERT" in q: + inserted.append(args) + + conn.execute.side_effect = cap + pairs = _valid_generic_http_pairs() + [("enabled", "on")] + req = _make_request(form_pairs=pairs) + + with patch("central.gui.routes.get_pool", return_value=pool), \ + patch("central.gui.routes.write_audit", new=AsyncMock()): + resp = await adapters_create_submit(req) + + assert resp.status_code == 302 + _name, _kind, _enabled, *_ = inserted[0] + assert _enabled is True + + @pytest.mark.asyncio + async def test_audit_record_written_on_create(self): + pool, conn = _make_pool(fetchval_returns=None, fetch_returns=[]) + conn.execute.return_value = None + audited: list = [] + + async def cap_audit(conn_, action, **kw): + audited.append((action, kw)) + + req = _make_request(form_pairs=_valid_generic_http_pairs()) + with patch("central.gui.routes.get_pool", return_value=pool), \ + patch("central.gui.routes.write_audit", side_effect=cap_audit): + await adapters_create_submit(req) + + assert len(audited) == 1 + action, kw = audited[0] + assert action == "adapter.create" + assert kw["target"] == "my_feed" + + +# --------------------------------------------------------------------------- +# Mock-DB: POST /adapters/new — validation errors +# --------------------------------------------------------------------------- + +async def _post_new(pairs, fetchval=None, fetch_returns=None): + """Helper: POST /adapters/new and return (response, template_call_args). + + fetchval=None means "adapter name does not exist" (duplicate check passes). + Pass fetchval=1 to simulate a duplicate. + """ + pool, conn = _make_pool( + fetchval_returns=fetchval, # None = not found; passed unconditionally + fetch_returns=fetch_returns or [], + ) + tmpl = MagicMock() + tmpl.TemplateResponse.return_value = MagicMock() + req = _make_request(form_pairs=pairs) + with patch("central.gui.routes._get_templates", return_value=tmpl), \ + patch("central.gui.routes.get_pool", return_value=pool), \ + patch("central.gui.routes.write_audit", new=AsyncMock()): + resp = await adapters_create_submit(req) + return resp, tmpl.TemplateResponse.call_args + + +class TestPostAdaptersNewValidationErrors: + @pytest.mark.asyncio + async def test_duplicate_name_returns_409(self): + pool, conn = _make_pool(fetchval_returns=1, fetch_returns=[]) + req = _make_request(form_pairs=_valid_generic_http_pairs()) + with patch("central.gui.routes.get_pool", return_value=pool), \ + patch("central.gui.routes.write_audit", new=AsyncMock()): + resp = await adapters_create_submit(req) + assert resp.status_code == 409 + + @pytest.mark.asyncio + async def test_bad_name_format_returns_422(self): + pairs = _valid_generic_http_pairs(name="Bad-Name!") + resp, ca = await _post_new(pairs) + assert ca.kwargs["status_code"] == 422 + assert "name" in ca.kwargs["context"]["errors"] + + @pytest.mark.asyncio + async def test_name_starts_with_digit_returns_422(self): + pairs = _valid_generic_http_pairs(name="1invalid") + resp, ca = await _post_new(pairs) + assert ca.kwargs["status_code"] == 422 + + @pytest.mark.asyncio + async def test_name_too_short_returns_422(self): + pairs = _valid_generic_http_pairs(name="a") # only 1 char + resp, ca = await _post_new(pairs) + assert ca.kwargs["status_code"] == 422 + + @pytest.mark.asyncio + async def test_name_shadows_kind_returns_422(self): + """Cannot use a registered kind name as an instance name.""" + pairs = _valid_generic_http_pairs(name="generic_http") + resp, ca = await _post_new(pairs) + assert ca.kwargs["status_code"] == 422 + assert "name" in ca.kwargs["context"]["errors"] + + @pytest.mark.asyncio + async def test_non_creatable_kind_returns_422(self): + # usgs_quake is a real built-in kind that is NOT operator_creatable + pairs = [ + ("kind", "usgs_quake"), + ("name", "my_quake"), + ("cadence_s", "300"), + ("url", "https://example.com"), + ("domain", "quake"), + ("id_path", "id"), + ] + resp, ca = await _post_new(pairs) + assert ca.kwargs["status_code"] == 422 + assert "kind" in ca.kwargs["context"]["errors"] + + @pytest.mark.asyncio + async def test_invalid_settings_missing_required_field_returns_422(self): + # Omit required 'url' field from generic_http settings + pairs = [ + ("kind", "generic_http"), + ("name", "my_feed"), + ("cadence_s", "300"), + # no 'url', no 'domain', no 'id_path' + ] + resp, ca = await _post_new(pairs) + assert ca.kwargs["status_code"] == 422 + + @pytest.mark.asyncio + async def test_cadence_below_10_returns_422(self): + pairs = _valid_generic_http_pairs() + # replace cadence_s + pairs = [(k, "5") if k == "cadence_s" else (k, v) for k, v in pairs] + resp, ca = await _post_new(pairs) + assert ca.kwargs["status_code"] == 422 + assert "cadence_s" in ca.kwargs["context"]["errors"] + + @pytest.mark.asyncio + async def test_invalid_domain_returns_422(self): + pairs = _valid_generic_http_pairs() + pairs = [(k, "notadomain") if k == "domain" else (k, v) for k, v in pairs] + resp, ca = await _post_new(pairs) + assert ca.kwargs["status_code"] == 422 + + +# --------------------------------------------------------------------------- +# Mock-DB: POST /adapters/{name}/delete +# --------------------------------------------------------------------------- + +class TestPostAdaptersDelete: + @pytest.mark.asyncio + async def test_operator_instance_is_deleted(self): + """Deleting an operator instance removes the row and audits.""" + pool, conn = _make_pool( + fetchrow_returns={"name": "my_feed", "kind": "generic_http"}, + ) + deleted: list = [] + + async def cap(q, *args): + if "DELETE FROM config.adapters" in q: + deleted.append(args) + + conn.execute.side_effect = cap + req = _make_request() + + with patch("central.gui.routes.get_pool", return_value=pool), \ + patch("central.gui.routes.write_audit", new=AsyncMock()): + resp = await adapters_delete(req, "my_feed") + + assert resp.status_code == 302 + assert resp.headers["location"] == "/adapters" + assert len(deleted) == 1 + assert deleted[0][0] == "my_feed" + + @pytest.mark.asyncio + async def test_builtin_adapter_returns_403(self): + """Attempting to delete a built-in adapter (name in registry) → 403.""" + pool, conn = _make_pool( + fetchrow_returns={"name": "usgs_quake", "kind": "usgs_quake"}, + ) + req = _make_request() + + with patch("central.gui.routes.get_pool", return_value=pool), \ + patch("central.gui.routes.write_audit", new=AsyncMock()): + resp = await adapters_delete(req, "usgs_quake") + + assert resp.status_code == 403 + assert "built-in" in resp.body.decode() + # DELETE must NOT have been called + for call in conn.execute.call_args_list: + assert "DELETE" not in str(call) + + @pytest.mark.asyncio + async def test_missing_adapter_returns_404(self): + pool, conn = _make_pool(fetchrow_returns=None) + req = _make_request() + + with patch("central.gui.routes.get_pool", return_value=pool), \ + patch("central.gui.routes.write_audit", new=AsyncMock()): + resp = await adapters_delete(req, "nonexistent") + + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_delete_audit_record_written(self): + pool, conn = _make_pool( + fetchrow_returns={"name": "my_feed", "kind": "generic_http"}, + ) + conn.execute.return_value = None + audited: list = [] + + async def cap_audit(conn_, action, **kw): + audited.append((action, kw)) + + req = _make_request() + with patch("central.gui.routes.get_pool", return_value=pool), \ + patch("central.gui.routes.write_audit", side_effect=cap_audit): + await adapters_delete(req, "my_feed") + + assert len(audited) == 1 + action, kw = audited[0] + assert action == "adapter.delete" + assert kw["target"] == "my_feed" + + @pytest.mark.asyncio + async def test_generic_http_kind_itself_is_protected(self): + """The class entry 'generic_http' IS in the registry → 403.""" + pool, conn = _make_pool( + fetchrow_returns={"name": "generic_http", "kind": "generic_http"}, + ) + req = _make_request() + + with patch("central.gui.routes.get_pool", return_value=pool), \ + patch("central.gui.routes.write_audit", new=AsyncMock()): + resp = await adapters_delete(req, "generic_http") + + assert resp.status_code == 403 + + +# --------------------------------------------------------------------------- +# Template smoke test: adapters_new.html renders without crashing +# --------------------------------------------------------------------------- + +class TestAdaptersNewTemplate: + def _render(self, ctx): + req = Request({ + "type": "http", "method": "GET", "path": "/", + "headers": [], "query_string": b"", + }) + return gui_templates.TemplateResponse( + request=req, name="adapters_new.html", context=ctx + ).body.decode() + + def _ctx(self, errors=None, form_data=None): + from central.gui.form_descriptors import describe_fields + from central.adapters.generic_http import GenericHttpSettings + fields = describe_fields(GenericHttpSettings, {}) + return { + "operator": SimpleNamespace(username="admin"), + "csrf_token": "x", + "creatable_kinds": [{"kind": "generic_http", "display_name": "Generic HTTP Source"}], + "selected_kind": "generic_http", + "default_cadence_s": 300, + "fields": fields, + "api_keys": [], + "errors": errors, + "form_data": form_data, + } + + def test_renders_kind_select(self): + out = self._render(self._ctx()) + assert "generic_http" in out + assert 'name="kind"' in out + + def test_renders_name_input(self): + out = self._render(self._ctx()) + assert 'name="name"' in out + + def test_renders_cadence_input_with_default(self): + out = self._render(self._ctx()) + assert 'name="cadence_s"' in out + assert "300" in out + + def test_renders_url_field_for_generic_http(self): + out = self._render(self._ctx()) + assert 'name="url"' in out + + def test_enabled_unchecked_by_default(self): + out = self._render(self._ctx()) + # The enabled checkbox must not be checked in default render + # (spec: ships disabled) + assert 'name="enabled"' in out + # Extract the enabled checkbox line and confirm no 'checked' attribute + for line in out.splitlines(): + if 'name="enabled"' in line: + assert "checked" not in line, f"enabled checkbox should be unchecked by default: {line}" + break + + def test_error_messages_displayed(self): + errors = {"name": "Name is invalid", "url": "URL is required"} + out = self._render(self._ctx(errors=errors)) + assert "Name is invalid" in out + assert "URL is required" in out + + def test_form_data_restores_values(self): + form_data = {"kind": "generic_http", "name": "restored_name", + "cadence_s": "600", "url": "https://example.com/data.json", + "domain": "fire", "id_path": "id", "enabled": False} + out = self._render(self._ctx(form_data=form_data)) + assert "restored_name" in out + assert "https://example.com/data.json" in out From dfcdda5ecf2a505087a41611b831712c78a6e179 Mon Sep 17 00:00:00 2001 From: malice Date: Wed, 1 Jul 2026 11:30:35 -0600 Subject: [PATCH 16/17] feat: wire GenericHttpAdapter into geocoder enrichment (v0.15.0 PR2.5) (#123) Set enrichment_locations = [("latitude", "longitude")] and surface extracted coords as top-level data keys so apply_enrichment can geocode generic_http events; non-Point geometries and coordless items degrade gracefully to region unknown. Co-authored-by: Ubuntu Co-authored-by: Claude Sonnet 4.6 --- src/central/adapters/generic_http.py | 28 +++-- tests/test_generic_http.py | 150 +++++++++++++++++++++++++++ 2 files changed, 172 insertions(+), 6 deletions(-) diff --git a/src/central/adapters/generic_http.py b/src/central/adapters/generic_http.py index 70b8dba..2de743e 100644 --- a/src/central/adapters/generic_http.py +++ b/src/central/adapters/generic_http.py @@ -191,7 +191,10 @@ class GenericHttpAdapter(SourceAdapter): default_cadence_s = 300 data_class = "event" wizard_order = None # not in the setup wizard; created via operator GUI - enrichment_locations = [] + # Generic instances surface lat/lon into event.data so the supervisor's + # geocoder enrichment can reach them. Coordless instances degrade to + # region unknown, which is fine — apply_enrichment handles it gracefully. + enrichment_locations = [("latitude", "longitude")] bypass_bbox_filter = False operator_creatable = True # GUI allows operators to create multiple instances @@ -365,6 +368,8 @@ class GenericHttpAdapter(SourceAdapter): # --- geo --- geo_kwargs: dict[str, Any] = {} + lat_val: float | None = None + lon_val: float | None = None # Try GeoJSON geometry first. if s.geometry_path: @@ -372,14 +377,15 @@ class GenericHttpAdapter(SourceAdapter): if isinstance(geom, dict): geo_kwargs["geometry"] = geom # Also set centroid when geometry is a simple Point. + # Non-Point geometries (LineString, Polygon) have no single + # representative point; lat_val/lon_val stay None → region unknown. if geom.get("type") == "Point": coords = geom.get("coordinates") or [] if len(coords) >= 2: try: - geo_kwargs["centroid"] = ( - float(coords[0]), - float(coords[1]), - ) + lon_val = float(coords[0]) + lat_val = float(coords[1]) + geo_kwargs["centroid"] = (lon_val, lat_val) except (TypeError, ValueError): pass @@ -389,7 +395,9 @@ class GenericHttpAdapter(SourceAdapter): raw_lon = _dig(item, s.lon_path) if raw_lat is not None and raw_lon is not None: try: - geo_kwargs["centroid"] = (float(raw_lon), float(raw_lat)) + lat_val = float(raw_lat) + lon_val = float(raw_lon) + geo_kwargs["centroid"] = (lon_val, lat_val) except (TypeError, ValueError): pass @@ -402,6 +410,14 @@ class GenericHttpAdapter(SourceAdapter): if title is not None: data["title"] = title + # Surface coordinates into data so the geocoder enrichment can reach them. + # Coordless instances degrade to region unknown — no special-casing needed. + # Written BEFORE field_mappings so an explicit operator mapping to + # latitude/longitude (rare) takes precedence. + if lat_val is not None and lon_val is not None: + data["latitude"] = lat_val + data["longitude"] = lon_val + for fm in s.field_mappings: data[fm.dest_key] = _dig(item, fm.source_path) diff --git a/tests/test_generic_http.py b/tests/test_generic_http.py index ad32b49..aca8364 100644 --- a/tests/test_generic_http.py +++ b/tests/test_generic_http.py @@ -664,3 +664,153 @@ class TestInstanceName: assert events[0].adapter == "my_source" await adapter.shutdown() + + +# --------------------------------------------------------------------------- +# Geocoder enrichment wiring +# --------------------------------------------------------------------------- + +class TestEnrichmentLocations: + def test_enrichment_locations_declared(self): + assert GenericHttpAdapter.enrichment_locations == [("latitude", "longitude")] + + @pytest.mark.asyncio + async def test_geojson_point_writes_latlon_to_data( + self, temp_db_path, mock_config_store + ): + """GeoJSON Point item: data["latitude"]/["longitude"] == geometry coords, + and geo.centroid == (lon, lat).""" + config = make_config( + domain="wx", + extra_settings={ + "id_path": "id", + "geometry_path": "geometry", + }, + ) + adapter = GenericHttpAdapter(config, mock_config_store, temp_db_path) + await adapter.startup() + + fixture = { + "features": [ + { + "id": "enrich-point-1", + "geometry": {"type": "Point", "coordinates": [-116.2, 43.7]}, + } + ] + } + with patch.object(adapter, "_fetch", new_callable=AsyncMock) as mf: + mf.return_value = fixture + events = [e async for e in adapter.poll()] + + assert len(events) == 1 + e = events[0] + # lat = coords[1], lon = coords[0] + assert e.data["latitude"] == 43.7 + assert e.data["longitude"] == -116.2 + assert e.geo.centroid == (-116.2, 43.7) + + await adapter.shutdown() + + @pytest.mark.asyncio + async def test_lat_lon_path_writes_latlon_to_data( + self, temp_db_path, mock_config_store + ): + """lat_path/lon_path item: data["latitude"]/["longitude"] populated.""" + config = make_config( + domain="fire", + extra_settings={ + "items_path": "alerts", + "id_path": "uid", + "geometry_path": "", + "lat_path": "lat", + "lon_path": "lon", + }, + ) + adapter = GenericHttpAdapter(config, mock_config_store, temp_db_path) + await adapter.startup() + + fixture = { + "alerts": [ + {"uid": "enrich-latlon-1", "lat": 43.5, "lon": -116.1}, + ] + } + with patch.object(adapter, "_fetch", new_callable=AsyncMock) as mf: + mf.return_value = fixture + events = [e async for e in adapter.poll()] + + assert len(events) == 1 + e = events[0] + assert e.data["latitude"] == 43.5 + assert e.data["longitude"] == -116.1 + + await adapter.shutdown() + + @pytest.mark.asyncio + async def test_non_point_geometry_no_latlon_in_data( + self, temp_db_path, mock_config_store + ): + """Non-Point geometry (LineString/Polygon) has no representative point; + latitude/longitude must NOT appear in data — degrades to region unknown.""" + config = make_config( + domain="wx", + extra_settings={ + "id_path": "id", + "geometry_path": "geometry", + }, + ) + adapter = GenericHttpAdapter(config, mock_config_store, temp_db_path) + await adapter.startup() + + fixture = { + "features": [ + { + "id": "enrich-linestring-1", + "geometry": { + "type": "LineString", + "coordinates": [[-116.0, 43.0], [-115.0, 44.0]], + }, + } + ] + } + with patch.object(adapter, "_fetch", new_callable=AsyncMock) as mf: + mf.return_value = fixture + events = [e async for e in adapter.poll()] + + assert len(events) == 1 + e = events[0] + assert "latitude" not in e.data + assert "longitude" not in e.data + + await adapter.shutdown() + + @pytest.mark.asyncio + async def test_coordless_item_no_latlon_in_data( + self, temp_db_path, mock_config_store + ): + """Item with no geometry and no lat/lon paths: latitude/longitude absent + from data — will degrade to region unknown.""" + config = make_config( + domain="wx", + extra_settings={ + "id_path": "id", + "geometry_path": "geometry", + }, + ) + adapter = GenericHttpAdapter(config, mock_config_store, temp_db_path) + await adapter.startup() + + fixture = { + "features": [ + {"id": "enrich-coordless-1", "geometry": None}, + ] + } + with patch.object(adapter, "_fetch", new_callable=AsyncMock) as mf: + mf.return_value = fixture + events = [e async for e in adapter.poll()] + + assert len(events) == 1 + e = events[0] + assert "latitude" not in e.data + assert "longitude" not in e.data + + await adapter.shutdown() From 3c8da28d80d17b81a8653807a45d68cc9f76f88f Mon Sep 17 00:00:00 2001 From: malice Date: Wed, 1 Jul 2026 11:33:12 -0600 Subject: [PATCH 17/17] chore: bump version to 0.15.0 (#124) Co-authored-by: Ubuntu Co-authored-by: Claude Sonnet 4.6 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bcc35c7..aeb3fa5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "central" -version = "0.14.9" +version = "0.15.0" requires-python = ">=3.12,<3.13" description = "Data hub spine — adapters, bus, archive." readme = "README.md"
Cadence Last Updated
{{ adapter.cadence_s }}s {{ adapter.updated_at.strftime('%Y-%m-%d %H:%M') if adapter.updated_at else '—' }} Edit + {% if adapter.deletable %} +
+ + +
+ {% endif %} +