feat(generic): config-driven REST/GeoJSON source adapter (ported from Central) (#78)

Universal, no-code data sources: one GenericHttpAdapter polls any public
REST/GeoJSON feed per config.generic_sources[] — dotted-path field mapping
(items/id/lat/lon/geometry/title/fields) → coverage-gated, persisted
(generic_events, v26), cold-start-silent, LLM-queryable events. Ports
Central's GenericHttpAdapter to meshai native. First real use case: Idaho
Power outages, configured (not hardcoded) — anyone can point it at their own
utility/feed. GUI editor is a follow-up.

Co-authored-by: Matt Johnson <mj@k7zvx.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

work/tests/test_generic_http.py

@@ -0,0 +1,248 @@
+"""Tests for the universal config-driven GenericHttpAdapter (env/generic_http.py).
+
+Ported-behavior coverage:
+  * _dig dotted-path walker (nested dict, list index, miss cases)
+  * _item_to_event field mapping (Idaho Power outage item) + _render template
+  * summary_template with a missing key does not crash
+  * cold-start-silent: first poll seeds + persists but broadcasts nothing
+  * geometry-path Point -> centroid extraction
+"""
+from __future__ import annotations
+
+import json
+
+from meshai.env.generic_http import GenericHttpAdapter, _dig
+from meshai.env.store import EnvironmentalStore
+from meshai.config import EnvironmentalConfig
+from meshai.notifications.pipeline.bus import EventBus
+from meshai.persistence import get_db
+
+
+# --- Idaho Power source config (CONFIGURED, not hardcoded) -------------------
+
+IDAHO_POWER_SOURCE = {
+    "name": "idaho_power",
+    "enabled": True,
+    "url": "https://apiedge.idahopower.com/api/Outage/GetCurrentOutageInformation",
+    "items_path": "object.outages",
+    "id_path": "omsOutageId",
+    "lat_path": "latitude",
+    "lon_path": "longitude",
+    "title_path": "probableCause",
+    "category": "power_outage",
+    "poll_seconds": 300,
+    "severity": "routine",
+    "field_mappings": [
+        {"source_path": "omsCustomerCount", "dest_key": "customers"},
+        {"source_path": "omsEstimatedTimeToRestorationMessage", "dest_key": "eta"},
+        {"source_path": "omsStatusDescription", "dest_key": "status"},
+    ],
+    "summary_template": "⚡ Power out — {customers} affected, ETA {eta} ({status})",
+    "emoji": "⚡",
+}
+
+IDAHO_POWER_ITEM = {
+    "omsOutageId": "123",
+    "latitude": 43.6,
+    "longitude": -116.2,
+    "omsCustomerCount": 250,
+    "probableCause": "Equipment",
+    "omsEstimatedTimeToRestorationMessage": "10:30 PM",
+    "omsStatusDescription": "Crew assigned",
+}
+
+
+# ===========================================================================
+# _dig
+# ===========================================================================
+
+def test_dig_nested_dict():
+    assert _dig({"a": {"b": 1}}, "a.b") == 1
+    assert _dig({"object": {"outages": [1, 2]}}, "object.outages") == [1, 2]
+
+
+def test_dig_list_index():
+    assert _dig({"a": [10, 20]}, "a.1") == 20
+    assert _dig({"geometry": {"coordinates": [-116.2, 43.6]}},
+                "geometry.coordinates.0") == -116.2
+
+
+def test_dig_miss_cases():
+    assert _dig({"a": 1}, "a.b") is None          # scalar then key
+    assert _dig(None, "anything") is None          # None root
+    assert _dig({"a": [1]}, "a.5") is None         # out-of-range index
+    assert _dig({"a": {}}, "a.missing") is None    # missing key
+    assert _dig({"a": [1]}, "a.x") is None         # non-int index on list
+
+
+# ===========================================================================
+# _item_to_event + _render
+# ===========================================================================
+
+def test_item_to_event_idaho_power_mapping():
+    adapter = GenericHttpAdapter([IDAHO_POWER_SOURCE])
+    evt = adapter._item_to_event(IDAHO_POWER_ITEM, IDAHO_POWER_SOURCE, now=1000.0)
+
+    assert evt is not None
+    assert evt["event_id"] == "123"
+    assert evt["category"] == "power_outage"
+    assert evt["latitude"] == 43.6
+    assert evt["longitude"] == -116.2
+    assert evt["title"] == "Equipment"
+    assert evt["data"]["customers"] == 250
+    assert evt["data"]["eta"] == "10:30 PM"
+    assert evt["data"]["status"] == "Crew assigned"
+    assert evt["data"]["latitude"] == 43.6
+    assert evt["data"]["longitude"] == -116.2
+    # source namespaced per configured name for independent dedup
+    assert evt["source"] == "generic:idaho_power"
+    assert evt["_source"] == "idaho_power"
+
+
+def test_render_summary_template():
+    adapter = GenericHttpAdapter([IDAHO_POWER_SOURCE])
+    evt = adapter._item_to_event(IDAHO_POWER_ITEM, IDAHO_POWER_SOURCE)
+    assert adapter._render(evt) == (
+        "⚡ Power out — 250 affected, ETA 10:30 PM (Crew assigned)"
+    )
+
+
+def test_render_template_missing_key_does_not_crash():
+    source = dict(IDAHO_POWER_SOURCE)
+    # Reference a key that no field mapping produces -> must render blank.
+    source["summary_template"] = "⚡ {customers} out, cause {nonexistent_key}!"
+    adapter = GenericHttpAdapter([source])
+    evt = adapter._item_to_event(IDAHO_POWER_ITEM, source)
+    rendered = adapter._render(evt)  # must not raise
+    assert "250" in rendered
+    assert "{nonexistent_key}" not in rendered   # token substituted (blank)
+    assert rendered == "⚡ 250 out, cause !"
+
+
+def test_render_default_no_template():
+    source = dict(IDAHO_POWER_SOURCE)
+    source.pop("summary_template")
+    adapter = GenericHttpAdapter([source])
+    evt = adapter._item_to_event(IDAHO_POWER_ITEM, source)
+    rendered = adapter._render(evt)
+    assert "Equipment" in rendered          # title
+    assert "customers: 250" in rendered      # mapped field
+
+
+def test_item_missing_id_is_skipped():
+    adapter = GenericHttpAdapter([IDAHO_POWER_SOURCE])
+    evt = adapter._item_to_event({"latitude": 43.6, "longitude": -116.2},
+                                 IDAHO_POWER_SOURCE)
+    assert evt is None
+
+
+# ===========================================================================
+# geometry-path Point -> centroid
+# ===========================================================================
+
+def test_geometry_point_centroid():
+    source = {
+        "name": "geo_feed",
+        "enabled": True,
+        "url": "https://example.com/feed.geojson",
+        "items_path": "features",
+        "id_path": "id",
+        "geometry_path": "geometry",
+        "title_path": "properties.name",
+        "category": "geo",
+    }
+    item = {
+        "id": "g1",
+        "geometry": {"type": "Point", "coordinates": [-116.2, 43.6]},
+        "properties": {"name": "Test Point"},
+    }
+    adapter = GenericHttpAdapter([source])
+    evt = adapter._item_to_event(item, source)
+    assert evt["latitude"] == 43.6
+    assert evt["longitude"] == -116.2
+    # the resolved geometry dict rides on data so the coverage gate can use it
+    assert evt["data"]["geometry"]["type"] == "Point"
+
+
+def test_to_event_carries_latlon_for_coverage_gate():
+    adapter = GenericHttpAdapter([IDAHO_POWER_SOURCE])
+    evt = adapter._item_to_event(IDAHO_POWER_ITEM, IDAHO_POWER_SOURCE)
+    event = adapter.to_event(evt)
+    assert event is not None
+    assert event.lat == 43.6
+    assert event.lon == -116.2
+    assert event.category == "power_outage"
+    assert event.summary == "⚡ Power out — 250 affected, ETA 10:30 PM (Crew assigned)"
+
+
+# ===========================================================================
+# cold-start-silent (full store path) + persistence
+# ===========================================================================
+
+def _payload(items):
+    return {"object": {"outages": items, "totalCustomersAffected": sum(
+        i.get("omsCustomerCount", 0) for i in items)}}
+
+
+def _make_store_with_generic():
+    bus = EventBus()
+    captured = []
+    bus.subscribe(lambda e: captured.append(e))
+    store = EnvironmentalStore(
+        EnvironmentalConfig(), event_bus=bus,
+        generic_sources=[dict(IDAHO_POWER_SOURCE)],
+    )
+    adapter = store._adapters["generic_http"]
+    return store, adapter, captured
+
+
+def test_cold_start_silent_first_poll_seeds_persists_no_emit():
+    store, adapter, captured = _make_store_with_generic()
+    # Stub the network fetch with one active outage.
+    adapter._fetch = lambda url: _payload([IDAHO_POWER_ITEM])
+
+    store.refresh()  # poll 1 == pre-existing backlog
+
+    # Nothing broadcast on the cold-start poll...
+    assert captured == [], "first poll must broadcast NOTHING (cold-start seed)"
+    # ...but the item IS persisted so the LLM sees it immediately.
+    row = get_db().execute(
+        "SELECT source, event_id, category, title, lat, lon FROM generic_events "
+        "WHERE source=? AND event_id=?", ("idaho_power", "123")).fetchone()
+    assert row is not None
+    assert row["category"] == "power_outage"
+    assert row["title"] == "Equipment"
+    assert abs(row["lat"] - 43.6) < 1e-6
+
+
+def test_later_poll_broadcasts_newly_received_item():
+    store, adapter, captured = _make_store_with_generic()
+    adapter._fetch = lambda url: _payload([IDAHO_POWER_ITEM])
+    store.refresh()  # poll 1 — seed silently
+    assert captured == []
+
+    # A genuinely NEW outage appears on a later poll -> it must broadcast.
+    new_item = dict(IDAHO_POWER_ITEM, omsOutageId="456", omsCustomerCount=99)
+    adapter._fetch = lambda url: _payload([IDAHO_POWER_ITEM, new_item])
+    adapter._last_poll.clear()  # force cadence to elapse
+    store.refresh()  # poll 2
+
+    assert len(captured) == 1, "only the newly-received outage broadcasts"
+    assert captured[0].category == "power_outage"
+    # both items persisted (dedup by source+event_id)
+    n = get_db().execute(
+        "SELECT COUNT(*) c FROM generic_events WHERE source=?",
+        ("idaho_power",)).fetchone()["c"]
+    assert n == 2
+
+
+def test_build_generic_detail_reader():
+    from meshai.notifications.env_reporter import EnvReporter
+    store, adapter, captured = _make_store_with_generic()
+    adapter._fetch = lambda url: _payload([IDAHO_POWER_ITEM])
+    store.refresh()
+
+    text = EnvReporter().build_generic_detail()
+    assert "idaho_power" in text
+    assert "power_outage" in text
+    assert "Equipment" in text

work/tests/test_observer_locations.py

@@ -15,7 +15,7 @@ from meshai.persistence.observer_locations import (
 # -- schema / migration -------------------------------------------------------
 
 def test_schema_version_is_25():
-    assert SCHEMA_VERSION == 25
+    assert SCHEMA_VERSION == 26
 
 
 def test_observer_locations_table_exists():
@@ -30,7 +30,7 @@ def test_schema_meta_at_current():
     conn = get_db()
     row = conn.execute(
         "SELECT value FROM schema_meta WHERE key='version'").fetchone()
-    assert int(row["value"]) == 25
+    assert int(row["value"]) == 26
 
 
 # -- accessors ----------------------------------------------------------------

work/tests/test_satpass_persisted_timer.py

@@ -99,9 +99,9 @@ def _ingest_envelope(norad_id=25544, observer="Boise", max_el=72.5,
 # ── schema / migration ───────────────────────────────────────────────
 
 def test_schema_version_is_current():
-    # Bumped to 25 by the LLM-persistence migrations (v24 avalanche_events,
-    # v25 ducting_events); was 23 at the native-satpass observer_locations (v23).
-    assert SCHEMA_VERSION == 25
+    # Bumped to 26 by the generic-source migration (v26 generic_events); v24
+    # avalanche_events, v25 ducting_events; was 23 at native-satpass (v23).
+    assert SCHEMA_VERSION == 26
 
 
 def test_v22_migration_applies_and_adds_due_at_column(tmp_path, monkeypatch):
@@ -114,7 +114,7 @@ def test_v22_migration_applies_and_adds_due_at_column(tmp_path, monkeypatch):
     close_thread_connection()
     conn = init_db()
     row = conn.execute("SELECT value FROM schema_meta WHERE key='version'").fetchone()
-    assert int(row["value"]) == 25
+    assert int(row["value"]) == 26
     cols = {r["name"] for r in conn.execute("PRAGMA table_info(satpass_pending)")}
     assert "due_at" in cols
     close_thread_connection()