From 63245b8fba2a0951cfec45932399e27bf3407bcf Mon Sep 17 00:00:00 2001 From: malice Date: Sun, 5 Jul 2026 10:10:09 -0600 Subject: [PATCH] fix: WZDx registry nested-url crash + composer "None" leak (dry-run findings) (#42) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs found running native adapters against real upstreams: - env/wzdx.py: _select_feeds() called .strip() on the registry url field, but Socrata "URL"-column values arrive as {"url": "..."} — crashed ALL native WZDx discovery with AttributeError. Added _unwrap_url() (dict/str/ None-robust), applied to url/apiurl/feed_url. - composer._context_segment: appended optional fields by key presence, so cause: None (set by native road adapters) leaked literal "None" onto the wire. Guard on value (cause/expires_at truthiness; containment_pct is-not- None so 0% still renders). Legacy Mode-B path — golden tests unchanged. +3 regression tests; golden/composer suites pass; full suite 10-failure baseline (1685 passed). Co-authored-by: Matt Johnson Co-authored-by: Claude Opus 4.8 (1M context) --- work/meshai/env/wzdx.py | 21 ++++++++++++++- .../notifications/renderers/composer.py | 6 ++--- work/tests/test_adapter_wzdx.py | 27 +++++++++++++++++++ work/tests/test_v052_dispatcher.py | 21 +++++++++++++++ 4 files changed, 71 insertions(+), 4 deletions(-) diff --git a/work/meshai/env/wzdx.py b/work/meshai/env/wzdx.py index 5e24c7f..f4825e4 100644 --- a/work/meshai/env/wzdx.py +++ b/work/meshai/env/wzdx.py @@ -75,6 +75,23 @@ _US_STATES = { } +def _unwrap_url(v) -> str: + """Coerce a registry URL field to a clean string. + + Socrata's "URL" column type serializes as a nested object + (``{"url": "https://…"}``) — e.g. Idaho's row in the live FHWA registry — + while plain-text columns come back as a bare string. Unwrap the nested + form, pass strings through, and treat anything else (None, numbers) as + empty so callers never trip over ``.strip()``. + """ + if isinstance(v, dict): + inner = v.get("url") + return inner.strip() if isinstance(inner, str) else "" + if isinstance(v, str): + return v.strip() + return "" + + class WZDxAdapter: """FHWA WZDx work-zone polling adapter (native ``work_zone`` source).""" @@ -199,7 +216,9 @@ class WZDxAdapter: fmt = str(row.get("format") or "").strip().lower() if fmt and "geojson" not in fmt: continue # only GeoJSON WZDx feeds; skip xml/other - url = (row.get("url") or row.get("apiurl") or row.get("feed_url") or "").strip() + url = (_unwrap_url(row.get("url")) + or _unwrap_url(row.get("apiurl")) + or _unwrap_url(row.get("feed_url"))) if url and url not in urls: urls.append(url) return urls diff --git a/work/meshai/notifications/renderers/composer.py b/work/meshai/notifications/renderers/composer.py index 91fab28..9ac1121 100644 --- a/work/meshai/notifications/renderers/composer.py +++ b/work/meshai/notifications/renderers/composer.py @@ -285,14 +285,14 @@ def _context_segment(event: Event) -> Optional[str]: """Optional context (dropped FIRST when over budget).""" data = event.data or {} bits: list[str] = [] - if "containment_pct" in data: + if data.get("containment_pct") is not None: try: bits.append(f"{int(float(data['containment_pct']))}% contained") except Exception: pass - if "cause" in data: + if data.get("cause"): bits.append(str(data["cause"])) - if "expires_at" in data: + if data.get("expires_at"): bits.append(f"exp {data['expires_at']}") return ", ".join(bits) if bits else None diff --git a/work/tests/test_adapter_wzdx.py b/work/tests/test_adapter_wzdx.py index c1a2b90..d4c4c83 100644 --- a/work/tests/test_adapter_wzdx.py +++ b/work/tests/test_adapter_wzdx.py @@ -122,6 +122,33 @@ def test_select_feeds_handles_garbage(adapter): assert adapter._select_feeds([None, 3, {"state": "ID"}]) == [] # no url +def test_select_feeds_unwraps_socrata_url_object(adapter): + """Live FHWA registry serializes Idaho's ``url`` as a Socrata "URL" column + object (``{"url": "…"}``), not a bare string. Selection must unwrap it and + never raise ``AttributeError: 'dict' object has no attribute 'strip'``.""" + rows = [ + {"state": "Idaho", "format": "geojson", + "url": {"url": "https://511.idaho.gov/api/wzdx"}}, + ] + feeds = adapter._select_feeds(rows) + assert feeds == ["https://511.idaho.gov/api/wzdx"] + + +def test_tick_survives_socrata_url_object(adapter, monkeypatch): + """End-to-end discovery path tolerates the nested-object ``url`` shape.""" + registry = [{"state": "Idaho", "format": "geojson", + "url": {"url": "https://511.idaho.gov/api/wzdx"}}] + fc = make_feature_collection(make_wzdx_feature(feat_id="A")) + + def fake_get(url, timeout=30): + return registry if "datahub" in url else fc + + monkeypatch.setattr(adapter, "_http_get_json", fake_get) + # Must not raise; the unwrapped feed URL is discovered and fetched. + assert adapter.tick() is True + assert {e["external_id"] for e in adapter.get_events()} == {"idot-1:A"} + + # ============================================================ # CATEGORY + CANONICAL DATA # ============================================================ diff --git a/work/tests/test_v052_dispatcher.py b/work/tests/test_v052_dispatcher.py index d75285e..4e6d9ee 100644 --- a/work/tests/test_v052_dispatcher.py +++ b/work/tests/test_v052_dispatcher.py @@ -232,6 +232,27 @@ def test_renderer_byte_budget_drops_optional_segments(): assert "lightning" not in s +def test_renderer_omits_none_context_fields(): + """Regression — native road adapters set optional context keys present but + None (``cause``/``expires_at``/``containment_pct``). The composer must + guard on value, not key presence, so the literal string 'None' never leaks + onto the wire.""" + e = make_event( + source="wzdx", category="wildfire_incident", severity="immediate", + title="Test Fire", region="Boise", timestamp=time.time(), + data={"cause": None, "expires_at": None, "containment_pct": None}, + ) + s = compose_mesh_message(e) + assert "None" not in s + # A real value on the same field still renders (guard doesn't over-suppress). + e2 = make_event( + source="wzdx", category="wildfire_incident", severity="immediate", + title="Test Fire", region="Boise", timestamp=time.time(), + data={"cause": "lightning", "expires_at": None, "containment_pct": None}, + ) + assert "lightning" in compose_mesh_message(e2) + + def test_renderer_never_mid_character_truncation(): """The composer must never emit a UTF-8 byte sequence that splits a codepoint. Even with required-only over budget, we drop wholesale or