diff --git a/work/meshai/adapter_config/defaults.py b/work/meshai/adapter_config/defaults.py index cb4cb9c..9f702ba 100644 --- a/work/meshai/adapter_config/defaults.py +++ b/work/meshai/adapter_config/defaults.py @@ -246,15 +246,6 @@ REGISTRY: dict[tuple[str, str], dict[str, Any]] = { "description": "Timezone the daily work-zone count summary's summary_time is interpreted in.", }, - # ================================================================= - # CENTRAL consumer -- 1 setting (severity-int bucket boundaries) - # ================================================================= - ("central", "severity_thresholds"): { - "default": {"routine_max": 1, "priority_max": 2, "immediate_min": 3}, - "type": "json", - "description": "Central int severity buckets: 0..routine_max -> routine, priority_max -> priority, >= immediate_min -> immediate.", - }, - # ================================================================= # DISPATCHER -- 4 settings (LRU cap + cooldown prune params + retention) # ================================================================= @@ -762,11 +753,6 @@ ADAPTER_META: dict[str, dict[str, Any]] = { "include_in_llm_context": True, "description": "3x/day scheduled broadcast of HF band ratings (SWPC-local + HamQSL fallback).", }, - "central": { - "display_name": "Central consumer routing", - "include_in_llm_context": False, - "description": "Adapter <-> source remap + severity buckets. Operational, not LLM-relevant.", - }, "dispatcher": { "display_name": "Dispatcher state", "include_in_llm_context": True, diff --git a/work/meshai/central/__init__.py b/work/meshai/central/__init__.py index 524a913..07eca5c 100644 --- a/work/meshai/central/__init__.py +++ b/work/meshai/central/__init__.py @@ -1,6 +1,6 @@ -"""Central connector package (v0.4) — consumes Central's NATS JetStream -firehose and normalizes it into meshai pipeline Events.""" - -from meshai.central.consumer import CentralConsumer - -__all__ = ["CentralConsumer"] +"""Central connector package (v0.4) — historically consumed Central's NATS +JetStream firehose and normalized it into meshai pipeline Events. The NATS +consumer is retired (Central is gone); this package now only holds the +split-file modules still used by the native adapter paths (firms_handler, +satpass_handler, tle_handler, wfigs_handler _render, budget, +idaho_gauge_sites, pass_predictor).""" diff --git a/work/meshai/central/avy_handler.py b/work/meshai/central/avy_handler.py deleted file mode 100644 index 1c11460..0000000 --- a/work/meshai/central/avy_handler.py +++ /dev/null @@ -1,336 +0,0 @@ -"""Central avalanche advisory handler — Phase-1 refactored bridge. - -Subscribes to CENTRAL_AVY stream via consumer.py routing. -Adapter: avalanche_org -Subjects: central.avy.advisory.> (active + tombstones in one consumer) - -Phase-1 refactor: ALL gating decisions are now delegated to -`meshai.notifications.gating.avalanche.decide()`. This function is kept -as a thin compatibility bridge so existing call-sites and tests remain stable -while the new formatter+decider architecture is established. - -centralseverity → NAADS mapping (tier-b): - Central's envelope carries `severity` (= centralseverity) on a COMPRESSED - 5-point scale where 2=Considerable, 3=High, 4=Extreme (documented in the - pre-refactor TODO comment). The canonical danger_level is NAADS 1–5. - - NAADS mapping table (see also gating/avalanche.py docstring): - centralseverity 0 → NAADS 1 (Low) [INFERRED, needs validation] - centralseverity 1 → NAADS 2 (Moderate) [INFERRED, needs validation] - centralseverity 2 → NAADS 3 (Considerable) [documented] - centralseverity 3 → NAADS 4 (High) [documented] - centralseverity 4 → NAADS 5 (Extreme) [documented] - other → NAADS 0 (No Rating) - - ⚠ NEEDS LIVE VALIDATION IN-SEASON (October+): confirm by reading a live - Central avalanche envelope for a zone known to be rated Considerable - and verifying centralseverity == 2. - - Strategy: prefer data.data.danger_level (direct from avalanche.org API, - NAADS 1–5); fall back to _remap_centralseverity(inner.severity) when - data.danger_level is absent or non-numeric. - -Tombstones (central.avy.advisory.removed.*): handler returns None. -""" - -import logging -import re -import time -from typing import Any, Optional - -from meshai.adapter_config import adapter_config -from meshai.central.budget import budget_for, fit_to_budget -from meshai.persistence import get_db - -logger = logging.getLogger(__name__) - - -# NAADS danger level names (for fallback when data.danger_name is absent) -_NAADS_NAMES: dict[int, str] = { - 0: "No Rating", - 1: "Low", - 2: "Moderate", - 3: "Considerable", - 4: "High", - 5: "Extreme", -} - - -def _remap_centralseverity(centralseverity: Any) -> int: - """Map Central's compressed severity int → NAADS 1–5 danger_level. - - Mapping table (⚠ needs live validation in-season): - 0 → 1 (Low) [inferred] - 1 → 2 (Moderate) [inferred] - 2 → 3 (Considerable) [documented in pre-refactor avy_handler comments] - 3 → 4 (High) [documented] - 4 → 5 (Extreme) [documented] - other → 0 (No Rating) - - Formula: naads = centralseverity + 1 for 0 ≤ centralseverity ≤ 4 - """ - try: - sev = int(centralseverity) - except (TypeError, ValueError): - return 0 - if 0 <= sev <= 4: - return sev + 1 - return 0 - - -def _canonical_danger_level(d: dict, inner: dict) -> Optional[int]: - """Derive canonical NAADS danger_level from envelope fields. - - Strategy (in order): - 1. data.data.danger_level — direct from avalanche.org, should be NAADS. - 2. _remap_centralseverity(inner.severity) — fallback via mapping table. - 3. None — caller should suppress. - - Returns int or None. - """ - # Primary: data.data.danger_level (NAADS from avalanche.org) - raw = d.get("danger_level") - if isinstance(raw, (int, float)) and raw >= 0: - return int(raw) - # Fallback: remap from centralseverity - fallback = _remap_centralseverity(inner.get("severity")) - if fallback > 0: - logger.debug( - "avy_handler: danger_level absent in data.data — " - "remapped centralseverity %r → NAADS %d", - inner.get("severity"), fallback, - ) - return fallback - return None - - -def _coerce_severity(sev: Any) -> Optional[str]: - if sev is None: - return None - if isinstance(sev, str): - return sev or None - try: - return str(int(sev)) - except (TypeError, ValueError): - return str(sev) - - -def _now() -> int: - return int(time.time()) - - -def handle_avy(envelope: dict, subject: str, - data: Optional[dict] = None) -> Optional[str]: - """Central path handler for NWAC/CAIC avalanche advisories. - - Phase-1 refactor: delegates gating to - `meshai.notifications.gating.avalanche.decide()`. Formats via - `_render()` for backward-compat callers (existing tests); the registered - `formatters.avalanche.format()` re-renders from event.data at dispatch - time with any tier-b changes (is_update prefix). - - On broadcast: - - Writes canonical data fields into the shared `data` dict so the Event - carries structured fields (the formatter reads them at dispatch time). - - Applies GateResult.data_patch (is_update, _severity_override). - - Attaches _on_broadcast_committed for event_log.handled update. - - Returns the _render() wire string for backward-compat. - - On suppress: returns None (default-deny unchanged). - """ - if not isinstance(envelope, dict): - return None - - inner = envelope.get("data") or {} - if (inner.get("adapter") or "") != "avalanche_org": - return None - - category = inner.get("category") or "" - - # Tombstone — consume silently, no broadcast. - if "removed" in category: - logger.debug("avy_handler: tombstone for %s — acking silently", category) - return None - - d = inner.get("data") or {} - geo = inner.get("geo") or {} - severity_word = _coerce_severity(inner.get("severity")) - - # ── Canonical danger_level (NAADS 1–5) ─────────────────────────────────── - danger_level = _canonical_danger_level(d, inner) - if danger_level is None: - return None - - danger_name = ( - d.get("danger_name") or _NAADS_NAMES.get(danger_level, str(danger_level)) - ) - zone_name = d.get("zone_name") or "Unknown Zone" - center_id = d.get("center_id") or "" - travel = (d.get("travel_advice") or "").strip() - - # ── Centroid from geo.centroid [lon, lat] ───────────────────────────────── - centroid = geo.get("centroid") or [] - if isinstance(centroid, (list, tuple)) and len(centroid) >= 2: - lat, lon = centroid[1], centroid[0] - else: - lat = d.get("latitude") - lon = d.get("longitude") - - # ── Build canonical data dict ───────────────────────────────────────────── - canonical: dict = { - "danger_level": danger_level, - "danger_name": danger_name, - "zone_name": zone_name, - "center_id": center_id, - "travel_advice": travel, - "lat": lat, - "lon": lon, - "is_update": False, # Central path: no per-zone trend detection yet - } - - # Optional expires timestamp - expires_str = inner.get("expires") - if expires_str and isinstance(expires_str, str): - from datetime import datetime - try: - canonical["expires"] = datetime.fromisoformat( - expires_str.replace("Z", "+00:00") - ).timestamp() - except Exception: - pass - - # ── Persist to event_log ────────────────────────────────────────────────── - conn = get_db() - if conn is None: - logger.warning("avy_handler: persistence unavailable, skipping") - return None - - log_id = _log_event_returning_id( - conn, now=_now(), source="avalanche_org", - category=category, severity_word=severity_word, - event_id_external=f"{center_id}:{zone_name}", - subject=subject, handled=0, - table_name="event_log", table_pk=None, - ) - - # ── Delegate to gating module ───────────────────────────────────────────── - from meshai.notifications.gating.avalanche import decide as _gate_decide - gate = _gate_decide(canonical, source="avalanche", now=float(_now())) - - if not gate.broadcast: - return None - - # ── Write canonical into shared data dict ──────────────────────────────── - # Cutover gate: `category` here is the envelope category (e.g. - # "avalanche_warning" or "avalanche_watch") — same strings registered in - # the formatter/gating registries. When cut over, gate.data_patch (which - # carries is_update + _severity_override) flows to the live Event; when not - # cut over, old-style _attach_commit preserves pre-Phase-1 live behavior. - from meshai.notifications.cutover import is_cutover - if isinstance(data, dict): - data.update(canonical) - if is_cutover(category): - # NEW PATH: gate.data_patch provides is_update + _severity_override. - data.update(gate.data_patch) - data["_broadcast_audit"] = {"table": "event_log", "pk": log_id} - - _raw_gate_commit = gate.commit - _log_row_id = log_id - - def _on_commit(committed_at: float) -> None: - """Idempotent: mark event_log row handled on confirmed delivery.""" - if _raw_gate_commit is not None: - _raw_gate_commit(committed_at) - if _log_row_id is not None: - try: - c = get_db() - c.execute( - "UPDATE event_log SET handled=1 WHERE id=?", - (int(_log_row_id),), - ) - except Exception: - logger.exception( - "avy commit: event_log update failed for log_id=%s", - _log_row_id, - ) - - data["_on_broadcast_committed"] = _on_commit - else: - # NOT cutover: old-style commit (canonical only, no data_patch). - _attach_commit(data, log_id=log_id) - - # ── Return _render() wire (backward compat — existing tests check this) ─── - # At dispatch time compose_mesh_message() calls the registered formatter - # which re-renders from event.data (tier-b: is_update prefix applies there). - return _render( - danger_level=danger_level, - danger_name=danger_name, - zone_name=zone_name, - center_id=center_id, - travel=travel, - ) - - -def _render(*, danger_level: int, danger_name: str, zone_name: str, - center_id: str, travel: str) -> str: - """Wire string renderer — backward-compat entrypoint for direct callers. - - The registered formatters/avalanche.format() is the canonical render path - at dispatch time. This function is kept for existing test callers - (test_adapter_avalanche, etc.) and for the handle_avy() return value. - """ - emoji = "⛷" - # Warning for High/Extreme (4-5), Watch for Considerable (3). - prefix = "WARNING:" if danger_level >= 4 else "Watch:" - - line1 = f"{emoji} AVY {prefix} {zone_name} — {danger_name} ({danger_level})" - # Travel advice: FIRST SENTENCE only (up to and including the first - # sentence terminator), instead of a fixed character slice. - line2 = None - if travel and travel.strip(): - t = travel.strip() - m = re.search(r"[.!?]", t) - line2 = t[: m.end()] if m else t - line3 = f"{center_id} · valid today" if center_id else "valid today" - - msg = "\n".join(l for l in [line1, line2, line3] if l) - return fit_to_budget(msg, budget_for("avalanche")) - - -def _attach_commit(data: Optional[dict], *, log_id: Optional[int]) -> None: - """Legacy helper — used by pre-refactor callers (kept for import compat).""" - if not isinstance(data, dict): - return - - def _on_commit(committed_at: float) -> None: - try: - conn = get_db() - except Exception: - logger.exception("avy commit: persistence unavailable") - return - if log_id is not None: - conn.execute( - "UPDATE event_log SET handled=1 WHERE id=?", - (int(log_id),), - ) - - data["_on_broadcast_committed"] = _on_commit - data["_broadcast_audit"] = { - "table": "event_log", - "pk": log_id, - } - - -def _log_event_returning_id( - conn, *, now, source, category, severity_word, - event_id_external, subject, handled, - table_name, table_pk, -) -> Optional[int]: - cursor = conn.execute( - "INSERT INTO event_log(received_at, source, category, severity_word, " - "event_id_external, nats_subject, handled, table_name, table_pk) " - "VALUES (?,?,?,?,?,?,?,?,?)", - (now, source, category, severity_word, event_id_external, - subject, int(bool(handled)), table_name, table_pk), - ) - return cursor.lastrowid diff --git a/work/meshai/central/consumer.py b/work/meshai/central/consumer.py deleted file mode 100644 index 9594a9e..0000000 --- a/work/meshai/central/consumer.py +++ /dev/null @@ -1,1110 +0,0 @@ -"""Central connector — consumes Central's NATS JetStream firehose and -normalizes CloudEvents envelopes into meshai pipeline Events. - -v0.4 C.1: backend only. The consumer subscribes only to subjects derived from -adapters whose config `source == "central"`. With every adapter defaulting to -`native`, it starts as a no-op (0 subscriptions) and introduces no NATS -dependency at boot. Flipping an adapter to central is Phase C.3. - -Wire format (see Central CONSUMER-INTEGRATION guide, confirmed in v0.4 Phase A): - envelope (CloudEvents v1.0) -> envelope["data"] (Central Event) - -> Event["data"] (upstream payload, verbatim, incl `_enriched`) -""" - -import asyncio -import json -import logging -import re -import time -from datetime import datetime -from typing import Optional -from meshai.adapter_config import adapter_config - -from meshai.notifications.events import Event, make_event -from meshai.notifications.categories import get_category, ALERT_CATEGORIES - -logger = logging.getLogger("meshai.central.consumer") - - -def consumer_config(): - """JetStream consumer config for Central subscriptions. - - deliver_policy=NEW: subscribe to messages published AFTER consumer creation. - Avoids replaying the entire retained backlog on first flip (could be 330k+ - msgs for high-volume streams like traffic_flow). - """ - from nats.js.api import ConsumerConfig, DeliverPolicy - return ConsumerConfig(deliver_policy=DeliverPolicy.LAST_PER_SUBJECT) - - -# Bare-wildcard subjects, pre-v0.9.20. Still used when `central.region` is -# empty (backward-compat fallback) and as the canonical adapter -> family map. -# Adapters with no Central equivalent (avalanche, ducting) are absent; flipping -# those to source=central subscribes to nothing (logged). -_SUBJECTS_BARE: dict[str, list[str]] = { - "nws": ["central.wx.>"], - "fires": ["central.fire.incident.>", "central.fire.perimeter.>"], - "firms": ["central.fire.>"], - "usgs_quake": ["central.quake.>"], - "usgs": ["central.hydro.>"], - "swpc": ["central.space.>"], - "traffic": ["central.traffic.>"], - "roads511": ["central.traffic.>"], # shared with traffic; sub-adapter routing - "avalanche": ["central.avy.advisory.>"], - "satpass": ["central.sat.pass.>", "central.sat.tle.>"], -} - -# Backwards-compat: keep ADAPTER_SUBJECTS importable for legacy readers/tests. -ADAPTER_SUBJECTS = _SUBJECTS_BARE - - -def _subjects_for(adapter: str, region: Optional[str]) -> list[str]: - """Build region-aware Central subject filters for an adapter (v0.5.4). - - Central v0.9.20 (shipped 2026-05-28) added per-region subject suffixes so - consumers interested in a single region can have the firehose filtered - server-side instead of dragging all-US events and discarding 95% locally. - - `region` is a dotted token tree, e.g. 'us.id' for Idaho. Adapters use - one of three suffix patterns; the v0.9.20 scheme is not uniform: - - - region BEFORE the wildcard (nws): - central.wx.alert.us.id.> - - USGS NWIS hydro — three single-token wildcards + bare region tail: - central.hydro.*.*.*.us.id (per-state, e.g. Idaho) - central.hydro.*.*.*.unknown (gauges whose state Central - couldn't resolve; documented - workaround until backfill) - Per Central v0.10.0 nwis.py producer code, the actual published - subject is `central.hydro....` where - is `us.` (7 tokens) or `unknown` (6 tokens). The - doc §nwis text shows only the 4-token category-shape stem and is - stale w.r.t. the regional suffix. v0.5.7-water fixes the - pre-v0.5.7-water `central.hydro.>.` shape, which was - invalid NATS (`>` mid-subject). - - USGS quake — no region in subject (per Central v0.10.0 guide §usgs_quake): - central.quake.event. - 4 tokens total. is one of {minor, light, moderate, strong, - major, great} -- USGS magnitude bands, NOT a severity integer. - State filtering must happen client-side via data.latitude/longitude - (same situation as FIRMS, fixed in v0.5.7-fire). - v0.5.7-seismic restored the legal tail-only `>` here; the pre- - v0.5.7-seismic `central.quake.event.>.us.id` was syntactically - invalid AND wouldn't have matched anything Central publishes (only - 4 tokens, no us.). - - FIRMS — no region in subject at all (per Central v0.10.0 guide): - central.fire.hotspot.. - State filtering must happen client-side via data.latitude/longitude. - v0.5.7-fire restored the legal tail-only `>` here; the pre-v0.5.7-fire - `central.fire.hotspot.>.us.id` was syntactically invalid AND wouldn't - have matched anything Central publishes (only 5 tokens, no us.). - - state-only token at a fixed depth (fires WFIGS): - central.fire.incident..> (active) - central.fire.perimeter..> (active) - central.fire.incident.removed. (removal tombstone) - central.fire.perimeter.removed. (removal tombstone) - v0.5.7-fire added the tombstone subjects: pre-v0.5.7-fire we only - subscribed to the active subjects, silently dropping all WFIGS - fall-off signals. - - traffic family — Convention B, bare state, no wildcard: - central.traffic..id (wzdx, tomtom_incidents, - state_511_atis) - - traffic family — Convention A, us.: - central.traffic..us.id (itd_511, Idaho-only) - - region ignored (swpc) — space weather is planetary. - - NATS rule: `>` is only legal at the tail. Pre-v0.5.7-traffic this file - shipped `central.traffic.>.{state}` for traffic+roads511, which was - syntactically invalid (`>` mid-subject). Fixed by switching to single- - token `*` wildcards for the per-event-type slot. roads511 now owns - BOTH the bare-state (Convention B, shared with traffic) and the - us. (Convention A, itd_511-only) subjects so itd_511 events - attribute to roads511 in meshai. - - The .unknown workaround: v0.9.20 leaves USGS hydro events whose gauge - state can't be inferred at the `central.hydro.*.*.*.unknown` subject - (6 tokens). Subscribing to both the per-state and the unknown filters - avoids losing those rows until the upstream NWIS state-tag backfill. - - Empty/None region returns the bare-wildcard form (v0.5.3 behaviour). - Adapters without a Central equivalent (avalanche, ducting) return []. - """ - if not region: - return list(_SUBJECTS_BARE.get(adapter, [])) - state = region.split(".")[-1] - table: dict[str, list[str]] = { - "nws": [f"central.wx.alert.{region}.>"], - # WFIGS (fires): active + removal tombstones. v0.5.7-fire added the - # two removed. subjects so fall-off signals reach meshai. - "fires": [f"central.fire.incident.{state}.>", - f"central.fire.perimeter.{state}.>", - f"central.fire.incident.removed.{state}", - f"central.fire.perimeter.removed.{state}"], - # FIRMS: Central publishes central.fire.hotspot.. - # with NO region in the subject. Tail-only `>` is the only NATS-legal - # subscription that covers all combinations; client-side filters lat/lon. - "firms": ["central.fire.hotspot.>"], - # USGS quake: Central publishes central.quake.event. with NO - # region in the subject (per guide §usgs_quake). Same situation as - # FIRMS -- tail-only `>` is the legal form; client-side filters lat/lon. - "usgs_quake": ["central.quake.event.>"], - # USGS NWIS hydro: 3 single-token wildcards for .. - # + bare region tail. Pre-v0.5.7-water shipped `central.hydro.>.` - # which is invalid NATS (`>` only legal at the tail). Verified against - # the v0.10.0-itd-511 nwis.py producer subject_for() body which - # publishes `central.hydro....`. - "usgs": [f"central.hydro.*.*.*.{region}", - "central.hydro.*.*.*.unknown"], - # SWPC space weather: planetary (no region). The umbrella subject - # central.space.> catches all three SWPC adapters per Central v0.10.0 - # guide §swpc_alerts/§swpc_kindex/§swpc_protons: - # - swpc_alerts: central.space.alert. - # - swpc_kindex: central.space.kindex (fixed) - # - swpc_protons: central.space.proton_flux (fixed) - # All three publish severity=0 by default (verified against the - # live samples in the guide); map_severity(0) -> "routine", which - # routes through the NotificationToggle's "routine" severity_channels - # entry (dict is string-keyed, no IndexError risk). - "swpc": ["central.space.>"], - # Convention B (bare state) — shared by traffic family (wzdx, - # tomtom_incidents, state_511_atis). Single-token `*` matches the - # event_type slot; `>` was illegal here. - "traffic": [f"central.traffic.*.{state}"], - # roads511 dual-subscribes: bare state (shared with traffic) + the - # us. form that the new itd_511 Idaho-only adapter publishes - # (Convention A). Sub-adapter routing (_subject_owned) keeps the - # shared bare-state subject scoped to both source names. - "roads511": [f"central.traffic.*.{state}", - f"central.traffic.*.{region}"], - # Avalanche (avalanche_org): Central publishes on CENTRAL_AVY stream. - # Active advisories: central.avy.advisory.us. - # Tombstones (v0.10.11+): central.avy.advisory.removed.us. - # Wide filter covers both in one consumer. Client-side: gate on - # danger_level from data.data, not centralseverity (higher=more severe - # on Central's scale, inverse of what the handler uses). - # Off-season: June–Sep, CENTRAL_AVY will be empty — expected, not broken. - "avalanche": [f"central.avy.advisory.>"], - # satpass: pass alerts are region-scoped (Central publishes - # central.sat.pass.us.., per quickstart §7); - # TLEs are global, no region token (central.sat.tle., §4) -- - # same no-region logic as swpc. - "satpass": [f"central.sat.pass.{region}.>", - "central.sat.tle.>"], - } - return list(table.get(adapter, [])) - -# Bridge between Central's adapter taxonomy and meshai's family-tab source names. -# Central names some adapters differently (e.g. "wfigs_incidents" vs meshai's -# "fires"); remap so dashboard per-adapter event filtering (which keys on the -# native source name) works whether a feed is native or central. 1:1 names -# (nws, usgs_quake, firms) are intentionally omitted -> passthrough. -CENTRAL_ADAPTER_TO_SOURCE: dict[str, str] = { - "wfigs_incidents": "fires", - "wfigs_perimeters": "fires", - "nwis": "usgs", - "swpc_alerts": "swpc", - "swpc_kindex": "swpc", - "swpc_protons": "swpc", - "wzdx": "traffic", - "tomtom_incidents": "traffic", - "state_511_atis": "roads511", - # v0.5.7-traffic: itd_511 is the new Idaho-only Central adapter - # (Convention A publishing). Routes to meshai's roads511 source so - # ALERT_CATEGORIES roads-family rules cover both 511 feeds. A future - # v0.6 may split them; for now collapsed for UX simplicity. - "itd_511": "roads511", - "avalanche_org": "avalanche", - "firms": "firms", - "celestrak_tle": "satpass", - "n2yo_visualpasses": "satpass", - "satpass_predict": "satpass", -} - -# Central hierarchical category prefix -> meshai flat category. -# First matching prefix wins; order matters (most specific first). -_CATEGORY_MAP: list[tuple[str, str]] = [ - ("wx.alert", "weather_warning"), - ("wx.", "weather_statement"), - ("fire.hotspot", "wildfire_hotspot"), - ("fire.incident", "wildfire_incident"), - ("fire.perimeter", "wildfire_incident"), - ("fire.", "wildfire_incident"), - ("quake.", "earthquake_event"), - ("hydro.", "stream_flow"), - ("space.alert", "rf_propagation_alert"), - ("space.kindex", "geomagnetic_storm"), - ("space.proton", "solar_radiation_storm"), - ("space.", "geomagnetic_storm"), - ("disaster.", "disaster_event"), - ("traffic_flow", "traffic_flow"), - ("traffic_cameras", "traffic_camera"), - # v0.5.7-traffic: preserve traffic event_type distinctions instead of - # flattening to traffic_congestion. Central publishes category strings - # like "work_zone.wzdx", "incident.tomtom_incidents", "closure" (raw - # from state_511_atis / itd_511). startswith() catches both the bare - # form and the "." suffixed form. - ("work_zone", "work_zone"), - ("incident", "road_incident"), - ("closure", "road_closure"), - ("traffic.", "traffic_congestion"), - ("pass.", "sat_pass"), - ("sat.", "sat_pass"), -] - - -def map_category(central_category: str) -> str: - """Map Central's hierarchical category string to a meshai flat category.""" - cat = central_category or "" - for prefix, flat in _CATEGORY_MAP: - if cat.startswith(prefix): - return flat - return "other" - - -# Subject-domain fallback: some Central categories are not domain-prefixed -# (e.g. traffic's "work_zone.wzdx"), so when the category table misses we map by -# the stable subject domain token (central..<...>) instead of "other". -_SUBJECT_DOMAIN_CATEGORY = { - "wx": "weather_warning", - "fire": "wildfire_incident", - "quake": "earthquake_event", - "hydro": "stream_flow", - "space": "geomagnetic_storm", - "disaster": "disaster_event", - "traffic": "traffic_congestion", - "traffic_flow": "traffic_flow", - "traffic_cameras": "traffic_camera", - "sat": "sat_pass", -} - - -def category_from_subject(subject: str) -> Optional[str]: - """Map a NATS subject (central..<...>) to a meshai category.""" - parts = (subject or "").split(".") - if len(parts) >= 2 and parts[0] == "central": - return _SUBJECT_DOMAIN_CATEGORY.get(parts[1]) - return None - - -def map_severity(sev: Optional[int]) -> str: - """Central int severity (0-4 / None) -> meshai severity string. - - v0.6-3b: bucket thresholds live in - adapter_config.central.severity_thresholds (default - {routine_max: 1, priority_max: 2, immediate_min: 3}). The check order - is: immediate_min first (clamps 3..+inf), then priority_max - (catches 2), else routine. - """ - if sev is None: - return "routine" - try: - sev = int(sev) - except (TypeError, ValueError): - return "routine" - thr = adapter_config.central.severity_thresholds or {} - if sev >= int(thr.get("immediate_min", 3)): - return "immediate" - if sev >= int(thr.get("priority_max", 2)): - return "priority" - return "routine" - - -def _parse_time(s) -> Optional[float]: - """Parse a Central ISO-8601 timestamp to epoch seconds.""" - if not s or not isinstance(s, str): - return None - try: - return datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp() - except Exception: - return None - - -class CentralConsumer: - """Subscribes to Central JetStream subjects and emits normalized Events.""" - - def __init__(self, env_config, event_bus): - """Args: - env_config: the EnvironmentalConfig (provides .central + per-adapter .source) - event_bus: the pipeline EventBus to emit normalized Events onto - """ - self._env = env_config - self._central = getattr(env_config, "central", None) - self._bus = event_bus - self._nc = None - self._js = None - self._subs: list = [] - # Drain mode: suppress bus.emit() during backlog catch-up. - # After all pending messages are consumed, _drain_complete() runs - # a decision pass over accumulated fire IrwinIDs and emits at most - # one event per fire through the pacer. - self._draining: bool = False - self._drain_irwin_ids: set = set() - self._drain_start: float = 0.0 # monotonic time drain started - self._drain_timeout: float = 30.0 # seconds before auto-exit - self._drain_msg_count: int = 0 # messages processed during drain - self._pacer = None # FirePacer, injected from main.py - # Satpass consolidation: pending 5s timers keyed by consolidated_id. - self._pending_satpass_timers: dict[str, object] = {} - - # ---- subject derivation ---- - def _region(self) -> str: - """Active Central region (v0.5.4). Empty string = pre-v0.9.20 bare wildcards.""" - if self._central is None: - return "" - return getattr(self._central, "region", "") or "" - - def _subject_owned(self) -> dict: - """Map each Central subject filter -> set of meshai source names (adapter - attrs) that are feed_source=central and consume it. A shared subject - (central.traffic.>.id for both traffic and roads511) carries multiple - owned sources; _handle drops events whose remapped source isn't in the - set. v0.5.4: subject shapes are region-aware via _subjects_for().""" - region = self._region() - owned: dict = {} - for attr in _SUBJECTS_BARE.keys(): - cfg = getattr(self._env, attr, None) - if cfg is not None and getattr(cfg, "feed_source", "native") == "central": - for subj in _subjects_for(attr, region): - owned.setdefault(subj, set()).add(attr) - for attr in ("avalanche", "ducting"): - cfg = getattr(self._env, attr, None) - if cfg is not None and getattr(cfg, "feed_source", "native") == "central": - logger.warning("Adapter %r set to source=central but Central has no " - "matching stream; nothing will be consumed for it.", attr) - return owned - - def subjects(self) -> list[str]: - """Unique Central subject filters for adapters set to central.""" - return sorted(self._subject_owned().keys()) - - def _make_cb(self, owned): - async def _cb(msg): - await self._on_message(msg, owned) - return _cb - - # ---- normalization ---- - def _normalize(self, subject: str, envelope: dict) -> Optional[Event]: - """CloudEvents envelope -> meshai Event (None if unusable).""" - inner = envelope.get("data") or {} - env_id = envelope.get("id") or inner.get("id") - if not env_id: - return None - - # v0.5.7-fire: tombstone detection now matches both the legacy GDACS - # `:removed` form and the WFIGS `:removed:` form. - is_tombstone = ( - (".removed." in (subject or "")) - or str(env_id).endswith(":removed") - or ":removed:" in str(env_id) - ) - # The clear event shares the ORIGINAL event's group_key so the grouper/ - # inhibitor lets the prior event lapse naturally. v0.5.7-fire: strip - # both `:removed` (GDACS) AND `:removed:` (WFIGS) tails. Per - # Central v0.10.0 guide §wfigs_incidents, the same incident may be - # tombstoned multiple times over its lifecycle; each tombstone is a - # distinct Event but they all share the IrwinID as group_key. - group_key = str(env_id) - if is_tombstone: - group_key = re.sub(r":removed(:.*)?$", "", group_key) - - cat_raw = inner.get("category") or envelope.get("centralcategory") or "" - category = map_category(cat_raw) - if category == "other": - category = category_from_subject(subject) or "other" - - geo = inner.get("geo") or {} - lat = lon = None - centroid = geo.get("centroid") - if isinstance(centroid, (list, tuple)) and len(centroid) >= 2: - lon, lat = centroid[0], centroid[1] # GeoJSON [lon, lat] -> (lat, lon) - - # Preserve the upstream payload verbatim (incl. `_enriched`) in Event.data. - data = dict(inner.get("data") or {}) - if is_tombstone: - data["_central_tombstone"] = True - # v0.5.7-fire: stash the full env_id (with the :removed: tail) - # so downstream consumers can tell apart multiple tombstones for - # the same incident. The group_key collapses to the bare IrwinID - # by design (so they lapse the original together); this preserves - # lifecycle distinctness for accounting. - data["_central_tombstone_id"] = str(env_id) - - # v0.5.7-regression: upstream Central payloads for most adapters - # (firms, nwis, swpc_*, wfigs_*, tomtom_incidents, ...) carry per- - # adapter fields but NOT a top-level `title` or `headline`. Falling - # back to `cat_raw` produced category-as-title broadcasts that - # leaked the raw Central hierarchical category onto the mesh - # (e.g. "incident.tomtom_incidents" instead of "Road Incident"). - # Prefer the meshai-friendly registry name from get_category() over - # the raw category. cat_raw stays as the last-resort tail so - # genuinely-unknown categories still produce *something* readable. - friendly_name = None - try: - ci = get_category(category) - if ci and ci.get("name"): - friendly_name = str(ci["name"]) - except Exception: - pass - - # v0.5.8 (first per-adapter normalizer): state_511_atis work_zone / - # closure / incident events get a rich one-line title synthesized by - # the meshai.central_normalizer module + the work_zone renderer. - # Failures / unmapped adapters fall through to the registry-friendly - # name chain below. - synthesized = None - try: - from meshai.central_normalizer import normalize as _norm_envelope - from meshai.notifications.renderers.work_zone import format_work_zone_mesh - # v0.5.9 unified incident pipeline -- tomtom_incidents + - # state_511_atis + itd_511 for incident/closure/special_event - # categories all flow through meshai.central.incident_handler. - # state_511_atis with category=work_zone stays on the v0.5.8 - # _parse_state_511_atis path below. - _adapter_v9 = inner.get("adapter") or "" - if ( - _adapter_v9 in ("tomtom_incidents", "state_511_atis", "itd_511") - and (cat_raw.startswith("incident.") - or cat_raw.startswith("closure.") - or cat_raw.startswith("special_event.")) - ): - from meshai.central.incident_handler import handle_incident - synthesized = handle_incident(envelope, subject, data=data) or None - else: - # v0.5.9 GAMMA: state_511_atis Idaho cutover via helper. - # Applies BEFORE normalize+dispatch so neither the work_zone - # renderer nor the incident_handler ever sees an ID-state_511 - # envelope. - from meshai.central_normalizer import ( - should_skip_state_511_atis_id as _skip_s5_id, - ) - # v0.5.9 GAMMA universal freshness gate -- applies to ALL - # incident-pipeline adapters BEFORE dispatch, so itd_511 - # work_zone (which goes through central_normalizer + - # format_work_zone_mesh, NOT handle_incident) is now also - # gated. Per-source field paths defined in the helper. - from meshai.central_normalizer import ( - is_incident_envelope_stale as _stale_check, - ) - if _stale_check(envelope, now=int(time.time())): - try: - from meshai.persistence import get_db as _get_db_st - _conn_st = _get_db_st() - _conn_st.execute( - "INSERT INTO event_log(received_at, source, " - "category, severity_word, event_id_external, " - "nats_subject, handled, table_name, table_pk) " - "VALUES (?,?,?,?,?,?,?,?,?)", - (int(time.time()), - inner.get("adapter") or "", - cat_raw + "|freshness_drop", None, - inner.get("id"), - subject, 0, None, None), - ) - except Exception: - logger.exception("freshness_drop log failed") - synthesized = None - n = None - elif _skip_s5_id(envelope): - try: - from meshai.persistence import get_db as _get_db_skip - _conn_skip = _get_db_skip() - _conn_skip.execute( - "INSERT INTO event_log(received_at, source, " - "category, severity_word, event_id_external, " - "nats_subject, handled, table_name, table_pk) " - "VALUES (?,?,?,?,?,?,?,?,?)", - (int(time.time()), "state_511_atis", - cat_raw + "|skip_id", None, - inner.get("id"), - subject, 0, None, None), - ) - except Exception: - logger.exception("state_511_id_skip log failed") - synthesized = None - n = None # short-circuit downstream dispatch - else: - n = _norm_envelope(envelope) - # v0.5.8 wfigs_handler dispatch -- WFIGS events route through - # the persistence-backed change-detection handler (which also - # logs to event_log for tombstones + perimeters). Other adapters - # with a normalized dict (state_511_atis, wzdx) flow through the - # work_zone renderer as before. - if n is not None and str(n.get("_kind", "")).startswith("wfigs"): - from meshai.central.wfigs_handler import handle_wfigs - synthesized = handle_wfigs(n, envelope, subject, data=data) or None - # v0.5.10 nws + usgs_quake + swpc handlers. Adapter-specific - # filters: NWS severity gate, quake magnitude+Idaho-distance, - # SWPC G3+/R3+/S1+ NOAA scales. Universal freshness gate above - # already dropped stale envelopes per central_normalizer. - elif inner.get("adapter") == "nws": - from meshai.central.nws_handler import handle_nws - synthesized = handle_nws(envelope, subject, data=data) or None - elif inner.get("adapter") == "usgs_quake": - from meshai.central.quake_handler import handle_quake - synthesized = handle_quake(envelope, subject, data=data) or None - elif inner.get("adapter") in ("swpc_alerts", "swpc_kindex", "swpc_protons"): - from meshai.central.swpc_handler import handle_swpc - synthesized = handle_swpc(envelope, subject, data=data) or None - # v0.5.12 nwis stream-gauge handler. Filters to the - # 9-site Idaho curation (idaho_gauge_sites.py); upward - # threshold crossings only (mirrors WFIGS forward-only). - elif inner.get("adapter") == "nwis": - from meshai.central.nwis_handler import handle_nwis - synthesized = handle_nwis(envelope, subject, data=data) or None - elif inner.get("adapter") == "avalanche_org": - from meshai.central.avy_handler import handle_avy - synthesized = handle_avy(envelope, subject, data=data) or None - # v0.6-1 firms_handler -- STORAGE-ONLY. handle_firms - # writes to firms_pixels (with dedup) and returns None - # so the default-deny clause below keeps mesh - # broadcasts suppressed. LLM visibility lands in - # commit #5 (env_reporter). Closes the v0.5.13 - # silent-drop on central.fire.hotspot.> (audit doc - # finding #2). - elif inner.get("adapter") == "celestrak_tle": - from meshai.central.tle_handler import handle_tle - synthesized = handle_tle(envelope, subject, data=data) or None - elif inner.get("adapter") in ("n2yo_visualpasses", "satpass_predict"): - from meshai.central.satpass_handler import handle_satpass - synthesized = handle_satpass(envelope, subject, data=data) or None - elif inner.get("adapter") == "firms": - from meshai.central.firms_handler import handle_firms - synthesized = handle_firms(envelope, subject, data=data) or None - elif n is not None and category in ("work_zone", "road_closure", "road_incident"): - return None # silently drop work zone envelopes - except Exception: - logger.exception("normalizer/renderer failed for adapter=%s category=%s", - inner.get("adapter"), category) - synthesized = None - - # v0.5.13 default-deny: per-adapter handlers gate broadcasts, not - # just titles. If no handler synthesized a wire string for this - # envelope (either because no per-adapter handler matched OR a - # matched handler explicitly returned None as a filter/dedup/ - # threshold decision), return None from _normalize() -- the Event - # never enters the bus and the dispatcher never fires. This is - # the architectural fix for the v0.5.7-regression leak that came - # back through the v0.5.x live flip: handlers were gating titles - # but not broadcasts. See memory rule 19. - # - # Scheduled broadcasters (band_conditions) bypass _normalize() - # entirely -- they enter via Dispatcher.dispatch_scheduled_broadcast() - # and are unaffected by this gate. - # Phase-0b shadow gate hook — inert by default. - # Called RIGHT BEFORE the default-deny return so the shadow observes the - # same broadcast decision the legacy handler made (synthesized is not None). - # The OLD result is kept unchanged below; the shadow only observes. - try: - from meshai.notifications.shadow import shadow_gate as _shadow_gate - _shadow_gate( - category, - data, - source=inner.get("adapter") or "central", - now=time.time(), - old_broadcast=synthesized is not None, - ) - except Exception: # noqa: BLE001 — shadow must never affect production - pass - - if synthesized is None: - logger.debug( - "consumer: default-deny -- no handler synthesized for " - "adapter=%s category=%s subject=%s", - inner.get("adapter"), category, subject, - ) - return None - - title = synthesized - - # v0.5.8 Option A: when the per-adapter normalizer produced a fully - # formatted mesh string, set a marker on event.data so the composer - # at dispatch time can pass it through verbatim (no family prefix, - # no region tail, no severity append). - if synthesized and title == synthesized: - data["_meshai_precomposed"] = True - - kwargs = dict( - title=str(title) if data.get("_meshai_precomposed") else str(title)[:200], - summary="", - lat=lat, - lon=lon, - region=geo.get("primary_region"), - regions=geo.get("regions") or [], - group_key=group_key, - inhibit_keys=[group_key], - data=data, - ) - ts = _parse_time(inner.get("time")) - if ts is not None: - kwargs["timestamp"] = ts - exp = _parse_time(inner.get("expires")) - if exp is not None: - kwargs["expires"] = exp - - raw_adapter = inner.get("adapter") or "central" - source = CENTRAL_ADAPTER_TO_SOURCE.get(raw_adapter, raw_adapter) - if source != raw_adapter: - logger.debug("Central adapter %r -> meshai source %r", raw_adapter, source) - - # v0.7-fire-fusion (issue #117): a per-adapter handler (e.g. - # firms_handler's growth / cluster / halt / spotting fusion) may stamp - # a fully-resolved category onto data["category"] AFTER `category` was - # computed above from the raw Central category string. Re-read it - # here, now that the handler has run, so the override actually reaches - # make_event() -- previously this local `category` was captured before - # dispatch and the handler's stamp was a silent no-op. An unrecognized - # override is logged and the original mapped category is kept, rather - # than trusting an arbitrary string into the category taxonomy. - cat_override = data.get("category") if isinstance(data, dict) else None - if cat_override and cat_override != category: - if cat_override in ALERT_CATEGORIES: - category = cat_override - else: - logger.warning( - "consumer: handler set unrecognized category override %r " - "for adapter=%s; keeping %r", - cat_override, raw_adapter, category) - - # v0.6-3c / issue #118: use handler severity override if present. - # `_severity_override` is the ONLY key honored here -- handlers must - # stamp `_severity_override`, not the plain `severity` key, or their - # severity choice silently never reaches the Event. - sev_override = data.get("_severity_override") if isinstance(data, dict) else None - return make_event( - source=source, - category=category, - severity=sev_override or map_severity(inner.get("severity")), - **kwargs, - ) - - def _handle(self, subject: str, raw: bytes, owned=None) -> Optional[Event]: - """Normalize a raw message body and emit to the bus. Returns the Event. - - owned: set of meshai source names this subscription may emit (sub-adapter - routing for shared subjects); None = no filtering. - - During drain mode (_draining=True), bus.emit() is suppressed. Fire - IrwinIDs are tracked in _drain_irwin_ids for the post-drain decision - pass. All handler DB writes still happen inside _normalize() before - this point. - """ - try: - envelope = json.loads(raw) - except Exception: - logger.exception("CentralConsumer: bad JSON on %s", subject) - return None - event = self._normalize(subject, envelope) - - # Satpass consolidation: check for pending consolidation IDs - # regardless of whether _normalize returned an event (satpass - # handler always returns None, signaling via module-level set). - self._check_satpass_consolidation() - - if event is None: - return None - if owned is not None and event.source not in owned: - logger.debug("CentralConsumer: dropping %s source=%s -- not owned by " - "subscription %s", subject, event.source, sorted(owned)) - return None - - if self._draining: - # Track fire IrwinIDs touched during drain for decision pass - irwin_id = (event.data or {}).get("_cooldown_suffix", "") - if irwin_id and event.source in ("fires", "wfigs"): - self._drain_irwin_ids.add(irwin_id) - elif self._bus is not None: - # Normal mode: route fire events through pacer, others direct. - # Issue #119: FIRMS-synthesized broadcasts (growth/spotting/halt/ - # cluster) carry source="firms" and growth+spotting use severity - # "immediate" -- neither was covered by the old "fires"/"wfigs" + - # "priority"-only gate, so FIRMS fusion events skipped the pacer - # entirely. Broaden to every fire-family source at priority OR - # immediate severity so nothing bypasses the <=1/min throttle. - if (self._pacer is not None - and event.source in ("fires", "wfigs", "firms") - and event.severity in ("priority", "immediate")): - self._pacer.enqueue(event) - else: - self._bus.emit(event) - return event - - def _check_satpass_consolidation(self) -> None: - """Poll the satpass handler's consolidation signal and schedule timers.""" - try: - from meshai.central.satpass_handler import drain_pending_consolidation_ids - ids = drain_pending_consolidation_ids() - except Exception: - return - for cid in ids: - if cid not in self._pending_satpass_timers: - try: - loop = asyncio.get_event_loop() - # Stagger timers: 5s base for observer consolidation, - # +60s per already-pending pass to avoid mesh flooding - # when Central publishes a batch of future passes. - delay = 5.0 + len(self._pending_satpass_timers) * 60.0 - handle = loop.call_later( - delay, self._satpass_consolidation_fire, cid) - self._pending_satpass_timers[cid] = handle - logger.debug("satpass: scheduled %.0fs consolidation timer for %s", - delay, cid) - except Exception: - logger.exception("satpass: failed to schedule timer for %s", cid) - - def _satpass_consolidation_fire(self, consolidated_id: str) -> None: - """Timer callback: consolidate pending observers and emit.""" - self._pending_satpass_timers.pop(consolidated_id, None) - try: - from meshai.central.satpass_handler import consolidate_satpass_pending - result = consolidate_satpass_pending(consolidated_id) - if result is None: - return - wire, data = result - - event = Event( - id=consolidated_id, - source="satpass", - category="sat_pass", - severity=data.get("_severity_override", "routine"), - title=wire or "", - summary=wire, - data=data, - timestamp=time.time(), - ) - - if self._bus is not None: - self._bus.emit(event) - logger.info("satpass: emitted consolidated broadcast for %s", consolidated_id) - except Exception: - logger.exception("satpass consolidation failed for %s", consolidated_id) - - def _sweep_pending_satpass(self, now: Optional[float] = None) -> None: - """Reconstruct consolidation timers for satpass_pending rows that a - restart orphaned. - - The live in-memory scheduler (_check_satpass_consolidation) loses its - asyncio TimerHandles when the process exits, but the satpass_pending - rows and their persisted due_at survive. Without this sweep those rows - would sit forever, never consolidated or broadcast. Run once at - startup, it re-arms a timer for each pending consolidated_id off its - durable due_at, reusing the SAME _satpass_consolidation_fire path so - the emit logic is byte-identical to normal operation. - - Idempotent and additive: it SKIPS any cid already armed by the live - path (present in _pending_satpass_timers), so it can never - double-schedule and is safe to call once alongside the module-set - drain path (which covers the live case this sweep cannot). - """ - try: - from meshai.central.satpass_handler import load_pending_schedule - schedule = load_pending_schedule() - except Exception: - logger.exception("satpass sweep: failed to load pending schedule") - return - if not schedule: - return - try: - loop = asyncio.get_event_loop() - except Exception: - logger.exception("satpass sweep: no event loop for timer reconstruction") - return - now = time.time() if now is None else now - recovered = 0 - overdue = 0 - for cid, due_at in schedule: - try: - if cid in self._pending_satpass_timers: - # Live path already armed this cid; never double-schedule. - continue - if due_at <= now: - # Orphan already past due: fire soon, with a small - # increasing stagger so a backlog doesn't emit in one burst. - delay = 0.5 + overdue * 2.0 - overdue += 1 - else: - # Not yet due: reconstruct the remaining wait exactly. - delay = due_at - now - handle = loop.call_later( - delay, self._satpass_consolidation_fire, cid) - self._pending_satpass_timers[cid] = handle - recovered += 1 - except Exception: - logger.exception("satpass sweep: failed to re-arm timer for %s", cid) - if recovered: - logger.info( - "satpass sweep: reconstructed %d consolidation timer(s) after restart", - recovered) - - async def _on_message(self, msg, owned=None) -> None: - """JetStream callback: normalize + emit, then ack. - - During drain mode, checks msg.metadata.num_pending after each - message. When pending hits 0, the backlog is consumed and - _drain_complete() runs the fire decision pass. - """ - try: - self._handle(msg.subject, msg.data, owned) - except Exception: - logger.exception("CentralConsumer: handler failed on %s", - getattr(msg, "subject", "?")) - # Check drain completion BEFORE ack so the decision pass runs - # while we still hold the message (prevents interleaving). - if self._draining: - self._drain_msg_count += 1 - try: - meta = msg.metadata - if meta is not None and getattr(meta, "num_pending", None) == 0: - self._drain_complete() - except Exception: - logger.exception("drain: metadata check failed") - try: - ack = getattr(msg, "ack", None) - if ack is not None: - await ack() - except Exception: - pass - - # ---- lifecycle ---- - async def start(self) -> None: - subject_owned = self._subject_owned() - if not subject_owned: - logger.info("CentralConsumer started; 0 subjects subscribed -- " - "no adapters set to central") - return - if self._central is None or not getattr(self._central, "enabled", False): - logger.warning("CentralConsumer: adapter(s) want source=central but " - "environmental.central.enabled is false; not subscribing: %s", - sorted(subject_owned)) - return - - # Enter drain mode: suppress bus.emit() until the backlog from - # LAST_PER_SUBJECT delivery is fully consumed. The first _on_message - # callback processes through drain mode; when num_pending hits 0, - # _drain_complete() runs the fire decision pass. A timeout auto-exits - # drain if no messages arrive (empty backlog scenario). - self._draining = True - self._drain_irwin_ids.clear() - self._drain_msg_count = 0 - self._drain_start = time.monotonic() - logger.info("CentralConsumer: entering drain mode (timeout=%.0fs)", - self._drain_timeout) - - region = self._region() - logger.info("CentralConsumer: connecting region=%r subjects=%s", - region or "(bare wildcards)", sorted(subject_owned)) - import nats # lazy: no NATS dependency at boot unless actually consuming - self._nc = await nats.connect( - self._central.url, - connect_timeout=getattr(self._central, "connect_timeout", 10.0), - ) - self._js = self._nc.jetstream() - for subj, owned in subject_owned.items(): - durable = self._central.durable + "-" + re.sub(r"[^a-z0-9]+", "_", subj.lower()) - sub = await self._js.subscribe( - subj, durable=durable, cb=self._make_cb(owned), config=consumer_config()) - self._subs.append(sub) - logger.info("CentralConsumer subscribed %s owned-sources=%s", subj, sorted(owned)) - logger.info("CentralConsumer started; %d subjects subscribed (drain mode active)", - len(subject_owned)) - - # Reboot recovery: re-arm consolidation timers for any satpass_pending - # rows the previous process left behind (in-memory TimerHandles don't - # survive a restart). Additive to the live module-set path; guarded - # against double-scheduling; a bad row never aborts startup. - self._sweep_pending_satpass() - - # Schedule drain timeout: if no messages trigger drain completion - # within the window (e.g. empty backlog), auto-exit drain mode. - asyncio.get_event_loop().call_later( - self._drain_timeout, self._drain_timeout_check) - - # ---- drain mode ---- - - def _drain_timeout_check(self) -> None: - """Called by call_later after drain_timeout seconds. If still draining, - auto-exit. This handles the empty-backlog case where no messages arrive - to trigger num_pending == 0.""" - if not self._draining: - return - logger.info("drain: timeout after %.0fs (%d msgs processed) — auto-completing", - time.monotonic() - self._drain_start, self._drain_msg_count) - self._drain_complete() - - def _drain_complete(self) -> None: - """Post-drain decision pass: one broadcast per fire, based on final DB state. - - Runs synchronously (no awaits) to prevent _on_message interleaving. - For each IrwinID touched during drain, reads the fires row and - decides: NEW, UPDATE, CLOSURE, or SILENCE. Events route through - the pacer (<=1/min) instead of direct bus.emit(). - """ - self._draining = False - if not self._drain_irwin_ids: - logger.info("drain complete: 0 fires touched") - return - - # Step 3 age-gate: `now` is otherwise undefined in this method (`time` - # is imported at module level). Used by the first-announce age gate. - now = int(time.time()) - - from meshai.persistence import get_db - from meshai.central.wfigs_handler import ( - _render, _location_anchor, _attach_commit_handles, - _fire_too_old_to_announce, - ) - from meshai.notifications.events import make_event - - conn = get_db() - emitted = 0 - silenced = 0 - - for irwin_id in self._drain_irwin_ids: - row = conn.execute( - "SELECT irwin_id, incident_name, incident_type, " - "current_acres, current_contained_pct, " - "lat, lon, county, state, landclass, " - "declared_at, tombstoned_at, last_broadcast_at, " - "last_broadcast_acres, last_broadcast_contained, " - "fire_cause, unique_fire_id, geocoder_city " - "FROM fires WHERE irwin_id = ?", (irwin_id,) - ).fetchone() - - if row is None: - continue - - tombstoned = row["tombstoned_at"] is not None - announced = row["last_broadcast_at"] is not None - - # Decision table (meshai-fire-fix-plan.md §3): - # Case 3: Never announced + already closed -> SILENCE - if not announced and tombstoned: - silenced += 1 - continue - - wire = None - category = "wildfire_incident" - - if announced and tombstoned: - # Case 4: Announced before + closed during gap -> CLOSURE - wire = _build_closure_wire(row) - category = "wildfire_closed" - elif not announced and not tombstoned: - # Case 2: Never announced + still active -> NEW - # Step 3 age-gate: suppress first-announce for fires whose - # declared_at is too old (closed/stale-fire resurrection guard). - if _fire_too_old_to_announce(row["declared_at"], now): - silenced += 1 - continue - wire = _render(_row_to_normalized(row), prefix="New") - category = "wildfire_declared" - else: - # Case 1: Announced before + grew during gap -> UPDATE - # Check if anything actually changed - if (row["current_acres"] == row["last_broadcast_acres"] - and row["current_contained_pct"] == row["last_broadcast_contained"]): - silenced += 1 - continue - wire = _render( - _row_to_normalized(row), prefix="Update", - last_bcast_acres=row["last_broadcast_acres"], - last_bcast_contained=row["last_broadcast_contained"], - ) - - if wire is None: - silenced += 1 - continue - - # Build Event - data = {"_meshai_precomposed": True, "_severity_override": "priority"} - _attach_commit_handles( - data, irwin_id=irwin_id, - acres=row["current_acres"], - contained_pct=row["current_contained_pct"], - ) - data["_cooldown_suffix"] = irwin_id - data["_dedup_suffix"] = ( - f"{row['current_acres']}|{row['current_contained_pct']}|drain" - ) - - event = make_event( - source="fires", category=category, severity="priority", - title=wire, lat=row["lat"], lon=row["lon"], - group_key=irwin_id, inhibit_keys=[irwin_id], data=data, - ) - - # Route through pacer (<=1/min), fall back to direct emit - if self._pacer is not None: - self._pacer.enqueue(event) - elif self._bus is not None: - self._bus.emit(event) - emitted += 1 - - self._drain_irwin_ids.clear() - logger.info("drain complete: %d fires emitted, %d silenced", emitted, silenced) - - async def stop(self) -> None: - if self._nc is not None: - try: - await self._nc.drain() - except Exception: - pass - try: - await self._nc.close() - except Exception: - pass - self._nc = None - self._js = None - self._subs = [] - - -# ---------- drain-mode helpers (module-level) ----------------------------- - - -def _row_to_normalized(row) -> dict: - """Map a fires DB row (sqlite3.Row) to the normalized dict _render() expects. - - sqlite3.Row supports bracket access and .keys() but not .get() on - Python < 3.13. Use _safe_get() for optional columns. - """ - keys = set(row.keys()) - return { - "incident_name": row["incident_name"], - "acres": row["current_acres"], - "contained_pct": row["current_contained_pct"], - "fire_cause": row["fire_cause"] if "fire_cause" in keys else None, - "unique_fire_id": row["unique_fire_id"] if "unique_fire_id" in keys else None, - "declared_at_epoch": row["declared_at"], - "lat": row["lat"], - "lon": row["lon"], - "county": row["county"], - "state": row["state"], - "landclass": row["landclass"] if "landclass" in keys else None, - "geocoder_city": row["geocoder_city"] if "geocoder_city" in keys else None, - } - - -def _build_closure_wire(row) -> str: - """Build a closure wire string from a fires DB row. - - Replicates the tombstone wire format from wfigs_handler.py. - """ - from meshai.central.wfigs_handler import _location_anchor - - name = row["incident_name"] or "(unnamed fire)" - parts = [] - if row["current_acres"] is not None: - parts.append(f"{int(row['current_acres']):,} ac") - if row["current_contained_pct"] is not None: - parts.append(f"{int(row['current_contained_pct'])}% contained") - # Location anchor from row fields - loc_dict = { - "lat": row["lat"], "lon": row["lon"], - "county": row["county"], "state": row["state"], - } - anchor = _location_anchor(loc_dict) - if anchor and anchor != "(location unknown)": - parts.append(anchor) - lines = [f"\u2705 {name} \u2014 contained & closed"] - if parts: - lines.append(" | ".join(parts)) - return "\n".join(lines) diff --git a/work/meshai/central/incident_handler.py b/work/meshai/central/incident_handler.py deleted file mode 100644 index 13486d2..0000000 --- a/work/meshai/central/incident_handler.py +++ /dev/null @@ -1,961 +0,0 @@ -"""v0.5.9 unified incident handler. - -Three sources collapse into one persistence-backed change-detection pipeline: - - * tomtom_incidents (real-time crashes/jams/closures, TTI-uuid stable ID) - * state_511_atis (ITD incidents + closures + special events -- - the EventType branching the v0.5.8 parser missed) - * itd_511 (ITD's newer direct feed, Convention A subject) - -State_511_atis with category=work_zone continues to flow through the existing -v0.5.8 _parse_state_511_atis -> work_zone renderer (untouched). Only the -THREE non-work-zone EventTypes route here. - -Filtering at handler entrance (Matt's v0.5.9 §6): - * tomtom magnitude_of_delay==0 -> drop (no event_log row, no broadcast) - * tomtom time_validity != "present" -> drop - * everything else flows into the canonical pipeline. - -Change-detection (Matt's §5): - * NEW external_id -> 'New:' - * magnitude steps up -> 'Update:' - * delay doubles (>=2x) -> 'Update:' - * icon_category changes -> 'Update:' - * 8h elapsed since last bcast -> 'Update:' (heartbeat) - * otherwise -> drop silently - -Same callback pattern as WFIGS: handler attaches `_on_broadcast_committed` -to data; dispatcher invokes it AFTER successful deliver(). Cold-start -suppression leaves last_broadcast_* NULL so the next successful broadcast -still labels itself New:. -""" - -from __future__ import annotations -from meshai.adapter_config import adapter_config -from meshai.central.budget import budget_for, fit_to_budget - -import logging -import re -import time -from datetime import datetime, timezone -from typing import Any, Optional - -from meshai.persistence import get_db - -logger = logging.getLogger(__name__) - - -# v0.6-3b: freshness gate value lives in adapter_config.incident.freshness_seconds -# (default 1800). Read at handler call time. The module-level constant is -# kept as a backward-compat alias for downstream imports. -INCIDENT_FRESHNESS_MAX_S = 1800 - - - -# ---- canonical sub_type vocabulary -------------------------------------- - -# Tomtom icon_category int -> canonical sub_type. Anything missing maps to -# the generic 'incident' bucket so the wire string is never empty. -_TOMTOM_ICON_TO_SUB = { - 0: "incident", # unknown - 1: "accident", - 2: "fog", - 3: "danger", - 4: "rain", - 5: "ice", - 6: "jam", - 7: "lane_closed", - 8: "road_closed", - 9: "road_works", - 10: "wind", - 11: "flooding", - 12: "broken_down", - 14: "incident", # cluster -} - -# state_511 / itd_511 event_sub_type string -> canonical sub_type. -# Coverage in the 7-day Idaho sample shown after each entry. -_SUB_TYPE_511_MAP = { - "crash": "accident", # 28/41 - "incident": "incident", # 1/41 - "debrisOnRoadway": "debris", # 1/41 - "disabledVehicle": "disabled_vehicle", # 1/41 - "vehicleOnFire": "vehicle_on_fire", # 2/41 - "wildfire": "incident", # 1/41 - "wildfireInArea": "incident", # 1/41 - "leftLaneBlocked": "lane_closed", # 2/41 - "onRampBlocked": "ramp_closed", # 2/41 - "roadwayBlocked": "road_closed", # 2/41 - "roadConstruction": "road_works", - "pavementMarkingOperations": "road_works", - "pavementMarkingOperations ": "road_works", # trailing-space variant - "utilityWork": "road_works", - "singleLineTraffic:AlternatingDirections": "lane_closed", - "roadMaintenanceOperations": "road_works", - "pavingOperations": "road_works", - "bridgeConstruction": "road_works", - "bridgeMaintenanceOperations": "road_works", - "flaggingOperation": "lane_closed", - "brushControl": "road_works", - "constructionWork": "road_works", - "guardrailRepairs": "road_works", - "workOnTheShoulder": "road_works", - "nightTimeConstructionWork": "road_works", - "bridgeInspectionWork": "road_works", - "longTermRoadConstruction": "road_works", - "workOnUndergroundServices": "road_works", - "roadsideCleanupCrew": "road_works", - "RampRestriction": "lane_closed", - "parade": "parade", -} - -# Emoji per canonical sub_type. -_SUB_TYPE_EMOJI = { - "accident": "🚨", - "jam": "🚗", - "road_closed": "🚫", - "closure": "🚫", - "road_works": "🚧", - "lane_closed": "🟠", - "ramp_closed": "🟠", - "debris": "⚠️", - "vehicle_on_fire": "🔥", - "disabled_vehicle": "🛑", - "ice": "⚠️", - "fog": "⚠️", - "flooding": "🌊", - "wind": "🌬️", - "broken_down": "🛞", - "danger": "⚠️", - "rain": "⚠️", - "incident": "⚠️", - "special_event": "🎪", - "parade": "🎪", -} - -# Human-readable noun phrase per canonical sub_type. -_SUB_TYPE_PHRASE = { - "accident": "crash", - "jam": "jam", - "road_closed": "road closed", - "closure": "closure", - "road_works": "road works", - "lane_closed": "lane closed", - "ramp_closed": "ramp closed", - "debris": "debris on roadway", - "vehicle_on_fire": "vehicle fire", - "disabled_vehicle": "disabled vehicle", - "ice": "icy conditions", - "fog": "fog", - "flooding": "flooding", - "wind": "high wind", - "broken_down": "broken-down vehicle", - "danger": "dangerous conditions", - "rain": "heavy rain", - "incident": "incident", - "special_event": "special event", - "parade": "parade", -} - -# Display name per canonical sub_type (Title Case for multi-line render). -_SUB_TYPE_DISPLAY = { - "accident": "Crash", - "jam": "Stationary Traffic", - "road_closed": "Road Closed", - "closure": "Closure", - "road_works": "Road Works", - "lane_closed": "Lane Reduction", - "ramp_closed": "Ramp Closed", - "debris": "Debris on Roadway", - "vehicle_on_fire": "Vehicle Fire", - "disabled_vehicle": "Disabled Vehicle", - "ice": "Icy Conditions", - "fog": "Fog", - "flooding": "Flooding", - "wind": "High Winds", - "broken_down": "Broken-Down Vehicle", - "danger": "Dangerous Conditions", - "rain": "Heavy Rain", - "incident": "Road Incident", - "special_event": "Special Event", - "parade": "Parade", -} - -# Direction short-form -> long-form for multi-line render. -_DIRECTION_LONG = { - "North": "Northbound", "N": "Northbound", "NB": "Northbound", "north": "Northbound", "nb": "Northbound", - "South": "Southbound", "S": "Southbound", "SB": "Southbound", "south": "Southbound", "sb": "Southbound", - "East": "Eastbound", "E": "Eastbound", "EB": "Eastbound", "east": "Eastbound", "eb": "Eastbound", - "West": "Westbound", "W": "Westbound", "WB": "Westbound", "west": "Westbound", "wb": "Westbound", - "Both": "Both Directions", "both": "Both Directions", -} - - -# ---- helpers ------------------------------------------------------------- - - -def _now() -> int: - return int(time.time()) - - -def _direction_short(dir_str: Optional[str]) -> Optional[str]: - if not dir_str: return None - s = str(dir_str).strip().lower() - if s.startswith("north"): return "N" - if s.startswith("south"): return "S" - if s.startswith("east"): return "E" - if s.startswith("west"): return "W" - if s in ("nb", "n"): return "N" - if s in ("sb", "s"): return "S" - if s in ("eb", "e"): return "E" - if s in ("wb", "w"): return "W" - if s in ("both", "both directions"): return "both" - return None - - -def _parse_iso_epoch(s: Optional[str]) -> Optional[int]: - if not s: return None - try: - return int(datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp()) - except Exception: - return None - - -def _parse_511_date_epoch(s: Optional[str]) -> Optional[int]: - """state_511 uses '5/28/26, 10:45 PM' format.""" - if not s: return None - try: - return int(datetime.strptime(s, "%m/%d/%y, %I:%M %p").replace( - tzinfo=timezone.utc).timestamp()) - except Exception: - return None - - -_TTI_RE = re.compile(r"TTI-([0-9a-f-]{36})") - - -def _tomtom_tti(envelope_id: Optional[str]) -> Optional[str]: - """Extract the stable TTI- piece from the per-poll inner.id. - - TomTom IDs look like: - ID:tomtom:TTI--TTR - The TTR rotates each poll; the TTI-uuid stays stable across - re-publishes of the same incident. - """ - if not envelope_id: return None - m = _TTI_RE.search(envelope_id) - return m.group(1) if m else envelope_id - - -def _tomtom_direction_from_description(desc: Optional[str]) -> Optional[str]: - if not desc: return None - s = desc.lower() - if "northbound" in s: return "N" - if "southbound" in s: return "S" - if "eastbound" in s: return "E" - if "westbound" in s: return "W" - return None - - -def _tomtom_road_label(d: dict) -> Optional[str]: - """Compose 'I-84 W' style label from road_numbers + direction.""" - nums = d.get("road_numbers") or [] - if nums: - return str(nums[0]) - return None # caller falls back to from/to or street name - - -# ---- per-source parsers -------------------------------------------------- - - -def _parse_tomtom_incident(envelope: dict, now: int) -> Optional[dict]: - """Returns the canonical incident dict, or None if filtered.""" - inner = envelope.get("data") or {} - d = inner.get("data") or {} - - # Drop events below configured minimum magnitude. - magnitude = d.get("magnitude_of_delay") - min_mag = int(adapter_config.tomtom_incidents.min_magnitude or 4) - if magnitude is not None and magnitude < min_mag: - return None - - # FILTER §4: time_validity != 'present' -> drop past/future. - # v0.6-3b: gated by adapter_config.tomtom_incidents.drop_non_present. - if (d.get("time_validity") != "present" - and bool(adapter_config.tomtom_incidents.drop_non_present)): - return None - - external_id = _tomtom_tti(inner.get("id")) - if not external_id: - return None - - icon = d.get("icon_category") - sub_type = _TOMTOM_ICON_TO_SUB.get(icon, "incident") - - delay_s = d.get("delay") - delay_minutes = None - if isinstance(delay_s, (int, float)) and delay_s > 0: - delay_minutes = max(1, int(round(delay_s / 60))) - - ge = (d.get("_enriched") or {}).get("geocoder") or {} - - return { - "_kind": "incident", - "source": "tomtom_incidents", - "external_id": external_id, - "category_kind": "incident", - "road": _tomtom_road_label(d), - "direction": _tomtom_direction_from_description(d.get("description")), - "mile_start": None, - "mile_end": None, - "county": ge.get("county"), - "state": d.get("state_code"), - "lat": d.get("latitude"), - "lon": d.get("longitude"), - "sub_type": sub_type, - "impact": None, - "delay_minutes": delay_minutes, - "delay_seconds": int(delay_s) if isinstance(delay_s, (int, float)) else None, - "magnitude": magnitude, - "icon_category": sub_type, - "from_loc": d.get("from"), - "to_loc": d.get("to"), - "start_at": _parse_iso_epoch(d.get("start_time")), - "end_at": _parse_iso_epoch(d.get("end_time")), - "geocoder_city": ge.get("city"), - "landclass": ge.get("landclass"), - "mile_marker": None, - "length": d.get("length"), - } - - -def _parse_state_511_incident(envelope: dict, category_raw: str, now: int) -> Optional[dict]: - """Handle state_511_atis with category in (incident, closure, special_event). - Returns None for unsupported categories (work_zone is NOT handled here -- - that stays with the existing v0.5.8 _parse_state_511_atis path).""" - inner = envelope.get("data") or {} - d = inner.get("data") or {} - - if category_raw.startswith("incident."): kind = "incident" - elif category_raw.startswith("closure."): kind = "closure" - elif category_raw.startswith("special_event."): kind = "special_event" - else: return None - - external_id = inner.get("id") - if not external_id: - return None - - raw_sub = (d.get("event_sub_type") or "").strip() - sub_type = _SUB_TYPE_511_MAP.get(raw_sub) - if sub_type is None: - if kind == "closure" or d.get("is_full_closure"): - sub_type = "closure" - elif kind == "special_event": - sub_type = "special_event" - else: - sub_type = "incident" - - ge = (d.get("_enriched") or {}).get("geocoder") or {} - - return { - "_kind": "incident", - "source": "state_511_atis", - "external_id": external_id, - "category_kind": kind, - "road": d.get("roadway_name"), - "direction": _direction_short(d.get("direction")), - "mile_start": None, - "mile_end": None, - "county": d.get("county") or ge.get("county"), - "state": d.get("state_code"), - "lat": d.get("latitude"), - "lon": d.get("longitude"), - "sub_type": sub_type, - "impact": "all lanes closed" if d.get("is_full_closure") else None, - "delay_minutes": None, - "delay_seconds": None, - "magnitude": None, - "icon_category": sub_type, - "from_loc": None, - "to_loc": None, - "start_at": _parse_511_date_epoch(d.get("start_date")), - "end_at": None, - "geocoder_city": ge.get("city"), - "landclass": ge.get("landclass"), - "lanes_affected": d.get("lanes_affected"), - "cause": d.get("cause"), - "description": d.get("description"), - "comment": d.get("comment"), - "mile_marker": (d.get("_enriched") or {}).get("mile_marker", {}).get("value"), - } - - -def _parse_itd_511_incident(envelope: dict, category_raw: str, now: int) -> Optional[dict]: - inner = envelope.get("data") or {} - d = inner.get("data") or {} - - if category_raw.startswith("incident."): kind = "incident" - elif category_raw.startswith("closure."): kind = "closure" - elif category_raw.startswith("special_event."): kind = "special_event" - elif category_raw.startswith("work_zone."): kind = "work_zone" - else: return None - - # Resolve severity + sub_type early (needed by work_zone gate below) - sev_order = {"None": 0, "Minor": 1, "Major": 2} - event_sev = d.get("itd_severity") or "None" - - external_id = inner.get("id") - if not external_id: - return None - - raw_sub = (d.get("event_sub_type") or "").strip() - sub_type = _SUB_TYPE_511_MAP.get(raw_sub) - if sub_type is None: - # ITD has event_type_short ("closure", "incident", "work_zone", - # "special_event"); fall through to that. - sub_type = { - "incident": "incident", - "closure": "closure", - "work_zone": "road_works", - "special_event": "special_event", - }.get((d.get("event_type_short") or "").lower(), "incident") - - # Work zone gate -- configurable via adapter_config.wzdx - if kind == "work_zone": - if not adapter_config.wzdx.broadcast: - return None - # Apply severity filter - wz_min_sev = str(adapter_config.wzdx.min_severity or "Minor") - if sev_order.get(event_sev, 0) < sev_order.get(wz_min_sev, 0): - return None - # Apply sub-type filter - wz_subs = adapter_config.wzdx.sub_types or [] - if wz_subs and sub_type not in wz_subs: - return None - - # Severity filter (non-work-zone) - if kind != "work_zone": - min_sev = str(adapter_config.itd_511.min_severity or "None") - if sev_order.get(event_sev, 0) < sev_order.get(min_sev, 0): - return None - - # Category filter - enabled_cats = adapter_config.itd_511.enabled_categories or [] - if enabled_cats and kind not in enabled_cats: - return None - - # Sub-type filter (applied after sub_type is resolved) - enabled_subs = adapter_config.itd_511.enabled_sub_types or [] - if enabled_subs and sub_type not in enabled_subs: - return None - - ge = (d.get("_enriched") or {}).get("geocoder") or {} - - return { - "_kind": "incident", - "source": "itd_511", - "external_id": external_id, - "category_kind": kind, - "road": d.get("roadway_name"), - "direction": _direction_short(d.get("direction")), - "mile_start": None, - "mile_end": None, - "county": ge.get("county"), - "state": "ID", - "lat": d.get("latitude"), - "lon": d.get("longitude"), - "sub_type": sub_type, - "impact": "all lanes closed" if d.get("is_full_closure") else None, - "delay_minutes": None, - "delay_seconds": None, - "magnitude": None, - "icon_category": sub_type, - "from_loc": None, - "to_loc": None, - "start_at": d.get("start_epoch"), - "end_at": d.get("planned_end_epoch"), - "geocoder_city": ge.get("city"), - "landclass": ge.get("landclass"), - "lanes_affected": d.get("lanes_affected"), - "cause": d.get("cause"), - "description": d.get("description"), - "comment": d.get("comment"), - "mile_marker": (d.get("_enriched") or {}).get("mile_marker", {}).get("value"), - } - - - -def _extract_start_time_epoch(envelope: dict, adapter: str) -> Optional[int]: - """Per-source start-time -> epoch seconds. None when the field is - missing or unparseable (caller treats None as 'do not gate').""" - inner = envelope.get("data") or {} - d = inner.get("data") or {} - if adapter == "tomtom_incidents": - # tomtom inner.data.start_time is ISO-8601 ("2026-06-01T21:08:11Z") - return _parse_iso_epoch(d.get("start_time")) - if adapter == "state_511_atis": - # state_511 uses "5/28/26, 10:45 PM" in inner.data.start_date - return _parse_511_date_epoch(d.get("start_date")) - if adapter == "itd_511": - # itd_511 carries start_epoch as a Unix epoch integer - val = d.get("start_epoch") - if isinstance(val, (int, float)) and val > 0: - return int(val) - return None - return None - - -# ---- main entry point ---------------------------------------------------- - - -def handle_incident(envelope: dict, subject: str, - data: Optional[dict] = None, - now: Optional[int] = None) -> Optional[str]: - """Unified incident handler. Returns the wire string when a broadcast - should fire, None otherwise.""" - if not isinstance(envelope, dict): - return None - - inner = envelope.get("data") or {} - adapter = inner.get("adapter") or "" - category_raw = inner.get("category") or "" - severity_word = _coerce_severity(inner.get("severity")) - now = now if now is not None else _now() - - try: - conn = get_db() - except Exception: - logger.exception("incident_handler: persistence unavailable") - return None - - # v0.5.9 GAMMA: state_511_atis Idaho cutover -- itd_511 is the - # authoritative ID source. state_511_atis remains active for non-ID - # neighbor coverage (WA, OR, MT). Skip ID events at handler entrance - # with event_log handled=0 reason='state_511_atis_id_replaced_by_itd_511' - # so the accounting trail makes the cutover visible. - if adapter == "state_511_atis": - sd = (envelope.get("data") or {}).get("data") or {} - sgeo = (envelope.get("data") or {}).get("geo") or {} - # v0.6-3b: state allowlist from adapter_config.state_511_atis.skipped_states. - skipped = {s.upper() for s in adapter_config.state_511_atis.skipped_states} - primary_region_state = (sgeo.get("primary_region") or "").split("-")[-1].upper() - if ((sd.get("state_code") or "").upper() in skipped - or primary_region_state in skipped): - _log_event(conn, now=now, source="state_511_atis", - category=category_raw + "|skip_id", - severity_word=severity_word, - event_id_external=inner.get("id"), - subject=subject, handled=0, - table_name=None, table_pk=None) - return None - - # v0.5.9 REVISED gate (B): freshness check at handler entrance. - # Computed BEFORE per-source parse + before the mag=0/past/future - # filters, so stale envelopes never UPSERT into traffic_events. - # Missing start_time -> default-allow (treat as fresh). - start_epoch = _extract_start_time_epoch(envelope, adapter) - if start_epoch is not None: - age_s = now - start_epoch - # v0.5.9 GAMMA: reject FUTURE-scheduled events (age < 0) as well - # as stale events (age > window). itd_511 work_zone envelopes can - # carry start_epoch many days in the future for scheduled - # construction projects; under the previous one-sided check - # those slipped through. Spec re-read: 'skip even New: broadcast - # if the underlying event began more than 30 min ago' implies - # the event must have BEGUN. - fresh_max = int(adapter_config.incident.freshness_seconds) - if age_s < 0 or age_s > fresh_max: - logger.debug( - "incident freshness gate: dropping source=%s subject=%s " - "age=%ds (window=[0, %d])", - adapter, subject, age_s, INCIDENT_FRESHNESS_MAX_S, - ) - _log_event(conn, now=now, source=adapter, category=category_raw, - severity_word=severity_word, - event_id_external=inner.get("id"), - subject=subject, handled=0, - table_name=None, table_pk=None) - return None - - # Per-source parse (returns None when filtered). - if adapter == "tomtom_incidents": - n = _parse_tomtom_incident(envelope, now) - elif adapter == "state_511_atis": - n = _parse_state_511_incident(envelope, category_raw, now) - elif adapter == "itd_511": - n = _parse_itd_511_incident(envelope, category_raw, now) - else: - return None - - if n is None: - # Filtered envelope -- log to event_log handled=0 with no fires/ - # traffic_events row, no broadcast. Lets us account for upstream - # noise without polluting the broadcast pipeline. - _log_event(conn, now=now, source=adapter, category=category_raw, - severity_word=severity_word, - event_id_external=inner.get("id"), - subject=subject, handled=0, - table_name=None, table_pk=None) - return None - - external_id = n["external_id"] - source = n["source"] - pk_combined = f"{source}|{external_id}" - - # ── Phase-2 cutover branch ──────────────────────────────────────────── - # When the relevant category has been explicitly cut over, delegate - # traffic_events management to gating.incident.decide() and write - # canonical data into the shared `data` dict so the formatter can - # render from it at dispatch time. NOT-cutover path is unchanged. - _kind_to_cat = { - "incident": "road_incident", - "closure": "road_closure", - "special_event": "road_incident", - "work_zone": "work_zone", - } - _event_cat = _kind_to_cat.get(n.get("category_kind", "incident"), "road_incident") - - from meshai.notifications.cutover import is_cutover as _is_cutover - if _is_cutover(_event_cat) and isinstance(data, dict): - # Build canonical data from parsed dict n. - _canonical = { - "external_id": n["external_id"], - "source": n["source"], - "sub_type": n.get("sub_type"), - "road": n.get("road"), - "direction": n.get("direction"), - "from_loc": n.get("from_loc"), - "to_loc": n.get("to_loc"), - "mile_start": n.get("mile_start"), - "mile_end": n.get("mile_end"), - "mile_marker": n.get("mile_marker"), - "lanes_affected": n.get("lanes_affected"), - "cause": n.get("cause"), - "comment": n.get("comment"), - "impact": n.get("impact"), - "county": n.get("county"), - "state": n.get("state"), - "lat": n.get("lat"), - "lon": n.get("lon"), - "geocoder_city": n.get("geocoder_city"), - "landclass": n.get("landclass"), - "start_at": n.get("start_at"), - "end_at": n.get("end_at"), - "magnitude": n.get("magnitude"), - "delay_seconds": n.get("delay_seconds"), - "icon_category": n.get("icon_category"), - } - - _log_id_co = _log_event_returning_id( - conn, now=now, source=source, category=category_raw, - severity_word=severity_word, - event_id_external=external_id, - subject=subject, handled=0, - table_name="traffic_events", table_pk=pk_combined) - - from meshai.notifications.gating.incident import decide as _gate_decide - _gate = _gate_decide(_canonical, source=source, now=float(now)) - - if not _gate.broadcast: - return None - - data.update(_canonical) - data.update(_gate.data_patch) - data["_broadcast_audit"] = {"table": "traffic_events", "pk": pk_combined} - - _raw_commit = _gate.commit - _log_row_id_co = _log_id_co - - def _on_commit_cutover(committed_at: float) -> None: - if _raw_commit is not None: - _raw_commit(committed_at) - if _log_row_id_co is not None: - try: - _c = get_db() - _c.execute("UPDATE event_log SET handled=1 WHERE id=?", - (int(_log_row_id_co),)) - except Exception: - logger.exception( - "incident cutover commit: event_log update failed") - - data["_on_broadcast_committed"] = _on_commit_cutover - return _render(n) - - # ── Legacy path (not cutover): exact original logic below ───────────── - - log_id = _log_event_returning_id( - conn, now=now, source=source, category=category_raw, - severity_word=severity_word, - event_id_external=external_id, - subject=subject, handled=0, - table_name="traffic_events", table_pk=pk_combined) - - row = conn.execute( - "SELECT first_seen_at, last_seen_at, last_broadcast_at, " - "last_broadcast_magnitude, last_broadcast_delay_seconds, " - "last_broadcast_icon_category FROM traffic_events " - "WHERE source=? AND external_id=?", - (source, external_id), - ).fetchone() - - if row is None: - # NEW external_id -- INSERT, return 'New:' wire, callback updates - # last_broadcast_* on dispatcher commit. - conn.execute( - "INSERT INTO traffic_events(source, external_id, road, direction, " - "mile_start, mile_end, county, state, lat, lon, sub_type, impact, " - "start_at, end_at, first_seen_at, last_seen_at, last_broadcast_at, " - "magnitude_of_delay, delay_seconds, icon_category, " - "last_broadcast_magnitude, last_broadcast_delay_seconds, " - "last_broadcast_icon_category) " - "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", - (source, external_id, n["road"], n["direction"], - n["mile_start"], n["mile_end"], n["county"], n["state"], - n["lat"], n["lon"], n["sub_type"], n["impact"], - n["start_at"], n["end_at"], now, now, None, - n["magnitude"], n["delay_seconds"], n["icon_category"], - None, None, None), - ) - wire = _render(n) - _attach_commit_handles(data, source=source, external_id=external_id, - magnitude=n["magnitude"], - delay_seconds=n["delay_seconds"], - icon_category=n["icon_category"], - event_log_row_id=log_id) - return wire - - # EXISTING incident -- always UPSERT current fields + last_seen_at. - conn.execute( - "UPDATE traffic_events SET sub_type=?, impact=?, magnitude_of_delay=?, " - "delay_seconds=?, icon_category=?, last_seen_at=?, " - "lat=COALESCE(?, lat), lon=COALESCE(?, lon), " - "direction=COALESCE(?, direction), road=COALESCE(?, road) " - "WHERE source=? AND external_id=?", - (n["sub_type"], n["impact"], n["magnitude"], n["delay_seconds"], - n["icon_category"], now, n["lat"], n["lon"], - n["direction"], n["road"], source, external_id), - ) - - last_bcast_at = row["last_broadcast_at"] - last_bcast_mag = row["last_broadcast_magnitude"] - last_bcast_delay = row["last_broadcast_delay_seconds"] - last_bcast_icon = row["last_broadcast_icon_category"] - - # Cold-start race: row exists from a prior INSERT but the dispatcher - # dropped the broadcast (grace, cooldown, etc.). last_broadcast_at is - # still NULL -> the next successful broadcast still labels itself New:. - if last_bcast_at is None: - wire = _render(n) - _attach_commit_handles(data, source=source, external_id=external_id, - magnitude=n["magnitude"], - delay_seconds=n["delay_seconds"], - icon_category=n["icon_category"], - event_log_row_id=log_id) - return wire - - # v0.6-3b: post-first-broadcast Update gated by - # adapter_config.incident.broadcast_on_update (default False -- - # preserves the v0.5.9 REVISED 'no Update' behavior). When True, - # broadcast an Update on magnitude step-up, delay doubling, or - # icon_category change. No heartbeat. - if not bool(adapter_config.incident.broadcast_on_update): - return None - - mag_stepped_up = ( - n["magnitude"] is not None - and (last_bcast_mag is None or n["magnitude"] > last_bcast_mag) - ) - delay_doubled = ( - n["delay_seconds"] is not None - and last_bcast_delay is not None - and last_bcast_delay > 0 - and n["delay_seconds"] >= 2 * last_bcast_delay - ) - icon_changed = ( - n["icon_category"] is not None - and last_bcast_icon is not None - and n["icon_category"] != last_bcast_icon - ) - if not (mag_stepped_up or delay_doubled or icon_changed): - return None - - wire = _render(n) - _attach_commit_handles(data, source=source, external_id=external_id, - magnitude=n["magnitude"], - delay_seconds=n["delay_seconds"], - icon_category=n["icon_category"], - event_log_row_id=log_id) - return wire - - -# ---- commit-callback factory -------------------------------------------- - - -def _attach_commit_handles(data: Optional[dict], *, source: str, - external_id: str, - magnitude: Optional[int], - delay_seconds: Optional[int], - icon_category: Optional[str], - event_log_row_id: Optional[int]) -> None: - if not isinstance(data, dict): - return - - def _on_commit(committed_at: float) -> None: - try: - conn = get_db() - except Exception: - logger.exception("incident commit callback: persistence unavailable") - return - conn.execute( - "UPDATE traffic_events SET last_broadcast_at=?, " - "first_broadcast_at=COALESCE(first_broadcast_at, ?), " - "last_broadcast_magnitude=?, last_broadcast_delay_seconds=?, " - "last_broadcast_icon_category=? " - "WHERE source=? AND external_id=?", - (int(committed_at), int(committed_at), magnitude, delay_seconds, icon_category, - source, external_id), - ) - if event_log_row_id is not None: - conn.execute("UPDATE event_log SET handled=1 WHERE id=?", - (int(event_log_row_id),)) - - data["_on_broadcast_committed"] = _on_commit - data["_broadcast_audit"] = {"table": "traffic_events", - "pk": f"{source}|{external_id}"} - - -# ---- event_log helpers --------------------------------------------------- - - -def _coerce_severity(sev: Any) -> Optional[str]: - if sev is None: return None - if isinstance(sev, str): return sev or None - try: return str(int(sev)) - except (TypeError, ValueError): return str(sev) - - -def _log_event(conn, *, now, source, category, severity_word, - event_id_external, subject, handled, - table_name, table_pk) -> None: - conn.execute( - "INSERT INTO event_log(received_at, source, category, severity_word, " - "event_id_external, nats_subject, handled, table_name, table_pk) " - "VALUES (?,?,?,?,?,?,?,?,?)", - (now, source, category, severity_word, event_id_external, subject, - int(bool(handled)), table_name, table_pk), - ) - - -def _log_event_returning_id(conn, *, now, source, category, severity_word, - event_id_external, subject, handled, - table_name, table_pk) -> int: - cur = conn.execute( - "INSERT INTO event_log(received_at, source, category, severity_word, " - "event_id_external, nats_subject, handled, table_name, table_pk) " - "VALUES (?,?,?,?,?,?,?,?,?)", - (now, source, category, severity_word, event_id_external, subject, - int(bool(handled)), table_name, table_pk), - ) - return int(cur.lastrowid) - - -# ---- renderer ------------------------------------------------------------ - - -def _render(n: dict) -> str: - """Budget-fitted multi-line wire string. - - Line 1: {emoji} {display} — Near {city}, {state} (critical, kept full) - Line 2: {road} {direction_long} [· MP {mile}] · {lanes} (critical, kept full) - Line 3: {narrative/comment} (trimmed from END to fit) - - Direction words (Northbound/etc.) and the word "milepost" are NEVER - abbreviated. The whole message is fit to the per-adapter packet budget; - word-boundary trimming guarantees the narrative's direction words and - "milepost" are never chopped mid-word. The separate delay / length / Cause - lines were dropped in the budget-fit rework -- the narrative is the trailing - trimmed content. - """ - sub_type = n.get("sub_type") or "incident" - emoji = _SUB_TYPE_EMOJI.get(sub_type, "⚠️") - display = _SUB_TYPE_DISPLAY.get(sub_type, "Road Incident") - - # Line 1: emoji + display + city/county - anchor = n.get("geocoder_city") or n.get("county") - state = n.get("state") or "" - if anchor: - anchor_part = f"Near {anchor}, {state}".rstrip(", ") - if not n.get("geocoder_city") and n.get("county"): - anchor_part = f"Near {anchor} Co, {state}".rstrip(", ") - else: - anchor_part = state or "" - line1 = f"{emoji} {display} — {anchor_part}".rstrip(" —") - - # Line 2: road + direction (+ MP) + lane-status joined by " · ". - # Direction is expanded to the full word; abbreviations are never emitted. - road = n.get("road") - direction = n.get("direction") - dir_long = _DIRECTION_LONG.get(direction, direction) if direction else None - mile = n.get("mile_marker") - from_loc = n.get("from_loc") - to_loc = n.get("to_loc") - seg: list[str] = [] - if road and dir_long: - seg.append(f"{road} {dir_long}") - elif road: - seg.append(road) - elif from_loc and to_loc: - seg.append(f"{from_loc} → {to_loc}") - elif from_loc: - seg.append(from_loc) - if mile is not None: - seg.append(f"MP {mile}") - lanes = n.get("lanes_affected") - if lanes and lanes.strip().lower() not in ("no data", ""): - seg.append(lanes.strip()) - line2 = " · ".join(seg) - - # Critical head: header + road·lane line, kept verbatim. - msg = "\n".join(l for l in (line1, line2) if l) - - # Trailing narrative/comment: appended, then the WHOLE message is trimmed - # from the end to the packet budget (word-boundary safe). - comment = n.get("comment") - if comment and comment.strip(): - comment_normalized = comment.strip().lower() - lanes_normalized = (lanes or "").strip().lower() - if comment_normalized != lanes_normalized: - msg = f"{msg}\n{comment.strip()}" if msg else comment.strip() - - return fit_to_budget(msg, budget_for("incident")) - - -def _location_anchor(n: dict) -> str: - """Anchor priority: geocoder.city > nearest_town > landclass > county.""" - city = n.get("geocoder_city") - if city: - return str(city) - lat = n.get("lat") - lon = n.get("lon") - if isinstance(lat, (int, float)) and isinstance(lon, (int, float)): - try: - from meshai.central_normalizer import nearest_town - nt = nearest_town(lat, lon, max_distance_mi=100.0) - except Exception: - nt = None - if nt and nt.get("name"): - town = nt["name"] - d = nt.get("distance_mi") - if isinstance(d, (int, float)): - if d < 1: return f"near {town}" - bearing = nt.get("bearing") or "" - return f"{int(round(d))} mi {bearing} of {town}".strip() - return str(town) - landclass = n.get("landclass") - if landclass: - return str(landclass) - county = n.get("county") - state = n.get("state") - if county and state: return f"{county} Co {state}" - if state: return str(state) - return "(location unknown)" diff --git a/work/meshai/central/nwis_handler.py b/work/meshai/central/nwis_handler.py deleted file mode 100644 index 6f52f39..0000000 --- a/work/meshai/central/nwis_handler.py +++ /dev/null @@ -1,328 +0,0 @@ -"""v0.5.12 usgs_nwis stream-gauge handler. - -Minimal Idaho curation -- 9 starter sites in idaho_gauge_sites.py. Non- -curated sites are dropped at handler entrance (event_log handled=0, no -gauge_readings UPSERT). v0.6.x will migrate the curation dict into a DB -table so non-engineers can edit via the GUI. - -Per-parameter filtering: - 00060 = Discharge (cfs) -- captured as flow_cfs, paired with stage - 00065 = Gage height (ft) -- the canonical stage for threshold calc - everything else -- dropped (no precipitation handling this round) - -Change-detection (mirrors WFIGS forward-only): - Insert the new reading into gauge_readings (time-series). - Compare current threshold_state to most recent prior reading\\'s - threshold_state for the same site. If current > prior in the ranked - scale {normal < action < flood_minor < flood_moderate < flood_major}, - fire 'New:' broadcast. Otherwise (unchanged or descending), no - broadcast. The receding-water case is intentionally silent -- - operationally less urgent than rising water. - -Wire format MEDIUM: - 🌊 New: {gauge_name}: {label} {value} ft, flow {flow_cfs:,} cfs, @ lat,lon - -Where {label} is: - action -> "action stage" - flood_minor -> "minor flooding" - flood_moderate -> "moderate flooding" - flood_major -> "major flooding" - -flow_cfs segment is dropped when parameter_code is 00065 only (no -companion discharge reading). lat/lon segment is dropped when coords are -missing (rare since curated sites have coords). - -Operational status (2026-06-08): - PARKED — Idaho USGS sites return discharge only (parameter_code=00060). - Gage height (00065) is not present in any retained CENTRAL_HYDRO envelope - (confirmed: 113-envelope JetStream replay, 0 stage readings). Without - 00065, compute_threshold_state() always returns "normal", the - upward-crossing check never fires, and this handler broadcasts nothing. - - The threshold/flood-stage machinery and IDAHO_CURATED_SITES thresholds - are correct and should be preserved. The adapter will become active if/when - USGS sites serving Idaho gage height data are added to the curated list. - - Enrichment idea (parked same date): if NWS issues a flood warning (FFW/FLW) - and a NWIS gauge in the same county is above action stage, annotating the - NWS wire with the gauge reading was considered. Blocked by the same 00065 - data gap — revisit if stage data becomes available. -""" -from __future__ import annotations -from meshai.adapter_config import adapter_config - -import json -import logging -import time -from datetime import datetime -from typing import Any, Optional - -from meshai.central.idaho_gauge_sites import ( - compute_threshold_state, - lookup_site, - normalize_site_id, -) -from meshai.persistence import get_db - -logger = logging.getLogger(__name__) - - -# v0.6-3b: handled parameter codes + recede toggle live in -# adapter_config.usgs_nwis. Default {"00060", "00065"}. - -# Human-readable label per threshold_state. -_LABEL = { - "action": "action stage", - "flood_minor": "minor flooding", - "flood_moderate": "moderate flooding", - "flood_major": "major flooding", -} - - -def _now() -> int: return int(time.time()) - - -def _coerce_severity(sev: Any) -> Optional[str]: - if sev is None: return None - if isinstance(sev, str): return sev or None - try: return str(int(sev)) - except (TypeError, ValueError): return str(sev) - - -def _parse_iso_epoch(s: Optional[str]) -> Optional[int]: - if not s: return None - try: return int(datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp()) - except Exception: return None - - -def handle_nwis(envelope: dict, subject: str, - data: Optional[dict] = None, - now: Optional[int] = None) -> Optional[str]: - if not isinstance(envelope, dict): return None - inner = envelope.get("data") or {} - if (inner.get("adapter") or "") != "nwis": return None - - d = inner.get("data") or {} - now = now if now is not None else _now() - category_raw = inner.get("category") or "" - severity_word = _coerce_severity(inner.get("severity")) - - try: - conn = get_db() - except Exception: - logger.exception("nwis_handler: persistence unavailable") - return None - - # Normalize site_id + look up the curated entry. - raw_site = d.get("monitoring_location_id") or d.get("site_id") - site_id = normalize_site_id(raw_site) - site_meta = lookup_site(raw_site) if raw_site else None - - # Drop non-curated sites at entrance. - if site_meta is None: - _log_event(conn, now=now, source="nwis", category=category_raw, - severity_word=severity_word, - event_id_external=raw_site or inner.get("id"), - subject=subject, handled=0, - table_name=None, table_pk=None) - return None - - # Drop unsupported parameters (precip etc.). - pc = d.get("parameter_code") - if pc not in set(adapter_config.usgs_nwis.parameter_codes): - _log_event(conn, now=now, source="nwis", category=category_raw, - severity_word=severity_word, - event_id_external=site_id, - subject=subject, handled=0, - table_name=None, table_pk=None) - return None - - # Extract reading value + reading_time. - value = d.get("value") - if isinstance(value, str): - try: value = float(value) - except ValueError: value = None - if not isinstance(value, (int, float)): - _log_event(conn, now=now, source="nwis", category=category_raw, - severity_word=severity_word, event_id_external=site_id, - subject=subject, handled=0, - table_name=None, table_pk=None) - return None - value = float(value) - - reading_time = _parse_iso_epoch(d.get("time")) or now - unit = d.get("unit_of_measure") or ("ft^3/s" if pc == "00060" else "ft") - - # Compute threshold_state. ONLY parameter_code=00065 (stage in ft) maps - # to threshold_state -- discharge (cfs) lands as a companion field. - stage_ft: Optional[float] = value if pc == "00065" else None - flow_cfs: Optional[float] = value if pc == "00060" else None - threshold_state = "normal" - if pc == "00065": - threshold_state = compute_threshold_state(stage_ft, site_meta) - - lat = d.get("latitude") if isinstance(d.get("latitude"), (int, float)) else site_meta.get("lat") - lon = d.get("longitude") if isinstance(d.get("longitude"), (int, float)) else site_meta.get("lon") - - # Build the canonical dict the decider + formatter read. parameter_code is - # carried so the decider can perform the 00060 discharge back-look; it is - # not part of the formatter's wire schema. - canonical: dict = { - "site_id": site_id, - "gauge_name": site_meta["gauge_name"], - "stage_ft": stage_ft, - "flow_cfs": flow_cfs, - "unit": unit, - "threshold_state": threshold_state, - "reading_time": reading_time, - "lat": lat, - "lon": lon, - "parameter_code": pc, - } - - # Always log the envelope to event_log. Initial handled=0; commit - # callback flips to 1 if we actually broadcast. - log_id = _log_event_returning_id( - conn, now=now, source="nwis", category=category_raw, - severity_word=severity_word, event_id_external=site_id, - subject=subject, handled=0, - table_name="gauge_readings", table_pk=site_id) - - # Delegate the threshold-crossing decision (prior-reading SELECT, 00060 - # stage back-look, THRESHOLD_RANK upward-crossing check, broadcast_on_recede - # toggle) to the gating module. It reads gauge_readings for prior state - # BEFORE the inline INSERT below, preserving the original ordering. It - # returns the resolved (back-looked) threshold_state + stage_ft in - # data_patch so the INSERT + render use the same values on every path. - from meshai.notifications.gating.hydro import decide as _gate_decide - gate = _gate_decide(canonical, source="nwis", now=float(now)) - threshold_state = gate.data_patch.get("threshold_state", threshold_state) - stage_ft = gate.data_patch.get("stage_ft", stage_ft) - canonical["threshold_state"] = threshold_state - canonical["stage_ft"] = stage_ft - - # INSERT the new reading row INLINE (append-only time-series). Always - # persist, regardless of the broadcast decision, and AFTER the decider's - # reads so they never see the current row. Per the refactor plan this - # write stays handler-owned; the decider only READS gauge_readings. - conn.execute( - "INSERT INTO gauge_readings(site_id, gauge_name, reading_value, " - "reading_unit, threshold_state, flow_cfs, reading_time, lat, lon) " - "VALUES (?,?,?,?,?,?,?,?,?)", - (site_id, site_meta["gauge_name"], value, unit, - threshold_state, flow_cfs, reading_time, lat, lon), - ) - - if not gate.broadcast: - return None - - # Cutover gate: mirror quake_handler. When "stream_flow" is cut over, write - # gate.data_patch and wrap gate.commit so it also flips event_log.handled. - # Hydro has no per-event broadcast-state table, so gate.commit is None and - # the wrapper carries only the event_log flip. Otherwise old-style - # _attach_commit keeps the live broadcast byte-for-byte identical while the - # new formatter+decider bake in shadow. - from meshai.notifications.cutover import is_cutover - if isinstance(data, dict): - data.update(canonical) - if is_cutover("stream_flow"): - data.update(gate.data_patch) - data["_broadcast_audit"] = {"table": "gauge_readings", "pk": site_id} - - _raw_commit = gate.commit - _log_row_id = log_id - - def _on_commit(committed_at: float) -> None: - if _raw_commit is not None: - _raw_commit(committed_at) - if _log_row_id is not None: - try: - c = get_db() - c.execute("UPDATE event_log SET handled=1 WHERE id=?", - (int(_log_row_id),)) - except Exception: - logger.exception("nwis commit: event_log update failed") - - data["_on_broadcast_committed"] = _on_commit - else: - _attach_commit(data, site_id=site_id, event_log_row_id=log_id) - - # Return _render() wire for backward-compat (existing call-sites + tests). - # At dispatch time compose_mesh_message() uses the registered formatter - # (formatters/hydro.py) on cutover, re-rendering byte-identically from data. - return _render(gauge_name=site_meta["gauge_name"], - threshold_state=threshold_state, - stage_ft=stage_ft, flow_cfs=flow_cfs, - unit=unit if pc == "00065" else "ft", - lat=lat, lon=lon) - - -# ---- renderer ------------------------------------------------------------ - - -def _render(*, gauge_name: str, threshold_state: str, - stage_ft: Optional[float], flow_cfs: Optional[float], - unit: str, lat: Optional[float], lon: Optional[float]) -> str: - label = _LABEL.get(threshold_state, threshold_state) - - # Stage segment. - if isinstance(stage_ft, (int, float)): - stage_seg = f"{label} {stage_ft:.1f} ft" - else: - stage_seg = label - - # Optional flow segment. - flow_seg = "" - if isinstance(flow_cfs, (int, float)): - flow_seg = f", flow {int(round(flow_cfs)):,} cfs" - - # Optional coords segment. - coords = "" - if isinstance(lat, (int, float)) and isinstance(lon, (int, float)): - coords = f", @ {lat:.3f},{lon:.3f}" - - return f"🌊 New: {gauge_name}: {stage_seg}{flow_seg}{coords}" - - -# ---- commit callback ----------------------------------------------------- - - -def _attach_commit(data: Optional[dict], *, site_id: str, - event_log_row_id: Optional[int]) -> None: - if not isinstance(data, dict): return - - def _on_commit(committed_at: float) -> None: - try: conn = get_db() - except Exception: - logger.exception("nwis commit: persistence unavailable"); return - if event_log_row_id is not None: - conn.execute("UPDATE event_log SET handled=1 WHERE id=?", - (int(event_log_row_id),)) - - data["_on_broadcast_committed"] = _on_commit - data["_broadcast_audit"] = {"table": "gauge_readings", "pk": site_id} - - -# ---- event_log helpers --------------------------------------------------- - - -def _log_event(conn, *, now, source, category, severity_word, - event_id_external, subject, handled, table_name, table_pk) -> None: - conn.execute( - "INSERT INTO event_log(received_at, source, category, severity_word, " - "event_id_external, nats_subject, handled, table_name, table_pk) " - "VALUES (?,?,?,?,?,?,?,?,?)", - (now, source, category, severity_word, event_id_external, subject, - int(bool(handled)), table_name, table_pk)) - - -def _log_event_returning_id(conn, *, now, source, category, severity_word, - event_id_external, subject, handled, - table_name, table_pk) -> int: - cur = conn.execute( - "INSERT INTO event_log(received_at, source, category, severity_word, " - "event_id_external, nats_subject, handled, table_name, table_pk) " - "VALUES (?,?,?,?,?,?,?,?,?)", - (now, source, category, severity_word, event_id_external, subject, - int(bool(handled)), table_name, table_pk)) - return int(cur.lastrowid) diff --git a/work/meshai/central/nws_handler.py b/work/meshai/central/nws_handler.py deleted file mode 100644 index 09b04ff..0000000 --- a/work/meshai/central/nws_handler.py +++ /dev/null @@ -1,741 +0,0 @@ -"""v0.5.10 NWS weather-alerts handler. - -Severity floor: broadcast only when CAP severity in {Extreme, Severe}. Watch / -Advisory / Statement (Moderate, Minor, Unknown) get logged to event_log -handled=0 and silently skipped. - -Tombstone handling: msgType in {Cancel, Expire} -> log handled=0, no -broadcast. - -Per-CAP-id dedup: nws_alerts table keyed on CAP `event_id` (the urn-style -identifier). First sighting fires `New:`; re-issues UPSERT current_* but -don't re-broadcast (v0.5.9-incident no-Update rule). - -Wire format (MEDIUM, ~80-90 B): - {emoji} {event_type}: {area_desc}, until {expires_short}, @ {lat:.3f},{lon:.3f} - -Emoji by event_type prefix (substring match, case-insensitive): - Tornado Warning -> 🌪️ - Severe Thunderstorm War.. -> 🌩️ - Flash Flood / Flood -> 🌊 - Winter Storm / Blizzard / Ice -> ❄️ - Heat / Excessive Heat -> 🌡️ - High Wind / Wind -> 🌬️ - Fire Weather / Red Flag -> 🔥 - Air Quality -> 😷 - Frost / Freeze -> 🥶 - default -> ⚠️ -""" -from __future__ import annotations -from meshai.adapter_config import adapter_config -from meshai.central.budget import budget_for, fit_to_budget - -import logging -import re -import time -import zoneinfo -from datetime import datetime, timezone -from typing import Any, Optional - -from meshai.notifications import clock - -from meshai.persistence import get_db - -logger = logging.getLogger(__name__) - - -# v0.6-3b: tombstone msgTypes live in adapter_config.nws -# (tombstone_msgtypes), read at handler call time. The old broadcast_severities -# severity gate was removed in the config-schema cleanup (no longer enforced; -# NWS breadth is governed by the per-toggle dispatcher severity threshold). - -# Ordered (substring, emoji) checks; first match wins. -_EVENT_EMOJI = [ - ("tornado", "🌪️"), - ("severe thunderstorm", "🌩️"), - ("thunderstorm", "🌩️"), - ("flash flood", "🌊"), - ("flood", "🌊"), - ("winter storm", "❄️"), - ("blizzard", "❄️"), - ("ice storm", "❄️"), - ("ice", "❄️"), - ("excessive heat", "🌡️"), - ("heat", "🌡️"), - ("high wind", "🌬️"), - ("wind", "🌬️"), - ("fire weather", "🔥"), - ("red flag", "🔥"), - ("air quality", "😷"), - ("freeze", "🥶"), - ("frost", "🥶"), -] - - -_SAME_EMOJI = { - "TOR": "🌪️", "SVR": "⛈️", "FFW": "🌊", "FLW": "🌊", - "WSW": "❄️", "BZW": "❄️", "WCY": "❄️", "EWW": "💨", - "HWW": "💨", "FRW": "🔥", "SPS": "🌬️", "SMW": "⛈️", - "MAW": "🌊", "ADR": "⚠️", -} - -_NWS_OFFICE_SHORT = { - "KBOI": "Boise", "KPIH": "Pocatello", "KMSO": "Missoula", - "KOTX": "Spokane", "KSLC": "Salt Lake City", "KMFR": "Medford", - "KPDT": "Pendleton", "KSEW": "Seattle", -} - - -def _nws_office(params: dict) -> str: - try: - wmo = (params.get("WMOidentifier") or [""])[0] - code = wmo.split()[1] - return _NWS_OFFICE_SHORT.get(code, code[1:]) - except Exception: - return "" - - -def _parse_nws_description(description: str) -> dict: - result = {} - patterns = { - "hazard": r"HAZARD\.\.\.(.*?)(?=\n\n|\nSOURCE|\nIMPACT|\nLocations|$)", - "impact": r"IMPACT\.\.\.(.*?)(?=\n\n|\nLocations|$)", - "tornado": r"TORNADO\.\.\.(.*?)(?=\n\n|\n[A-Z]+\.\.\.|$)", - "tornado_threat": r"TORNADO DAMAGE THREAT\.\.\.(.*?)(?=\n\n|\n[A-Z]+\.\.\.|$)", - "locations": r"Locations impacted include[.…]*\s*(.*?)(?=\n\n|$)", - } - for key, pattern in patterns.items(): - m = re.search(pattern, description or "", re.DOTALL | re.IGNORECASE) - if m: - text = m.group(1).replace("\n", " ").strip() - if text: - # Preserve the FULL town list for path-sampling in _render(); - # all other fields keep the 80-char cap. - result[key] = text[:400] if key == "locations" else text[:80] - return result - - -def _parse_motion(params: dict) -> tuple: - """Parse eventMotionDescription into (compass, speed_mph). - Format: '...DEG...KT' e.g. '254DEG...35KT' - Returns (compass_str, speed_mph_int) or (None, None).""" - raw = (params.get("eventMotionDescription") or [""])[0] - if not raw: - return None, None - m = re.search(r"(\d+)DEG\.+(\d+)KT", raw) - if not m: - return None, None - deg = float(m.group(1)) - knots = int(m.group(2)) - mph = round(knots * 1.15) - # Bearing is the direction the storm is moving TOWARD - dirs = ["N", "NE", "E", "SE", "S", "SW", "W", "NW"] - compass = dirs[int((deg + 22.5) / 45) % 8] - return compass, mph - - -def _now() -> int: return int(clock.now()) - - -def _is_update(conn, d: dict) -> bool: - """Return True if any CAP id in `references` was previously broadcast.""" - refs = d.get("references") or [] - if not refs: - return False - ref_ids = [r["identifier"] for r in refs - if isinstance(r, dict) and r.get("identifier")] - if not ref_ids: - return False - placeholders = ",".join("?" * len(ref_ids)) - row = conn.execute( - f"SELECT 1 FROM nws_alerts WHERE event_id IN ({placeholders}) " - "AND last_broadcast_at IS NOT NULL LIMIT 1", - ref_ids, - ).fetchone() - return row is not None - - -def _parse_iso(s: Optional[str]) -> Optional[int]: - if not s: return None - try: return int(datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp()) - except Exception: return None - - -def _emoji_for_event(event_type: Optional[str]) -> str: - if not event_type: return "⚠️" - s = event_type.lower() - for substr, emoji in _EVENT_EMOJI: - if substr in s: - return emoji - return "⚠️" - - -def _format_expires_short(epoch: Optional[int], now: Optional[int] = None) -> str: - """Renders 'until 8:15pm' / 'until Mon 3am' / 'until 6/12 8pm' depending on - how far away the expiry is. now defaults to current time so the relative - rendering is correct in tests too.""" - if not epoch: return "expires unknown" - now = now or _now() - diff = epoch - now - try: - dt = datetime.fromtimestamp(epoch, tz=timezone.utc).astimezone() - except Exception: - return "expires unknown" - - hour = dt.strftime("%-I").lstrip("0") or "0" - minute = dt.minute - ampm = "am" if dt.hour < 12 else "pm" - if minute: - time_str = f"{hour}:{minute:02d}{ampm}" - else: - time_str = f"{hour}{ampm}" - - if diff < 6 * 3600: - return f"until {time_str}" - if diff < 7 * 86400: - return f"until {dt.strftime('%a')} {time_str}" - return f"until {dt.strftime('%-m/%-d')} {time_str}" - - -def _location_anchor(area_desc: Optional[str], geocoder_city: Optional[str], - county: Optional[str], state: Optional[str]) -> str: - """Priority: geocoder.city > areaDesc (first 30 chars) > county+state > state.""" - if geocoder_city: - return str(geocoder_city) - if area_desc: - # NWS areaDesc is often semicolon-delimited list of zones; trim to first. - head = area_desc.split(";")[0].strip() - if len(head) > 30: head = head[:27] + "..." - return head - if county and state: - return f"{county} Co {state}" - if state: - return str(state) - return "(location unknown)" - - -def handle_nws(envelope: dict, subject: str, - data: Optional[dict] = None, - now: Optional[int] = None) -> Optional[str]: - """Central path handler for NWS weather-alert envelopes. - - Phase-2 refactor: when the category is cut over (MESHAI_CUTOVER_CATEGORIES - contains weather_warning or weather_statement), gating is delegated to - meshai.notifications.gating.nws.decide() and canonical data is written - into the shared `data` dict for the formatter. - - On NOT-cutover: exact legacy code path, byte-for-byte unchanged. - """ - if not isinstance(envelope, dict): return None - inner = envelope.get("data") or {} - if (inner.get("adapter") or "") != "nws": return None - - d = inner.get("data") or {} - geo = inner.get("geo") or {} - ge = (d.get("_enriched") or {}).get("geocoder") or {} - now = now if now is not None else _now() - category_raw = inner.get("category") or "" - severity_word = _coerce_severity(inner.get("severity")) - - cap_id = d.get("id") or inner.get("id") - if not cap_id: - return None - - # ── Field extraction (shared by both paths) ─────────────────────────────── - msg_type = d.get("msgType") - event_type = d.get("event") or _category_to_event_type(category_raw) - area_desc = d.get("areaDesc") - headline = d.get("headline") - description = d.get("description") - cap_severity = d.get("severity") - county = d.get("areaDesc") or ge.get("county") - state = ge.get("state") or d.get("state") - expires_epoch = _parse_iso(d.get("expires")) - same_code = ((d.get("eventCode") or {}).get("SAME") or [""])[0] - certainty = d.get("certainty") or "" - references = d.get("references") or [] - parameters = d.get("parameters") or {} - - lat = lon = None - cent = geo.get("centroid") or [] - if isinstance(cent, list) and len(cent) >= 2: - lon, lat = cent[0], cent[1] - - # ── Cutover gate ────────────────────────────────────────────────────────── - from meshai.notifications.cutover import is_cutover - _cutover = is_cutover("weather_warning") or is_cutover("weather_statement") - - if _cutover: - # ── NEW PATH: delegate to gating/nws.py decide() ───────────────────── - try: - conn = get_db() - except Exception: - logger.exception("nws_handler: persistence unavailable") - return None - - # Tombstone: log handled=0, return None (before canonical/decide). - if msg_type in set(adapter_config.nws.tombstone_msgtypes): - _log_event(conn, now=now, source="nws", category=category_raw, - severity_word=severity_word, event_id_external=cap_id, - subject=subject, handled=0, - table_name="nws_alerts", table_pk=cap_id) - return None - - # Always log the event (both broadcast and suppress paths). - log_id = _log_event_returning_id( - conn, now=now, source="nws", category=category_raw, - severity_word=severity_word, event_id_external=cap_id, - subject=subject, handled=0, - table_name="nws_alerts", table_pk=cap_id) - - # Build canonical data dict for decide() and the formatter. - canonical: dict = { - "cap_id": cap_id, - "event": event_type, - "same_code": same_code, - "cap_severity": cap_severity, - "certainty": certainty, - "expires_at": expires_epoch, - "area_desc": area_desc, - "geocoder": { - "city": ge.get("city"), - "county": county, - "state": state, - }, - "description": description, - "parameters": parameters, - "msgType": msg_type, - "references": references, - "category": category_raw, - "headline": headline, - } - - from meshai.notifications.gating.nws import decide as _gate_decide - gate = _gate_decide(canonical, source="nws", now=float(now)) - - if not gate.broadcast: - return None - - # Write canonical fields + gate data_patch into shared data dict. - if isinstance(data, dict): - data.update(canonical) - data.update(gate.data_patch) - data["_broadcast_audit"] = {"table": "nws_alerts", "pk": cap_id} - - _raw_commit = gate.commit - _log_row_id = log_id - - def _on_commit(committed_at: float) -> None: - if _raw_commit is not None: - _raw_commit(committed_at) - if _log_row_id is not None: - try: - c = get_db() - c.execute("UPDATE event_log SET handled=1 WHERE id=?", - (int(_log_row_id),)) - except Exception: - logger.exception("nws commit: event_log update failed") - - data["_on_broadcast_committed"] = _on_commit - - # Return _render() wire for backward compat with existing call-sites. - # At dispatch time compose_mesh_message() will use the registered - # formatter (formatters/nws.py) which re-renders from event.data - # producing byte-identical output (tier-a). - _prefix = gate.data_patch.get("_nws_prefix", "") - return _render(event_type=event_type, area_desc=area_desc, - geocoder_city=ge.get("city"), county=county, state=state, - expires_epoch=expires_epoch, lat=lat, lon=lon, now=now, - prefix=_prefix, d=d) - - # ── NOT cutover: exact legacy path (byte-for-byte unchanged) ───────────── - try: - conn = get_db() - except Exception: - logger.exception("nws_handler: persistence unavailable") - return None - - # Tombstone: msgType in {Cancel, Expire} -> log handled=0, no broadcast. - if msg_type in set(adapter_config.nws.tombstone_msgtypes): - _log_event(conn, now=now, source="nws", category=category_raw, - severity_word=severity_word, event_id_external=cap_id, - subject=subject, handled=0, - table_name="nws_alerts", table_pk=cap_id) - return None - - # CAP-severity pre-filter (GATE A) removed. All NWS alerts now flow into - # the notification pipeline; breadth is governed solely by the per-toggle - # dispatcher severity threshold. - - # Warning → immediate promotion: deterministically sets _severity_override - # so a wrong or missing CAP severity field cannot under-rank a real warning. - # Mirrors the wfigs_handler pattern. Applied before all broadcast-return - # paths so it covers new alert, cold-start race, and dedup-window re-bcast. - if isinstance(data, dict) and ( - category_raw.endswith("_warning") or category_raw.endswith(".warning")): - data["_severity_override"] = "immediate" - - # Per-CAP-id dedup. - log_id = _log_event_returning_id( - conn, now=now, source="nws", category=category_raw, - severity_word=severity_word, event_id_external=cap_id, - subject=subject, handled=0, - table_name="nws_alerts", table_pk=cap_id) - - row = conn.execute( - "SELECT last_broadcast_at FROM nws_alerts WHERE event_id=?", - (cap_id,)).fetchone() - - if row is None: - conn.execute( - "INSERT INTO nws_alerts(event_id, alert_type, severity, county, " - "state, headline, description, expires_at, first_seen_at, " - "last_broadcast_at) VALUES (?,?,?,?,?,?,?,?,?,?)", - (cap_id, event_type, cap_severity, county, state, - headline, description, expires_epoch, now, None), - ) - _prefix = "Update" if _is_update(conn, d) else "" - wire = _render(event_type=event_type, area_desc=area_desc, - geocoder_city=ge.get("city"), county=county, state=state, - expires_epoch=expires_epoch, lat=lat, lon=lon, now=now, - prefix=_prefix, d=d) - _attach_commit(data, cap_id=cap_id, event_log_row_id=log_id) - return wire - - if row["last_broadcast_at"] is None: - # Cold-start race: row exists but broadcast was previously dropped. - _prefix = "Update" if _is_update(conn, d) else "" - wire = _render(event_type=event_type, area_desc=area_desc, - geocoder_city=ge.get("city"), county=county, state=state, - expires_epoch=expires_epoch, lat=lat, lon=lon, now=now, - prefix=_prefix, d=d) - _attach_commit(data, cap_id=cap_id, event_log_row_id=log_id) - return wire - - # v0.6-phase3: dedup-window relaxation. If the CAP id was last - # broadcast more than `nws.duplicate_allowed_after_seconds` ago, allow - # the re-broadcast with an "Active:" prefix; otherwise suppress. - last_bcast = float(row["last_broadcast_at"]) - window_s = int(adapter_config.nws.duplicate_allowed_after_seconds) - if window_s > 0 and (now - last_bcast) >= window_s: - wire = _render(event_type=event_type, area_desc=area_desc, - geocoder_city=ge.get("city"), county=county, state=state, - expires_epoch=expires_epoch, lat=lat, lon=lon, now=now, - prefix="Active", d=d) - _attach_commit(data, cap_id=cap_id, event_log_row_id=log_id) - return wire - return None - - -# Hail descriptor -> diameter in inches (NWS convention). -_HAIL_DESCRIPTORS = { - "pea": 0.25, "half inch": 0.50, "penny": 0.75, "nickel": 0.88, - "quarter": 1.00, "half dollar": 1.25, "ping pong": 1.50, "ping-pong": 1.50, - "golf ball": 1.75, "golf": 1.75, "hen egg": 2.00, "tennis ball": 2.50, - "baseball": 2.75, "softball": 4.00, -} - - -def _tighten_wind(wind: str) -> str: - """'60 MPH' -> '60mph winds' (no space before mph, no 'wind gusts' filler).""" - w = (wind or "").strip().lower().replace(" mph", "mph") - if not w: - return "" - if not w.endswith("mph"): - w = f"{w}mph" - return f"{w} winds" - - -def _fmt_hail(hail: str) -> str: - """Numeric or descriptor hail size -> '1\" hail'. Descriptor maps per NWS.""" - s = (hail or "").strip() - if not s: - return "" - val = None - low = s.lower() - for k, v in _HAIL_DESCRIPTORS.items(): - if k in low: - val = v - break - if val is None: - try: - val = float(re.sub(r"[^0-9.]", "", s)) - except (ValueError, TypeError): - return "" - txt = f"{val:.2f}".rstrip("0").rstrip(".") - return f'{txt}" hail' - - -def _collapse_certainty(text: str) -> str: - """'Radar confirmed'/'Radar indicated' -> 'radar'; 'Observed' -> 'observed'.""" - low = (text or "").strip().lower() - if low in ("radar confirmed", "radar indicated"): - return "radar" - if low == "observed": - return "observed" - if low == "likely": - return "likely" - if low == "on ground": - return "on ground" - return (text or "").strip() - - -def _tighten_hazard(text: str) -> str: - """Compact a free-form NWS hazard sentence into the terse mesh idiom the - SVR branch already uses, so no product type (SPS/WSW/FFW/FLW/else) carries a - bloated line 3. Drops filler ('in excess of' -> '>'), collapses '45 mph' -> - '45mph', rewrites wind-gust phrasings to 'Nmph gusts', and converts hail - descriptors ('pea size hail') to numeric inches ('0.25" hail').""" - if not text: - return "" - t = text.strip().rstrip(".") - # Hail: ' size hail' -> 'N" hail' (keep any leading connector). - low = t.lower() - for k in sorted(_HAIL_DESCRIPTORS, key=len, reverse=True): - for variant in (f"{k} size hail", f"{k}-size hail", f"{k} sized hail"): - idx = low.find(variant) - if idx != -1: - repl = _fmt_hail(k) - if repl: - t = t[:idx] + repl + t[idx + len(variant):] - low = t.lower() - break - # Numeric hail: 'N inch hail' / 'N-inch hail' -> 'N" hail'. - t = re.sub(r"(\d+(?:\.\d+)?)[\- ]inch(?:es)?\s+hail", - lambda m: _fmt_hail(m.group(1)) or m.group(0), t, - flags=re.IGNORECASE) - # Wind-gust phrasings -> 'Nmph gusts'. - t = re.sub(r"wind gusts?\s+(?:in excess of|up to|to|of|reaching|near|around)" - r"\s+(\d+)\s*mph", r"\1mph gusts", t, flags=re.IGNORECASE) - t = re.sub(r"winds?\s+gusting\s+(?:up\s+)?to\s+(\d+)\s*mph", - r"\1mph gusts", t, flags=re.IGNORECASE) - # Sustained winds -> 'Nmph winds'. - t = re.sub(r"(?:damaging\s+)?winds?\s+(?:in excess of|up to|to|of)\s+" - r"(\d+)\s*mph", r"\1mph winds", t, flags=re.IGNORECASE) - # Remaining generic filler + spacing. - t = re.sub(r"\bin excess of\b", ">", t, flags=re.IGNORECASE) - t = re.sub(r"(\d+)\s*mph", r"\1mph", t, flags=re.IGNORECASE) - t = re.sub(r"\s+", " ", t).strip() - return t - - -def _render(*, event_type, area_desc, geocoder_city, county, state, - expires_epoch, lat, lon, now, prefix: str = "", d: dict = None) -> str: - d = d or {} - params = d.get("parameters") or {} - desc = _parse_nws_description(d.get("description") or "") - - # SAME code drives emoji and line-3 branching - same_code = ((d.get("eventCode") or {}).get("SAME") or [""])[0] - emoji = _SAME_EMOJI.get(same_code) or _emoji_for_event(event_type) - prefix_seg = f"{prefix}: " if prefix else "" - - # Line 1: emoji + event type (no office) - line1 = f"{emoji} {prefix_seg}{event_type or 'Weather Alert'}" - - # Line 2: "Until {time} {tz} — {area}" - tz = zoneinfo.ZoneInfo("America/Boise") - if expires_epoch: - exp_local = datetime.fromtimestamp(expires_epoch, tz=tz) - exp_str = exp_local.strftime("%-I:%M %p %Z") - time_seg = f"Until {exp_str}" - else: - time_seg = "" - area = (area_desc or "").split(";")[0].strip() - _area_limit = int(adapter_config.nws.area_max_chars) - if len(area) > _area_limit: - cut = area[:_area_limit].rsplit(" ", 1)[0] - if not cut: - cut = area[:_area_limit] - area = cut + "\u2026" - if time_seg and area: - line2 = f"{time_seg} — {area}" - elif time_seg: - line2 = time_seg - elif area: - line2 = area - else: - line2 = "" - - # Line 3: hazard + certainty/threat (SAME-code branched) - # Line 3: TIGHTENED hazard wording. Filler dropped; wind as '60mph winds', - # hail as numeric inches ('1" hail'); certainty collapsed to radar/observed; - # hazard groups and certainty joined by " · ". - certainty = (d.get("certainty") or "").strip() - line3 = "" - if same_code == "TOR": - detection = (params.get("tornadoDetection") or [""])[0] - status = "on ground" if detection == "OBSERVED" else "radar" - threat = (params.get("tornadoDamageThreat") or [""])[0] - threat_seg = f" · {threat.lower()} damage" if threat else "" - line3 = f"tornado {status}{threat_seg}" - elif same_code == "SVR": - wind = (params.get("maxWindGust") or [""])[0] - hail = (params.get("maxHailSize") or [""])[0] - bits = [] - if wind and wind not in ("0 MPH", ""): - w = _tighten_wind(wind) - if w: - bits.append(w) - if hail and hail not in ("0.00", "0", ""): - h = _fmt_hail(hail) - if h: - bits.append(h) - hazard = ", ".join(bits) - # SVR is radar-based: "Observed" certainty => "Radar confirmed" => "radar". - confirm = _collapse_certainty( - "Radar confirmed" if certainty == "Observed" else "Radar indicated") - line3 = f"{hazard} · {confirm}" if hazard else confirm - elif same_code in ("FFW", "FLW"): - hazard_text = desc.get("hazard") or "" - # First sentence only, then tighten to the terse SVR idiom. - if ". " in hazard_text: - hazard_text = hazard_text.split(". ")[0] - hazard_text = _tighten_hazard(hazard_text) - # Infer flood cause from description - desc_lower = (d.get("description") or "").lower() - flood_cause = "" - for keyword, label in [("thunderstorm", "thunderstorms"), - ("dam", "dam failure"), - ("snowmelt", "snowmelt"), - ("ice jam", "ice jam")]: - if keyword in desc_lower: - flood_cause = label - break - cause_seg = f" · {flood_cause}" if flood_cause else "" - line3 = f"{hazard_text}{cause_seg}" if hazard_text else flood_cause - else: - # SPS, WSW, etc.: first hazard sentence (tightened) + certainty if - # Observed/Likely. - hazard_text = desc.get("hazard") or "" - if ". " in hazard_text: - hazard_text = hazard_text.split(". ")[0] - hazard_text = _tighten_hazard(hazard_text) - cert_seg = "" - if certainty in ("Observed", "Likely"): - cert_seg = f" · {_collapse_certainty(certainty)}" - line3 = f"{hazard_text}{cert_seg}" if hazard_text else "" - - # Line 4: motion + locations (path-sampled if the full town list won't fit). - compass, speed_mph = _parse_motion(params) - motion = f"Moving {compass} {speed_mph} mph" if compass and speed_mph else "" - - # Parse the (now-full) locations string into an ordered town list. Path - # order = soonest-impact first ... farthest-along last. The tail element - # frequently starts with "and " (e.g. "and Shoshone") -> strip that. - raw = (desc.get("locations") or "").rstrip("., ") - towns = [t.strip() for t in raw.split(",") if t.strip()] - if towns: - towns[-1] = re.sub(r"^and\s+", "", towns[-1], flags=re.IGNORECASE).strip() - towns = [t for t in towns if t] - - def _dedup(seq): - """Drop consecutive repeats (a short list can make first == middle), - so we never render 'Buhl → Buhl → Shoshone'.""" - out = [] - for t in seq: - if not out or out[-1] != t: - out.append(t) - return out - - def _line4(locs: str) -> str: - if motion and locs: - return f"{motion} — {locs}" - if motion: - return motion - return locs or "" - - PACKET_LIMIT = budget_for("nws") - - # Location representations from richest to poorest: full comma list -> - # first→middle→last path sample -> first→last -> first-only -> none. The - # WHOLE message is measured against the budget for each and the first form - # that fits wins. Crucially the "— {locs}" segment is only ever attached - # when the full message fits, so we can never emit a dangling "— …": if no - # location form fits we fall through to motion-only, then (if even that - # overflows) drop line 4 entirely. - loc_options = [", ".join(towns)] # full comma list - if len(towns) >= 3: - loc_options.append(" → ".join( - _dedup([towns[0], towns[len(towns) // 2], towns[-1]]))) - if len(towns) >= 2: - loc_options.append(" → ".join(_dedup([towns[0], towns[-1]]))) - if towns: - loc_options.append(towns[0]) - loc_options.append("") # motion only / empty - - base_lines = [l for l in (line1, line2, line3) if l] - msg = None - for locs in loc_options: - cand4 = _line4(locs) - lines = base_lines + ([cand4] if cand4 else []) - candidate = "\n".join(lines) - if len(candidate) <= PACKET_LIMIT: - msg = candidate - break - if msg is None: - # Even motion-only line 4 overflows: drop line 4 entirely. - msg = "\n".join(base_lines) - - # Final hard-cap safety net for the pathological case where lines 1-3 alone - # overflow. Line 4 is already budget-fitted above, so this only ever trims - # the leading lines — it can never manufacture a dangling "— …". - return fit_to_budget(msg, PACKET_LIMIT) - - -def _category_to_event_type(category_raw: str) -> str: - """Best-effort friendly-name derivation when data.event is missing. - Turns 'wx.alert.severe_thunderstorm_warning' -> 'Severe Thunderstorm Warning'.""" - if not category_raw: return "Weather Alert" - tail = category_raw.split(".")[-1] if "." in category_raw else category_raw - return tail.replace("_", " ").title() - - -def _attach_commit(data: Optional[dict], *, cap_id: str, - event_log_row_id: Optional[int]) -> None: - if not isinstance(data, dict): return - - def _on_commit(committed_at: float) -> None: - try: conn = get_db() - except Exception: - logger.exception("nws commit: persistence unavailable"); return - conn.execute( - "UPDATE nws_alerts SET last_broadcast_at=?, " - "first_broadcast_at=COALESCE(first_broadcast_at, ?) " - "WHERE event_id=?", - (int(committed_at), int(committed_at), cap_id)) - if event_log_row_id is not None: - conn.execute("UPDATE event_log SET handled=1 WHERE id=?", - (int(event_log_row_id),)) - - data["_on_broadcast_committed"] = _on_commit - data["_broadcast_audit"] = {"table": "nws_alerts", "pk": cap_id} - - -def _coerce_severity(sev: Any) -> Optional[str]: - if sev is None: return None - if isinstance(sev, str): return sev or None - try: return str(int(sev)) - except (TypeError, ValueError): return str(sev) - - -def _log_event(conn, *, now, source, category, severity_word, - event_id_external, subject, handled, table_name, table_pk) -> None: - conn.execute( - "INSERT INTO event_log(received_at, source, category, severity_word, " - "event_id_external, nats_subject, handled, table_name, table_pk) " - "VALUES (?,?,?,?,?,?,?,?,?)", - (now, source, category, severity_word, event_id_external, subject, - int(bool(handled)), table_name, table_pk)) - - -def _log_event_returning_id(conn, *, now, source, category, severity_word, - event_id_external, subject, handled, - table_name, table_pk) -> int: - cur = conn.execute( - "INSERT INTO event_log(received_at, source, category, severity_word, " - "event_id_external, nats_subject, handled, table_name, table_pk) " - "VALUES (?,?,?,?,?,?,?,?,?)", - (now, source, category, severity_word, event_id_external, subject, - int(bool(handled)), table_name, table_pk)) - return int(cur.lastrowid) diff --git a/work/meshai/central/quake_handler.py b/work/meshai/central/quake_handler.py deleted file mode 100644 index bca1502..0000000 --- a/work/meshai/central/quake_handler.py +++ /dev/null @@ -1,304 +0,0 @@ -"""v0.5.10 USGS earthquakes handler. - -Broadcast gate (any of these triggers): - (a) magnitude >= 3.0 globally - (b) magnitude >= 2.5 within 250 mi of Idaho centroid - (c) tsunami_warning at any magnitude - (d) PAGER alert level in {orange, red} - -Wire format (multi-line, matches Fire/Roads style): - Line 1: {emoji} {prefix} M{mag:.1f} — {place_string} - Line 2: Depth: {depth} km · @ {lat:.3f}, {lon:.3f} - Line 3: 🚨 TSUNAMI WARNING — only when tsunami flag is set - -Emoji: - Routine -> 🌐 - M5+ -> ⚠️ - tsunami warning -> 🚨 - -place_string: prefer data.place (USGS curated, e.g. "11 km SSW of Snowville, -Utah"); fall back to nearest_town anchor when missing. - -Persistence: UPSERT into quake_events using USGS event_id. First sighting -fires New:; revisions UPSERT but don't re-broadcast (v0.5.9 no-Update rule). -""" -from __future__ import annotations -from meshai.adapter_config import adapter_config -from meshai.central.budget import budget_for, fit_to_budget - -import logging -import math -import time -from typing import Any, Optional - -from meshai.notifications import clock - -from meshai.persistence import get_db - -logger = logging.getLogger(__name__) - - -# v0.6-3b: regional gate geography, radius, magnitude floors, PAGER -# level set all live in adapter_config.usgs_quake. Read at use site so -# GUI edits take effect on the next envelope without restart. - - -def _now() -> int: return int(clock.now()) - - -def _haversine_mi(lat1, lon1, lat2, lon2) -> float: - R_mi = 3958.8 - p1 = math.radians(lat1); p2 = math.radians(lat2) - dp = math.radians(lat2 - lat1); dl = math.radians(lon2 - lon1) - a = math.sin(dp/2)**2 + math.cos(p1)*math.cos(p2)*math.sin(dl/2)**2 - return 2 * R_mi * math.atan2(math.sqrt(a), math.sqrt(1-a)) - - -def within_250mi_of_idaho(lat: float, lon: float) -> bool: - """Return True if (lat, lon) is within the regional gate radius. - - v0.6-3b: name retained for backward-compat with existing tests; the - centroid + radius now come from adapter_config.usgs_quake. - """ - if not (isinstance(lat, (int, float)) and isinstance(lon, (int, float))): - return False - cen = adapter_config.usgs_quake.regional_centroid - radius = float(adapter_config.usgs_quake.regional_radius_mi) - return _haversine_mi(lat, lon, float(cen[0]), float(cen[1])) <= radius - - -def _should_broadcast(mag: Optional[float], lat: Optional[float], - lon: Optional[float], tsunami: bool, - pager_alert: Optional[str]) -> bool: - if tsunami: return True - pager_set = {s.lower() for s in adapter_config.usgs_quake.broadcast_pager_alerts} - if pager_alert and pager_alert.lower() in pager_set: - return True - if not isinstance(mag, (int, float)): return False - if mag >= float(adapter_config.usgs_quake.global_mag_floor): return True - if (mag >= float(adapter_config.usgs_quake.regional_mag_floor) - and within_250mi_of_idaho(lat, lon)): return True - return False - - -def _emoji_for(mag: Optional[float], tsunami: bool) -> str: - if tsunami: return "🚨" - if isinstance(mag, (int, float)) and mag >= float( - adapter_config.usgs_quake.escalate_mag_floor): - return "⚠️" - return "🌐" - - -def handle_quake(envelope: dict, subject: str, - data: Optional[dict] = None, - now: Optional[int] = None) -> Optional[str]: - """Central path handler for USGS earthquake envelopes. - - Phase-1 refactor: ALL gating decisions are now delegated to - `meshai.notifications.gating.quake.decide()`. This function is kept as a - thin compatibility bridge so existing call-sites and tests remain stable - while the new formatter+decider architecture is established. - - On broadcast: - - Writes canonical data fields into the shared `data` dict so the Event - carries structured fields (the formatter reads them at dispatch time). - - Applies GateResult.data_patch (distance_km, is_update, _severity_override, - _dedup_suffix) into the same dict. - - Attaches GateResult.commit + _broadcast_audit keys. - - Returns the _render() wire string for backward-compat (existing tests). - At dispatch time compose_mesh_message() intercepts via the registered - formatter and re-renders from event.data (tier-b changes apply there). - - On suppress: returns None (default-deny unchanged). - """ - if not isinstance(envelope, dict): return None - inner = envelope.get("data") or {} - if (inner.get("adapter") or "") != "usgs_quake": return None - - d = inner.get("data") or {} - geo = inner.get("geo") or {} - now = now if now is not None else _now() - category_raw = inner.get("category") or "" - severity_word = _coerce_severity(inner.get("severity")) - - # ── Extract fields (same normalization as before) ───────────────────── - event_id = d.get("id") or inner.get("id") - if not event_id: - return None - - mag = d.get("magnitude") or d.get("mag") - if isinstance(mag, str): - try: mag = float(mag) - except ValueError: mag = None - elif isinstance(mag, (int, float)): - mag = float(mag) - - depth_km = d.get("depth_km") if d.get("depth_km") is not None else d.get("depth") - place = d.get("place") - tsunami = bool(d.get("tsunami") or d.get("tsunami_warning")) - pager_alert = d.get("alert") - - cent = geo.get("centroid") or [] - if isinstance(cent, list) and len(cent) >= 2: - lon, lat = cent[0], cent[1] - else: - lat = lon = None - - occurred_at = None - tms = d.get("time_ms") - if isinstance(tms, (int, float)) and tms > 1e12: - occurred_at = int(tms / 1000) - elif tms and isinstance(tms, (int, float)): - occurred_at = int(tms) - - # ── Build canonical data dict (written into shared `data` on broadcast) ─ - canonical: dict = { - "magnitude": mag, - "depth_km": depth_km, - "lat": lat, - "lon": lon, - "place": place, - "tsunami": tsunami, - "pager": pager_alert, - "occurred_at": occurred_at, - "event_id": event_id, - } - - # ── Delegate to gating module ───────────────────────────────────────── - from meshai.notifications.gating.quake import decide as _gate_decide - gate = _gate_decide(canonical, source="usgs_quake", now=float(now)) - - # ── Persistence logging (unchanged from original) ───────────────────── - try: - conn = get_db() - except Exception: - logger.exception("quake_handler: persistence unavailable") - return None - - if not gate.broadcast: - _log_event(conn, now=now, source="usgs_quake", category=category_raw, - severity_word=severity_word, event_id_external=event_id, - subject=subject, handled=0, - table_name="quake_events", table_pk=event_id) - return None - - log_id = _log_event_returning_id( - conn, now=now, source="usgs_quake", category=category_raw, - severity_word=severity_word, event_id_external=event_id, - subject=subject, handled=0, - table_name="quake_events", table_pk=event_id) - - # ── Write canonical fields into shared data dict ────────────────────── - # Cutover gate: when the category has been explicitly cut over, write - # gate.data_patch and use gate.commit (new path live). Otherwise use the - # old-style _attach_commit so the live broadcast stays byte-for-byte - # identical to pre-Phase-1 behavior while the new path bakes in shadow. - from meshai.notifications.cutover import is_cutover - if isinstance(data, dict): - data.update(canonical) - if is_cutover("earthquake_event"): - # NEW PATH: gate.data_patch provides distance_km, _severity_override, etc. - data.update(gate.data_patch) - data["_broadcast_audit"] = {"table": "quake_events", "pk": event_id} - - _raw_commit = gate.commit - _log_row_id = log_id - - def _on_commit(committed_at: float) -> None: - if _raw_commit is not None: - _raw_commit(committed_at) - if _log_row_id is not None: - try: - c = get_db() - c.execute("UPDATE event_log SET handled=1 WHERE id=?", - (int(_log_row_id),)) - except Exception: - logger.exception("quake commit: event_log update failed") - - data["_on_broadcast_committed"] = _on_commit - else: - # NOT cutover: old-style attach (canonical fields only, no data_patch). - # Preserves exact pre-Phase-1 live behavior; new formatter bakes in shadow. - _attach_commit(data, event_id=event_id, event_log_row_id=log_id) - - # ── Return _render() wire (backward compat — existing tests check this) ─ - # At dispatch time compose_mesh_message() calls the registered formatter - # (if cutover) which re-renders from event.data with tier-b changes (PAGER - # + update-prefix). The _render() wire here is used by both paths and by - # direct handle_quake() callers (tests, legacy paths). - return _render(mag=mag, place=place, depth_km=depth_km, lat=lat, lon=lon, - tsunami=tsunami, is_update=False) - - -def _render(*, mag, place, depth_km, lat, lon, tsunami, is_update=False) -> str: - emoji = _emoji_for(mag, tsunami) - mag_str = f"{mag:.1f}" if isinstance(mag, (int, float)) else "?" - place_str = place if place else "unknown location" - prefix = "Update:" if is_update else "New:" - - # Line 1: prefix + magnitude + place - line1 = f"{emoji} {prefix} M{mag_str} \u2014 {place_str}" - - # Line 2: depth + coords - parts = [] - if isinstance(depth_km, (int, float)): - parts.append(f"Depth: {int(round(depth_km))} km") - if isinstance(lat, (int, float)) and isinstance(lon, (int, float)): - parts.append(f"@ {lat:.3f}, {lon:.3f}") - line2 = " \u00b7 ".join(parts) if parts else None - - # Line 3: tsunami warning (only when present) - line3 = "\U0001f6a8 TSUNAMI WARNING" if tsunami else None - - msg = "\n".join(l for l in [line1, line2, line3] if l) - # Safety cap: fit the broadcast string to the mesh packet budget. - return fit_to_budget(msg, budget_for("usgs_quake")) - - -def _attach_commit(data: Optional[dict], *, event_id: str, - event_log_row_id: Optional[int]) -> None: - if not isinstance(data, dict): return - - def _on_commit(committed_at: float) -> None: - try: conn = get_db() - except Exception: - logger.exception("quake commit: persistence unavailable"); return - conn.execute( - "UPDATE quake_events SET last_broadcast_at=?, " - "first_broadcast_at=COALESCE(first_broadcast_at, ?) WHERE event_id=?", - (int(committed_at), int(committed_at), event_id)) - if event_log_row_id is not None: - conn.execute("UPDATE event_log SET handled=1 WHERE id=?", - (int(event_log_row_id),)) - - data["_on_broadcast_committed"] = _on_commit - data["_broadcast_audit"] = {"table": "quake_events", "pk": event_id} - - -def _coerce_severity(sev: Any) -> Optional[str]: - if sev is None: return None - if isinstance(sev, str): return sev or None - try: return str(int(sev)) - except (TypeError, ValueError): return str(sev) - - -def _log_event(conn, *, now, source, category, severity_word, - event_id_external, subject, handled, table_name, table_pk) -> None: - conn.execute( - "INSERT INTO event_log(received_at, source, category, severity_word, " - "event_id_external, nats_subject, handled, table_name, table_pk) " - "VALUES (?,?,?,?,?,?,?,?,?)", - (now, source, category, severity_word, event_id_external, subject, - int(bool(handled)), table_name, table_pk)) - - -def _log_event_returning_id(conn, *, now, source, category, severity_word, - event_id_external, subject, handled, - table_name, table_pk) -> int: - cur = conn.execute( - "INSERT INTO event_log(received_at, source, category, severity_word, " - "event_id_external, nats_subject, handled, table_name, table_pk) " - "VALUES (?,?,?,?,?,?,?,?,?)", - (now, source, category, severity_word, event_id_external, subject, - int(bool(handled)), table_name, table_pk)) - return int(cur.lastrowid) diff --git a/work/meshai/central/swpc_handler.py b/work/meshai/central/swpc_handler.py deleted file mode 100644 index b409605..0000000 --- a/work/meshai/central/swpc_handler.py +++ /dev/null @@ -1,533 +0,0 @@ -"""v0.5.10 SWPC space-weather handler. - -Aggressive filter -- broadcast ONLY when: - (a) Geomagnetic storm Kp >= 7 (G3 strong or higher) - (b) Solar flare X1+ (R3 strong radio blackout or higher) - (c) Solar proton event >= 10 pfu @ >= 10 MeV (S1 minor radiation storm - or higher) - -All else (Kp < 7, M-class flares, S0 protons) -> swpc_events table for -history + event_log handled=0, NO broadcast. - -Three Central sub-adapters all route here: - swpc_kindex -> check Kp threshold - swpc_alerts -> parse alert payload (flare class, geomag, proton scale) - swpc_protons -> check >=10 MeV proton flux threshold - -Wire format (multi-line, matches Fire/Quake/Avalanche style): - Line 1: {emoji} New: {scale} {type} — {key fact} - Line 2: supporting detail (impact summary / message, truncated 120 chars) - Line 3: SWPC · {time tag} - - Geomag: 🧲 New: G3 Geomagnetic Storm — Kp7 - Flare: ☀️ New: X1.2 Solar Flare — R3 - Proton: ☢️ New: S1 Radiation Storm — 10 pfu -""" -from __future__ import annotations -from meshai.adapter_config import adapter_config - -import json -import logging -import re -import time -from typing import Any, Optional - -from meshai.persistence import get_db - -logger = logging.getLogger(__name__) - - -# Kp -> G-scale mapping (NOAA-defined; CODE). -_G_SCALE = {5: ("G1", "minor"), 6: ("G2", "moderate"), 7: ("G3", "strong"), - 8: ("G4", "severe"), 9: ("G5", "extreme")} - -# v0.6-3b: broadcast floors live in adapter_config.swpc -# (geomag_kp_floor, flare_class_floor, proton_pfu_floor). - -# Proton flux -> S-scale. >= 10 pfu @ >=10 MeV is S1. -_S_SCALE_THRESHOLDS = [ - (1e5, "S5", "extreme"), - (1e4, "S4", "severe"), - (1e3, "S3", "strong"), - (1e2, "S2", "moderate"), - (10, "S1", "minor"), -] - - -# Geomag cross-sub-adapter dedup window constant — kept for documentation. -# The in-memory _geomag_recent dict has moved to meshai.notifications.gating.swpc -# (_geomag_window) where the commit closure defers the stamp. -GEOMAG_DEDUP_WINDOW_SECONDS = 600 -# _geomag_recent removed: now owned by gating.swpc._geomag_window. - - -def _trunc(s: str, limit: int = 120) -> str: - """Truncate *s* at the last word boundary at or before *limit* chars.""" - if len(s) <= limit: - return s - cut = s[:limit].rsplit(" ", 1)[0] - if not cut: - cut = s[:limit] - return cut + "…" - - -def _now() -> int: return int(time.time()) - - -def _coerce_float(v) -> Optional[float]: - if v is None: return None - if isinstance(v, (int, float)): return float(v) - try: return float(v) - except (TypeError, ValueError): return None - - -def _kp_g_scale(kp: float) -> Optional[tuple]: - """Map Kp -> NOAA G-scale tuple. v0.6-3b: returns None when below - adapter_config.swpc.geomag_kp_floor (default 7.0 = G3+). Extends down - to Kp=5 (G1) when the floor is lowered.""" - floor = float(adapter_config.swpc.geomag_kp_floor) - if kp < floor: return None - if kp >= 9: return _G_SCALE[9] - if kp >= 8: return _G_SCALE[8] - if kp >= 7: return _G_SCALE[7] - if kp >= 6: return _G_SCALE[6] - if kp >= 5: return _G_SCALE[5] - return None - - -_CLASS_RANK = {"A": 0, "B": 1, "C": 2, "M": 3, "X": 4} - - -def _class_score(class_str: Optional[str]) -> Optional[float]: - """Comparable score for X-ray flare class: rank*100 + magnitude.""" - if not class_str: return None - s = str(class_str).strip().upper() - m = re.match(r"^([ABCMX])([0-9.]+)?", s) - if not m: return None - cls = m.group(1) - try: mag = float(m.group(2)) if m.group(2) else 1.0 - except ValueError: mag = 1.0 - return _CLASS_RANK[cls] * 100 + min(mag, 99.9) - - -def _flare_r_scale(flare_class: Optional[str]) -> Optional[tuple]: - """Parse 'X1.2', 'M5.5', 'C3.1' etc. Return (R-code, label, class_str). - - v0.6-3b: filters to class at-or-above adapter_config.swpc.flare_class_floor - (default 'X1'). Default keeps prior X-only behavior. Lowered floors - accept M-class -> R1/R2.""" - obs_score = _class_score(flare_class) - if obs_score is None: return None - floor_str = str(adapter_config.swpc.flare_class_floor) - floor_score = _class_score(floor_str) - if floor_score is None: floor_score = _CLASS_RANK["X"] * 100 + 1.0 # X1 default - if obs_score < floor_score: return None - - s = str(flare_class).strip().upper() - m = re.match(r"^([ABCMX])([0-9.]+)?", s) - cls = m.group(1) - try: mag = float(m.group(2)) if m.group(2) else 1.0 - except ValueError: mag = 1.0 - if cls == "X": - if mag >= 20: return ("R5", "extreme", s) - if mag >= 10: return ("R4", "severe", s) - return ("R3", "strong", s) - if cls == "M": - if mag >= 5: return ("R2", "moderate", s) - return ("R1", "minor", s) - # B/C/A: no NOAA R-code defined -- skip even if floor allowed entry. - return None - - -def _proton_s_scale(pfu: float) -> Optional[tuple]: - """Return (S-code, label, pfu_value) for proton flux at-or-above the - NOAA S-scale threshold. - - v0.6-3b: gated by adapter_config.swpc.proton_pfu_floor (default 10 = S1). - The S-scale lookup itself is CODE.""" - if pfu < float(adapter_config.swpc.proton_pfu_floor): - return None - for thr, code, label in _S_SCALE_THRESHOLDS: - if pfu >= thr: - return (code, label, pfu) - return None - - -def _extract_kp(d: dict) -> Optional[float]: - for k in ("kp_index", "kp", "k_index", "kindex", "value", "estimated_kp"): - v = d.get(k) - f = _coerce_float(v) - if f is not None: return f - return None - - -# S1+ NOAA scale is calibrated for the >=10 MeV proton channel. Lower- -# energy channels (>=1 MeV, >=5 MeV) have much higher baseline flux and -# would trigger spurious 'storm' events. Only honor these energy labels. -_S_SCALE_RELEVANT_ENERGIES = ("10", ">=10", ">10", ">=10 MeV", ">=10MeV", - "30", ">=30", ">=30 MeV", ">=50 MeV", - ">=100 MeV", ">=100") - - -def _is_relevant_proton_energy(energy) -> bool: - if energy is None: - return False # missing energy label -> can't validate; safer to skip - if isinstance(energy, (int, float)): - return energy >= 10 - s = str(energy).strip() - return s in _S_SCALE_RELEVANT_ENERGIES - - -def _extract_proton_flux(d: dict) -> Optional[float]: - """Match the 'flux at >=10 MeV' channel (or higher). Field names vary; - explicit channel labels win. Envelopes with `energy='>=1 MeV'` or - `'>=5 MeV'` are ALWAYS rejected -- different background floor.""" - # Explicit per-channel field names (already named after the energy). - for k in ("p10mev", "proton_flux_10mev", "flux_10mev", "p_geq_10MeV"): - v = d.get(k) - f = _coerce_float(v) - if f is not None: return f - # Generic flux/value -- require the `energy` field to validate channel. - energy = d.get("energy_mev") or d.get("energy") - if _is_relevant_proton_energy(energy): - for k in ("flux", "value", "proton_flux"): - v = d.get(k) - f = _coerce_float(v) - if f is not None: return f - return None - - -def _extract_flare_class(d: dict) -> Optional[str]: - for k in ("flare_class", "class", "magnitude_class", "x_ray_class"): - v = d.get(k) - if v: return str(v) - # The product_id sometimes encodes the class (e.g. "X1.2 FLARE"). - pid = d.get("product_id") or d.get("message") or "" - m = re.search(r"\b([MX][0-9.]+)\b", str(pid).upper()) - return m.group(0) if m else None - - -def handle_swpc(envelope: dict, subject: str, - data: Optional[dict] = None, - now: Optional[int] = None) -> Optional[str]: - if not isinstance(envelope, dict): return None - inner = envelope.get("data") or {} - adapter = inner.get("adapter") or "" - if adapter not in ("swpc_alerts", "swpc_kindex", "swpc_protons"): - return None - - d = inner.get("data") or {} - now = now if now is not None else _now() - category_raw = inner.get("category") or "" - severity_word = _coerce_severity(inner.get("severity")) - - try: - conn = get_db() - except Exception: - logger.exception("swpc_handler: persistence unavailable") - return None - - event_id = d.get("id") or inner.get("id") or d.get("product_id") - if not event_id: - return None - - # Classify the event + decide. - event_kind = None # "geomag" | "flare" | "proton" - scale_code = None - label = None - scalar_str = None - - if adapter == "swpc_kindex": - kp = _extract_kp(d) - if kp is not None: - g = _kp_g_scale(kp) - if g: - event_kind = "geomag" - scale_code, label = g - scalar_str = f"Kp{int(round(kp))}" - - elif adapter == "swpc_protons": - pfu = _extract_proton_flux(d) - if pfu is not None: - s = _proton_s_scale(pfu) - if s: - event_kind = "proton" - scale_code, label, val = s - scalar_str = f"{int(val) if float(val) >= 1 else val:.0f} pfu" if val >= 1 else f"{val:.1f} pfu" - - elif adapter == "swpc_alerts": - # swpc_alerts can carry any kind. Try Kp first, flare next, proton last. - kp = _extract_kp(d) - if kp is not None: - g = _kp_g_scale(kp) - if g: - event_kind = "geomag"; scale_code, label = g - scalar_str = f"Kp{int(round(kp))}" - if event_kind is None: - fcls = _extract_flare_class(d) - r = _flare_r_scale(fcls) - if r: - event_kind = "flare"; scale_code, label, cls_str = r - scalar_str = cls_str - if event_kind is None: - pfu = _extract_proton_flux(d) - if pfu is not None: - s = _proton_s_scale(pfu) - if s: - event_kind = "proton"; scale_code, label, val = s - scalar_str = f"{int(val)} pfu" if val >= 1 else f"{val:.1f} pfu" - - # Persist + filter. - payload_json = None - try: payload_json = json.dumps(d, default=str)[:8000] - except Exception: payload_json = None - occurred_at = None - t = d.get("time") or d.get("issued_at") or d.get("issue_time") - if isinstance(t, str): - try: - from datetime import datetime as _dt - occurred_at = int(_dt.fromisoformat(t.replace("Z", "+00:00")).timestamp()) - except Exception: pass - elif isinstance(t, (int, float)): - occurred_at = int(t / 1000) if t > 1e12 else int(t) - - if event_kind is None: - # Below threshold (routine Kp, M-class flare, S0 protons, etc). - # Persist for history; log handled=0; no broadcast. - _upsert_swpc(conn, event_id=event_id, adapter=adapter, - payload_json=payload_json, occurred_at=occurred_at or now, - first_seen_at=now, set_last_broadcast=False) - _log_event(conn, now=now, source="swpc", category=category_raw, - severity_word=severity_word, event_id_external=event_id, - subject=subject, handled=0, - table_name="swpc_events", table_pk=event_id) - return None - - # ── Extract detail + time tag ───────────────────────────────────────────── - _detail = d.get("message") or d.get("description") or "" - if isinstance(_detail, str): - _detail = _trunc(_detail.strip()) - else: - _detail = "" - _time_tag = "" - _t_raw = d.get("time") or d.get("issued_at") or d.get("issue_time") or "" - if isinstance(_t_raw, str) and _t_raw: - _time_tag = _t_raw[:16].replace("T", " ") - - # ── NEW ARCH: geomag + flare delegate to gating.swpc.decide() ──────────── - if event_kind in ("geomag", "flare"): - # Build canonical data dict for the decider + formatter. - if event_kind == "geomag": - _kp_val = _extract_kp(d) # idempotent re-extract - canonical: dict = { - "event_id": event_id, - "driver": "kp", - "scalar": _kp_val, - "scale_code": scale_code, - "message": _detail, - "issued_at": _t_raw if _t_raw else None, - } - else: # flare — scalar_str is the class string set by classification - canonical = { - "event_id": event_id, - "driver": "flare", - "scalar": scalar_str, - "scale_code": scale_code, - "message": _detail, - "issued_at": _t_raw if _t_raw else None, - } - - from meshai.notifications.gating.swpc import decide as _swpc_decide - gate = _swpc_decide(canonical, source="swpc", now=float(now)) - - if not gate.broadcast: - logger.debug( - "swpc_handler: geomag/flare suppressed by gating.swpc: %s", - gate.reason, - ) - _upsert_swpc(conn, event_id=event_id, adapter=adapter, - payload_json=payload_json, occurred_at=occurred_at or now, - first_seen_at=now, set_last_broadcast=False) - _log_event(conn, now=now, source="swpc", category=category_raw, - severity_word=severity_word, event_id_external=event_id, - subject=subject, handled=0, - table_name="swpc_events", table_pk=event_id) - return None - - log_id = _log_event_returning_id( - conn, now=now, source="swpc", category=category_raw, - severity_word=severity_word, event_id_external=event_id, - subject=subject, handled=0, - table_name="swpc_events", table_pk=event_id) - - # _upsert_swpc fills in event_type=adapter + payload_json via the - # UPDATE path (decide() already INSERT-OR-IGNOREd the row). - _upsert_swpc(conn, event_id=event_id, adapter=adapter, - payload_json=payload_json, occurred_at=occurred_at or now, - first_seen_at=now, set_last_broadcast=False) - - wire = _render(event_kind, scale_code, label, scalar_str, - is_update=False, detail=_detail, time_tag=_time_tag) - - # Cutover gate: geomag → geomagnetic_storm; flare → rf_propagation_alert. - # Per-derived-category so geomag and flare can be cut over independently. - from meshai.notifications.cutover import is_cutover - _derived_cat = "geomagnetic_storm" if event_kind == "geomag" else "rf_propagation_alert" - - if isinstance(data, dict): - data.update(canonical) - if is_cutover(_derived_cat): - # NEW PATH: gate.data_patch provides _severity_override, _cooldown_suffix. - data.update(gate.data_patch) - data["_broadcast_audit"] = {"table": "swpc_events", "pk": event_id} - _raw_commit = gate.commit - _log_row_id = log_id - - def _on_commit(committed_at: float, - _rc=_raw_commit, _lr=_log_row_id) -> None: - if _rc is not None: - _rc(committed_at) - if _lr is not None: - try: - c = get_db() - c.execute("UPDATE event_log SET handled=1 WHERE id=?", - (int(_lr),)) - except Exception: - logger.exception("swpc commit: event_log update failed") - - data["_on_broadcast_committed"] = _on_commit - else: - # NOT cutover: old-style attach (canonical only, no data_patch). - # gate.commit is intentionally not called; geomag window does not - # tick in shadow-bake mode (acceptable — worst case one extra - # broadcast per restart, caught by shadow_gate diff). - _attach_commit(data, event_id=event_id, event_log_row_id=log_id) - - return wire - - # ── LEGACY path: proton events (solar_radiation_storm) ─────────────────── - # solar_radiation_storm is NOT registered in the gating/formatter - # registries; it stays on this inline legacy path unchanged. - if event_kind != "proton": - logger.warning("swpc_handler: unexpected event_kind=%r; suppressing", event_kind) - return None - - log_id = _log_event_returning_id( - conn, now=now, source="swpc", category=category_raw, - severity_word=severity_word, event_id_external=event_id, - subject=subject, handled=0, - table_name="swpc_events", table_pk=event_id) - - row = conn.execute( - "SELECT last_broadcast_at FROM swpc_events WHERE event_id=?", - (event_id,)).fetchone() - - if row is None: - _upsert_swpc(conn, event_id=event_id, adapter=adapter, - payload_json=payload_json, occurred_at=occurred_at or now, - first_seen_at=now, set_last_broadcast=False) - wire = _render(event_kind, scale_code, label, scalar_str, - is_update=False, detail=_detail, time_tag=_time_tag) - _attach_commit(data, event_id=event_id, event_log_row_id=log_id) - return wire - - if row["last_broadcast_at"] is None: - wire = _render(event_kind, scale_code, label, scalar_str, - is_update=False, detail=_detail, time_tag=_time_tag) - _attach_commit(data, event_id=event_id, event_log_row_id=log_id) - return wire - - # Already broadcast — no Update re-broadcast for SWPC point-in-time events. - return None - - -def _render(event_kind, scale_code, label, scalar_str, - *, is_update: bool = False, detail: str = "", - time_tag: str = "") -> str: - prefix = "Update:" if is_update else "New:" - - if event_kind == "geomag": - line1 = f"🧲 {prefix} {scale_code} Geomagnetic Storm — {scalar_str}" - line2 = _trunc(detail) if detail else "HF degraded, aurora possible" - line3 = f"SWPC · {time_tag}" if time_tag else "SWPC" - elif event_kind == "flare": - line1 = f"☀️ {prefix} {scalar_str} Solar Flare — {scale_code}" - line2 = _trunc(detail) if detail else "HF radio fading, GPS may glitch" - line3 = f"SWPC · {time_tag}" if time_tag else "SWPC" - elif event_kind == "proton": - line1 = f"☢️ {prefix} {scale_code} Radiation Storm — {scalar_str}" - line2 = _trunc(detail) if detail else "Polar HF radio affected" - line3 = f"SWPC · {time_tag}" if time_tag else "SWPC" - else: - line1 = f"⚠️ {prefix} Space Weather Event — {scale_code or '?'}" - line2 = _trunc(detail) if detail else None - line3 = f"SWPC · {time_tag}" if time_tag else "SWPC" - - return "\n".join(l for l in [line1, line2, line3] if l) - - -def _upsert_swpc(conn, *, event_id, adapter, payload_json, occurred_at, - first_seen_at, set_last_broadcast=False, broadcast_at=None) -> None: - existing = conn.execute( - "SELECT 1 FROM swpc_events WHERE event_id=?", (event_id,)).fetchone() - if existing is None: - conn.execute( - "INSERT INTO swpc_events(event_id, event_type, severity_int, " - "payload_json, occurred_at, first_seen_at, last_broadcast_at) " - "VALUES (?,?,?,?,?,?,?)", - (event_id, adapter, None, payload_json, occurred_at, - first_seen_at, broadcast_at if set_last_broadcast else None)) - else: - conn.execute( - "UPDATE swpc_events SET event_type=?, payload_json=?, occurred_at=? " - "WHERE event_id=?", - (adapter, payload_json, occurred_at, event_id)) - - -def _attach_commit(data: Optional[dict], *, event_id: str, - event_log_row_id: Optional[int]) -> None: - if not isinstance(data, dict): return - - def _on_commit(committed_at: float) -> None: - try: conn = get_db() - except Exception: - logger.exception("swpc commit: persistence unavailable"); return - conn.execute( - "UPDATE swpc_events SET last_broadcast_at=?, " - "first_broadcast_at=COALESCE(first_broadcast_at, ?) WHERE event_id=?", - (int(committed_at), int(committed_at), event_id)) - if event_log_row_id is not None: - conn.execute("UPDATE event_log SET handled=1 WHERE id=?", - (int(event_log_row_id),)) - - data["_on_broadcast_committed"] = _on_commit - data["_broadcast_audit"] = {"table": "swpc_events", "pk": event_id} - - -def _coerce_severity(sev: Any) -> Optional[str]: - if sev is None: return None - if isinstance(sev, str): return sev or None - try: return str(int(sev)) - except (TypeError, ValueError): return str(sev) - - -def _log_event(conn, *, now, source, category, severity_word, - event_id_external, subject, handled, table_name, table_pk) -> None: - conn.execute( - "INSERT INTO event_log(received_at, source, category, severity_word, " - "event_id_external, nats_subject, handled, table_name, table_pk) " - "VALUES (?,?,?,?,?,?,?,?,?)", - (now, source, category, severity_word, event_id_external, subject, - int(bool(handled)), table_name, table_pk)) - - -def _log_event_returning_id(conn, *, now, source, category, severity_word, - event_id_external, subject, handled, - table_name, table_pk) -> int: - cur = conn.execute( - "INSERT INTO event_log(received_at, source, category, severity_word, " - "event_id_external, nats_subject, handled, table_name, table_pk) " - "VALUES (?,?,?,?,?,?,?,?,?)", - (now, source, category, severity_word, event_id_external, subject, - int(bool(handled)), table_name, table_pk)) - return int(cur.lastrowid) diff --git a/work/meshai/config.py b/work/meshai/config.py index b9e184b..ab9b229 100644 --- a/work/meshai/config.py +++ b/work/meshai/config.py @@ -596,24 +596,6 @@ class IPAWSConfig(_SourcedFeed): status_actual_only: bool = True -@dataclass -class CentralConsumerConfig: - """Connection settings for the Central NATS JetStream consumer (v0.4). - - v0.5.4 adds `region` — a dotted v0.9.20 region token (e.g. 'us.id' for - Idaho) appended to each subscribed Central subject so the firehose is - filtered server-side. Empty string falls back to bare wildcards (pre- - v0.9.20 behaviour). One region applies to all central adapters; per- - adapter overrides can land in v0.6. - """ - - enabled: bool = False - url: str = "nats://central.echo6.mesh:4222" - durable: str = "meshai-consumer" - connect_timeout: float = 10.0 - region: str = "us.id" - - @dataclass class GeocoderConfig: """Photon reverse geocoder settings.""" @@ -643,7 +625,6 @@ class EnvironmentalConfig: firms: FIRMSConfig = field(default_factory=FIRMSConfig) satpass: SatpassConfig = field(default_factory=SatpassConfig) ipaws: IPAWSConfig = field(default_factory=IPAWSConfig) - central: CentralConsumerConfig = field(default_factory=CentralConsumerConfig) geocoder: GeocoderConfig = field(default_factory=GeocoderConfig) diff --git a/work/meshai/main.py b/work/meshai/main.py index 82c6a9e..c3b85bc 100644 --- a/work/meshai/main.py +++ b/work/meshai/main.py @@ -49,8 +49,6 @@ class MeshAI: self.event_bus = None # Notification pipeline EventBus (v0.3) self._pipeline_scheduler = None # DigestScheduler from start_pipeline() self.env_store = None # Environmental feeds store - self._central_consumer = None # Central NATS consumer (v0.4) - self._central_retry_task = None # Background retry for Central boot-connect self._fire_pacer = None # FirePacer for rate-limited fire broadcasts self.router: Optional[MessageRouter] = None self.responder: Optional[Responder] = None @@ -142,11 +140,6 @@ class MeshAI: if self.env_store is not None: self.env_store._fire_pacer = self._fire_pacer - from .central.consumer import CentralConsumer - self._central_consumer = CentralConsumer(self.config.environmental, self.event_bus) - self._central_consumer._pacer = self._fire_pacer - await self._start_central_consumer_guarded() - logger.info("MeshAI started successfully") # Keep running @@ -294,64 +287,6 @@ class MeshAI: await asyncio.sleep(1.0) logger.info("watchdog loop exited") - async def _start_central_consumer_guarded(self) -> None: - """Start the Central NATS consumer, degrading gracefully on failure. - - If Central is unreachable at boot, logs a WARNING and schedules a - background retry task instead of propagating the exception. The NATS - client's own allow_reconnect handles runtime drops once the initial - connect succeeds, so the retry loop is only for the boot-time window. - """ - try: - await self._central_consumer.start() - except Exception as exc: - logger.warning( - "Central unreachable at startup (%s); continuing without hazard " - "firehose, will retry in background", exc, - ) - # Only spin the retry loop when there are subjects to subscribe to; - # mirrors the no-op guard in CentralConsumer.start() so we never - # retry a no-op (all-native or disabled) configuration. - if self._central_consumer.subjects(): - self._central_retry_task = asyncio.create_task( - self._central_retry_loop() - ) - - async def _central_retry_loop(self) -> None: - """Background task: retry the Central NATS boot-connect with exponential backoff. - - Delay sequence: 30s → 60s → 120s → 240s → 300s (capped). Exits as - soon as connect succeeds or the bot shuts down. Double-start is guarded - by checking _nc before each attempt. - """ - delay = 30.0 - max_delay = 300.0 - while self._running: - try: - await asyncio.sleep(delay) - except asyncio.CancelledError: - return - if not self._running: - return - # Guard against a racing success (e.g. two retries overlapping). - if self._central_consumer._nc is not None: - logger.info("Central retry: already connected, stopping retry loop") - return - try: - await self._central_consumer.start() - logger.info( - "Central connected after delayed boot (retry backoff was %.0fs)", delay - ) - return - except asyncio.CancelledError: - return - except Exception as exc: - next_delay = min(delay * 2, max_delay) - logger.warning( - "Central retry failed (%s); next attempt in %.0fs", exc, next_delay, - ) - delay = next_delay - async def stop(self) -> None: """Stop the bot.""" logger.info("Stopping MeshAI...") @@ -361,16 +296,6 @@ class MeshAI: from .notifications.pipeline import stop_pipeline await stop_pipeline(self._pipeline_scheduler) - if self._central_retry_task is not None: - self._central_retry_task.cancel() - try: - await self._central_retry_task - except (asyncio.CancelledError, Exception): - pass - - if self._central_consumer is not None: - await self._central_consumer.stop() - if self._fire_pacer is not None: await self._fire_pacer.stop() diff --git a/work/pyproject.toml b/work/pyproject.toml index 7220daf..16c56a1 100644 --- a/work/pyproject.toml +++ b/work/pyproject.toml @@ -39,7 +39,6 @@ dependencies = [ "fastapi>=0.110.0", "uvicorn[standard]>=0.27.0", "aiomqtt>=2.0.0", - "nats-py>=2.0.0", ] [project.optional-dependencies] diff --git a/work/tests/test_adapter_avalanche.py b/work/tests/test_adapter_avalanche.py index 5022b5d..e319bae 100644 --- a/work/tests/test_adapter_avalanche.py +++ b/work/tests/test_adapter_avalanche.py @@ -194,49 +194,3 @@ def test_missing_event_id_returns_none(adapter): def test_does_not_raise_on_corrupted_dict(adapter): """Corrupted dict returns None without raising.""" assert adapter.to_event({"garbage": True}) is None - - -# ============================================================================ -# Central avy_handler._render (mesh broadcast wire) -- budget-fit format. -# advice -> FIRST SENTENCE only; zone/level/source kept FULL; fits 140. -# ============================================================================ - -from meshai.central.avy_handler import _render as _avy_render - - -def test_avy_render_advice_first_sentence_only(): - wire = _avy_render( - danger_level=4, danger_name="High", - zone_name="Western Mountains", - center_id="SNFAC", - travel=("Avoid all avalanche terrain today. Natural and human-triggered " - "avalanches are likely on steep slopes."), - ) - # first sentence retained (with its period), the rest dropped - assert "Avoid all avalanche terrain today." in wire - assert "Natural and human-triggered" not in wire - # zone / level / source kept FULL (never abbreviated) - assert "Western Mountains" in wire - assert "High (4)" in wire - assert "SNFAC" in wire - - -def test_avy_render_worst_case_fits_140(): - # Long multi-sentence advice paragraph -- only the first sentence is kept, - # which lets the full zone / level / source survive under the 140 budget. - wire = _avy_render( - danger_level=5, danger_name="Extreme", - zone_name="Western Mountains", - center_id="Sawtooth Avalanche Center", - travel=("Avoid all avalanche terrain today! Very dangerous conditions " - "exist across all elevations and aspects with widespread natural " - "avalanche activity likely through the afternoon and overnight."), - ) - assert len(wire) <= 140, f"{len(wire)} chars:\n{wire!r}" - # zone, level, source, and the first-sentence advice all present - assert "Western Mountains" in wire - assert "Extreme (5)" in wire - assert "Sawtooth Avalanche Center" in wire - assert "Avoid all avalanche terrain today!" in wire - # first sentence terminates at the '!' -> the rest is gone - assert "Very dangerous conditions" not in wire diff --git a/work/tests/test_adapter_config_api.py b/work/tests/test_adapter_config_api.py index 67101c6..8e66188 100644 --- a/work/tests/test_adapter_config_api.py +++ b/work/tests/test_adapter_config_api.py @@ -208,11 +208,16 @@ def test_put_json_accepts_list(client): def test_put_json_accepts_dict(client): + # central/severity_thresholds was removed with the Central handler path; + # any surviving "json"-type key accepts dict values equally (the API + # only checks JSON-serializability, not shape) -- use reminders_wfigs/ + # terminate_when (default is a list) to exercise the dict-value path. r = client.put( - "/api/adapter-config/central/severity_thresholds", + "/api/adapter-config/reminders_wfigs/terminate_when", json={"value": {"routine_max": 0, "priority_max": 1, "immediate_min": 2}}, ) assert r.status_code == 200 + assert r.json()["value"] == {"routine_max": 0, "priority_max": 1, "immediate_min": 2} def test_put_json_accepts_none(client): @@ -291,8 +296,8 @@ def test_list_meta(client): body = r.json() assert "wfigs" in body assert body["wfigs"]["include_in_llm_context"] is True - # central / geocoder default to False - assert body["central"]["include_in_llm_context"] is False + # geocoder defaults to False (central adapter meta was removed with the + # Central handler path) assert body["geocoder"]["include_in_llm_context"] is False diff --git a/work/tests/test_adapter_config_foundation.py b/work/tests/test_adapter_config_foundation.py index 71ce557..ec05635 100644 --- a/work/tests/test_adapter_config_foundation.py +++ b/work/tests/test_adapter_config_foundation.py @@ -153,9 +153,9 @@ def test_registry_has_no_duplicate_keys(): def test_adapter_meta_at_19(fresh_db): - # Count sentinel — bump when an adapter row is added. 23 -> 24 with the - # IPAWS civil-alert adapter (adapter_config/defaults.py ADAPTER_META["ipaws"]). - assert len(ADAPTER_META) == 24 + # Count sentinel — bump when an adapter row is added/removed. 24 -> 23 + # with the removal of ADAPTER_META["central"] (dead NATS consumer excised). + assert len(ADAPTER_META) == 23 # ---------- seed ---------------------------------------------------------- @@ -303,12 +303,6 @@ def test_accessor_returns_json_list(fresh_db): assert adapter_config.nws.tombstone_msgtypes == ["Cancel", "Expire"] -def test_accessor_returns_json_dict(fresh_db): - invalidate_cache() - v = adapter_config.central.severity_thresholds - assert v == {"routine_max": 1, "priority_max": 2, "immediate_min": 3} - - def test_accessor_returns_json_none(fresh_db): invalidate_cache() assert adapter_config.firms.bbox is None diff --git a/work/tests/test_avalanche_refactor.py b/work/tests/test_avalanche_refactor.py index 73e190a..eaef994 100644 --- a/work/tests/test_avalanche_refactor.py +++ b/work/tests/test_avalanche_refactor.py @@ -1,17 +1,20 @@ """Phase-1 avalanche refactor tests — formatter+decider architecture. -Four test groups: +The Central `avy_handler` module (and its centralseverity→NAADS remap, +`handle_avy()`, `_render()`) has been deleted — the native path is the only +production path now. Pure old-vs-new parity tests and tests of +Central-only logic (`_remap_centralseverity`, `handle_avy`) have been +removed; original diffs are preserved in git history. What remains +exercises native code directly (hand-written expected strings are kept as +regression pins on the current wire format). -1. Parity (tier-b): formatter renders from canonical data. - Expected strings are hand-written (the new correct format). - OLD _render() output for the same fixture is captured in comments so the - intended tier-b diff is explicit and reviewable. - The centralseverity=2 (Considerable) case is OLD-vs-NEW identical. - The is_update=True case shows the Update: prefix diff. +Three test groups: -2. Cross-source identity: native AvalancheAdapter.to_event() builds the same - canonical data as the Central path for fixture 0000. Both render - byte-identically via the registered formatter. +1. Parity (tier-b): formatter renders from canonical data. Expected + strings are hand-written (the current correct format). + +2. Cross-source identity: native AvalancheAdapter.to_event() canonical data + renders correctly via the registered formatter. 3. Gate-sequence: replay canonical data through gating.avalanche.decide(). Verify danger-level gate (below / at / above threshold) and first→update @@ -66,16 +69,6 @@ def _canonical_from_fixture(fixture: dict, *, is_update: bool = False) -> dict: } -def _avy_render_old(*, danger_level: int, danger_name: str, zone_name: str, - center_id: str, travel: str) -> str: - """Capture OLD _render() output from avy_handler for diff comments.""" - from meshai.central.avy_handler import _render - return _render( - danger_level=danger_level, danger_name=danger_name, - zone_name=zone_name, center_id=center_id, travel=travel, - ) - - # ───────────────────────────────────────────────────────────────────────────── # 1. Parity (tier-b) — formatter renders from canonical data # ───────────────────────────────────────────────────────────────────────────── @@ -117,17 +110,6 @@ class TestFormatterParity: assert_byte_identical(result, expected) - # Confirm old _render() is identical for this case (no tier-b diff) - old_wire = _avy_render_old( - danger_level=3, danger_name="Considerable", - zone_name="Sawtooth Mountains", center_id="SNFAC", - travel="Dangerous conditions on steep slopes. Conservative decision-making is advised.", - ) - assert_byte_identical(result, old_wire), ( - "For fixture 0000 (Considerable, is_update=False) old and new " - "outputs must be identical — tier-b diff only appears for is_update=True." - ) - def test_fixture_0001_high_warning_prefix(self): """Fixture 0001 (High, NAADS 4) → formatter uses 'WARNING:' prefix. @@ -157,15 +139,6 @@ class TestFormatterParity: assert_byte_identical(result, expected) - old_wire = _avy_render_old( - danger_level=4, danger_name="High", - zone_name="Banner Summit", center_id="SNFAC", - travel="Avoid all avalanche terrain today. Natural avalanches are likely on steep slopes.", - ) - assert_byte_identical(result, old_wire), ( - "For High (level=4, is_update=False) old and new must be identical." - ) - def test_fixture_0002_extreme_warning_prefix(self): """Fixture 0002 (Extreme, NAADS 5) → formatter uses 'WARNING:' prefix. @@ -196,17 +169,7 @@ class TestFormatterParity: assert_byte_identical(result, expected) def test_tier_b_update_prefix_rendered(self): - """Tier-b: is_update=True produces 'AVY Update:' prefix. - - OLD _render() output (no is_update path): - '⛷ AVY Watch: Sawtooth Mountains — Considerable (3) - Dangerous conditions on steep slopes. - SNFAC · valid today' - NEW formatter output (is_update=True): - '⛷ AVY Update: Sawtooth Mountains — Considerable (3) - Dangerous conditions on steep slopes. - SNFAC · valid today' - """ + """Tier-b: is_update=True produces 'AVY Update:' prefix.""" from meshai.notifications.formatters.avalanche import format as avyfmt fixtures = load_fixtures("avalanche") @@ -214,17 +177,6 @@ class TestFormatterParity: # Extract with is_update=True canonical = _canonical_from_fixture(fx, is_update=True) - # OLD _render() output (no is_update support) - old_wire = _avy_render_old( - danger_level=3, danger_name="Considerable", - zone_name="Sawtooth Mountains", center_id="SNFAC", - travel="Dangerous conditions on steep slopes. Conservative decision-making is advised.", - ) - expected_old = ( - "⛷ AVY Watch: Sawtooth Mountains — Considerable (3)" - "\nDangerous conditions on steep slopes." - "\nSNFAC · valid today" - ) expected_new = ( "⛷ AVY Update: Sawtooth Mountains — Considerable (3)" "\nDangerous conditions on steep slopes." @@ -234,7 +186,6 @@ class TestFormatterParity: with pinned_time(_AT): result = avyfmt(_make_fake_event(canonical), now=_AT, budget=140) - assert_byte_identical(old_wire, expected_old) assert_byte_identical(result, expected_new) assert "Update:" in result assert "Watch:" not in result @@ -435,39 +386,6 @@ class TestCrossSourceIdentity: "the formatter registry governs rendering" ) - def test_handle_avy_writes_canonical_into_data(self): - """handle_avy() writes canonical fields into the shared data dict on broadcast.""" - from meshai.central.avy_handler import handle_avy - - envelope = { - "id": "avy-snfac-sawtooth-test", - "data": { - "adapter": "avalanche_org", - "category": "advisory.us.id", - "severity": 2, - "geo": {"centroid": [-114.9, 43.8]}, - "data": { - "danger_level": 3, - "danger_name": "Considerable", - "zone_name": "Sawtooth Mountains", - "center_id": "SNFAC", - "travel_advice": "Dangerous conditions.", - }, - }, - } - data: dict = {} - wire = handle_avy(envelope, "central.avy.advisory.us.id", data=data) - - assert wire is not None, "handle_avy must return a wire string on broadcast" - # Canonical fields must be in the shared data dict - assert data.get("danger_level") == 3 - assert data.get("danger_name") == "Considerable" - assert data.get("zone_name") == "Sawtooth Mountains" - assert data.get("center_id") == "SNFAC" - assert "_on_broadcast_committed" in data - assert "_broadcast_audit" in data - - # ───────────────────────────────────────────────────────────────────────────── # 3. Gate-sequence — danger-level gate + first→update trend # ───────────────────────────────────────────────────────────────────────────── @@ -541,86 +459,6 @@ class TestGateSequence: result = decide({"zone_name": "test"}, source="avalanche", now=_AT) assert result.broadcast is False - def test_centralseverity_remap_considerable(self): - """centralseverity=2 → NAADS 3 (Considerable) → broadcast at min_level=3. - - This tests the remap logic in _remap_centralseverity() as used by - handle_avy() when data.data.danger_level is absent. - - ⚠ Mapping table (needs live validation in-season Oct+): - centralseverity 2 → NAADS 3 (Considerable) [documented] - centralseverity 3 → NAADS 4 (High) [documented] - centralseverity 4 → NAADS 5 (Extreme) [documented] - centralseverity 0 → NAADS 1 (Low) [inferred] - centralseverity 1 → NAADS 2 (Moderate) [inferred] - """ - from meshai.central.avy_handler import _remap_centralseverity - - # Documented values - assert _remap_centralseverity(2) == 3, "centralseverity 2 → NAADS 3 (Considerable)" - assert _remap_centralseverity(3) == 4, "centralseverity 3 → NAADS 4 (High)" - assert _remap_centralseverity(4) == 5, "centralseverity 4 → NAADS 5 (Extreme)" - # Inferred values - assert _remap_centralseverity(0) == 1, "centralseverity 0 → NAADS 1 (Low) [inferred]" - assert _remap_centralseverity(1) == 2, "centralseverity 1 → NAADS 2 (Moderate) [inferred]" - # Out-of-range → 0 (No Rating) - assert _remap_centralseverity(5) == 0, "centralseverity 5 → 0 (No Rating)" - assert _remap_centralseverity(-1) == 0 - assert _remap_centralseverity("?") == 0 - - def test_centralseverity_gate_sequence_via_fixtures(self): - """Gate-sequence via fixtures: centralseverity 2/3/4 all broadcast; 1 suppresses. - - Uses handle_avy to exercise the full Central ingestion path including - the centralseverity → NAADS remap and the decide() call. - """ - from meshai.central.avy_handler import handle_avy - - def _make_envelope(centralseverity: int, naads_level: int, - danger_name: str) -> dict: - return { - "data": { - "adapter": "avalanche_org", - "category": "advisory.us.id", - "severity": centralseverity, - "geo": {"centroid": [-114.9, 43.8]}, - "data": { - "danger_level": naads_level, - "danger_name": danger_name, - "zone_name": "Test Zone", - "center_id": "SNFAC", - "travel_advice": "Test advice.", - }, - } - } - - # centralseverity=1 → NAADS 2 (Moderate) → suppressed (below min_level=3) - env_moderate = _make_envelope(1, 2, "Moderate") - data_m: dict = {} - wire_m = handle_avy(env_moderate, "central.avy.advisory.us.id", data=data_m) - assert wire_m is None, "Moderate (NAADS 2) must be suppressed" - - # centralseverity=2 → NAADS 3 (Considerable) → broadcast - env_considerable = _make_envelope(2, 3, "Considerable") - data_c: dict = {} - wire_c = handle_avy(env_considerable, "central.avy.advisory.us.id", data=data_c) - assert wire_c is not None, "Considerable (NAADS 3) must broadcast" - assert "Watch" in wire_c - - # centralseverity=3 → NAADS 4 (High) → broadcast with WARNING - env_high = _make_envelope(3, 4, "High") - data_h: dict = {} - wire_h = handle_avy(env_high, "central.avy.advisory.us.id", data=data_h) - assert wire_h is not None, "High (NAADS 4) must broadcast" - assert "WARNING" in wire_h - - # centralseverity=4 → NAADS 5 (Extreme) → broadcast with WARNING - env_extreme = _make_envelope(4, 5, "Extreme") - data_e: dict = {} - wire_e = handle_avy(env_extreme, "central.avy.advisory.us.id", data=data_e) - assert wire_e is not None, "Extreme (NAADS 5) must broadcast" - assert "WARNING" in wire_e - def test_is_update_propagated_into_data_patch(self): """decide() data_patch.is_update reflects the incoming is_update flag.""" from meshai.notifications.gating.avalanche import decide diff --git a/work/tests/test_avalanche_v057.py b/work/tests/test_avalanche_v057.py deleted file mode 100644 index 7bff45b..0000000 --- a/work/tests/test_avalanche_v057.py +++ /dev/null @@ -1,140 +0,0 @@ -"""v0.5.7-avalanche: Central avalanche check + categories audit. - -Covers two things shipped in v0.5.7-avalanche: - -1. Central avalanche adapter check -- VERIFIED ABSENT in Central v0.10.0. - The guide (docs/CONSUMER-INTEGRATION.md at v0.10.0-itd-511) has zero - `avalanche` / `NWAC` / `CAIC` references, and the producer source tree - (src/central/adapters/) has no avalanche-named adapter files. meshai's - consumer already documents this explicitly: _subjects_for("avalanche", *) - returns [], and _subject_owned() logs a warning if someone flips - avalanche.feed_source=central. This phase pins those invariants so a - future refactor that introduces an avalanche Central wire breaks - loudly here. - -2. ALERT_CATEGORIES avalanche-family audit. Native avalanche.py emits two - categories based on NWAC/CAIC danger_level: - danger_level >= 4 (High, Extreme) -> avalanche_warning - danger_level == 3 (Considerable) -> avalanche_watch - danger_level <= 2 (Low, Moderate) -> silently dropped - Pre-v0.5.7-avalanche the registry had avalanche_warning + - avalanche_considerable. avalanche_considerable was a legacy name for - the Considerable-danger tier; native code now emits avalanche_watch - for the same semantic. Added avalanche_watch in v0.5.7-avalanche; - kept avalanche_considerable as a forward-compat target (no migration - churn). -""" - -import inspect -import re - -import pytest - -from meshai.central.consumer import ( - CENTRAL_ADAPTER_TO_SOURCE, - CentralConsumer, - _SUBJECTS_BARE, - _subjects_for, -) -from meshai.config import EnvironmentalConfig -from meshai.notifications.categories import ALERT_CATEGORIES - - -# ---------- FIX 1: Central has no avalanche adapter ----------------------- - - -def test_avalanche_has_no_central_subscription(): - """_subjects_for returns empty for the avalanche source regardless of - region (no Central counterpart exists in v0.10.0).""" - for region in ("us.id", "us.mt", "us.co", "", None): - # Avalanche was added to central pipeline; verify it has subjects. - assert _subjects_for("avalanche", region) != [], \ - f"expected subjects for region={region!r}" - - -def test_avalanche_absent_from_subjects_bare(): - """The bare-wildcard table also has no avalanche entry.""" - assert "avalanche" in _SUBJECTS_BARE - - -def test_avalanche_absent_from_central_adapter_remap(): - """No Central adapter name remaps to meshai's 'avalanche' source.""" - assert "avalanche" in CENTRAL_ADAPTER_TO_SOURCE.values(), \ - f"avalanche should have a remap entry: {CENTRAL_ADAPTER_TO_SOURCE}" - - -def test_avalanche_feed_source_central_subscribes_nothing(): - """If a user accidentally sets avalanche.feed_source=central, the - subject_owned() builder must not emit a subscription (and the - consumer logs a warning -- documented in consumer.py).""" - env = EnvironmentalConfig() - env.avalanche.feed_source = "central" - so = CentralConsumer(env, None)._subject_owned() - # No subjects added for avalanche; nothing to subscribe to. - assert not any("avalanche" in s.lower() for s in so.keys()) - - -# ---------- FIX 2: ALERT_CATEGORIES avalanche-family audit --------------- - - -def test_avalanche_watch_in_registry(): - """v0.5.7-avalanche: avalanche_watch is now registry-present so the - Advanced Rules editor can target Considerable-tier emissions.""" - assert "avalanche_watch" in ALERT_CATEGORIES - info = ALERT_CATEGORIES["avalanche_watch"] - assert info["toggle"] == "avalanche" - assert info["default_severity"] == "routine" - assert info["name"] - assert info["description"] - assert info["example_message"] - - -def test_avalanche_warning_still_in_registry(): - """Pre-v0.5.7-avalanche entry survives the edit.""" - assert "avalanche_warning" in ALERT_CATEGORIES - assert ALERT_CATEGORIES["avalanche_warning"]["toggle"] == "avalanche" - - -def test_avalanche_considerable_legacy_kept(): - """avalanche_considerable kept as forward-compat / legacy target even - though no current code path emits it. Documented in the commit body - and categories.py inline note for future cleanup.""" - assert "avalanche_considerable" in ALERT_CATEGORIES - assert ALERT_CATEGORIES["avalanche_considerable"]["toggle"] == "avalanche" - - -def _native_emitted_avalanche_categories() -> set[str]: - """Walk avalanche.py for category= literals routing to toggle=avalanche.""" - from meshai.env import avalanche as aval_mod - src = inspect.getsource(aval_mod) - emitted = set(re.findall(r'category\s*=\s*"([a-z_]+)"', src)) - return {c for c in emitted if c in ALERT_CATEGORIES - and ALERT_CATEGORIES[c].get("toggle") == "avalanche"} - - -def test_alert_categories_avalanche_complete(): - """Every category native avalanche.py emits must have a registry entry - under toggle='avalanche'. Legacy entries without an emitter are - allowed (subset assertion, not equality).""" - registry_avalanche = { - cid for cid, info in ALERT_CATEGORIES.items() - if info.get("toggle") == "avalanche" - } - native = _native_emitted_avalanche_categories() - missing = native - registry_avalanche - assert not missing, f"avalanche emit set missing from ALERT_CATEGORIES: {missing}" - # Sanity: the two v0.5.7-avalanche-recognized categories are both there. - assert "avalanche_warning" in native, "native should emit avalanche_warning" - assert "avalanche_watch" in native, "native should emit avalanche_watch" - - -@pytest.mark.parametrize( - "cat", ["avalanche_warning", "avalanche_watch", "avalanche_considerable"], -) -def test_avalanche_categories_have_required_fields(cat): - info = ALERT_CATEGORIES[cat] - assert info["toggle"] == "avalanche" - assert info["name"] - assert info["description"] - assert info["default_severity"] in {"routine", "priority", "immediate"} - assert info["example_message"] diff --git a/work/tests/test_central_boot_guard.py b/work/tests/test_central_boot_guard.py deleted file mode 100644 index a5e7c26..0000000 --- a/work/tests/test_central_boot_guard.py +++ /dev/null @@ -1,310 +0,0 @@ -"""Tests for Central NATS boot-time grace + retry (fix/central-boot-guard). - -meshai.main cannot be imported in the test environment (missing runtime deps: -openai, aiosqlite, meshtastic, …). The spec allows unit-testing the guard -helper in isolation. We do this by: - - 1. Embedding the exact method bodies from main.py into a minimal async class - (BootGuard) that exposes only what the methods need. If the method body - in main.py changes, the test will naturally drift — it exists to catch - regressions in the guarded-connect-and-retry contract. - 2. Separately testing CentralConsumer.start() no-op guard (already - exercised in test_central_consumer.py; duplicated here as a sanity check - that the NATS connect path is never reached when nothing is configured). - -The methods under test (copied verbatim from meshai/main.py): - • _start_central_consumer_guarded - • _central_retry_loop -""" - -import asyncio -import logging -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest - -logger = logging.getLogger(__name__) - - -# --------------------------------------------------------------------------- -# Minimal class that replicates the two new methods from MeshAI, verbatim. -# This is intentional: if the logic in main.py changes, the copyed body here -# drifts and the test surfaces the mismatch. -# --------------------------------------------------------------------------- - -class BootGuard: - """Thin stand-in for the two guard methods on MeshAI.""" - - def __init__(self, consumer, running=True): - self._central_consumer = consumer - self._central_retry_task = None - self._running = running - - async def _start_central_consumer_guarded(self) -> None: - try: - await self._central_consumer.start() - except Exception as exc: - logger.warning( - "Central unreachable at startup (%s); continuing without hazard " - "firehose, will retry in background", exc, - ) - if self._central_consumer.subjects(): - self._central_retry_task = asyncio.create_task( - self._central_retry_loop() - ) - - async def _central_retry_loop(self) -> None: - delay = 30.0 - max_delay = 300.0 - while self._running: - try: - await asyncio.sleep(delay) - except asyncio.CancelledError: - return - if not self._running: - return - if self._central_consumer._nc is not None: - logger.info("Central retry: already connected, stopping retry loop") - return - try: - await self._central_consumer.start() - logger.info( - "Central connected after delayed boot (retry backoff was %.0fs)", delay - ) - return - except asyncio.CancelledError: - return - except Exception as exc: - next_delay = min(delay * 2, max_delay) - logger.warning( - "Central retry failed (%s); next attempt in %.0fs", exc, next_delay, - ) - delay = next_delay - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -def _consumer(subjects=None, nc=None): - c = MagicMock() - c.subjects.return_value = subjects if subjects is not None else [] - c._nc = nc - return c - - -# --------------------------------------------------------------------------- -# _start_central_consumer_guarded: success path -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_guarded_start_success_does_not_raise(): - """When start() succeeds, no exception propagates and no retry is created.""" - c = _consumer(subjects=["central.quake.>"]) - c.start = AsyncMock() - g = BootGuard(c) - - await g._start_central_consumer_guarded() - - c.start.assert_awaited_once() - assert g._central_retry_task is None - - -# --------------------------------------------------------------------------- -# _start_central_consumer_guarded: failure paths -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_guarded_start_failure_does_not_raise(): - """Exception from start() is swallowed — boot continues.""" - c = _consumer(subjects=["central.wx.alert.us.id.>"]) - c.start = AsyncMock(side_effect=Exception("nats: no servers")) - g = BootGuard(c) - - await g._start_central_consumer_guarded() # must NOT raise - - -@pytest.mark.asyncio -async def test_guarded_start_failure_schedules_retry_when_subjects_nonempty(): - """Exception + non-empty subjects → retry task is created.""" - c = _consumer(subjects=["central.wx.alert.us.id.>"]) - c.start = AsyncMock(side_effect=Exception("nats: no servers")) - g = BootGuard(c) - - await g._start_central_consumer_guarded() - - assert g._central_retry_task is not None - g._central_retry_task.cancel() - try: - await g._central_retry_task - except (asyncio.CancelledError, Exception): - pass - - -@pytest.mark.asyncio -async def test_guarded_start_failure_no_retry_when_no_subjects(): - """Exception + empty subjects (all-native/disabled config) → no retry task.""" - c = _consumer(subjects=[]) - c.start = AsyncMock(side_effect=Exception("unexpected")) - g = BootGuard(c) - - await g._start_central_consumer_guarded() - - assert g._central_retry_task is None - - -# --------------------------------------------------------------------------- -# _central_retry_loop -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_retry_loop_exits_on_success(): - """Loop calls start(), which succeeds by setting _nc, then exits.""" - c = _consumer(subjects=["central.quake.>"]) - - async def succeed(): - c._nc = MagicMock() - - c.start = AsyncMock(side_effect=succeed) - g = BootGuard(c) - - with patch("asyncio.sleep", new=AsyncMock()): - await g._central_retry_loop() - - c.start.assert_awaited_once() - - -@pytest.mark.asyncio -async def test_retry_loop_exits_on_cancel_during_sleep(): - """CancelledError from sleep causes clean exit without calling start().""" - c = _consumer(subjects=["central.quake.>"]) - c.start = AsyncMock() - g = BootGuard(c) - - async def raise_cancel(*_): - raise asyncio.CancelledError - - with patch("asyncio.sleep", new=AsyncMock(side_effect=raise_cancel)): - await g._central_retry_loop() # must not raise - - c.start.assert_not_awaited() - - -@pytest.mark.asyncio -async def test_retry_loop_skips_if_already_connected(): - """If _nc is already set when the retry fires, loop exits without start().""" - c = _consumer(subjects=["central.quake.>"], nc=MagicMock()) - c.start = AsyncMock() - g = BootGuard(c) - - with patch("asyncio.sleep", new=AsyncMock()): - await g._central_retry_loop() - - c.start.assert_not_awaited() - - -@pytest.mark.asyncio -async def test_retry_loop_stops_when_not_running(): - """_running=False causes the loop to exit after the first sleep.""" - c = _consumer(subjects=["central.quake.>"]) - c.start = AsyncMock() - g = BootGuard(c, running=False) - - with patch("asyncio.sleep", new=AsyncMock()): - await g._central_retry_loop() - - c.start.assert_not_awaited() - - -@pytest.mark.asyncio -async def test_retry_loop_backoff_accumulates(): - """Delay doubles each cycle, capped at 300s.""" - c = _consumer(subjects=["central.quake.>"]) - call_count = 0 - - async def start_on_third(): - nonlocal call_count - call_count += 1 - if call_count < 3: - raise Exception("still down") - c._nc = MagicMock() - - c.start = AsyncMock(side_effect=start_on_third) - g = BootGuard(c) - - sleep_delays = [] - - async def record_sleep(d): - sleep_delays.append(d) - - with patch("asyncio.sleep", new=AsyncMock(side_effect=record_sleep)): - await g._central_retry_loop() - - assert call_count == 3 - assert sleep_delays == [30.0, 60.0, 120.0] - - -@pytest.mark.asyncio -async def test_retry_loop_caps_delay_at_max(): - """Delay is capped at 300s after enough failures.""" - c = _consumer(subjects=["central.quake.>"]) - delays_seen = [] - call_count = 0 - - async def always_fail(): - nonlocal call_count - call_count += 1 - if call_count >= 8: - # Eventually succeed so the test terminates - c._nc = MagicMock() - return - raise Exception("still down") - - c.start = AsyncMock(side_effect=always_fail) - g = BootGuard(c) - - async def record_sleep(d): - delays_seen.append(d) - - with patch("asyncio.sleep", new=AsyncMock(side_effect=record_sleep)): - await g._central_retry_loop() - - # After several doublings, delay must be capped at 300s, not grow unbounded - assert max(delays_seen) == 300.0 - - -# --------------------------------------------------------------------------- -# CentralConsumer.start() no-op guard (component-level sanity check) -# --------------------------------------------------------------------------- - - -def test_consumer_start_is_noop_when_unconfigured(): - """start() must not attempt NATS connect when no adapter is central-sourced. - - This is the direct regression guard: if CentralConsumer.start() were to - call nats.connect() unconditionally it would fail in this environment - (no real NATS server), proving the guard works. - - Note: the conftest seeds adapter_config from the DB which may flip satpass - to feed_source=central. We override all adapters to native explicitly so - subjects() is empty and start() must be a pure no-op. - """ - from meshai.config import EnvironmentalConfig - from meshai.central.consumer import CentralConsumer, _SUBJECTS_BARE - from meshai.notifications.pipeline.bus import EventBus - - env = EnvironmentalConfig() - # Force all known adapters to native so subjects() returns [] - for attr in list(_SUBJECTS_BARE.keys()) + ["avalanche", "ducting"]: - cfg = getattr(env, attr, None) - if cfg is not None and hasattr(cfg, "feed_source"): - cfg.feed_source = "native" - - bus = EventBus() - c = CentralConsumer(env, bus) - assert c.subjects() == [], f"Expected no subjects, got: {c.subjects()}" - asyncio.run(c.start()) # must not raise, must not touch NATS - assert c._nc is None diff --git a/work/tests/test_central_consumer.py b/work/tests/test_central_consumer.py deleted file mode 100644 index 32a9db8..0000000 --- a/work/tests/test_central_consumer.py +++ /dev/null @@ -1,221 +0,0 @@ -"""v0.4 C.1: Central connector backend — normalization, lifecycle, source gate.""" - -import asyncio -import json - -import pytest - -from meshai.config import EnvironmentalConfig -from meshai.central.consumer import CentralConsumer, map_category, map_severity -from meshai.notifications.pipeline.bus import EventBus - -pytestmark = pytest.mark.skip( - reason="v0.5.13 default-deny: consumer-level tests assumed envelopes without a handler-synthesized wire still emit an Event with title fallback. New architecture (test_consumer_default_deny.py) verifies the inverse: default-deny when no handler synthesized. v0.6 will rebuild source-remap tests.") - - - -def make_consumer(): - env = EnvironmentalConfig() - bus = EventBus() - rec = [] - bus.subscribe(rec.append) - return CentralConsumer(env, bus), env, rec - - -def envelope(adapter="usgs_quake", category="quake.event", severity=2, - eid="us6000abcd", centroid=(-114.5, 42.6), upstream=None, - time="2026-05-27T12:00:00Z", expires=None): - return { - "id": eid, "source": "central.echo6.co", - "type": f"central.{category}.v1", "time": time, - "centralcategory": category, "centralseverity": severity, - "specversion": "1.0", "datacontenttype": "application/json", - "data": { - "id": eid, "adapter": adapter, "category": category, - "time": time, "expires": expires, "severity": severity, - "geo": {"centroid": list(centroid), "bbox": None, - "regions": ["US-ID"], "primary_region": "US-ID"}, - "data": upstream if upstream is not None else {"magnitude": 4.2, "place": "near Twin Falls"}, - }, - } - - -class FakeMsg: - def __init__(self, subject, env): - self.subject = subject - self.data = json.dumps(env).encode() - self.acked = False - - async def ack(self): - self.acked = True - - -# ---- subject derivation / source gate ---- - -def test_no_subjects_when_all_native(): - c, env, rec = make_consumer() - assert c.subjects() == [] - - -def test_subjects_when_central(): - # v0.5.4: assert the legacy bare-wildcard form by clearing region. - # Region-aware subject shapes are covered by test_central_region_routing.py. - c, env, rec = make_consumer() - env.central.region = "" - env.usgs_quake.feed_source = "central" - assert "central.quake.>" in c.subjects() - - -def test_source_central_skips_native_instantiation(): - from meshai.env.store import EnvironmentalStore - env = EnvironmentalConfig() - env.enabled = True - env.usgs_quake.enabled = True - env.usgs_quake.feed_source = "central" # should be skipped natively - env.nws.enabled = True # native -> present - store = EnvironmentalStore(config=env, region_anchors=[], event_bus=None) - assert "usgs_quake" not in store._adapters - assert "nws" in store._adapters - - -# ---- normalization ---- - -def test_normalize_and_emit(): - c, env, rec = make_consumer() - ev = c._handle("central.quake.event.moderate", json.dumps(envelope()).encode()) - assert ev is not None - assert len(rec) == 1 - e = rec[0] - assert e.source == "usgs_quake" - assert e.category == "earthquake_event" - assert e.severity == "priority" # central severity 2 - assert e.lat == 42.6 and e.lon == -114.5 # [lon,lat] -> (lat,lon) - assert e.group_key == "us6000abcd" - assert e.region == "US-ID" - assert e.data.get("magnitude") == 4.2 # upstream preserved verbatim - - -def test_enriched_preserved_verbatim(): - c, env, rec = make_consumer() - up = {"magnitude": 5.1, "_enriched": {"geocoder": {"state": "Idaho"}, "usgs_stats": {"x": 1}}} - ev = c._handle("central.quake.event.strong", json.dumps(envelope(severity=4, upstream=up)).encode()) - assert ev.severity == "immediate" - assert ev.data["_enriched"]["geocoder"]["state"] == "Idaho" - assert ev.data["_enriched"]["usgs_stats"] == {"x": 1} - - -def test_tombstone_translates_to_clear(): - c, env, rec = make_consumer() - msg = envelope(adapter="gdacs", category="disaster.fl.removed", severity=0, eid="FL1103885:removed") - ev = c._handle("central.disaster.fl.removed.austria", json.dumps(msg).encode()) - assert ev is not None - assert ev.group_key == "FL1103885" # ':removed' stripped -> matches original - assert ev.data.get("_central_tombstone") is True - - -def test_severity_mapping(): - assert map_severity(0) == "routine" - assert map_severity(1) == "routine" - assert map_severity(2) == "priority" - assert map_severity(3) == "immediate" - assert map_severity(4) == "immediate" - assert map_severity(None) == "routine" - - -def test_category_mapping(): - assert map_category("wx.alert.severe_thunderstorm_warning") == "weather_warning" - assert map_category("quake.event") == "earthquake_event" - assert map_category("fire.hotspot.viirs_noaa20.high") == "wildfire_hotspot" - assert map_category("hydro.00060.usgs.06901250") == "stream_flow" - - -# ---- async callback path ---- - -def test_on_message_emits_and_acks(): - c, env, rec = make_consumer() - msg = FakeMsg("central.quake.event.moderate", envelope()) - asyncio.run(c._on_message(msg)) - assert msg.acked is True - assert len(rec) == 1 - - -def test_start_no_op_when_all_native(): - """start() is a no-op (no NATS connect) when no adapter is central.""" - c, env, rec = make_consumer() - asyncio.run(c.start()) # must not raise / must not require NATS - assert c._nc is None - - -def test_consumer_config_uses_deliver_policy_new(): - """C.3.1: Central subscriptions use deliver_policy=NEW (no full-backlog replay).""" - from meshai.central.consumer import consumer_config - from nats.js.api import DeliverPolicy - assert consumer_config().deliver_policy == DeliverPolicy.NEW - - -def test_subject_domain_fallback_for_unmapped_category(): - """D.1: an unmapped category falls back to the subject domain instead - of returning 'other'. - - v0.5.7-traffic note: 'work_zone.wzdx' is now MAPPED (-> 'work_zone'), - so we use a genuinely-unmapped category string here to exercise the - fallback path. The subject-domain fallback for central.traffic.* is - still 'traffic_congestion'. - """ - import json - from meshai.central.consumer import CentralConsumer, category_from_subject - from meshai.config import EnvironmentalConfig - from meshai.notifications.pipeline.bus import EventBus - assert category_from_subject("central.traffic.work_zone.ok") == "traffic_congestion" - rec = [] - bus = EventBus(); bus.subscribe(rec.append) - c = CentralConsumer(EnvironmentalConfig(), bus) - env = {"id": "wz1", "data": {"id": "wz1", "adapter": "wzdx", - "category": "telematics.unknown_thing", "time": "2026-05-28T00:00:00Z", "severity": 1, - "geo": {"centroid": [-96.2, 36.15], "primary_region": "US-OK", "regions": ["US-OK"]}, - "data": {"road": "I-44"}}} - ev = c._handle("central.traffic.work_zone.ok", json.dumps(env).encode()) - assert ev is not None and ev.category == "traffic_congestion" - - -def test_v057_traffic_work_zone_now_mapped(): - """v0.5.7-traffic: 'work_zone.wzdx' maps to the new 'work_zone' meshai - category (not flattened to traffic_congestion).""" - import json - from meshai.central.consumer import CentralConsumer - from meshai.config import EnvironmentalConfig - from meshai.notifications.pipeline.bus import EventBus - rec = [] - bus = EventBus(); bus.subscribe(rec.append) - c = CentralConsumer(EnvironmentalConfig(), bus) - env = {"id": "wz2", "data": {"id": "wz2", "adapter": "wzdx", - "category": "work_zone.wzdx", "time": "2026-05-28T00:00:00Z", "severity": 1, - "geo": {"centroid": [-114.0, 42.0], "primary_region": "US-ID", "regions": ["US-ID"]}, - "data": {"road": "I-84"}}} - ev = c._handle("central.traffic.work_zone.id", json.dumps(env).encode()) - assert ev is not None and ev.category == "work_zone" - - -@pytest.mark.parametrize("adapter,expected", [ - ("wfigs_incidents", "fires"), - ("nwis", "usgs"), - ("swpc_alerts", "swpc"), - ("wzdx", "traffic"), - ("nws", "nws"), # 1:1 passthrough - ("experimental_foo", "experimental_foo"), # unknown -> passthrough -]) -def test_central_adapter_source_remap(adapter, expected): - """D.2: Central adapter names map to meshai source names (unknown passes through).""" - import json - from meshai.central.consumer import CentralConsumer - from meshai.config import EnvironmentalConfig - from meshai.notifications.pipeline.bus import EventBus - rec = [] - bus = EventBus(); bus.subscribe(rec.append) - c = CentralConsumer(EnvironmentalConfig(), bus) - env = {"id": "e1", "data": {"id": "e1", "adapter": adapter, "category": "wx.alert.x", - "time": "2026-05-28T00:00:00Z", "severity": 1, - "geo": {"centroid": [-114.0, 42.0], "primary_region": "US-ID", "regions": ["US-ID"]}, - "data": {}}} - ev = c._handle("central.wx.alert.x", json.dumps(env).encode()) - assert ev is not None and ev.source == expected diff --git a/work/tests/test_central_envelope_to_wire_v057.py b/work/tests/test_central_envelope_to_wire_v057.py deleted file mode 100644 index 0ac6246..0000000 --- a/work/tests/test_central_envelope_to_wire_v057.py +++ /dev/null @@ -1,289 +0,0 @@ -"""v0.5.7-regression: end-to-end Central envelope -> mesh wire string. - -Closes the seam between consumer/composer/renderer that the v0.5.7 staged -flip exposed. Pre-v0.5.7-regression two pre-existing bugs were dormant: - - 1. consumer._normalize() fell back to `cat_raw` (the raw Central - hierarchical category like "incident.tomtom_incidents") when the - upstream payload lacked `title`/`headline`. That string ended up as - event.title and the composer's primary identifier. - 2. MeshRenderer._format_one_line() prepended "[] " to every - payload.message -- including composer output that already starts - with the family label (e.g. "🚨 ROADS:"). Produced the visually- - broken duplicate "[Roads] 🚨 ROADS: ..." that Matt observed. - -Both bugs predate the v0.5.7 campaign but only manifested when v0.5.7 -was the first to flip Central live with master ON. Both were unit-tested -in isolation (composer with clean titles, renderer with legacy messages) -but no integration test exercised the full envelope -> wire path with a -realistic Central payload. This file fills that gap. - -For five representative Central adapter envelopes (one per stream family -that produces user-facing broadcasts), assert the rendered wire string: - - - Does NOT start with "[" (no [Family] legacy prefix). - - Does NOT contain raw Central category tokens like ".tomtom_incidents", - ".firms", ".kindex", ".proton_flux" -- those would indicate the - category-as-title fallback fired. - - DOES start with the composer's emoji + family label (e.g. "🚨 ", - "🔥 ", "⚠ ", "🌐 "). - - Contains the meshai-friendly registry name from ALERT_CATEGORIES - when the upstream payload lacks a useful title/headline. -""" - -import json - -import pytest - -from meshai.central.consumer import CentralConsumer -from meshai.config import EnvironmentalConfig -from meshai.notifications.events import make_payload_from_event -from meshai.notifications.pipeline.bus import EventBus -from meshai.notifications.renderers.composer import compose_mesh_message -from meshai.notifications.renderers.mesh import MeshRenderer -from meshai.notifications.categories import ALERT_CATEGORIES - -pytestmark = pytest.mark.skip( - reason="v0.5.13 default-deny removed the v0.5.7-regression title fallback chain. These tests guard the OLD behavior (envelopes without a per-adapter handler still got broadcast with legacy family-prefix format). The new architecture: handler must synthesize a wire string for a broadcast to fire. This entire file is obsolete in v0.5.13.") - - - -# ---------- Envelope -> Event helper --------------------------------------- - - -def _envelope_to_event(subject: str, envelope: dict): - """Run a CloudEvents envelope through CentralConsumer._normalize/_handle - the way it would in production, returning the emitted Event.""" - rec = [] - bus = EventBus(); bus.subscribe(rec.append) - c = CentralConsumer(EnvironmentalConfig(), bus) - ev = c._handle(subject, json.dumps(envelope).encode()) - assert ev is not None, f"_handle returned None for subject {subject!r}" - return ev - - -def _render_to_wire(event) -> str: - """Run an Event through the dispatcher's composer + renderer path the way - _dispatch_toggles does for mesh_broadcast / mesh_dm, returning the final - wire-format string the renderer would hand to the connector.""" - friendly = compose_mesh_message(event) - assert friendly, "composer returned empty" - payload = make_payload_from_event(event, message=friendly) - chunks = MeshRenderer().render(payload) - assert chunks, "renderer returned no chunks" - return chunks[0] - - -# ---------- Five-adapter representative envelopes ------------------------- - - -# 1. tomtom_incidents -- the exact failure mode Matt observed live -TOMTOM_ENV = { - "id": "tt-12345", - "data": { - "id": "tt-12345", - "adapter": "tomtom_incidents", - "category": "incident.tomtom_incidents", - "time": "2026-06-04T15:40:00+00:00", - "severity": 3, # immediate per map_severity (>=3) - "geo": {"centroid": [-114.0, 42.5], "primary_region": "US-ID", - "regions": ["US-ID"]}, - # NOTE: tomtom_incidents upstream payload carries per-incident fields - # like roadway / event_type but NO top-level title or headline. That's - # the trigger for the v0.5.7-regression cat_raw fallback bug. - "data": {"roadway": "I-84 EB", "event_type": "crash", - "delay_seconds": 1800}, - }, -} - -# 2. FIRMS hotspot -- VIIRS NOAA-20, high confidence -FIRMS_ENV = { - "id": "viirs_noaa20:2026-06-04:0530:43.123:-115.456", - "data": { - "id": "viirs_noaa20:2026-06-04:0530:43.123:-115.456", - "adapter": "firms", - "category": "fire.hotspot.viirs_noaa20.high", - "time": "2026-06-04T05:30:00+00:00", - "severity": 2, - "geo": {"centroid": [-115.456, 43.123], "primary_region": "US-ID", - "regions": ["US-ID"]}, - "data": {"latitude": 43.123, "longitude": -115.456, - "confidence": "high", "frp": 22.5, "satellite": "N20"}, - }, -} - -# 3. NWS alert -- explicitly carries headline (positive control) -NWS_ENV = { - "id": "urn:oid:2.49.0.1.840.0.abc", - "data": { - "id": "urn:oid:2.49.0.1.840.0.abc", - "adapter": "nws", - "category": "wx.alert.us.id.severe_thunderstorm_warning", - "time": "2026-06-04T15:40:00+00:00", - "severity": 3, - "geo": {"centroid": [-116.2, 43.6], "primary_region": "US-ID", - "regions": ["US-ID"]}, - "data": { - "headline": "Severe Thunderstorm Warning issued June 4 by NWS Boise", - "description": "

The NWS in Boise has issued a Severe Thunderstorm Warning...

", - "areaDesc": "Ada, ID", - }, - }, -} - -# 4. USGS quake -- carries title (positive control) -QUAKE_ENV = { - "id": "us8000mc12", - "data": { - "id": "us8000mc12", - "adapter": "usgs_quake", - "category": "quake.event.moderate", - "time": "2026-06-04T12:00:00+00:00", - "severity": 2, - "geo": {"centroid": [-114.5, 44.2], "primary_region": "US-ID", - "regions": ["US-ID"]}, - "data": {"title": "M 4.2 - 23 km ESE of Stanley, ID", - "magnitude": 4.2, "place": "23 km ESE of Stanley, ID", - "depth": 8.0, "magType": "ml"}, - }, -} - -# 5. SWPC alert -- no title/headline, just message body -SWPC_ENV = { - "id": "A20F|2026-04-24 23:50:43.280", - "data": { - "id": "A20F|2026-04-24 23:50:43.280", - "adapter": "swpc_alerts", - "category": "space.alert", - "time": "2026-04-24T23:50:43.280Z", - "severity": 0, - "geo": {"centroid": None, "primary_region": None, "regions": []}, - "data": {"product_id": "A20F", - "issue_datetime": "2026-04-24 23:50:43.280", - "message": "WATCH: Geomagnetic Storm Category G1 Predicted ..."}, - }, -} - -CASES = [ - pytest.param( - "central.traffic.incident.id", TOMTOM_ENV, - "road_incident", "Road Incident", - id="tomtom_incidents-no-title-cat-fallback", - ), - pytest.param( - "central.fire.hotspot.viirs_noaa20.high", FIRMS_ENV, - "wildfire_hotspot", "Wildfire Hotspot", - id="firms-hotspot-no-title-cat-fallback", - ), - pytest.param( - "central.wx.alert.us.id.severe_thunderstorm_warning", NWS_ENV, - "weather_warning", None, # NWS supplies headline; friendly name not used - id="nws-with-headline", - ), - pytest.param( - "central.quake.event.moderate", QUAKE_ENV, - "earthquake_event", None, # USGS supplies title - id="quake-with-title", - ), - pytest.param( - "central.space.alert.a20f", SWPC_ENV, - "rf_propagation_alert", "Space Weather Alert", - id="swpc-alert-no-title-cat-fallback", - ), -] - - -@pytest.mark.parametrize("subject,envelope,expected_cat,expected_friendly_name", CASES) -def test_wire_string_no_legacy_family_prefix(subject, envelope, expected_cat, expected_friendly_name): - """No payload should produce a wire string starting with '[' -- the v0.5.0 - debug-format prefix the MeshRenderer used to add and now no longer does.""" - ev = _envelope_to_event(subject, envelope) - wire = _render_to_wire(ev) - assert not wire.startswith("["), ( - f"wire string still starts with legacy [Family] prefix: {wire!r}" - ) - - -@pytest.mark.parametrize("subject,envelope,expected_cat,expected_friendly_name", CASES) -def test_wire_string_no_raw_central_category_leaks(subject, envelope, expected_cat, expected_friendly_name): - """No wire string should contain a raw Central hierarchical category token - like '.tomtom_incidents', '.firms', '.kindex', '.proton_flux'. Those would - indicate the cat_raw fallback fired and the title-fallback fix didn't take.""" - ev = _envelope_to_event(subject, envelope) - wire = _render_to_wire(ev) - for leak in ( - ".tomtom_incidents", ".firms", - ".kindex", ".proton_flux", - "fire.hotspot.viirs", "incident.tomtom", - ): - assert leak not in wire, ( - f"raw Central category token {leak!r} leaked to wire: {wire!r}" - ) - - -@pytest.mark.parametrize("subject,envelope,expected_cat,expected_friendly_name", CASES) -def test_event_category_is_meshai_flat(subject, envelope, expected_cat, expected_friendly_name): - """The consumer must produce a meshai-flat category (not the raw Central - hierarchical string) so downstream filtering + UI selectability work.""" - ev = _envelope_to_event(subject, envelope) - assert ev.category == expected_cat, ( - f"expected event.category={expected_cat!r} got {ev.category!r}" - ) - assert ev.category in ALERT_CATEGORIES, ( - f"event.category {ev.category!r} not in ALERT_CATEGORIES -- audit gap" - ) - - -@pytest.mark.parametrize("subject,envelope,expected_cat,expected_friendly_name", CASES) -def test_friendly_name_used_when_upstream_has_no_title(subject, envelope, expected_cat, expected_friendly_name): - """For Central adapters whose upstream payload lacks `title`/`headline`, - the consumer's title fallback must use the meshai-friendly registry name - (`ALERT_CATEGORIES[category]['name']`) instead of `cat_raw`. NWS / USGS - quake carry their own title; this assertion skips those (expected_friendly_name=None).""" - if expected_friendly_name is None: - pytest.skip("adapter supplies its own title -- registry fallback not exercised") - ev = _envelope_to_event(subject, envelope) - assert ev.title == expected_friendly_name, ( - f"expected title={expected_friendly_name!r} got {ev.title!r}" - ) - - -@pytest.mark.parametrize("subject,envelope,expected_cat,expected_friendly_name", CASES) -def test_wire_string_starts_with_composer_label(subject, envelope, expected_cat, expected_friendly_name): - """The wire string should start with an emoji + family label like - '🚨 ROADS:', '🔥 FIRE:', '⚠ WX:', '🌐 RF:', '⛷ AVY:'. Confirms the - composer is what produces the formatting (not the renderer).""" - ev = _envelope_to_event(subject, envelope) - wire = _render_to_wire(ev) - # Find ":" within the first ~20 chars: that's the label terminator. - head = wire[:30] - assert ":" in head, ( - f"wire string head {head!r} has no composer label terminator ':'" - ) - - -# ---------- Specific Matt-saw regression ---------------------------------- - - -def test_matt_smoking_gun_no_longer_reproduces(): - """The exact regression Matt saw at 15:40:30 on 2026-06-04: - [Roads] 🚨 ROADS: incident.tomtom_incidents, US-ID. immediate - must NEVER reproduce. Strong-form assertion combining all three failure - modes: no '[Roads]' prefix, no raw category leak, no missing friendly name.""" - ev = _envelope_to_event("central.traffic.incident.id", TOMTOM_ENV) - wire = _render_to_wire(ev) - - assert not wire.startswith("[Roads]"), ( - f"the exact regression reproduced: {wire!r}" - ) - assert "incident.tomtom_incidents" not in wire, ( - f"raw central category still leaks to wire: {wire!r}" - ) - # Friendly name in primary slot - assert "Road Incident" in wire, ( - f"friendly registry name not in wire: {wire!r}" - ) - # Severity tail present - assert "immediate" in wire, ( - f"severity tail missing: {wire!r}" - ) diff --git a/work/tests/test_central_region_routing.py b/work/tests/test_central_region_routing.py deleted file mode 100644 index 8fa6d98..0000000 --- a/work/tests/test_central_region_routing.py +++ /dev/null @@ -1,126 +0,0 @@ -"""v0.5.4: Central v0.9.20 region-aware subject building. - -Exercises `_subjects_for(adapter, region)` and the wiring through -`CentralConsumer._subject_owned()`. The spec is hard-coded in the test -strings on purpose so a future drift in the v0.9.20 subject scheme -fails noisily here instead of silently shipping wrong filters. -""" - -from meshai.central.consumer import ( - CentralConsumer, - _subjects_for, - _SUBJECTS_BARE, -) -from meshai.config import EnvironmentalConfig - - -# --------------------------------------------------------------------- per-adapter - -def test_subjects_for_nws_us_id(): - """NWS: region BEFORE wildcard (matches alert..<...>).""" - assert _subjects_for("nws", "us.id") == ["central.wx.alert.us.id.>"] - - -def test_subjects_for_usgs_quake_us_id_uses_tail_only_wildcard(): - """v0.5.7-seismic: USGS quake publishes `central.quake.event.` with - NO region in the subject (per Central v0.10.0 guide §usgs_quake; same - situation as FIRMS). The pre-v0.5.7-seismic `central.quake.event.>.us.id` - was syntactically invalid (`>` mid-subject) AND wouldn't have matched - anything Central publishes (only 4 tokens, no us.). Region - filtering for quakes now happens client-side via data.latitude/longitude. - Subscription uses tail-only `>` (NATS-legal).""" - assert _subjects_for("usgs_quake", "us.id") == ["central.quake.event.>"] - - -def test_subjects_for_firms_us_id_uses_tail_only_wildcard(): - """v0.5.7-fire: FIRMS publishes `central.fire.hotspot..` - with NO region in the subject (per Central v0.10.0 guide §firms). The - pre-v0.5.7-fire `central.fire.hotspot.>.us.id` was syntactically invalid - (`>` mid-subject) AND wouldn't have matched anything Central actually - publishes. Region filtering for FIRMS now happens client-side via - data.latitude/longitude. Subscription uses tail-only `>` (NATS-legal).""" - assert _subjects_for("firms", "us.id") == ["central.fire.hotspot.>"] - - -def test_subjects_for_fires_us_id_includes_tombstones(): - """v0.5.7-fire: WFIGS subjects -- active state-token at depth-3 + the - removal-tombstone subjects (`central.fire.{incident,perimeter}.removed.`) - per Central v0.10.0 guide §wfigs_incidents §wfigs_perimeters. Pre-v0.5.7-fire - we only subscribed to active subjects, silently dropping fall-off signals.""" - assert _subjects_for("fires", "us.id") == [ - "central.fire.incident.id.>", - "central.fire.perimeter.id.>", - "central.fire.incident.removed.id", - "central.fire.perimeter.removed.id", - ] - - -def test_subjects_for_traffic_uses_convention_b(): - """v0.5.7-traffic: traffic adapter -> bare-state Convention B with `*` - in the event_type slot. Pre-v0.5.7-traffic this was `>.{state}` which - is invalid NATS (`>` must be at the tail). The bare-state subject is - shared with roads511 (sub-adapter routing picks the right meshai source).""" - assert _subjects_for("traffic", "us.id") == ["central.traffic.*.id"] - - -def test_subjects_for_roads511_dual_subscribes_convention_a_and_b(): - """v0.5.7-traffic: roads511 owns BOTH the shared bare-state subject - (Convention B, shared with traffic) AND the us. subject - (Convention A) where the new Idaho-only itd_511 adapter publishes.""" - assert _subjects_for("roads511", "us.id") == [ - "central.traffic.*.id", - "central.traffic.*.us.id", - ] - - -def test_subjects_for_usgs_includes_unknown_workaround(): - """v0.5.7-water: USGS NWIS hydro subscribes to BOTH the region-tagged - filter and the .unknown filter. Per the v0.10.0-itd-511 nwis.py - producer, the actual published subject is - `central.hydro....` where is - either `us.` (7 tokens) or `unknown` (6 tokens). The - pre-v0.5.7-water shape `central.hydro.>.` was invalid NATS - (`>` mid-subject). Fixed by using three single-token `*` wildcards - in the parameter/agency/site slots.""" - assert _subjects_for("usgs", "us.id") == [ - "central.hydro.*.*.*.us.id", - "central.hydro.*.*.*.unknown", - ] - - -def test_subjects_for_swpc_stays_global(): - """SWPC: space weather is planetary; region argument is ignored.""" - assert _subjects_for("swpc", "us.id") == ["central.space.>"] - assert _subjects_for("swpc", "us.mt") == ["central.space.>"] # same regardless - assert _subjects_for("swpc", "") == ["central.space.>"] - - -# --------------------------------------------------------------------- backward compat - -def test_subjects_for_empty_region_falls_back_to_bare_wildcards(): - """Empty/None region = pre-v0.9.20 behaviour for every adapter, byte-identical - to the legacy _SUBJECTS_BARE map. Adapters absent from the map return [].""" - for adapter, expected in _SUBJECTS_BARE.items(): - assert _subjects_for(adapter, "") == expected, f"empty region mismatch for {adapter}" - assert _subjects_for(adapter, None) == expected, f"None region mismatch for {adapter}" - # Unknown adapters return empty regardless of region. - assert _subjects_for("ducting", "us.id") == [] - assert _subjects_for("avalanche", "") != [] # avalanche now in central pipeline - - -# --------------------------------------------------------------------- integration - -def test_central_region_default_propagates_to_consumer_subjects(): - """Default region = 'us.id': flipping nws to central → consumer subscribes - to the region-aware subject, not the bare wildcard.""" - env = EnvironmentalConfig() - assert env.central.region == "us.id" # spec default - env.nws.feed_source = "central" - so = CentralConsumer(env, None)._subject_owned() - # satpass also defaults to feed_source='central', so it appears too - assert "central.wx.alert.us.id.>" in so - assert so["central.wx.alert.us.id.>"] == {"nws"} - assert "central.sat.pass.us.id.>" in so - assert so["central.sat.pass.us.id.>"] == {"satpass"} - assert "central.sat.tle.>" in so - assert so["central.sat.tle.>"] == {"satpass"} diff --git a/work/tests/test_central_sub_adapter_routing.py b/work/tests/test_central_sub_adapter_routing.py deleted file mode 100644 index c905945..0000000 --- a/work/tests/test_central_sub_adapter_routing.py +++ /dev/null @@ -1,94 +0,0 @@ -"""v0.5.1: sub-adapter (owned-sources) routing for shared Central subjects.""" - -import json - -from meshai.config import EnvironmentalConfig -from meshai.central.consumer import CentralConsumer -from meshai.notifications.pipeline.bus import EventBus -import pytest - -pytestmark = pytest.mark.skip( - reason="v0.5.13 default-deny: sub-adapter routing tests asserted that envelopes without a wire-string-returning handler still emit an Event. New architecture: no handler-wire = no Event. v0.6 will rebuild these tests around the new default-deny model.") - - - -def _envelope(adapter, category="x.y", eid="e1"): - return {"id": eid, "data": { - "id": eid, "adapter": adapter, "category": category, - "time": "2026-05-28T00:00:00Z", "severity": 1, - "geo": {"centroid": [-114.0, 42.0], "primary_region": "US-ID", "regions": ["US-ID"]}, - "data": {}}} - - -def _route(central, adapter, subject, category="x.y"): - """Simulate a message arriving on the subscription that matches `subject`, - with that subscription's owned-sources, and return the emitted Event (or None). - - v0.5.4: this helper deliberately clears central.region so sub-adapter - routing is exercised against bare wildcards (its concern is the - owned-sources filter, not the region-aware subject shape — those are - tested in test_central_region_routing.py). - """ - env = EnvironmentalConfig() - env.central.region = "" - for a in central: - getattr(env, a).feed_source = "central" - rec = [] - bus = EventBus(); bus.subscribe(rec.append) - c = CentralConsumer(env, bus) - so = c._subject_owned() - owned = None - for filt, o in so.items(): - prefix = filt[:-1] if filt.endswith(">") else filt - if subject == filt or subject.startswith(prefix): - owned = o - break - ev = c._handle(subject, json.dumps(_envelope(adapter, category)).encode(), owned) - return ev, rec - - -def test_roads511_only_drops_wzdx(): - ev, rec = _route(["roads511"], "wzdx", "central.traffic.work_zone.ok") - assert ev is None and rec == [] - - -def test_roads511_only_emits_state_511_atis(): - ev, rec = _route(["roads511"], "state_511_atis", "central.traffic.event.id.1") - assert ev is not None and ev.source == "roads511" and len(rec) == 1 - - -def test_both_central_wzdx_routes_to_traffic(): - ev, rec = _route(["traffic", "roads511"], "wzdx", "central.traffic.work_zone.ok") - assert ev is not None and ev.source == "traffic" - - -def test_both_central_state511_routes_to_roads511(): - ev, rec = _route(["traffic", "roads511"], "state_511_atis", "central.traffic.event.id.1") - assert ev is not None and ev.source == "roads511" - - -def test_firms_only_drops_wfigs(): - ev, rec = _route(["firms"], "wfigs_incidents", "central.fire.incident.mt.x") - assert ev is None and rec == [] - - -def test_firms_only_emits_firms(): - ev, rec = _route(["firms"], "firms", "central.fire.hotspot.viirs_noaa20.high") - assert ev is not None and ev.source == "firms" and len(rec) == 1 - - -def test_tomtom_incidents_remaps_to_traffic(): - ev, rec = _route(["traffic"], "tomtom_incidents", "central.traffic.incident.x") - assert ev is not None and ev.source == "traffic" - - -def test_subject_owned_shares_traffic_subject(): - # v0.5.4: assert the legacy bare-wildcard shape by clearing region. - # Region-aware shared-subject behaviour ('central.traffic.>.id' for both - # traffic and roads511) is covered in test_central_region_routing.py. - env = EnvironmentalConfig() - env.central.region = "" - env.traffic.feed_source = "central" - env.roads511.feed_source = "central" - so = CentralConsumer(env, None)._subject_owned() - assert so.get("central.traffic.>") == {"traffic", "roads511"} diff --git a/work/tests/test_clock_seam.py b/work/tests/test_clock_seam.py index 4922ef2..688c82a 100644 --- a/work/tests/test_clock_seam.py +++ b/work/tests/test_clock_seam.py @@ -2,8 +2,7 @@ Tests: 1. clock.now() returns a float close to time.time() at runtime. - 2. Monkeypatching clock.now propagates into the three refactored handlers - (quake_handler._now, nws_handler._now, wfigs_handler._now). + 2. Monkeypatching clock.now propagates into wfigs_handler._now. """ import time @@ -11,8 +10,6 @@ import time import pytest import meshai.notifications.clock as clock_mod -import meshai.central.quake_handler as quake_handler -import meshai.central.nws_handler as nws_handler import meshai.central.wfigs_handler as wfigs_handler @@ -32,24 +29,6 @@ def test_clock_now_is_monkeypatchable(monkeypatch): assert clock_mod.now() == _FROZEN_TS -def test_quake_handler_now_uses_clock_seam(monkeypatch): - """quake_handler._now() must reflect a monkeypatched clock.now.""" - monkeypatch.setattr(clock_mod, "now", lambda: _FROZEN_TS) - result = quake_handler._now() - assert result == int(_FROZEN_TS), ( - f"quake_handler._now() returned {result!r}, expected {int(_FROZEN_TS)}" - ) - - -def test_nws_handler_now_uses_clock_seam(monkeypatch): - """nws_handler._now() must reflect a monkeypatched clock.now.""" - monkeypatch.setattr(clock_mod, "now", lambda: _FROZEN_TS) - result = nws_handler._now() - assert result == int(_FROZEN_TS), ( - f"nws_handler._now() returned {result!r}, expected {int(_FROZEN_TS)}" - ) - - def test_wfigs_handler_now_uses_clock_seam(monkeypatch): """wfigs_handler._now() must reflect a monkeypatched clock.now.""" monkeypatch.setattr(clock_mod, "now", lambda: _FROZEN_TS) diff --git a/work/tests/test_config_source_field.py b/work/tests/test_config_source_field.py index 028701c..4ac39be 100644 --- a/work/tests/test_config_source_field.py +++ b/work/tests/test_config_source_field.py @@ -1,10 +1,10 @@ -"""v0.4 C.1: per-adapter `source` field + CentralConsumerConfig.""" +"""v0.4 C.1: per-adapter `source` field.""" import pytest from meshai.config import ( NWSConfig, FIRMSConfig, USGSQuakeConfig, - EnvironmentalConfig, CentralConsumerConfig, + EnvironmentalConfig, ) _ADAPTERS = ("nws", "swpc", "ducting", "fires", "avalanche", @@ -35,13 +35,6 @@ def test_source_garbage_rejects(): FIRMSConfig(feed_source="") -def test_environmental_has_central_default(): - env = EnvironmentalConfig() - assert isinstance(env.central, CentralConsumerConfig) - assert env.central.enabled is False - assert env.central.url.startswith("nats://") - - def test_source_field_survives_dict_coercion(): """A `source` in yaml/dict is coerced onto the adapter config.""" from meshai.config import Config, _dict_to_dataclass diff --git a/work/tests/test_consumer_default_deny.py b/work/tests/test_consumer_default_deny.py deleted file mode 100644 index 9dce44d..0000000 --- a/work/tests/test_consumer_default_deny.py +++ /dev/null @@ -1,231 +0,0 @@ -"""v0.5.13 tests for consumer._normalize() default-deny gate. - -The consumer must return None when the per-adapter handler dispatch -returns synthesized=None -- regardless of what data.title / data.headline -say. Conversely, when a handler returns a wire string, _normalize must -return an Event with that exact title + _meshai_precomposed=True so the -composer bypass kicks in. - -Covers four cases: - (a) envelope with NO matching handler (adapter='avalanche' has no - Central adapter wired) -> _normalize returns None - (b) envelope hits handler, handler returns None (e.g. sub-G3 swpc, - stale tomtom) -> _normalize returns None - (c) envelope hits handler, handler returns wire string - -> _normalize returns Event with title=wire and - data['_meshai_precomposed'] = True - (d) envelope with data.title and data.headline set, but no handler match - -> _normalize STILL returns None (no title fallback) -""" -import pytest -from unittest.mock import patch, MagicMock - -from meshai.config import Config -from meshai.central.consumer import CentralConsumer -from meshai.persistence import close_thread_connection, init_db -from meshai.persistence import db as persistence_db - - -@pytest.fixture -def mem_db(monkeypatch, tmp_path): - db_path = str(tmp_path / "v0513-test.sqlite") - monkeypatch.setenv("MESHAI_DB_PATH", db_path) - persistence_db._initialised.clear() - close_thread_connection() - yield init_db() - close_thread_connection() - persistence_db._initialised.discard(db_path) - - -@pytest.fixture -def consumer(): - """CentralConsumer with mocked bus (we test _normalize only). - CentralConsumer.__init__(env_config, event_bus) where env_config - is the EnvironmentalConfig (provides .central + per-adapter source). - """ - cfg = Config() - cfg.notifications.cold_start_grace_seconds = 0 - bus = MagicMock() - c = CentralConsumer(cfg.environmental, bus) - return c - - -# ---------- envelope builders ---------------------------------------------- - - -def _make_envelope(adapter, category, *, inner_id="test_001", - title=None, headline=None, severity="routine", - extra_data=None): - inner_data = dict(extra_data or {}) - if title is not None: - inner_data["title"] = title - if headline is not None: - inner_data["headline"] = headline - return { - "subject": f"central.{adapter}.test", - "id": f"env_{inner_id}", - "data": { - "id": inner_id, - "adapter": adapter, - "category": category, - "severity": severity, - "time": "2026-06-05T15:00:00Z", - "geo": {"primary_region": "US-ID"}, - "data": inner_data, - }, - } - - -# ============================================================================ -# (a) envelope with NO matching handler -> default-deny -# ============================================================================ - - -def test_no_handler_match_returns_none(consumer, mem_db): - """Avalanche has no handler (no Central adapter). envelope must - drop at consumer._normalize as the default-deny baseline.""" - env = _make_envelope("avalanche", "avalanche.forecast", - inner_id="aval_001") - out = consumer._normalize(env["subject"], env) - assert out is None - - -def test_unknown_adapter_returns_none(consumer, mem_db): - """Any future adapter that meshai doesn't know about must default-deny.""" - env = _make_envelope("future_adapter", "some.category.v1", - inner_id="future_001", - title="Some Title", headline="Some Headline") - out = consumer._normalize(env["subject"], env) - assert out is None - - -# ============================================================================ -# (b) handler returns None -> default-deny (regardless of data.title) -# ============================================================================ - - -def test_handler_returns_none_drops_event(consumer, mem_db, monkeypatch): - """Stale tomtom incident -> incident_handler returns None -> drop.""" - env = _make_envelope("tomtom_incidents", "incident.tomtom_incidents", - inner_id="ID:tomtom:TTI-stale", - title="Old Jam", headline="Headline Jam", - extra_data={ - "id": "ID:tomtom:TTI-stale", - "magnitude_of_delay": 4, - "icon_category": 6, - "time_validity": "past", # filtered - "start_time": "2024-01-01T00:00:00Z", - "latitude": 43.5, "longitude": -116.0, - }) - out = consumer._normalize(env["subject"], env) - assert out is None, "default-deny: handler None -> no Event" - - -def test_data_title_does_not_rescue_handler_none(consumer, mem_db): - """v0.5.13: even when envelope has data.title set, if no handler\ - synthesized, the broadcast is denied.""" - env = _make_envelope("swpc_kindex", "space.kindex", - inner_id="kp_sub_threshold", - title="Kp Update", - extra_data={ - "id": "kp_sub_threshold", - "kp_index": 2.0, # well below G3 (Kp>=7) - "time": "2026-06-05T15:00:00Z", - }) - out = consumer._normalize(env["subject"], env) - assert out is None - assert out is None # double-check - - -# ============================================================================ -# (c) handler returns wire string -> Event emitted with precomposed marker -# ============================================================================ - - -def test_handler_returns_wire_event_emitted(consumer, mem_db, monkeypatch): - """Fresh tomtom envelope passes the handler gate -> Event created.""" - # Disable Photon to avoid network calls in test. - import meshai.central_normalizer as cn - monkeypatch.setattr(cn, "_photon_reverse_places", lambda lat, lon: []) - if hasattr(cn, "_H3_NEAREST_CACHE"): - cn._H3_NEAREST_CACHE.clear() - - import time - now_iso = "2026-06-05T15:00:00Z" - # Build a fresh envelope (start_time = now-300s would require dynamic - # clock control; we instead set the freshness window via the envelope - # built relative to a fixed time and mock the handler to bypass freshness). - env = _make_envelope( - "tomtom_incidents", "incident.tomtom_incidents", - inner_id="ID:tomtom:TTI-aaaa1111-2222-3333-4444-555555555555-TTR1", - extra_data={ - "id": "ID:tomtom:TTI-aaaa1111-2222-3333-4444-555555555555-TTR1", - "magnitude_of_delay": 4, - "icon_category": 6, - "time_validity": "present", - "start_time": now_iso, - "latitude": 43.6, "longitude": -116.2, - "delay": 180, "from": "A St", "to": "B St", - "road_numbers": ["I-84"], - "state_code": "ID", - "_enriched": {"geocoder": {"city": "Boise", "county": "Ada", - "state": "ID"}}, - }, - ) - - # Use a "now" that aligns with start_time so freshness gate passes. - import datetime as _dt - now_epoch = int(_dt.datetime.fromisoformat( - now_iso.replace("Z","+00:00")).timestamp()) + 60 # 1 min after start - - with patch("time.time", return_value=now_epoch): - out = consumer._normalize(env["subject"], env) - - assert out is not None, "fresh tomtom should produce an Event" - assert out.data.get("_meshai_precomposed") is True - assert out.title.startswith("🚗") # jam emoji - assert "Boise" in out.title - - -# ============================================================================ -# (d) envelope with title but no handler still drops (no title fallback) -# ============================================================================ - - -def test_envelope_with_title_still_drops_without_handler(consumer, mem_db): - """Regression guard: the v0.5.7-fallback path (data.title -> headline ->\ - friendly_name -> cat_raw) is GONE in v0.5.13. Uses an unhandled adapter - (avalanche) since v0.6-1 added the FIRMS handler.""" - env = _make_envelope("avalanche", "avalanche.forecast", - inner_id="aval_with_title", - title="Avalanche Warning", - headline="Backcountry advisory") - out = consumer._normalize(env["subject"], env) - assert out is None, ( - "v0.5.13 default-deny: data.title and data.headline must NOT rescue\n" - "an envelope that no handler synthesized for." - ) - - -# ============================================================================ -# (e) memory rule 19 -- confirms _normalize ENTRY logging behavior -# ============================================================================ - - -def test_default_deny_path_is_silent_at_INFO(consumer, mem_db, caplog): - """Default-deny paths log at DEBUG, not INFO/WARNING. We don't want - millions of DEBUG-noise to feel like errors at default log levels.""" - import logging - caplog.set_level(logging.INFO, logger="meshai.central.consumer") - env = _make_envelope("firms", "fire.hotspot.viirs", - inner_id="silent_check") - out = consumer._normalize(env["subject"], env) - assert out is None - # No INFO/WARNING/ERROR for normal default-deny. - info_or_higher = [r for r in caplog.records - if r.levelno >= logging.INFO - and r.name == "meshai.central.consumer"] - assert len(info_or_higher) == 0, ( - f"default-deny should be silent at INFO+; got: " - f"{[(r.levelname, r.message) for r in info_or_higher]}" - ) diff --git a/work/tests/test_fire_v057.py b/work/tests/test_fire_v057.py deleted file mode 100644 index 8692bdb..0000000 --- a/work/tests/test_fire_v057.py +++ /dev/null @@ -1,262 +0,0 @@ -"""v0.5.7-fire: FIRMS NATS pattern + WFIGS tombstone dedup + categories audit. - -Covers four things shipped in v0.5.7-fire: - -1. FIRMS subject pattern -- per Central v0.10.0 guide, FIRMS publishes - `central.fire.hotspot..` with NO region in the - subject. The pre-v0.5.7-fire `central.fire.hotspot.>.us.id` was - syntactically invalid (`>` mid-subject) AND wouldn't have matched - anything. NOTE on user-prompt discrepancy: the v0.5.7-fire prompt - specified `central.fire.hotspot.*.*.us.id` (7 tokens with us. - tail) but the actual Central v0.10.0 guide shows exactly 5 tokens with - no region. We follow the guide -- following the prompt verbatim would - produce a subscription that matches zero messages in production. -2. WFIGS subjects -- active state-token subjects + the four removal - tombstone subjects per guide §wfigs_incidents §wfigs_perimeters. -3. WFIGS tombstone dedup -- env_id form `:removed:` must - strip to the bare IrwinID for group_key so all tombstones for the same - incident share the group_key (per guide §wfigs_incidents removal - semantics: "the same incident can have one or more removal tombstones - over its lifecycle"). Two tombstones with the same IrwinID but different - :removed: tails: both must propagate through _handle as distinct - Events; both must share group_key == IrwinID. -4. ALERT_CATEGORIES fire-family audit -- fire_proximity and - wildfire_proximity removed (Matt: parametric, can't set "near" - threshold in UI); new_ignition, wildfire_hotspot, wildfire_incident kept - / added. -""" - -import inspect -import json -import re - -import pytest - -from meshai.central.consumer import ( - CentralConsumer, - _SUBJECTS_BARE, - _subjects_for, - map_category, -) -from meshai.config import EnvironmentalConfig -from meshai.notifications.categories import ALERT_CATEGORIES -from meshai.notifications.pipeline.bus import EventBus - -pytestmark = pytest.mark.skip( - reason="v0.5.13 default-deny: WFIGS tombstones now correctly return None from wfigs_handler (logged to event_log handled=0, no Event). These tests asserted the legacy clear-event-emission. New behavior is covered by tests/test_wfigs_handler.py.") - - - -def _assert_legal_nats(subject: str) -> None: - """Assert NATS multi-level wildcard `>` only appears at the tail token.""" - tokens = subject.split(".") - if ">" in tokens: - assert tokens[-1] == ">", f"`>` not at tail in {subject!r}" - assert tokens.count(">") == 1, f"multiple `>` in {subject!r}" - for tok in tokens: - assert tok, f"empty token in {subject!r}" - if tok not in {"*", ">"}: - assert "*" not in tok and ">" not in tok, f"mixed wildcard in token {tok!r}" - - -# ---------- FIRMS subject pattern ----------------------------------------- - - -def test_firms_subject_uses_tail_only_wildcard(): - """FIRMS publishes . only -- no us..""" - subs = _subjects_for("firms", "us.id") - assert subs == ["central.fire.hotspot.>"] - for s in subs: - _assert_legal_nats(s) - - -def test_firms_subject_has_no_mid_string_wildcard(): - """Belt-and-braces: `>` only at tail, no mid-subject placement.""" - for s in _subjects_for("firms", "us.id"): - tokens = s.split(".") - for tok in tokens[:-1]: - assert tok != ">", f"`>` mid-subject in {s!r}" - - -# ---------- WFIGS subjects (fires) ---------------------------------------- - - -def test_fires_subjects_cover_active_and_tombstones(): - """v0.5.7-fire: tombstone subjects are now subscribed alongside active.""" - subs = _subjects_for("fires", "us.id") - assert subs == [ - "central.fire.incident.id.>", - "central.fire.perimeter.id.>", - "central.fire.incident.removed.id", - "central.fire.perimeter.removed.id", - ] - for s in subs: - _assert_legal_nats(s) - - -def test_fires_subjects_no_mid_subject_wildcard(): - for s in _subjects_for("fires", "us.id"): - tokens = s.split(".") - for tok in tokens[:-1]: - assert tok != ">", f"`>` mid-subject in {s!r}" - - -# ---------- WFIGS tombstone dedup ----------------------------------------- - - -def _envelope(adapter, eid, category="fire.incident.removed"): - """Build a CloudEvents-shaped envelope for a single WFIGS tombstone.""" - return {"id": eid, "data": { - "id": eid, "adapter": adapter, "category": category, - "time": "2026-05-19T02:50:39+00:00", "severity": 0, - "geo": {"centroid": None, "primary_region": None, "regions": []}, - "data": {"irwin_id": "{01AAC875-E26E-49E4-9DB0-80B5965A7B9F}", - "state": "US-ID", "county": "Custer", - "reason": "fallen_off_current_service", - "last_observed_at": "2026-05-19T02:50:00+00:00"}}} - - -def test_wfigs_tombstone_strips_removed_iso_suffix(): - """Single WFIGS tombstone -- group_key recovers the bare IrwinID.""" - rec = [] - bus = EventBus(); bus.subscribe(rec.append) - c = CentralConsumer(EnvironmentalConfig(), bus) - irwin = "{01AAC875-E26E-49E4-9DB0-80B5965A7B9F}" - eid = f"{irwin}:removed:2026-05-19T02:50:39.843049+00:00" - env = _envelope("wfigs_incidents", eid) - ev = c._handle("central.fire.incident.removed.id", json.dumps(env).encode()) - assert ev is not None - assert ev.data.get("_central_tombstone") is True - assert ev.group_key == irwin, f"group_key did not strip :removed: tail: {ev.group_key!r}" - - -def test_wfigs_two_tombstones_same_irwin_both_propagate(): - """Per guide §wfigs_incidents: the same incident can have multiple - removal tombstones over its lifecycle. Both tombstones with the same - IrwinID but different :removed: tails must: - - both be emitted by _handle (not collapsed at consumer layer) - - share the same group_key (== IrwinID) so they signal lapse - against the same original event - """ - rec = [] - bus = EventBus(); bus.subscribe(rec.append) - c = CentralConsumer(EnvironmentalConfig(), bus) - irwin = "{01AAC875-E26E-49E4-9DB0-80B5965A7B9F}" - eid1 = f"{irwin}:removed:2026-05-19T02:50:39.843049+00:00" - eid2 = f"{irwin}:removed:2026-05-20T14:22:17.111222+00:00" - env1 = _envelope("wfigs_incidents", eid1) - env2 = _envelope("wfigs_incidents", eid2) - ev1 = c._handle("central.fire.incident.removed.id", json.dumps(env1).encode()) - ev2 = c._handle("central.fire.incident.removed.id", json.dumps(env2).encode()) - # Both emitted -- no consumer-layer dedup collapsing. - assert ev1 is not None and ev2 is not None - assert len(rec) == 2, f"expected 2 events on bus, got {len(rec)}" - # Both share the bare IrwinID as group_key (so they lapse the original - # incident's accumulator entry by the same key). - assert ev1.group_key == irwin - assert ev2.group_key == irwin - # Event.id is intentionally deterministic from (source, category, - # group_key, lat, lon) — two tombstones for the same incident produce - # the same Event.id by design. Distinctness is preserved on - # data['_central_tombstone_id'] which carries the full :removed: - # tail so downstream consumers can tell the two fall-off events apart - # if they want to. - assert ev1.data.get("_central_tombstone_id") == eid1 - assert ev2.data.get("_central_tombstone_id") == eid2 - assert ev1.data["_central_tombstone_id"] != ev2.data["_central_tombstone_id"] - - -def test_legacy_gdacs_tombstone_still_strips_plain_suffix(): - """Regression guard: the legacy GDACS `:removed` shape (no : - tail) must still strip cleanly. The v0.5.7-fire regex is a superset - of the pre-v0.5.7-fire regex, not a replacement.""" - rec = [] - bus = EventBus(); bus.subscribe(rec.append) - c = CentralConsumer(EnvironmentalConfig(), bus) - env = {"id": "FL1103885:removed", "data": { - "id": "FL1103885:removed", "adapter": "gdacs", "category": "disaster.fl.removed", - "time": "2026-05-28T00:00:00Z", "severity": 0, - "geo": {"centroid": None, "primary_region": None, "regions": []}, - "data": {}}} - ev = c._handle("central.disaster.fl.removed.austria", json.dumps(env).encode()) - assert ev is not None - assert ev.data.get("_central_tombstone") is True - assert ev.group_key == "FL1103885" - - -# ---------- ALERT_CATEGORIES fire-family audit ---------------------------- - - -def test_fire_proximity_removed_from_registry(): - """Matt: 'fire near mesh has its own set of parameters that I don't even - know what they could be. like how far is near mesh? I don't know I - can't set that.' -- removed in v0.5.7-fire; parametric distance is - queued for v0.5.8.""" - assert "fire_proximity" not in ALERT_CATEGORIES - - -def test_wildfire_proximity_removed_from_registry(): - """Duplicate 'Fire Near Mesh' name w/ fire_proximity; same parametric - issue; removed in v0.5.7-fire.""" - assert "wildfire_proximity" not in ALERT_CATEGORIES - - -def test_no_duplicate_fire_near_mesh_names(): - """No two fire-family registry entries share the 'Fire Near Mesh' name.""" - names = [info["name"] for cid, info in ALERT_CATEGORIES.items() - if info.get("toggle") == "fire"] - assert names.count("Fire Near Mesh") == 0 - assert len(set(names)) == len(names), f"duplicate fire-family names: {names}" - - -def _native_emitted_fire_categories() -> set[str]: - """Walk firms.py and fires.py for category= literals.""" - from meshai.env import firms as firms_mod, fires as fires_mod - emitted: set[str] = set() - for mod in (firms_mod, fires_mod): - src = inspect.getsource(mod) - emitted |= set(re.findall(r'category="([a-z_]+)"', src)) - # Also pick up `category = "..."` ternary forms. - emitted |= set(re.findall(r'category\s*=\s*"([a-z_]+)"\s+if', src)) - emitted |= set(re.findall(r'else\s+"([a-z_]+)"', src)) - # Filter to known fire-family ids (other ternary branches may surface - # non-fire strings; we only care about ones routed through toggle=fire). - return {c for c in emitted if c in ALERT_CATEGORIES - and ALERT_CATEGORIES[c].get("toggle") == "fire"} - - -def _central_path_fire_categories() -> set[str]: - central_inputs = [ - "fire.hotspot.viirs_noaa20.high", - "fire.incident.id.ada", - "fire.incident.removed", - "fire.perimeter.id.ada", - "fire.perimeter.removed", - "fire.unknown_subtype", - ] - return {map_category(c) for c in central_inputs} - - -def test_alert_categories_fire_complete(): - """Native + central-path emit must equal registry's fire-family set.""" - registry_fire = { - cid for cid, info in ALERT_CATEGORIES.items() - if info.get("toggle") == "fire" - } - emitted = _native_emitted_fire_categories() | _central_path_fire_categories() - missing = emitted - registry_fire - orphans = registry_fire - emitted - assert not missing, f"fire emit set missing from ALERT_CATEGORIES: {missing}" - assert not orphans, f"ALERT_CATEGORIES has orphan fire entries: {orphans}" - - -@pytest.mark.parametrize( - "cat", ["new_ignition", "wildfire_hotspot", "wildfire_incident"], -) -def test_fire_categories_have_required_fields(cat): - info = ALERT_CATEGORIES[cat] - assert info["toggle"] == "fire" - assert info["name"] - assert info["description"] - assert info["default_severity"] in {"routine", "priority", "immediate"} - assert info["example_message"] diff --git a/work/tests/test_firms_fusion_event_contract.py b/work/tests/test_firms_fusion_event_contract.py index 09531a3..58acd80 100644 --- a/work/tests/test_firms_fusion_event_contract.py +++ b/work/tests/test_firms_fusion_event_contract.py @@ -1,319 +1,49 @@ -"""Regression tests for the FIRMS fire-fusion Event contract (issues #117-#119). +"""Regression tests for the FIRMS FirePacer contract (issue #119). -Three independent bugs in the path from firms_handler's growth / spotting / -halt / cluster fusion decisions to the actual meshai Event that reaches the -dispatcher + pacer: +Originally three sections (A/B/C) guarding issues #117-#119 in the path +from firms_handler's growth/spotting/halt/cluster fusion decisions to the +actual meshai Event that reaches the dispatcher + pacer. Sections A and B, +plus one test in section C, drove that path exclusively through +`meshai.central.consumer.CentralConsumer._normalize()`/`._handle()` -- the +Central NATS-consumer bridge, which has been deleted (production runs the +native env/firms.py -> firms_handler.ingest_hotspot_pixel fusion path +exclusively; see meshai.env.firms.FirmsAdapter._make_fusion_event, which +independently applies the same `_severity_override`-over-`severity` +resolution). Deleting CentralConsumer makes those tests uncollectable, and +since the mechanism they guarded (issues #117/#118) lived entirely inside +the now-dead consumer, they can no longer exist as tests of live behavior +-- git history preserves them. - #117 category overrides dead: consumer._normalize() computed `category` - from the raw Central category BEFORE the per-adapter handler ran, and - the final make_event() call never re-read data["category"], so every - firms_handler category stamp (wildfire_growth / wildfire_halted / - wildfire_spotting / unattributed_hotspot_cluster) was a silent no-op. +The #117/#118 category+severity contract they exercised at the data_patch +level is independently covered against the LIVE native gating path in +tests/test_firms_refactor.py (asserts `data_patch["category"]` / +`data_patch["_severity_override"]` for growth/spotting/halt/cluster) and at +the Event/category level in tests/test_firms_native_fusion.py (drives the +real adapter tick() -> to_event() chain and asserts `ev.category`). - #118 severity overrides dead for 3 of 4 fusion kinds: consumer.py only ever - honored data["_severity_override"], but firms_handler's halt / spotting - / cluster sites stamped the plain data["severity"] key instead (only - growth used the correct key), so those events fell back to whatever - map_severity(inner.get("severity")) produced from the raw envelope. - - #119 FirePacer didn't cover FIRMS: the pacer gate only matched - source in ("fires", "wfigs") and severity == "priority", but FIRMS - fusion broadcasts carry source="firms" and growth/spotting are - severity="immediate" -- so none of them were ever paced. The fix - broadens the gate to source="firms" + {"priority","immediate"}, and - adds head-of-line insertion so an "immediate" event is never stuck - behind already-queued "priority" events. - -Cluster detection itself is a deliberate, always-on feature of main (PR #73: -curated new-fire cluster broadcasts, cold-start silent-seeded). Nothing here -enables or disables it -- the cluster case below only asserts that its -severity override reaches the Event (the #118 fix). - -Sections: - A. Category overrides survive to the emitted Event (growth/halt/spotting). - B. Severity overrides survive to the emitted Event, using the shared - `_severity_override` contract (spotting=immediate, halt=routine, - cluster=priority). - C. FirePacer routes FIRMS broadcasts; immediate jumps the queue; nothing - is ever dropped. +What remains here (issue #119, section C) is two tests that exercise the +FirePacer class directly with no dependency on CentralConsumer -- these are +native, standalone FirePacer unit tests (head-of-line ordering, no-drop +guarantee) and survive unchanged. The third section-C test (routing a real +FIRMS growth broadcast into a mocked pacer via CentralConsumer._handle) is +deleted along with A/B for the same reason; the equivalent native-path +routing guarantee (store._emit_event() -> FirePacer, for source="firms") +is already covered end-to-end in tests/test_native_fire_pacer.py. """ from __future__ import annotations import asyncio -import math -import time -import uuid -import pytest - -from meshai.config import Config -from meshai.central.consumer import CentralConsumer from meshai.notifications.events import make_event from meshai.notifications.pipeline.pacer import FirePacer -from meshai.persistence import close_thread_connection, init_db -from meshai.persistence import db as persistence_db - -_MI_PER_DEG_LAT = 69.0 -_SUBJECT = "central.fire.hotspot.N20.high.us.id" - - -# ── isolation ──────────────────────────────────────────────────────────────── - -@pytest.fixture(autouse=True) -def _isolate_db(tmp_path, monkeypatch): - db_path = str(tmp_path / f"meshai-{uuid.uuid4().hex}.sqlite") - monkeypatch.setenv("MESHAI_DB_PATH", db_path) - persistence_db._initialised.clear() - close_thread_connection() - init_db() - try: - from meshai.adapter_config import adapter_config as _ac - _ac.invalidate() - except Exception: - pass - yield db_path - close_thread_connection() - persistence_db._initialised.discard(db_path) - - -@pytest.fixture(autouse=True) -def _no_cutover(monkeypatch): - """Default deploy state: nothing cut over -> firms_handler's legacy - stamps are what actually reach `data`.""" - monkeypatch.delenv("MESHAI_CUTOVER_CATEGORIES", raising=False) - from meshai.notifications.cutover import _clear_cache - _clear_cache() - yield - _clear_cache() - - -# firms_handler.handle_firms defaults `now` to real wall-clock time -# (int(time.time())) whenever it is called without an explicit `now` -# kwarg -- which is exactly how consumer._normalize()/_handle() call it (no -# `now` is threaded through from the envelope). The growth/spotting/halt -# fixtures below use fixed 2026-06-06 acq_date/acq_time values (matching the -# rest of the FIRMS test suite), so pin the wall clock to a fixed reference -# in the same window; otherwise the halt detector opportunistically fires on -# every pixel once the real host clock is weeks past the canned acq_times. -_FIXED_NOW = 1780768800.0 # 2026-06-06 18:00 UTC - - -@pytest.fixture(autouse=True) -def _fixed_clock(monkeypatch): - monkeypatch.setattr("time.time", lambda: _FIXED_NOW) - yield - - -@pytest.fixture -def consumer(): - """CentralConsumer with a mocked bus, mirroring - test_consumer_default_deny.py's `consumer` fixture.""" - from unittest.mock import MagicMock - cfg = Config() - cfg.notifications.cold_start_grace_seconds = 0 - bus = MagicMock() - c = CentralConsumer(cfg.environmental, bus) - return c, bus - - -# ── envelope + fire-seeding helpers (mirror test_firms_refactor.py) ───────── - -def _seed_fire(*, irwin_id, lat, lon, name="Stub Fire", **cols): - from meshai.persistence import get_db - conn = get_db() - base = {"irwin_id": irwin_id, "incident_name": name, "lat": lat, "lon": lon, - "last_event_at": int(time.time())} - base.update(cols) - keys = ",".join(base) - ph = ",".join("?" * len(base)) - conn.execute(f"INSERT INTO fires({keys}) VALUES ({ph})", tuple(base.values())) - - -def _envelope(*, lat, lon, acq_date="2026-06-06", acq_time="1200", - frp=20.0, satellite="N20", eid=None): - eid = eid or f"firms-{lat}-{lon}-{acq_time}" - return { - "id": f"env_{eid}", - "data": { - "id": eid, - "adapter": "firms", - "category": "wildfire_hotspot", - "severity": "routine", - "geo": {"primary_region": "US-ID"}, - "data": { - "latitude": lat, "longitude": lon, "frp": frp, - "bright_ti4": 320.0, "satellite": satellite, - "instrument": "VIIRS", "confidence": "high", - "acq_date": acq_date, "acq_time": acq_time, - "daynight": "D", "version": "2.0NRT", - }, - }, - } - - -def _offset_mi(lat, lon, north_mi, east_mi): - dlat = north_mi / _MI_PER_DEG_LAT - dlon = east_mi / (_MI_PER_DEG_LAT * math.cos(math.radians(lat))) - return lat + dlat, lon + dlon # ═════════════════════════════════════════════════════════════════════════════ -# A. Category overrides reach the emitted Event (issue #117) -# ═════════════════════════════════════════════════════════════════════════════ - -class TestCategoryOverrideReachesEvent: - def test_growth_event_category_is_wildfire_growth(self, consumer): - c, _bus = consumer - center_lat, center_lon = 42.0, -114.0 - _seed_fire(irwin_id="ID-CAT-G", lat=center_lat, lon=center_lon, - name="Pine Gulch") - # Pass A: 5 pixels build the baseline (no broadcast yet). - for i in range(5): - evt = c._normalize(_SUBJECT, _envelope( - lat=center_lat + 0.0001 * i, lon=center_lon + 0.0001 * (i - 2), - acq_time=f"12{i:02d}", frp=20.0 + i, eid=f"ga{i}")) - assert evt is None, "pass-A pixels must not broadcast" - # Pass B: 1 mi N, later pass bucket -> growth boundary. - pass_b_lat = center_lat + (1.0 / _MI_PER_DEG_LAT) - evt = c._normalize(_SUBJECT, _envelope( - lat=pass_b_lat, lon=center_lon, acq_time="1800", frp=22.0, eid="gb")) - assert evt is not None - assert evt.category == "wildfire_growth", ( - "category override must survive to the Event, not the generic " - "wildfire_hotspot/wildfire_incident fallback") - assert evt.source == "firms" - - def test_halt_event_category_is_wildfire_halted(self, consumer): - c, _bus = consumer - now = 1780768800 - idle_at = now - 14 * 3600 - from meshai.persistence import get_db - get_db().execute( - "INSERT INTO fires(irwin_id, incident_name, lat, lon, " - "last_event_at, last_pass_id, last_pass_at) VALUES (?,?,?,?,?,?,?)", - ("ID-CAT-H", "Cold Fire", 42.5, -114.5, int(idle_at), - "N20-329627", float(idle_at))) - # A fresh unattributed pixel far away triggers the opportunistic - # halt detector for the idle fire. - evt = c._normalize(_SUBJECT, _envelope( - lat=45.0, lon=-118.0, acq_time="1800", eid="halt1")) - assert evt is not None - assert evt.category == "wildfire_halted" - - def test_spotting_event_category_is_wildfire_spotting(self, consumer): - c, _bus = consumer - center_lat, center_lon = 43.0, -115.0 - _seed_fire(irwin_id="ID-CAT-S", lat=center_lat, lon=center_lon, - name="Spot Fire") - for i in range(6): - angle = i * math.pi / 3 - la = center_lat + (0.5 / _MI_PER_DEG_LAT) * math.sin(angle) - cos_lat = math.cos(math.radians(center_lat)) - lo = center_lon + (0.5 / (_MI_PER_DEG_LAT * cos_lat)) * math.cos(angle) - evt = c._normalize(_SUBJECT, _envelope( - lat=la, lon=lo, acq_time=f"12{i * 2:02d}", eid=f"sa{i}")) - assert evt is None - sp_lat, sp_lon = _offset_mi(center_lat, center_lon, - north_mi=2.0 / math.sqrt(2), - east_mi=2.0 / math.sqrt(2)) - evt = c._normalize(_SUBJECT, _envelope( - lat=sp_lat, lon=sp_lon, acq_time="1800", eid="sb")) - assert evt is not None - assert evt.category == "wildfire_spotting" - - -# ═════════════════════════════════════════════════════════════════════════════ -# B. Severity overrides reach the emitted Event via `_severity_override` -# (issue #118) -# ═════════════════════════════════════════════════════════════════════════════ - -class TestSeverityOverrideReachesEvent: - def test_spotting_event_severity_is_immediate(self, consumer): - c, _bus = consumer - center_lat, center_lon = 44.0, -116.0 - _seed_fire(irwin_id="ID-SEV-S", lat=center_lat, lon=center_lon) - for i in range(6): - angle = i * math.pi / 3 - la = center_lat + (0.5 / _MI_PER_DEG_LAT) * math.sin(angle) - cos_lat = math.cos(math.radians(center_lat)) - lo = center_lon + (0.5 / (_MI_PER_DEG_LAT * cos_lat)) * math.cos(angle) - c._normalize(_SUBJECT, _envelope( - lat=la, lon=lo, acq_time=f"12{i * 2:02d}", eid=f"ssa{i}")) - sp_lat, sp_lon = _offset_mi(center_lat, center_lon, - north_mi=2.0 / math.sqrt(2), - east_mi=2.0 / math.sqrt(2)) - evt = c._normalize(_SUBJECT, _envelope( - lat=sp_lat, lon=sp_lon, acq_time="1800", eid="ssb")) - assert evt is not None - assert evt.severity == "immediate", ( - "spotting must reach the pacer/dispatcher as immediate severity, " - "not fall back to map_severity() of the raw envelope") - - def test_halt_event_severity_is_routine(self, consumer): - c, _bus = consumer - now = 1780768800 - idle_at = now - 14 * 3600 - from meshai.persistence import get_db - get_db().execute( - "INSERT INTO fires(irwin_id, incident_name, lat, lon, " - "last_event_at, last_pass_id, last_pass_at) VALUES (?,?,?,?,?,?,?)", - ("ID-SEV-H", "Cold Fire", 42.5, -114.5, int(idle_at), - "N20-329627", float(idle_at))) - evt = c._normalize(_SUBJECT, _envelope( - lat=45.0, lon=-118.0, acq_time="1800", eid="sevhalt")) - assert evt is not None - assert evt.severity == "routine" - - def test_cluster_event_severity_is_priority(self, consumer): - c, _bus = consumer - base_lat, base_lon = 43.500, -114.500 - pixels = [ - (base_lat, base_lon, "1200"), - (base_lat + 0.001, base_lon + 0.001, "1210"), - (base_lat - 0.001, base_lon - 0.002, "1220"), - ] - events = [] - for i, (la, lo, t) in enumerate(pixels): - evt = c._normalize("central.fire.hotspot.N20.high.unknown", _envelope( - lat=la, lon=lo, acq_time=t, eid=f"clu{i}")) - if evt is not None: - events.append(evt) - assert len(events) == 1, f"expected exactly one cluster event: {events}" - assert events[0].category == "unattributed_hotspot_cluster" - assert events[0].severity == "priority" - - -# ═════════════════════════════════════════════════════════════════════════════ -# C. FirePacer covers FIRMS; immediate jumps the queue; nothing is dropped -# (issue #119) +# FirePacer covers FIRMS; immediate jumps the queue; nothing is dropped +# (issue #119) # ═════════════════════════════════════════════════════════════════════════════ class TestPacerCoversFirms: - def test_firms_growth_broadcast_routes_through_pacer(self, consumer): - """A real FIRMS growth broadcast (source=firms, severity=immediate) - must be handed to the pacer, not emitted straight to the bus.""" - from unittest.mock import MagicMock - c, bus = consumer - pacer = MagicMock() - c._pacer = pacer - - center_lat, center_lon = 42.0, -114.0 - _seed_fire(irwin_id="ID-PACE-G", lat=center_lat, lon=center_lon, - name="Pine Gulch") - for i in range(5): - c._handle(_SUBJECT, _raw(_envelope( - lat=center_lat + 0.0001 * i, lon=center_lon + 0.0001 * (i - 2), - acq_time=f"12{i:02d}", frp=20.0 + i, eid=f"pga{i}"))) - pass_b_lat = center_lat + (1.0 / _MI_PER_DEG_LAT) - event = c._handle(_SUBJECT, _raw(_envelope( - lat=pass_b_lat, lon=center_lon, acq_time="1800", frp=22.0, - eid="pgb"))) - - assert event is not None - assert event.category == "wildfire_growth" - pacer.enqueue.assert_called_once_with(event) - bus.emit.assert_not_called() - def test_immediate_event_emitted_before_already_queued_priority_events(self): """Two 'priority' events are queued first; a later 'immediate' event must still be emitted BEFORE them (head-of-line), not after.""" @@ -376,8 +106,3 @@ class TestPacerCoversFirms: assert len(emitted) == total, ( f"pacer must never drop events: expected {total}, got {len(emitted)}") assert pacer.pending_count() == 0 - - -def _raw(envelope: dict) -> bytes: - import json - return json.dumps(envelope).encode() diff --git a/work/tests/test_firms_handler.py b/work/tests/test_firms_handler.py index 8543114..a86c83e 100644 --- a/work/tests/test_firms_handler.py +++ b/work/tests/test_firms_handler.py @@ -347,38 +347,3 @@ def test_short_acq_time_zero_padded(mem_db): assert out is None assert _row_count(mem_db, "firms_pixels") == 1 - -# ============================================================================ -# Integration: envelope through the full consumer -> handler -> SQLite path -# ============================================================================ - - -def test_end_to_end_envelope_through_consumer(mem_db, monkeypatch): - """Confirm: envelope enters consumer._normalize, handle_firms is invoked, - firms_pixels row is inserted, and consumer returns None (default-deny - keeps the broadcast suppressed). mesh_broadcasts_out MUST stay empty.""" - from unittest.mock import MagicMock - from meshai.config import Config - from meshai.central.consumer import CentralConsumer - - cfg = Config() - cfg.notifications.cold_start_grace_seconds = 0 - bus = MagicMock() - consumer = CentralConsumer(cfg.environmental, bus) - - env = _firms_env(envelope_id="e2e_001") - out = consumer._normalize(env["subject"], env) - - # consumer.normalize returns None -> Event never reaches bus. - assert out is None - bus.emit.assert_not_called() - - # firms_pixels MUST have the row; mesh_broadcasts_out MUST be empty. - assert _row_count(mem_db, "firms_pixels") == 1 - assert _row_count(mem_db, "mesh_broadcasts_out") == 0 - - # event_log records the storage. - log = _last_event_log(mem_db) - assert log["source"] == "firms" - assert log["handled"] == 1 - assert log["table_name"] == "firms_pixels" diff --git a/work/tests/test_firms_refactor.py b/work/tests/test_firms_refactor.py index 6eb2b40..231bf32 100644 --- a/work/tests/test_firms_refactor.py +++ b/work/tests/test_firms_refactor.py @@ -450,83 +450,3 @@ class TestClusterBelowThreshold: assert out is None assert data == {} - -# ───────────────────────────────────────────────────────────────────────────── -# 6. issue #121 — cutover severity must reach the Event (regression guard) -# ───────────────────────────────────────────────────────────────────────────── -# Spotting and halt used to stamp a plain "severity" key in data_patch. -# central/consumer.py only ever promotes data["_severity_override"] onto -# Event.severity (see consumer.py's issue #118 comment) -- the plain key was -# a silent no-op, the same class of bug fixed for firms_handler.py's own -# inline stamps in PR #120. These drive the REAL cutover path end-to-end -# through CentralConsumer._normalize (the actual production entry point, -# adapter=="firms" dispatch) and assert the emitted Event's severity, so a -# regression back to a plain "severity" key fails loudly instead of silently. - -class TestCutoverSeverityReachesEvent: - def _consumer(self): - from unittest.mock import MagicMock - from meshai.config import Config - from meshai.central.consumer import CentralConsumer - cfg = Config() - cfg.notifications.cold_start_grace_seconds = 0 - return CentralConsumer(cfg.environmental, MagicMock()) - - def test_spotting_cutover_event_severity_is_immediate(self, monkeypatch): - _cutover(monkeypatch, "wildfire_spotting") - try: - _seed_pass_a_hex_then_close("ID-SEV-S", 43.0, -115.0) - sp_lat, sp_lon = _offset_mi(43.0, -115.0, - north_mi=2.0 / math.sqrt(2), - east_mi=2.0 / math.sqrt(2)) - env = _envelope(lat=sp_lat, lon=sp_lon, acq_time="1800") - env["id"] = "spot-sev-test" - env["data"]["id"] = "spot-sev-test" - env["data"]["geo"] = {"centroid": [sp_lon, sp_lat]} - - event = self._consumer()._normalize(_SUBJECT, env) - assert event is not None, "expected a broadcast Event, got None" - assert event.severity == "immediate", ( - f"issue #121 regression: expected 'immediate', got " - f"{event.severity!r} -- gating.firms.decide's spotting " - f"data_patch must use _severity_override, not the plain " - f"'severity' key" - ) - finally: - from meshai.notifications.cutover import _clear_cache - _clear_cache() - - def test_halt_cutover_event_severity_is_routine(self, monkeypatch): - _cutover(monkeypatch, "wildfire_halted") - try: - from meshai.central import firms_handler as _fh - fixed_now = 1780768800 - monkeypatch.setattr(_fh.time, "time", lambda: float(fixed_now)) - _seed_stale_fire("ID-SEV-H", now_epoch=fixed_now, idle_hours=14) - - # Any unrelated pixel arrival opportunistically triggers the halt - # scan (_maybe_emit_halt runs on every pixel as a fallback); use - # one far away so it isn't attributed to (and doesn't grow) the - # stale fire instead. - env = _envelope(lat=10.0, lon=10.0, acq_time="1800") - env["id"] = "halt-sev-test" - env["data"]["id"] = "halt-sev-test" - env["data"]["geo"] = {"centroid": [10.0, 10.0]} - # Raw envelope severity maps to "immediate" (>= immediate_min=3), - # deliberately NOT "routine" -- so the assertion below can only - # pass if the halt data_patch's _severity_override actually - # overrides it down to "routine". A plain "severity" key (the - # bug) would silently leave this at "immediate" instead. - env["data"]["severity"] = 3 - - event = self._consumer()._normalize(_SUBJECT, env) - assert event is not None, "expected a broadcast Event, got None" - assert event.severity == "routine", ( - f"issue #121 regression: expected 'routine', got " - f"{event.severity!r} -- gating.firms.decide's halt " - f"data_patch must use _severity_override, not the plain " - f"'severity' key" - ) - finally: - from meshai.notifications.cutover import _clear_cache - _clear_cache() diff --git a/work/tests/test_hydro_refactor.py b/work/tests/test_hydro_refactor.py index 0d53cbe..34556e0 100644 --- a/work/tests/test_hydro_refactor.py +++ b/work/tests/test_hydro_refactor.py @@ -1,19 +1,34 @@ """Phase-3 hydro (USGS NWIS) refactor tests. -Verifies the source-agnostic formatter+decider migration for the stream-gauge -hazard, mirroring test_quake_refactor.py: +Verifies the formatter+decider for the stream-gauge hazard (gating/hydro.py, +formatters/hydro.py), mirroring test_quake_refactor.py: -1. Golden byte-identical: formatters.hydro.format() reproduces the old - nwis_handler._render() wire exactly, for a stage-only crossing, a paired - flow(00060)+stage(00065) reading, and every threshold label. +1. Golden: formatters.hydro.format() renders the expected wire, for a + stage-only crossing, a paired flow(00060)+stage(00065) reading, and every + threshold label. -2. Gate-sequence parity: an explicit `now`-timeline of readings driven through - the NEW gating.hydro.decide() matches the OLD handle_nwis broadcast/suppress - behavior (upward crossing broadcasts; same-rank + receding suppress unless - broadcast_on_recede). +2. Gate-sequence: an explicit `now`-timeline of readings driven through + gating.hydro.decide() (upward crossing broadcasts; same-rank + receding + suppress unless broadcast_on_recede). + +The Central `nwis_handler` module (`_render()`, `handle_nwis()`) has been +deleted along with the rest of the Central NATS consumer path. Pure +old-vs-new parity assertions have been removed (original diffs are +preserved in git history); what remains asserts against hand-written +expected strings / broadcast outcomes. + +decide() is READ-ONLY over gauge_readings by design (the append-only INSERT +was always caller-owned -- previously the Central handler, inline, +immediately after calling decide()). With the handler gone there is no +current producer for the "stream_flow" category in production (no native +env/ adapter emits it -- meshai.env.usgs.USGSStreamsAdapter is a separate, +older stream-gauge pipeline with different categories/schema). Tests below +that need prior-reading state seed gauge_readings directly via a local SQL +helper that mirrors the deleted handler's INSERT shape, so the gate logic +itself stays under direct, native-only test coverage. The real registry / cutover key is "stream_flow" — the flat category the -Central nwis path produces for every central.hydro.* envelope. +Central nwis path used to produce for every central.hydro.* envelope. """ from __future__ import annotations @@ -21,7 +36,7 @@ import pytest from meshai.persistence import close_thread_connection, init_db from meshai.persistence import db as persistence_db -from tests.harness.goldens import assert_byte_identical, run_gate_sequence +from tests.harness.goldens import assert_byte_identical _AT = 1_783_200_000.0 # pinned epoch (unused by hydro render/gate, kept for parity) @@ -53,11 +68,7 @@ def _make_fake_event(data: dict): # ───────────────────────────────────────────────────────────────────────────── class TestFormatterGolden: - """formatters.hydro.format() == nwis_handler._render() for the same inputs.""" - - def _render_old(self, **kw): - from meshai.central.nwis_handler import _render - return _render(**kw) + """formatters.hydro.format() renders the expected wire for canonical data.""" def _fmt_new(self, canonical: dict) -> str: from meshai.notifications.formatters.hydro import format as hfmt @@ -74,13 +85,10 @@ class TestFormatterGolden: "lat": 43.612, "lon": -111.654, } - old = self._render_old( - gauge_name="Snake River at Heise", threshold_state="action", - stage_ft=12.5, flow_cfs=None, unit="ft", lat=43.612, lon=-111.654, - ) new = self._fmt_new(canonical) - assert_byte_identical(new, old) - assert new == "🌊 New: Snake River at Heise: action stage 12.5 ft, @ 43.612,-111.654" + assert_byte_identical( + new, "🌊 New: Snake River at Heise: action stage 12.5 ft, @ 43.612,-111.654" + ) def test_paired_flow_and_stage(self): """00060 discharge back-looked onto a 00065 stage: flow segment present.""" @@ -93,14 +101,11 @@ class TestFormatterGolden: "lat": 43.600, "lon": -116.200, } - old = self._render_old( - gauge_name="Boise River", threshold_state="flood_minor", - stage_ft=14.5, flow_cfs=8400, unit="ft", lat=43.600, lon=-116.200, - ) new = self._fmt_new(canonical) - assert_byte_identical(new, old) - assert "flow 8,400 cfs" in new - assert "minor flooding 14.5 ft" in new + assert_byte_identical( + new, + "🌊 New: Boise River: minor flooding 14.5 ft, flow 8,400 cfs, @ 43.600,-116.200", + ) @pytest.mark.parametrize( "state,label", @@ -112,7 +117,7 @@ class TestFormatterGolden: ], ) def test_every_threshold_label(self, state, label): - """Each threshold_state maps to the correct label — byte-identical to _render.""" + """Each threshold_state maps to the correct label.""" canonical = { "gauge_name": "Test Gauge", "threshold_state": state, @@ -122,16 +127,13 @@ class TestFormatterGolden: "lat": 44.0, "lon": -114.0, } - old = self._render_old( - gauge_name="Test Gauge", threshold_state=state, stage_ft=20.0, - flow_cfs=None, unit="ft", lat=44.0, lon=-114.0, - ) new = self._fmt_new(canonical) - assert_byte_identical(new, old) - assert f"{label} 20.0 ft" in new + assert_byte_identical( + new, f"🌊 New: Test Gauge: {label} 20.0 ft, @ 44.000,-114.000" + ) def test_missing_coords_drops_at_tail(self): - """No coords → no @ segment (byte-identical to _render).""" + """No coords → no @ segment.""" canonical = { "gauge_name": "No Coords Gauge", "threshold_state": "action", @@ -141,13 +143,8 @@ class TestFormatterGolden: "lat": None, "lon": None, } - old = self._render_old( - gauge_name="No Coords Gauge", threshold_state="action", - stage_ft=10.0, flow_cfs=None, unit="ft", lat=None, lon=None, - ) new = self._fmt_new(canonical) - assert_byte_identical(new, old) - assert "@" not in new + assert_byte_identical(new, "🌊 New: No Coords Gauge: action stage 10.0 ft") # ───────────────────────────────────────────────────────────────────────────── @@ -183,33 +180,35 @@ def _parse_iso_epoch(s): return int(datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp()) +def _insert_reading(conn, *, site_id, gauge_name, value, unit, + threshold_state, flow_cfs, reading_time, lat, lon): + """Directly seed a gauge_readings row. + + Mirrors the schema the deleted Central nwis_handler used to INSERT + inline, immediately after calling decide(). decide() is read-only over + this table by design (see gating/hydro.py docstring) -- the INSERT was + always caller-owned, so tests seed state directly instead of reaching + into the deleted handler. + """ + conn.execute( + "INSERT INTO gauge_readings(site_id, gauge_name, reading_value, " + "reading_unit, threshold_state, flow_cfs, reading_time, lat, lon) " + "VALUES (?,?,?,?,?,?,?,?,?)", + (site_id, gauge_name, value, unit, threshold_state, flow_cfs, + reading_time, lat, lon), + ) + + class TestGateSequenceParity: - """New decide() decisions match old handle_nwis broadcast/suppress.""" + """gating.hydro.decide() broadcast/suppress across a reading timeline.""" @pytest.fixture(autouse=True) def _db(self, mem_db): self.db = mem_db - def _old_gate(self, fixture, *, now): - """OLD path: handle_nwis returning non-None = broadcast. - - handle_nwis owns the append-only gauge_readings INSERT, so replaying - through it advances the persisted time-series exactly as production - would — the decider (below) then reads that same state. - """ - from meshai.central.nwis_handler import handle_nwis - env = fixture["envelope"] - wire = handle_nwis(env, env["subject"], data={}, now=int(now)) - return wire is not None - - def _new_gate(self, fixture, *, now): - """NEW path: build canonical (as the handler does) then decide(). - - We do NOT insert here — the old-gate replay already advances - gauge_readings; the decider only READS prior state. This mirrors the - production ordering where decide() runs before the inline INSERT. - """ - from meshai.notifications.gating.hydro import decide + def _canonical(self, fixture): + """Build canonical data from a Central-style fixture (as the deleted + handler used to) via the still-live idaho_gauge_sites helpers.""" from meshai.central.idaho_gauge_sites import ( compute_threshold_state, lookup_site, normalize_site_id, ) @@ -238,10 +237,34 @@ class TestGateSequenceParity: "lon": d.get("longitude"), "parameter_code": pc, } + return canonical, value + + def _decide(self, fixture, *, now): + """decide() only (no persist) — for assertions on a single reading.""" + from meshai.notifications.gating.hydro import decide + canonical, _value = self._canonical(fixture) return decide(canonical, source="nwis", now=float(now)) + def _decide_and_persist(self, fixture, *, now): + """decide() then INSERT the resolved reading — mirrors the deleted + handler's decide-then-insert ordering, so later steps in a sequence + see accumulated prior state exactly as production would.""" + from meshai.notifications.gating.hydro import decide + canonical, value = self._canonical(fixture) + gate = decide(canonical, source="nwis", now=float(now)) + threshold_state = gate.data_patch.get("threshold_state", canonical["threshold_state"]) + stage_ft = gate.data_patch.get("stage_ft", canonical["stage_ft"]) + _insert_reading( + self.db, + site_id=canonical["site_id"], gauge_name=canonical["gauge_name"], + value=value, unit=canonical["unit"], threshold_state=threshold_state, + flow_cfs=canonical["flow_cfs"], reading_time=canonical["reading_time"], + lat=canonical["lat"], lon=canonical["lon"], + ) + return gate + def test_gate_sequence_matches(self): - """Timeline of Heise readings: old and new gates agree on every step. + """Timeline of Heise readings: decide() agrees with expectations at every step. Heise (USGS-13186000): action=12.0ft. [0] 8.0 ft normal (first reading, no prior) → suppress @@ -264,22 +287,16 @@ class TestGateSequenceParity: ] timeline = [float(base + i * 900) for i in range(len(specs))] - results = run_gate_sequence(self._old_gate, self._new_gate, ordered, - timeline=timeline) - mismatches = [r for r in results if not r["match"]] - assert not mismatches, ( - "Gate sequence mismatch old handle_nwis vs new decide():\n" - + "\n".join( - f" step {r['fixture_n']}: old={r['old_broadcast']} " - f"new={r['new_broadcast']} diffs={r['diffs']}" - for r in mismatches - ) - ) - assert results[0]["old_broadcast"] is False, "normal first reading suppressed" - assert results[1]["old_broadcast"] is True, "normal→action broadcasts" - assert results[2]["old_broadcast"] is False, "action→action suppressed" - assert results[3]["old_broadcast"] is True, "action→flood_minor broadcasts" - assert results[4]["old_broadcast"] is False, "receding suppressed (no toggle)" + results = [ + self._decide_and_persist(fx, now=t) + for fx, t in zip(ordered, timeline) + ] + + assert results[0].broadcast is False, "normal first reading suppressed" + assert results[1].broadcast is True, "normal→action broadcasts" + assert results[2].broadcast is False, "action→action suppressed" + assert results[3].broadcast is True, "action→flood_minor broadcasts" + assert results[4].broadcast is False, "receding suppressed (no toggle)" def test_00060_backlook_inherits_stage_band(self, mem_db): """A 00060 discharge reading inherits the last 00065 stage band. @@ -288,16 +305,14 @@ class TestGateSequenceParity: decider's back-look must resolve threshold_state=action + the prior stage_ft, and (same rank as the seeded action) suppress the discharge. """ - from meshai.notifications.gating.hydro import decide - # Seed a 00065 stage reading at action via the old handler (writes row). env_stage = _nwis_env(parameter_code="00065", value=12.5, time_iso="2026-06-05T10:00:00Z", envelope_id="seed") - self._old_gate({"envelope": env_stage}, now=1_000_000) + self._decide_and_persist({"envelope": env_stage}, now=1_000_000) # Now decide on a 00060 discharge — should back-look the action band. env_flow = _nwis_env(parameter_code="00060", value=8400, unit="ft^3/s", time_iso="2026-06-05T10:05:00Z", envelope_id="q") - gate = self._new_gate({"envelope": env_flow}, now=1_000_300) + gate = self._decide({"envelope": env_flow}, now=1_000_300) assert gate.data_patch["threshold_state"] == "action" assert gate.data_patch["stage_ft"] == 12.5 # action → action (same rank) → suppress @@ -306,11 +321,10 @@ class TestGateSequenceParity: def test_recede_toggle_enables_broadcast(self, mem_db): """With broadcast_on_recede set, a receding crossing broadcasts.""" from meshai.adapter_config._accessor import set_runtime_override, _overrides - from meshai.notifications.gating.hydro import decide # Seed an action reading. env_high = _nwis_env(parameter_code="00065", value=12.5, time_iso="2026-06-05T10:00:00Z", envelope_id="hi") - self._old_gate({"envelope": env_high}, now=1_000_000) + self._decide_and_persist({"envelope": env_high}, now=1_000_000) # Force the recede toggle on for the decision only (runtime override, # since adapter_config accessors are read-only). @@ -318,7 +332,7 @@ class TestGateSequenceParity: try: env_low = _nwis_env(parameter_code="00065", value=8.0, time_iso="2026-06-05T11:00:00Z", envelope_id="lo") - gate = self._new_gate({"envelope": env_low}, now=1_003_600) + gate = self._decide({"envelope": env_low}, now=1_003_600) finally: _overrides.pop(("usgs_nwis", "broadcast_on_recede"), None) assert gate.broadcast is True, "receding must broadcast when toggle is on" diff --git a/work/tests/test_incident_handler.py b/work/tests/test_incident_handler.py deleted file mode 100644 index 7b59543..0000000 --- a/work/tests/test_incident_handler.py +++ /dev/null @@ -1,828 +0,0 @@ -"""Tests for meshai.central.incident_handler (v0.5.9). - -Coverage: - Tomtom parsing + rendering (a/b/c/d): - (a) jam, accident, road_closed, lane_closed, road_works each render with - correct emoji + phrase + delay segment - (b) magnitude_of_delay == 0 events filtered at entrance - (c) delay == null events render WITHOUT the delay segment - (d) time_validity past/future events filtered - - Per-incident change-detection (e-i): - (e) republish with no change -> drop silently, no new audit - (f) magnitude bump up -> Update - (g) delay double (>=2x) -> Update - (h) icon change -> Update - - state_511 / itd_511 EventType branching (j-m): - (j) state_511_atis incident parses - (k) state_511_atis closure parses - (l) state_511_atis special_event parses (synthetic) - (m) itd_511 incident parses - - Decoupled callback (n) and traffic_events UPSERT (o): - (n) cold-start scenario -- handler runs but callback never fires; - second pass still emits New: (not Update:) - (o) existing traffic_events row gets UPSERTed across passes -""" - -import re -import time - -import pytest - -from meshai.central.incident_handler import ( - handle_incident, - _render as _incident_render, -) -from meshai.persistence import close_thread_connection, init_db -from meshai.persistence import db as persistence_db - - -# ---------- fixtures ------------------------------------------------------ - - -@pytest.fixture -def mem_db(monkeypatch, tmp_path): - db_path = str(tmp_path / "incident-test.sqlite") - monkeypatch.setenv("MESHAI_DB_PATH", db_path) - persistence_db._initialised.clear() - close_thread_connection() - conn = init_db() - yield conn - close_thread_connection() - persistence_db._initialised.discard(db_path) - - -@pytest.fixture -def no_photon(monkeypatch): - import meshai.central_normalizer as cn - monkeypatch.setattr(cn, "_photon_reverse_places", lambda lat, lon: []) - if hasattr(cn, "_H3_NEAREST_CACHE"): - cn._H3_NEAREST_CACHE.clear() - - -# ---------- tomtom envelope builder -------------------------------------- - - -_TTI_A = "cfb0c03f-9ab9-46f9-ac21-9b17d0f715f2" -_TTI_B = "13ca4176-4eea-428e-a807-49bee662159a" -_TTI_C = "3573b54b-9e55-4aff-83d3-253048825e77" - - -def _tomtom_env(*, tti=_TTI_A, - icon_category=6, - magnitude=4, - delay=412, - time_validity="present", - description="Queuing traffic on I-84 Westbound from Orchard St to ID-55. ", - road_numbers=("I-84",), - from_loc="Orchard St/Exit 52 (I-84)", - to_loc="ID-55/Exit 46 (I-84)", - start_time=None, - end_time=None, - lat=43.5833926533, lon=-116.2598321532, - state_code="ID", bbox_name="treasure_valley_ext", - geocoder_city="Boise"): - inner_id = f"ID:tomtom:TTI-{tti}-TTR{int(time.time()*1000)}" - geocoder = {"city": geocoder_city, "county": "Ada", "state": "ID", - "country": "United States", "landclass": None} - return { - "id": inner_id, - "subject": "central.traffic.incident.id", - "data": { - "id": inner_id, "adapter": "tomtom_incidents", - "category": "incident.tomtom_incidents", "severity": 1, - "geo": {"centroid": [lon, lat], "primary_region": "US-ID"}, - "data": { - "id": inner_id, - "description": description, - "event_code": 108, - "from": from_loc, "to": to_loc, - "magnitude_of_delay": magnitude, - "icon_category": icon_category, - "length": 6112.13, - "delay": delay, - "road_numbers": list(road_numbers), - "start_time": start_time, "end_time": end_time, - "time_validity": time_validity, - "state_code": state_code, "bbox_name": bbox_name, - "latitude": lat, "longitude": lon, - "_enriched": {"geocoder": geocoder}, - }, - }, - } - - -def _state_511_env(*, layer="Incidents", category_prefix="incident", - event_sub_type="crash", - roadway="US-95", direction="Both", - is_full_closure=False, - external_id="ID:Incidents:33948", - lat=48.5295, lon=-116.4293, - geocoder_city="Naples", county="Boundary"): - return { - "id": external_id, - "subject": f"central.traffic.{category_prefix}.id", - "data": { - "id": external_id, "adapter": "state_511_atis", - "category": f"{category_prefix}.state_511_atis", "severity": 1, - "geo": {"centroid": [lon, lat], "primary_region": "US-WA"}, - "data": { - "roadway_name": roadway, "direction": direction, - "event_sub_type": event_sub_type, - "description": "test event", - "is_full_closure": is_full_closure, - "layer": layer, - # v0.5.9 GAMMA: default to non-ID neighbor state because - # state_511_atis no longer covers Idaho (itd_511 took over). - # Tests that want to exercise the ID-skip path override these. - "county": county, "state": "Washington", "state_code": "WA", - "start_date": None, - "last_updated": None, - "latitude": lat, "longitude": lon, - "_enriched": {"geocoder": { - "city": geocoder_city, "county": county, "state": "ID", - "country": "United States", "landclass": None, - }}, - }, - }, - } - - -def _itd_511_env(*, category_prefix="incident", - event_type_short="incident", - event_sub_type="crash", - roadway="I-84", direction="East", - is_full_closure=False, - external_id="ITD:469:17", - lat=43.6486, lon=-116.4870, - geocoder_city="Caldwell"): - return { - "id": external_id, - "subject": f"central.traffic.{category_prefix}.us.id", - "data": { - "id": external_id, "adapter": "itd_511", - "category": f"{category_prefix}.itd_511", "severity": 1, - "geo": {"centroid": [lon, lat], "primary_region": "US-ID"}, - "data": { - "event_type_short": event_type_short, - "event_sub_type": event_sub_type, - "roadway_name": roadway, "direction": direction, - "description": "test itd event", - "lanes_affected": "All lanes affected", - "is_full_closure": is_full_closure, - "itd_severity": "None", - "comment": "", "cause": "roadwork", - "organization": "ERS", - "recurrence_text": "", "recurrence_schedules": [], - "restrictions": {}, "encoded_polyline": "", - "id_internal": 17, "source_id": "469", - "reported_epoch": None, - "last_updated_epoch": None, - "start_epoch": None, - "planned_end_epoch": None, - "latitude": lat, "longitude": lon, - "_enriched": {"geocoder": { - "city": geocoder_city, "county": "Canyon", "state": "ID", - }}, - }, - }, - } - - -def _commit(data, committed_at): - cb = data.get("_on_broadcast_committed") - assert callable(cb), "handler must attach commit callback" - cb(committed_at) - - -# ============================================================================ -# (a) tomtom parsing -- all five icon categories render correctly -# ============================================================================ - - -@pytest.mark.parametrize("icon, expected_emoji, expected_phrase", [ - (1, "🚨", "Crash"), - (6, "🚗", "Stationary Traffic"), - (7, "🟠", "Lane Reduction"), - (8, "🚫", "Road Closed"), - (9, "🚧", "Road Works"), -]) -def test_a_tomtom_icon_renders(mem_db, no_photon, icon, expected_emoji, expected_phrase): - env = _tomtom_env(icon_category=icon, delay=300) - data = {} - wire = handle_incident(env, env["subject"], data=data, now=1_000_000) - assert wire is not None - assert wire.startswith(f"{expected_emoji} {expected_phrase}") - assert "Near Boise, ID" in wire - assert "I-84" in wire - # Budget-fit rework: the separate "N min delay" line was dropped; the road - # segment carries road (+ direction/lanes), and the message fits 140. - assert "min delay" not in wire - assert len(wire) <= 140 - - -# ============================================================================ -# (b) tomtom magnitude_of_delay == 0 events filtered -# ============================================================================ - - -def test_b_tomtom_magnitude_zero_filtered(mem_db, no_photon): - env = _tomtom_env(icon_category=6, magnitude=0, delay=10) - data = {} - wire = handle_incident(env, env["subject"], data=data, now=1_000_000) - assert wire is None - # filtered envelope leaves no traffic_events row - n_rows = mem_db.execute("SELECT COUNT(*) AS n FROM traffic_events").fetchone()["n"] - assert n_rows == 0 - # but the event IS logged to event_log handled=0 for accounting - n_log = mem_db.execute( - "SELECT COUNT(*) AS n FROM event_log WHERE source='tomtom_incidents'" - ).fetchone()["n"] - assert n_log == 1 - assert "_on_broadcast_committed" not in data - - -# ============================================================================ -# (c) tomtom delay == null events render WITHOUT delay segment -# ============================================================================ - - -def test_c_tomtom_delay_null_no_delay_segment(mem_db, no_photon): - env = _tomtom_env(icon_category=1, delay=None) - data = {} - wire = handle_incident(env, env["subject"], data=data, now=1_000_000) - assert wire is not None - assert "Crash" in wire - assert "min delay" not in wire # no delay segment - - -# ============================================================================ -# (d) tomtom time_validity past/future filtered -# ============================================================================ - - -@pytest.mark.parametrize("validity", ["past", "future"]) -def test_d_tomtom_time_validity_filtered(mem_db, no_photon, validity): - env = _tomtom_env(icon_category=6, magnitude=2, delay=300, - time_validity=validity) - data = {} - wire = handle_incident(env, env["subject"], data=data, now=1_000_000) - assert wire is None - n_rows = mem_db.execute( - "SELECT COUNT(*) AS n FROM traffic_events").fetchone()["n"] - assert n_rows == 0 - - -# ============================================================================ -# (e) per-incident dedup -- republish with no change drops silently -# ============================================================================ - - -def test_e_per_incident_dedup_no_change(mem_db, no_photon): - env = _tomtom_env(icon_category=6, delay=300) - data1 = {} - wire1 = handle_incident(env, env["subject"], data=data1, now=1_000_000) - assert wire1 is not None - _commit(data1, 1_000_001) - - # Re-publish 5 minutes later, same magnitude/delay/icon. - data2 = {} - wire2 = handle_incident(env, env["subject"], data=data2, now=1_000_300) - assert wire2 is None # no change, no broadcast - - -# ============================================================================ -# (f) magnitude bump triggers Update -# ============================================================================ - - -def test_f_magnitude_bump_triggers_update(mem_db, no_photon): - env1 = _tomtom_env(icon_category=6, delay=300) - data1 = {} - handle_incident(env1, env1["subject"], data=data1, now=1_000_000) - _commit(data1, 1_000_001) - - # v0.5.9 REVISED gate (A): magnitude bump no longer fires Update. - # State still flips in traffic_events, but no wire string returns. - env2 = _tomtom_env(icon_category=6, magnitude=5, delay=300) - data2 = {} - wire2 = handle_incident(env2, env2["subject"], data=data2, now=1_000_300) - assert wire2 is None - # Current magnitude tracked in the row. - row = mem_db.execute( - "SELECT magnitude_of_delay FROM traffic_events " - "WHERE source='tomtom_incidents'").fetchone() - assert row["magnitude_of_delay"] == 5 - - -# ============================================================================ -# (g) delay double triggers Update -# ============================================================================ - - -def test_g_delay_double_triggers_update(mem_db, no_photon): - env1 = _tomtom_env(icon_category=6, delay=300) - data1 = {} - handle_incident(env1, env1["subject"], data=data1, now=1_000_000) - _commit(data1, 1_000_001) - - # v0.5.9 REVISED gate (A): delay double no longer fires Update. - env2 = _tomtom_env(icon_category=6, delay=700) - data2 = {} - wire2 = handle_incident(env2, env2["subject"], data=data2, now=1_000_300) - assert wire2 is None - row = mem_db.execute( - "SELECT delay_seconds FROM traffic_events " - "WHERE source='tomtom_incidents'").fetchone() - assert row["delay_seconds"] == 700 - - -def test_g_delay_below_double_no_update(mem_db, no_photon): - """delay 300 -> 500 (1.67x) should NOT trigger broadcast.""" - env1 = _tomtom_env(icon_category=6, delay=300) - data1 = {} - handle_incident(env1, env1["subject"], data=data1, now=1_000_000) - _commit(data1, 1_000_001) - - env2 = _tomtom_env(icon_category=6, delay=500) - data2 = {} - wire2 = handle_incident(env2, env2["subject"], data=data2, now=1_000_300) - assert wire2 is None - - -# ============================================================================ -# (h) icon change triggers Update -# ============================================================================ - - -def test_h_icon_change_triggers_update(mem_db, no_photon): - env1 = _tomtom_env(icon_category=6, delay=300) - data1 = {} - handle_incident(env1, env1["subject"], data=data1, now=1_000_000) - _commit(data1, 1_000_001) - - # v0.5.9 REVISED gate (A): icon change no longer fires Update. - env2 = _tomtom_env(icon_category=8, delay=300) - data2 = {} - wire2 = handle_incident(env2, env2["subject"], data=data2, now=1_000_300) - assert wire2 is None - row = mem_db.execute( - "SELECT icon_category FROM traffic_events " - "WHERE source='tomtom_incidents'").fetchone() - assert row["icon_category"] == "road_closed" - - -# ============================================================================ -# (j) state_511_atis incident parses -# ============================================================================ - - -def test_j_state_511_incident_parses(mem_db, no_photon): - env = _state_511_env(layer="Incidents", category_prefix="incident", - event_sub_type="crash") - data = {} - wire = handle_incident(env, env["subject"], data=data, now=1_000_000) - assert wire is not None - assert wire.startswith("🚨 Crash") # crash -> 🚨 - assert "US-95" in wire - assert "Near Naples" in wire - row = mem_db.execute( - "SELECT source, sub_type, state FROM traffic_events " - "WHERE source='state_511_atis'").fetchone() - assert row["sub_type"] == "accident" - # v0.5.9 GAMMA: state_511_atis is non-ID only -- helper now defaults to WA. - assert row["state"] == "WA" - - -# ============================================================================ -# (k) state_511_atis closure parses -# ============================================================================ - - -def test_k_state_511_closure_parses(mem_db, no_photon): - env = _state_511_env(layer="Closures", category_prefix="closure", - event_sub_type="roadConstruction", - is_full_closure=True, - external_id="ID:Closures:33950") - data = {} - wire = handle_incident(env, env["subject"], data=data, now=1_000_000) - assert wire is not None - # roadConstruction -> road_works -> 🚧 - assert wire.startswith("🚧 Road Works") - assert "US-95" in wire - - -# ============================================================================ -# (l) state_511_atis special_event parses -# ============================================================================ - - -def test_l_state_511_special_event_parses(mem_db, no_photon): - env = _state_511_env(layer="Special Events", - category_prefix="special_event", - event_sub_type="parade", - external_id="ID:SpecialEvents:42") - data = {} - wire = handle_incident(env, env["subject"], data=data, now=1_000_000) - assert wire is not None - # parade -> 🎪 - assert wire.startswith("🎪 Parade") - assert "US-95" in wire - - -# ============================================================================ -# (m) itd_511 incident parses -# ============================================================================ - - -def test_m_itd_511_incident_parses(mem_db, no_photon): - env = _itd_511_env(category_prefix="incident", - event_type_short="incident", - event_sub_type="crash") - data = {} - wire = handle_incident(env, env["subject"], data=data, now=1_000_000) - assert wire is not None - assert wire.startswith("🚨 Crash") - row = mem_db.execute( - "SELECT source, state FROM traffic_events " - "WHERE source='itd_511'").fetchone() - assert row["state"] == "ID" - - -# ============================================================================ -# (n) decoupled callback -- cold-start scenario still emits New: on second pass -# ============================================================================ - - -def test_n_cold_start_then_resume_still_new(mem_db, no_photon): - env = _tomtom_env(icon_category=6, delay=300) - data1 = {} - wire1 = handle_incident(env, env["subject"], data=data1, now=1_000_000) - assert wire1.startswith("🚗 Stationary Traffic") - # Cold-start grace drops the broadcast -- DO NOT call _commit(). - - # 5 minutes later, same incident republishes. - data2 = {} - wire2 = handle_incident(env, env["subject"], data=data2, now=1_000_300) - assert wire2 is not None - assert wire2.startswith("🚗 Stationary Traffic"), \ - "must still be New: until commit callback fires" - - row = mem_db.execute( - "SELECT last_broadcast_at, last_broadcast_magnitude " - "FROM traffic_events WHERE source='tomtom_incidents'").fetchone() - assert row["last_broadcast_at"] is None - assert row["last_broadcast_magnitude"] is None - - -# ============================================================================ -# (o) traffic_events row gets UPSERTed on each pass; event_log handled flips -# ============================================================================ - - -def test_o_traffic_events_upsert_and_event_log_handled_flip(mem_db, no_photon): - env1 = _tomtom_env(icon_category=6, delay=300) - data1 = {} - handle_incident(env1, env1["subject"], data=data1, now=1_000_000) - - # event_log row exists with handled=0 BEFORE callback. - el_pre = mem_db.execute( - "SELECT handled FROM event_log " - "WHERE source='tomtom_incidents' ORDER BY id DESC LIMIT 1" - ).fetchone() - assert el_pre["handled"] == 0 - - _commit(data1, 1_000_001) - - el_post = mem_db.execute( - "SELECT handled FROM event_log " - "WHERE source='tomtom_incidents' ORDER BY id DESC LIMIT 1" - ).fetchone() - assert el_post["handled"] == 1 - - fr_post = mem_db.execute( - "SELECT last_broadcast_at, last_broadcast_magnitude, " - "last_broadcast_delay_seconds, last_broadcast_icon_category " - "FROM traffic_events WHERE source='tomtom_incidents'" - ).fetchone() - assert fr_post["last_broadcast_at"] == 1_000_001 - assert fr_post["last_broadcast_magnitude"] == 4 - assert fr_post["last_broadcast_delay_seconds"] == 300 - assert fr_post["last_broadcast_icon_category"] == "jam" - - # Re-publish: UPSERT updates current_* but doesn't touch last_broadcast_*. - env2 = _tomtom_env(icon_category=6, delay=500) - data2 = {} - wire2 = handle_incident(env2, env2["subject"], data=data2, now=1_000_300) - # v0.5.9 REVISED: no Update broadcasts regardless of delta size -- - # under the OLD rule this was 'delay 1.67x not enough', now it's - # 'we never re-broadcast'. - assert wire2 is None - fr2 = mem_db.execute( - "SELECT delay_seconds, last_broadcast_delay_seconds " - "FROM traffic_events WHERE source='tomtom_incidents'" - ).fetchone() - assert fr2["delay_seconds"] == 500 # UPSERT happened - assert fr2["last_broadcast_delay_seconds"] == 300 # last broadcast unchanged - - -# ============================================================================ -# v0.5.9 REVISED -- conservative gates -# ============================================================================ - - -def test_p_known_id_all_changed_no_broadcast(mem_db, no_photon): - """Regression guard for the new no-Update rule. Republish the SAME - external_id with magnitude AND delay AND icon all changed. The old rule - would have triggered Update on any one of those; the new rule fires - nothing.""" - env1 = _tomtom_env(icon_category=6, delay=300) - data1 = {} - wire1 = handle_incident(env1, env1["subject"], data=data1, now=1_000_000) - assert wire1.startswith("🚗 Stationary Traffic") - _commit(data1, 1_000_001) - - env2 = _tomtom_env(icon_category=8, magnitude=5, delay=700) # ALL changed - data2 = {} - wire2 = handle_incident(env2, env2["subject"], data=data2, now=1_000_300) - assert wire2 is None, "no Update broadcasts under the v0.5.9 REVISED rule" - - # State still tracks the latest field values. - row = mem_db.execute( - "SELECT magnitude_of_delay, delay_seconds, icon_category " - "FROM traffic_events WHERE source='tomtom_incidents'").fetchone() - assert row["magnitude_of_delay"] == 5 - assert row["delay_seconds"] == 700 - assert row["icon_category"] == "road_closed" - - -# ============================================================================ -# Freshness gate (q/r/s) -# ============================================================================ - - -def _now_anchor_relative(start_age_seconds: int): - """Choose (now, start_time_iso) so that `now - parse(start_time) == - start_age_seconds`. Used by the freshness-gate tests.""" - import datetime as _dt - now = 2_000_000_000 # arbitrary fixed epoch >>1970 - start_iso = _dt.datetime.fromtimestamp( - now - start_age_seconds, tz=_dt.timezone.utc - ).strftime("%Y-%m-%dT%H:%M:%SZ") - return now, start_iso - - -def test_q_fresh_event_15min_ago_broadcasts(mem_db, no_photon): - """Event started 15 min ago -- WITHIN the 30-min fresh window. New: fires.""" - now, start_iso = _now_anchor_relative(15 * 60) - env = _tomtom_env(icon_category=6, delay=300, - start_time=start_iso) - data = {} - wire = handle_incident(env, env["subject"], data=data, now=now) - assert wire is not None - assert wire.startswith("🚗 Stationary Traffic") - n_rows = mem_db.execute( - "SELECT COUNT(*) AS n FROM traffic_events").fetchone()["n"] - assert n_rows == 1 - - -def test_r_stale_event_45min_ago_dropped_no_row(mem_db, no_photon): - """Event started 45 min ago -- OUTSIDE the 30-min window. Drop AT - handler entrance, no UPSERT into traffic_events, event_log handled=0.""" - now, start_iso = _now_anchor_relative(45 * 60) - env = _tomtom_env(icon_category=6, magnitude=2, delay=300, - start_time=start_iso) - data = {} - wire = handle_incident(env, env["subject"], data=data, now=now) - assert wire is None - - n_rows = mem_db.execute( - "SELECT COUNT(*) AS n FROM traffic_events").fetchone()["n"] - assert n_rows == 0, "no UPSERT for stale incidents" - - n_log = mem_db.execute( - "SELECT COUNT(*) AS n FROM event_log WHERE handled=0 " - "AND source='tomtom_incidents'").fetchone()["n"] - assert n_log == 1, "stale envelope must still be logged (handled=0)" - - -def test_s_null_start_time_default_allow(mem_db, no_photon): - """start_time missing -> default-allow (treat as fresh and broadcast).""" - env = _tomtom_env(icon_category=6, delay=300, - start_time=None) - data = {} - wire = handle_incident(env, env["subject"], data=data, now=1_000_000) - assert wire is not None - assert wire.startswith("🚗 Stationary Traffic") - - -# ============================================================================ -# Per-source startTime field path (t) -# ============================================================================ - - -def test_t_state_511_start_date_path(mem_db, no_photon): - """state_511_atis pulls start_time from inner.data.start_date ('5/28/26, - 10:45 PM' format). Construct fresh (within 30 min) and stale (>30 min) - cases and confirm gate behavior is per-source-correct.""" - # 5 min ago in the 511-date format. - import datetime as _dt - now = int(_dt.datetime(2026, 6, 4, 12, 0, tzinfo=_dt.timezone.utc).timestamp()) - fresh_date = _dt.datetime.fromtimestamp(now - 5 * 60, - tz=_dt.timezone.utc).strftime( - "%-m/%-d/%y, %-I:%M %p") - env = _state_511_env(layer="Incidents", category_prefix="incident", - event_sub_type="crash") - # Replace the default start_date with a fresh one. - env["data"]["data"]["start_date"] = fresh_date - wire = handle_incident(env, env["subject"], data={}, now=now) - assert wire is not None - assert wire.startswith("🚨 Crash") - - # Stale variant (45 min ago). - stale_date = _dt.datetime.fromtimestamp(now - 45 * 60, - tz=_dt.timezone.utc).strftime( - "%-m/%-d/%y, %-I:%M %p") - env2 = _state_511_env(layer="Incidents", category_prefix="incident", - event_sub_type="crash", - external_id="ID:Incidents:99999") - env2["data"]["data"]["start_date"] = stale_date - wire2 = handle_incident(env2, env2["subject"], data={}, now=now) - assert wire2 is None - - -def test_t_itd_511_start_epoch_path(mem_db, no_photon): - """itd_511 pulls start_time from inner.data.start_epoch (Unix int).""" - now = 1_700_000_000 - # Fresh (5 min ago) variant. - env = _itd_511_env(category_prefix="incident", - event_type_short="incident", - event_sub_type="crash") - env["data"]["data"]["start_epoch"] = now - 5 * 60 - wire = handle_incident(env, env["subject"], data={}, now=now) - assert wire is not None - assert wire.startswith("🚨 Crash") - - # Stale variant (45 min ago). - env2 = _itd_511_env(category_prefix="incident", - event_type_short="incident", - event_sub_type="crash", - external_id="ITD:469:99999") - env2["data"]["data"]["start_epoch"] = now - 45 * 60 - wire2 = handle_incident(env2, env2["subject"], data={}, now=now) - assert wire2 is None - - -def test_t_tomtom_start_time_path(mem_db, no_photon): - """tomtom pulls start_time from inner.data.start_time (ISO-8601).""" - now, fresh_iso = _now_anchor_relative(5 * 60) - env = _tomtom_env(icon_category=6, delay=300, - start_time=fresh_iso) - wire = handle_incident(env, env["subject"], data={}, now=now) - assert wire is not None - assert wire.startswith("🚗 Stationary Traffic") - - _, stale_iso = _now_anchor_relative(45 * 60) - env2 = _tomtom_env(icon_category=6, delay=300, - start_time=stale_iso, - tti="11111111-2222-3333-4444-555555555555") - wire2 = handle_incident(env2, env2["subject"], data={}, now=now) - assert wire2 is None - - -# ============================================================================ -# v0.5.9 GAMMA -- two-sided freshness gate + state_511 ID skip -# ============================================================================ - - -def test_u_future_scheduled_event_dropped(mem_db, no_photon): - """itd_511 work_zone envelopes can carry start_epoch in the future - (scheduled construction). The v0.5.9 REVISED one-sided gate let those - slip through. The GAMMA fix rejects negative ages too.""" - import datetime as _dt - now = 2_000_000_000 - # start_epoch 8 hours in the future - env = _itd_511_env(category_prefix="incident", - event_type_short="incident", - event_sub_type="crash", - external_id="ITD:future:1") - env["data"]["data"]["start_epoch"] = now + 8 * 3600 - wire = handle_incident(env, env["subject"], data={}, now=now) - assert wire is None - n_rows = mem_db.execute( - "SELECT COUNT(*) AS n FROM traffic_events").fetchone()["n"] - assert n_rows == 0 - n_log = mem_db.execute( - "SELECT COUNT(*) AS n FROM event_log WHERE handled=0 " - "AND source='itd_511'").fetchone()["n"] - assert n_log == 1 - - -def test_v_state_511_id_skipped_at_handler_entrance(mem_db, no_photon): - """state_511_atis with state_code='ID' is skipped at handler entrance -- - no parse, no traffic_events row, event_log records the skip.""" - env = _state_511_env(layer="Incidents", category_prefix="incident", - event_sub_type="crash") - # Override defaults (post-GAMMA helper defaults to WA) to ID for this test. - env["data"]["data"]["state_code"] = "ID" - env["data"]["data"]["state"] = "Idaho" - env["data"]["geo"]["primary_region"] = "US-ID" - wire = handle_incident(env, env["subject"], data={}, now=1_000_000) - assert wire is None - n_rows = mem_db.execute( - "SELECT COUNT(*) AS n FROM traffic_events").fetchone()["n"] - assert n_rows == 0 - row = mem_db.execute( - "SELECT category, handled FROM event_log " - "WHERE source='state_511_atis' ORDER BY id DESC LIMIT 1" - ).fetchone() - assert row is not None - assert "|skip_id" in row["category"] - assert row["handled"] == 0 - - -def test_v_state_511_non_id_still_processed(mem_db, no_photon): - """Regression guard: state_511_atis with state_code='WA' (or anything - that's not ID) keeps going through the handler. After Idaho cutover - we still need state_511 for neighbor coverage.""" - env = _state_511_env(layer="Incidents", category_prefix="incident", - event_sub_type="crash", county="Spokane") - # Override state_code to WA. - env["data"]["data"]["state_code"] = "WA" - env["data"]["geo"]["primary_region"] = "US-WA" - wire = handle_incident(env, env["subject"], data={}, now=1_000_000) - assert wire is not None - assert wire.startswith("🚨 Crash") - # traffic_events row written for WA event. - row = mem_db.execute( - "SELECT state FROM traffic_events WHERE source='state_511_atis'" - ).fetchone() - assert row is not None - assert row["state"] == "WA" - - -def test_w_itd_511_future_scheduled_dropped_via_start_epoch(mem_db, no_photon): - """Direct exercise of the itd_511 start_epoch field path with a - future-scheduled value (the Phase-1 leak source). Confirms the gate - really uses start_epoch and the new two-sided check catches it.""" - import meshai.central.incident_handler as h - env = _itd_511_env(category_prefix="incident", - event_type_short="incident", - event_sub_type="crash", - external_id="ITD:future:work") - env["data"]["data"]["start_epoch"] = 99_000_000_000 # year ~5108 -- definitely future - se = h._extract_start_time_epoch(env, "itd_511") - assert se == 99_000_000_000 - now = 2_000_000_000 - wire = handle_incident(env, env["subject"], data={}, now=now) - assert wire is None - - -# ============================================================================ -# Budget-fit worst case: longest plausible traffic payload must fit 140 chars -# with critical fields (type, location, road, FULL direction word, lane -# status, trimmed narrative with intact direction word + "milepost") present. -# ============================================================================ - - -def test_incident_worst_case_fits_140(): - n = { - "sub_type": "accident", - "geocoder_city": None, - "county": "Minidoka", - "state": "ID", - "road": "SH-27", - "direction": "north", # abbreviation/short form -> MUST expand - "mile_marker": None, - "lanes_affected": "1 Right lane blocked", - "comment": ( - "Southbound right lane at milepost 24 and westbound onramp to I-84 " - "blocked due to a multi-vehicle collision, expect major delays " - "through the evening commute and seek alternate routes tonight" - ), - } - wire = _incident_render(n) - - # (a) fits one mesh packet - assert len(wire) <= 140, f"{len(wire)} chars:\n{wire!r}" - - # (b) critical fields present - assert wire.startswith("🚨 Crash") # type - assert "Near Minidoka Co, ID" in wire # location - assert "SH-27" in wire # road - assert "Northbound" in wire # FULL direction (expanded) - assert "1 Right lane blocked" in wire # lane status - assert "SH-27 Northbound · 1 Right lane blocked" in wire # road·lane line - - # (c) direction is NEVER abbreviated anywhere in the wire - for abbr in (" NB", " N ", "Sbound", "N/B"): - assert abbr not in wire - - # (d) narrative present and word-boundary trimmed (no mid-word chop): the - # intact direction word "Southbound" and the intact word "milepost" survive. - assert "Southbound" in wire - assert "milepost" in wire - # trimmed from the END -> ends with the ellipsis, not raw text - assert wire.endswith("…") diff --git a/work/tests/test_incident_refactor.py b/work/tests/test_incident_refactor.py index d0d9eda..811a2f0 100644 --- a/work/tests/test_incident_refactor.py +++ b/work/tests/test_incident_refactor.py @@ -1,27 +1,37 @@ """Phase-2 incident/roads refactor tests — TIER-A (byte-identical goldens). -Test strategy -------------- -Golden (byte-identical) tests call the old per-source parsers and the legacy -renderer directly, then compare byte-for-byte against the new -formatters.incident.format() output. This bypasses the handle_incident -freshness gate (which drops all captured fixtures as stale) and isolates the -rendering logic — exactly the same pattern as test_quake_refactor.py. +The Central `incident_handler` module (`_parse_tomtom_incident()`, +`_parse_itd_511_incident()`, `_render()`) has been deleted along with the +rest of the Central NATS consumer path. The TomTom-incidents and ITD-511 +golden-fixture parity groups that called those deleted parsers directly to +build a "golden" wire and compare it byte-for-byte against +formatters.incident.format() have been removed in full (46 tests across +TestTomtomGolden, TestItd511IncidentGolden, and +TestSchemaConformance::test_incident_canonical_keys_from_parser) — every +assertion in those tests was generated FROM the deleted parser output, with +no independent hand-written expected content to fall back to, and no +native adapter exists that parses the *TomTom Incident Details* or Central +ITD-511-envelope raw shapes those fixtures capture (env/traffic.py is the +TomTom traffic-*flow* adapter, a different feed; env/roads511.py parses +ITD 511's own REST API shape, not the Central envelope fixtures here). +Original diffs are preserved in git history. This is a real production gap +flagged for Matt: TomTom road-incident ingestion in particular has no +native replacement. + +Work-zone parity is unaffected — `meshai.central_normalizer` (a top-level, +non-Central-NATS module; note the name is legacy) and +`meshai.notifications.renderers.work_zone` were never part of the deleted +consumer path and remain live. Groups ------ -1. Tomtom incident golden byte-parity (all 40 traffic/*.json fixtures with - min_magnitude override=0 so the parser doesn't filter them; fixtures that - the parser still rejects for other reasons are skipped gracefully). -2. ITD-511 incident golden byte-parity (traffic/0032.json + the non-work-zone - fixtures in traffic_last/). -3. Work-zone golden byte-parity (traffic_last/0002 itd_511, traffic_last/0003 +1. Work-zone golden byte-parity (traffic_last/0002 itd_511, traffic_last/0003 wzdx) — calls normalize() directly, same as the production consumer. -4. Gate sequence: decide() lifecycle transitions (new → cold-dup → +2. Gate sequence: decide() lifecycle transitions (new → cold-dup → suppress-on-update-False → magnitude-up → suppress-no-change). -5. _anchor.resolve_anchor: DB hit and Photon fallback path. -6. Schema conformance: canonical data dict has all expected keys. -7. Cross-source identity: same render fields → same output regardless of +3. _anchor.resolve_anchor: DB hit and Photon fallback path. +4. Schema conformance: canonical data dict has all expected keys. +5. Cross-source identity: same render fields → same output regardless of source string. """ from __future__ import annotations @@ -45,15 +55,6 @@ _FIXTURE_DIR = pathlib.Path(__file__).parent / "fixtures" # Using captured_epoch of the traffic_last fixtures (1783206522). _AT_WZ = 1_783_206_522.0 -# Expected canonical data keys for incident events. -_INCIDENT_CANONICAL_KEYS = frozenset({ - "external_id", "source", "sub_type", "road", "direction", - "from_loc", "to_loc", "mile_start", "mile_end", "mile_marker", - "lanes_affected", "cause", "comment", "impact", - "county", "state", "lat", "lon", "geocoder_city", "landclass", - "start_at", "end_at", "magnitude", "delay_seconds", "icon_category", -}) - # Expected canonical data keys for work-zone events. _WZ_CANONICAL_KEYS = frozenset({ "road", "direction", "mile_start", "mile_end", "sub_type", "impact", @@ -75,42 +76,10 @@ def _load_dir(hazard: str): return out -_TRAFFIC_FX = _load_dir("traffic") # 40 files (mostly tomtom, one itd_511) _TRAFFIC_LAST_FX = _load_dir("traffic_last") # 6 files (mixed adapters) -# ── Helper: build canonical incident dict from parser output ───────────────── - -def _n_to_canonical_incident(n: dict) -> dict: - """Mirror the cutover-branch canonical extraction in handle_incident.""" - return { - "external_id": n.get("external_id"), - "source": n.get("source"), - "sub_type": n.get("sub_type"), - "road": n.get("road"), - "direction": n.get("direction"), - "from_loc": n.get("from_loc"), - "to_loc": n.get("to_loc"), - "mile_start": n.get("mile_start"), - "mile_end": n.get("mile_end"), - "mile_marker": n.get("mile_marker"), - "lanes_affected": n.get("lanes_affected"), - "cause": n.get("cause"), - "comment": n.get("comment"), - "impact": n.get("impact"), - "county": n.get("county"), - "state": n.get("state"), - "lat": n.get("lat"), - "lon": n.get("lon"), - "geocoder_city": n.get("geocoder_city"), - "landclass": n.get("landclass"), - "start_at": n.get("start_at"), - "end_at": n.get("end_at"), - "magnitude": n.get("magnitude"), - "delay_seconds": n.get("delay_seconds"), - "icon_category": n.get("icon_category"), - } - +# ── Helper: build canonical work-zone dict from normalize() output ─────────── def _n_to_canonical_workzone(n: dict) -> dict: """Build canonical work-zone data dict from a normalize() result. @@ -153,20 +122,6 @@ def _make_event(category: str, data: dict): # ── pytest fixtures: adapter_config overrides ──────────────────────────────── -@pytest.fixture() -def tomtom_min_mag_zero(): - """Set tomtom_incidents.min_magnitude=-999 via runtime override for one test. - - The parser uses `int(adapter_config.tomtom_incidents.min_magnitude or 4)` - which treats 0 as falsy and falls back to 4. Using -999 (truthy) forces - all magnitudes to pass the `magnitude < min_mag` filter. - """ - from meshai.adapter_config._accessor import set_runtime_override, _overrides - set_runtime_override("tomtom_incidents", "min_magnitude", -999) - yield - _overrides.pop(("tomtom_incidents", "min_magnitude"), None) - - @pytest.fixture() def broadcast_on_update_on(): """Enable incident.broadcast_on_update for gate-sequence tests.""" @@ -176,143 +131,13 @@ def broadcast_on_update_on(): _overrides.pop(("incident", "broadcast_on_update"), None) -# ── Helpers: determine adapter type and call the right parser ──────────────── +# ── Helper: determine adapter type ──────────────────────────────────────────── def _adapter_for(fx: dict) -> str: return (fx["envelope"]["data"] or {}).get("adapter") or "" -def _category_for(fx: dict) -> str: - return (fx["envelope"]["data"] or {}).get("category") or "" - - -# ── 1. Tomtom incident golden byte-parity ──────────────────────────────────── - -class TestTomtomGolden: - """All traffic/*.json tomtom fixtures rendered via old parser + _render() - must produce byte-identical output from the new formatters.incident.format(). - - min_magnitude is overridden to 0 so all fixtures (including mag=3 ones) - exercise the renderer. The one itd_511 fixture (0032) is skipped here. - """ - - @pytest.mark.parametrize("name,fx", _TRAFFIC_FX) - def test_golden_parity(self, name, fx, tomtom_min_mag_zero): - from meshai.central.incident_handler import _parse_tomtom_incident, _render - from meshai.notifications.formatters.incident import format as fmt - from meshai.notifications.formatters._budget import budget_for - - adapter = _adapter_for(fx) - if adapter != "tomtom_incidents": - pytest.skip(f"{name}: adapter={adapter!r}, not tomtom") - - envelope = fx["envelope"] - now = int(fx.get("captured_epoch", time.time())) - - n = _parse_tomtom_incident(envelope, now) - if n is None: - pytest.skip(f"{name}: parser returned None (filtered)") - - golden = _render(n) - canonical = _n_to_canonical_incident(n) - - # Determine event category from category_kind - kind_map = { - "incident": "road_incident", - "closure": "road_closure", - "special_event": "road_incident", - "work_zone": "work_zone", - } - cat = kind_map.get(n.get("category_kind", "incident"), "road_incident") - event = _make_event(cat, canonical) - - budget = budget_for("incident") - new_out = fmt(event, now=float(now), budget=budget) - - assert_byte_identical(new_out, golden) - - -# ── 2. ITD-511 incident golden byte-parity ─────────────────────────────────── - -class TestItd511IncidentGolden: - """itd_511 incident/closure/special_event fixtures rendered via - _parse_itd_511_incident + _render() must be byte-identical from formatter. - - Covers traffic/0032.json (itd_511) + non-work-zone traffic_last fixtures. - """ - - def _run_single(self, name: str, fx: dict): - from meshai.central.incident_handler import _parse_itd_511_incident, _render - from meshai.notifications.formatters.incident import format as fmt - from meshai.notifications.formatters._budget import budget_for - - envelope = fx["envelope"] - category_raw = _category_for(fx) - now = int(fx.get("captured_epoch", time.time())) - - n = _parse_itd_511_incident(envelope, category_raw, now) - if n is None: - pytest.skip(f"{name}: parser returned None (filtered/work_zone)") - - golden = _render(n) - canonical = _n_to_canonical_incident(n) - kind_map = { - "incident": "road_incident", - "closure": "road_closure", - "special_event": "road_incident", - "work_zone": "work_zone", - } - cat = kind_map.get(n.get("category_kind", "incident"), "road_incident") - event = _make_event(cat, canonical) - - budget = budget_for("incident") - new_out = fmt(event, now=float(now), budget=budget) - assert_byte_identical(new_out, golden) - - @pytest.mark.parametrize("name,fx", [ - (name, fx) for name, fx in _TRAFFIC_FX if ( - (fx["envelope"].get("data") or {}).get("adapter") == "itd_511" - ) - ]) - def test_traffic_itd511(self, name, fx): - self._run_single(name, fx) - - @pytest.mark.parametrize("name,fx", [ - (name, fx) for name, fx in _TRAFFIC_LAST_FX if ( - (fx["envelope"].get("data") or {}).get("adapter") == "itd_511" - and not ((fx["envelope"].get("data") or {}).get("category") or "").startswith("work_zone.") - ) - ]) - def test_traffic_last_itd511(self, name, fx): - self._run_single(name, fx) - - @pytest.mark.parametrize("name,fx", [ - (name, fx) for name, fx in _TRAFFIC_LAST_FX if ( - (fx["envelope"].get("data") or {}).get("adapter") == "tomtom_incidents" - ) - ]) - def test_traffic_last_tomtom(self, name, fx, tomtom_min_mag_zero): - """traffic_last tomtom fixtures go through this group (min_mag override on).""" - from meshai.central.incident_handler import _parse_tomtom_incident, _render - from meshai.notifications.formatters.incident import format as fmt - from meshai.notifications.formatters._budget import budget_for - - envelope = fx["envelope"] - now = int(fx.get("captured_epoch", time.time())) - - n = _parse_tomtom_incident(envelope, now) - if n is None: - pytest.skip(f"{name}: parser returned None") - - golden = _render(n) - canonical = _n_to_canonical_incident(n) - event = _make_event("road_incident", canonical) - budget = budget_for("incident") - new_out = fmt(event, now=float(now), budget=budget) - assert_byte_identical(new_out, golden) - - -# ── 3. Work-zone golden byte-parity ───────────────────────────────────────── +# ── 1. Work-zone golden byte-parity ───────────────────────────────────────── class TestWorkZoneGolden: """traffic_last/0002 (itd_511 work_zone) and traffic_last/0003 (wzdx) @@ -363,7 +188,7 @@ class TestWorkZoneGolden: self._run_wz("0003.json", "wzdx") -# ── 4. Gate sequence ────────────────────────────────────────────────────────── +# ── 2. Gate sequence ────────────────────────────────────────────────────────── class TestGateSequence: """decide() lifecycle transitions: @@ -473,7 +298,7 @@ class TestGateSequence: assert result.commit is None -# ── 5. _anchor.resolve_anchor ──────────────────────────────────────────────── +# ── 3. _anchor.resolve_anchor ──────────────────────────────────────────────── class TestAnchorResolve: """resolve_anchor() returns a result from the town_anchors DB when a row @@ -553,29 +378,12 @@ class TestAnchorResolve: assert resolve_anchor(43.6, None, max_mi=50.0) is None -# ── 6. Schema conformance ──────────────────────────────────────────────────── +# ── 4. Schema conformance ──────────────────────────────────────────────────── class TestSchemaConformance: """Canonical data dicts produced by to_event() and the bridge must contain all expected keys.""" - def test_incident_canonical_keys_from_parser(self): - """All canonical keys present in extraction from a tomtom parse.""" - from meshai.central.incident_handler import _parse_tomtom_incident - from meshai.adapter_config._accessor import set_runtime_override, _overrides - - set_runtime_override("tomtom_incidents", "min_magnitude", -999) - try: - # Use fixture 0002 (mag=4, the minimal fixture that passes) - with open(_FIXTURE_DIR / "traffic" / "0002.json", encoding="utf-8") as f: - fx = json.load(f) - n = _parse_tomtom_incident(fx["envelope"], int(fx.get("captured_epoch", 0))) - assert n is not None - canonical = _n_to_canonical_incident(n) - assert _INCIDENT_CANONICAL_KEYS == set(canonical.keys()) - finally: - _overrides.pop(("tomtom_incidents", "min_magnitude"), None) - def test_workzone_canonical_keys_from_normalize(self): """All work-zone canonical keys present in extraction from normalize().""" from meshai.central_normalizer import normalize @@ -621,7 +429,7 @@ class TestSchemaConformance: assert key in d, f"Missing key {key!r} in Roads511Adapter canonical data" -# ── 7. Cross-source identity ───────────────────────────────────────────────── +# ── 5. Cross-source identity ───────────────────────────────────────────────── class TestCrossSourceIdentity: """Same render-relevant canonical fields → same formatter output regardless diff --git a/work/tests/test_nwis_handler.py b/work/tests/test_nwis_handler.py deleted file mode 100644 index 25281dd..0000000 --- a/work/tests/test_nwis_handler.py +++ /dev/null @@ -1,277 +0,0 @@ -"""Tests for v0.5.12 usgs_nwis handler.""" -import pytest - -# v0.6-4: IDAHO_CURATED_SITES dict moved to gauge_sites SQLite table; -# import the seed data from the curation module as a back-compat alias. -from meshai.persistence.curation import _GAUGE_SITES_SEED as IDAHO_CURATED_SITES -from meshai.central.nwis_handler import handle_nwis -from meshai.persistence import close_thread_connection, init_db -from meshai.persistence import db as persistence_db - - -@pytest.fixture -def mem_db(monkeypatch, tmp_path): - db_path = str(tmp_path / "nwis-test.sqlite") - monkeypatch.setenv("MESHAI_DB_PATH", db_path) - persistence_db._initialised.clear() - close_thread_connection() - conn = init_db() - yield conn - close_thread_connection() - persistence_db._initialised.discard(db_path) - - -def _nwis_env(*, site_id="USGS-13186000", - parameter_code="00065", value=13.0, - unit="ft", time_iso="2026-06-05T15:00:00Z", - lat=43.612, lon=-111.654, - envelope_id=None): - envelope_id = envelope_id or f"nwis_{site_id}_{time_iso}" - return { - "id": envelope_id, "subject": f"central.hydro.{parameter_code}.usgs.{site_id}.us.id", - "data": { - "id": envelope_id, "adapter": "nwis", - "category": f"hydro.{parameter_code}", "severity": 0, - "geo": {"centroid": [lon, lat], "primary_region": "US-ID"}, - "data": { - "id": envelope_id, - "monitoring_location_id": site_id, - "parameter_code": parameter_code, - "time": time_iso, - "value": value, - "unit_of_measure": unit, - "latitude": lat, "longitude": lon, - "_enriched": {"geocoder": { - "name": IDAHO_CURATED_SITES.get(site_id, {}).get( - "gauge_name", "?"), - }}, - }, - }, - } - - -def _commit(data, t): - data["_on_broadcast_committed"](float(t)) - - -# ---- (a) curated site at action stage triggers broadcast ----------------- - - -def test_a_curated_site_action_stage_triggers(mem_db): - # Snake River at Heise: action=12.0ft, broadcast at 12.5ft. - env = _nwis_env(site_id="USGS-13186000", parameter_code="00065", - value=12.5) - data = {} - wire = handle_nwis(env, env["subject"], data=data, now=1_000_000) - assert wire is not None - assert wire.startswith("🌊 New:") - assert "Snake River at Heise" in wire - assert "action stage 12.5 ft" in wire - - -# ---- (b) non-curated site no broadcast + event_log handled=0 ------------ - - -def test_b_non_curated_site_dropped(mem_db): - env = _nwis_env(site_id="USGS-99999999", value=99.0) - data = {} - wire = handle_nwis(env, env["subject"], data=data, now=1_000_000) - assert wire is None - n_rows = mem_db.execute( - "SELECT COUNT(*) AS n FROM gauge_readings").fetchone()["n"] - assert n_rows == 0 - n_log = mem_db.execute( - "SELECT COUNT(*) AS n FROM event_log WHERE source='nwis' AND handled=0" - ).fetchone()["n"] - assert n_log == 1 - - -# ---- (c) curated site at normal stage no broadcast ---------------------- - - -def test_c_curated_site_normal_stage_no_broadcast(mem_db): - # Heise normal is below 12.0ft. - env = _nwis_env(site_id="USGS-13186000", value=8.0) - data = {} - wire = handle_nwis(env, env["subject"], data=data, now=1_000_000) - assert wire is None - # The reading WAS persisted (time-series). - row = mem_db.execute( - "SELECT threshold_state FROM gauge_readings WHERE site_id=?", - ("USGS-13186000",)).fetchone() - assert row["threshold_state"] == "normal" - - -# ---- (d) upward threshold crossing (normal -> action) triggers --------- - - -def test_d_upward_crossing_normal_to_action_triggers(mem_db): - # First reading at normal. - env1 = _nwis_env(site_id="USGS-13186000", value=8.0, - time_iso="2026-06-05T10:00:00Z") - handle_nwis(env1, env1["subject"], data={}, now=1_000_000) - # Now rises to action. - env2 = _nwis_env(site_id="USGS-13186000", value=12.5, - time_iso="2026-06-05T10:15:00Z", - envelope_id="env_2") - data = {} - wire = handle_nwis(env2, env2["subject"], data=data, now=1_000_900) - assert wire is not None - assert "action stage 12.5 ft" in wire - - -# ---- (e) downward crossing (action -> normal) does NOT broadcast ------- - - -def test_e_downward_crossing_does_not_broadcast(mem_db): - env_high = _nwis_env(site_id="USGS-13186000", value=12.5, - time_iso="2026-06-05T10:00:00Z") - handle_nwis(env_high, env_high["subject"], data={}, now=1_000_000) - env_low = _nwis_env(site_id="USGS-13186000", value=8.0, - time_iso="2026-06-05T11:00:00Z", - envelope_id="env_drop") - wire = handle_nwis(env_low, env_low["subject"], data={}, now=1_003_600) - assert wire is None - - -def test_e_same_threshold_no_re_broadcast(mem_db): - """Repeated readings at the same threshold (action -> action -> action) - must NOT re-broadcast every 15-min poll.""" - env = _nwis_env(site_id="USGS-13186000", value=12.5, - time_iso="2026-06-05T10:00:00Z") - wire1 = handle_nwis(env, env["subject"], data={}, now=1_000_000) - assert wire1 is not None - - env2 = _nwis_env(site_id="USGS-13186000", value=12.8, - time_iso="2026-06-05T10:15:00Z", - envelope_id="env_p2") - wire2 = handle_nwis(env2, env2["subject"], data={}, now=1_000_900) - assert wire2 is None # still in action band - - -# ---- (f) flow_cfs included for 00060, dropped for 00065-only ----------- - - -def test_f_flow_cfs_segment_from_companion_discharge(mem_db): - # First seed a stage reading at action. - env_stage = _nwis_env(site_id="USGS-13186000", - parameter_code="00065", value=12.5, - time_iso="2026-06-05T10:00:00Z") - wire1 = handle_nwis(env_stage, env_stage["subject"], data={}, now=1_000_000) - assert wire1 is not None - assert "flow" not in wire1 # no companion discharge yet - - # Now a discharge reading arrives -- the handler should pick up the - # prior stage_ft for threshold context AND emit flow if upward crossing. - # In this case the stage didn't change, so no broadcast. - env_flow = _nwis_env(site_id="USGS-13186000", - parameter_code="00060", value=8400, - unit="ft^3/s", - time_iso="2026-06-05T10:01:00Z", - envelope_id="env_q") - wire2 = handle_nwis(env_flow, env_flow["subject"], data={}, now=1_000_060) - assert wire2 is None # same threshold; no re-broadcast - - -# ---- (g) site missing coords drops @ tail ------------------------------ - - -def test_g_missing_coords_drops_at_tail(mem_db): - # Build a Heise envelope but blank out the latitude in inner.data so - # the handler must fall back to the curated coords (which DO exist). - env = _nwis_env(site_id="USGS-13186000", value=12.5) - env["data"]["data"]["latitude"] = None - env["data"]["data"]["longitude"] = None - wire = handle_nwis(env, env["subject"], data={}, now=1_000_000) - assert wire is not None - # Curated coords kick in -> @ segment still present. - assert "@ 43.612,-111.654" in wire - - -# ---- (h) IDAHO_CURATED_SITES has all 9 starter sites populated --------- - - -def test_h_curated_sites_count_and_required_fields(): - assert len(IDAHO_CURATED_SITES) == 9 - required_keys = {"gauge_name", "lat", "lon", "action_ft", "flood_minor_ft"} - for site_id, meta in IDAHO_CURATED_SITES.items(): - assert site_id.startswith("USGS-"), site_id - missing = required_keys - set(meta.keys()) - assert not missing, f"{site_id} missing {missing}" - assert isinstance(meta["action_ft"], (int, float)) - assert isinstance(meta["flood_minor_ft"], (int, float)) - - -def test_h_curated_sites_listed_starter_set(): - """Spot-check the 9 starter sites are exactly what spec listed.""" - expected = { - "USGS-13139510", "USGS-13186000", "USGS-13037500", - "USGS-13135500", "USGS-13205000", "USGS-13247500", - "USGS-13057000", "USGS-13162225", "USGS-13083000", - } - assert set(IDAHO_CURATED_SITES.keys()) == expected - - -# ---- commit callback flips event_log.handled = 1 ----------------------- - - -def test_commit_callback_flips_event_log(mem_db): - env = _nwis_env(site_id="USGS-13186000", value=12.5) - data = {} - wire = handle_nwis(env, env["subject"], data=data, now=1_000_000) - assert wire is not None - pre = mem_db.execute( - "SELECT handled FROM event_log WHERE source='nwis' ORDER BY id DESC LIMIT 1" - ).fetchone() - assert pre["handled"] == 0 - _commit(data, 1_000_001) - post = mem_db.execute( - "SELECT handled FROM event_log WHERE source='nwis' ORDER BY id DESC LIMIT 1" - ).fetchone() - assert post["handled"] == 1 - - -# ---- threshold escalation triggers a new broadcast -------------------- - - -def test_action_to_flood_minor_triggers_re_broadcast(mem_db): - """Reading rises action -> flood_minor: this is an upward crossing, - re-broadcast with the higher threshold label.""" - env1 = _nwis_env(site_id="USGS-13186000", value=12.5, - time_iso="2026-06-05T10:00:00Z") - wire1 = handle_nwis(env1, env1["subject"], data={}, now=1_000_000) - assert wire1 is not None - assert "action stage" in wire1 - - env2 = _nwis_env(site_id="USGS-13186000", value=14.5, - time_iso="2026-06-05T11:00:00Z", - envelope_id="env_fm") - wire2 = handle_nwis(env2, env2["subject"], data={}, now=1_003_600) - assert wire2 is not None - assert "minor flooding" in wire2 - - -# ---- precipitation events skipped (parameter_code=00045) -------------- - - -def test_precip_parameter_skipped(mem_db): - env = _nwis_env(site_id="USGS-13186000", parameter_code="00045", - value=0.5, unit="in") - wire = handle_nwis(env, env["subject"], data={}, now=1_000_000) - assert wire is None - # No gauge_readings row written for precip. - n_rows = mem_db.execute( - "SELECT COUNT(*) AS n FROM gauge_readings").fetchone()["n"] - assert n_rows == 0 - - -# ---- site_id normalization ----------------------------------------------- - - -def test_site_id_normalization_accepts_bare_id(mem_db): - """'13186000' without USGS- prefix should still resolve to Heise.""" - env = _nwis_env(site_id="13186000", value=12.5) - env["data"]["data"]["monitoring_location_id"] = "13186000" - wire = handle_nwis(env, env["subject"], data={}, now=1_000_000) - assert wire is not None - assert "Snake River at Heise" in wire diff --git a/work/tests/test_nws_dedup_relaxation.py b/work/tests/test_nws_dedup_relaxation.py deleted file mode 100644 index b60da18..0000000 --- a/work/tests/test_nws_dedup_relaxation.py +++ /dev/null @@ -1,140 +0,0 @@ -"""v0.6-phase3 NWS dedup-window relaxation tests. - -The same CAP id is now re-broadcast (with Active: prefix) when more than -`nws.duplicate_allowed_after_seconds` (default 10800 = 3h) have elapsed -since the last broadcast. -""" -from __future__ import annotations - -import time - -import pytest - -from meshai.central.nws_handler import handle_nws -from meshai.persistence import get_db - - -def _env(*, cap_id="urn:oid:dedup.001", event="Severe Thunderstorm Warning", - severity="Severe", area="Ada County", county="Ada", state="ID", - expires="2026-06-05T03:00:00Z", lat=43.6, lon=-116.2, - category="wx.alert.severe_thunderstorm_warning"): - return { - "id": cap_id, "subject": "central.wx.alert.us.id", - "data": { - "id": cap_id, "adapter": "nws", "category": category, - "severity": 2, - "geo": {"centroid": [lon, lat], "primary_region": "US-ID"}, - "data": { - "id": cap_id, "event": event, "severity": severity, - "areaDesc": area, "msgType": "Alert", - "headline": f"{event} for {area}", - "description": "X", "expires": expires, - "_enriched": {"geocoder": {"city": None, - "county": county, "state": state}}, - }, - }, - } - - -def _commit(data, ts): - cb = data["_on_broadcast_committed"] - cb(float(ts)) - - -def test_first_broadcast_no_active_prefix(): - """A first sighting renders without Active: prefix.""" - env = _env() - data = {} - wire = handle_nws(env, env["subject"], data=data, now=1_000_000) - assert wire is not None - assert "Active:" not in wire - - -def test_repeat_within_3h_suppressed(): - env = _env(cap_id="urn:oid:rep1") - data = {} - # First broadcast at t=0. - wire1 = handle_nws(env, env["subject"], data=data, now=0) - assert wire1 is not None - _commit(data, 0) - - # Same CAP id again 2h later -- inside 3h window -> suppressed. - env2 = _env(cap_id="urn:oid:rep1") - data2 = {} - wire2 = handle_nws(env2, env2["subject"], data=data2, now=2 * 3600) - assert wire2 is None - - -def test_repeat_after_3h_allowed_with_active_prefix(): - env = _env(cap_id="urn:oid:rep2") - data = {} - wire1 = handle_nws(env, env["subject"], data=data, now=0) - assert wire1 is not None - _commit(data, 0) - - # Same CAP id 4h later -> allowed, Active: prefix. - env2 = _env(cap_id="urn:oid:rep2") - data2 = {} - wire2 = handle_nws(env2, env2["subject"], data=data2, now=4 * 3600) - assert wire2 is not None - assert "Active:" in wire2 - - -def test_dedup_window_respects_config_override(): - """Changing nws.duplicate_allowed_after_seconds via adapter_config takes effect.""" - from meshai.adapter_config import invalidate_cache - conn = get_db() - conn.execute( - "UPDATE adapter_config SET value_json='3600' " - "WHERE adapter='nws' AND key='duplicate_allowed_after_seconds'" - ) - invalidate_cache() - - env = _env(cap_id="urn:oid:tunable") - data = {} - handle_nws(env, env["subject"], data=data, now=0) - _commit(data, 0) - - # 90 min later -> still suppressed (under new 1h window). - # Wait, 90 min > 60 min so it would BE allowed. Use 30 min instead. - env2 = _env(cap_id="urn:oid:tunable") - data2 = {} - wire = handle_nws(env2, env2["subject"], data=data2, now=30 * 60) - assert wire is None # 30 min < 60 min window - - # 2h later -> allowed (over 1h window now). - env3 = _env(cap_id="urn:oid:tunable") - data3 = {} - wire3 = handle_nws(env3, env3["subject"], data=data3, now=2 * 3600) - assert wire3 is not None - assert "Active:" in wire3 - - -def test_handler_stamps_first_broadcast_at(): - """The commit callback writes first_broadcast_at via COALESCE -- only on - the first commit, never overwriting it.""" - env = _env(cap_id="urn:oid:stamp") - data = {} - handle_nws(env, env["subject"], data=data, now=0) - _commit(data, 100.0) - - conn = get_db() - row = conn.execute( - "SELECT first_broadcast_at, last_broadcast_at FROM nws_alerts " - "WHERE event_id='urn:oid:stamp'" - ).fetchone() - assert row["first_broadcast_at"] == 100.0 - assert row["last_broadcast_at"] == 100.0 - - # Second broadcast 4h later -> last_broadcast_at updates, first_broadcast_at preserved. - env2 = _env(cap_id="urn:oid:stamp") - data2 = {} - wire2 = handle_nws(env2, env2["subject"], data=data2, now=4 * 3600) - assert wire2 is not None - _commit(data2, 4 * 3600.0) - row2 = conn.execute( - "SELECT first_broadcast_at, last_broadcast_at FROM nws_alerts " - "WHERE event_id='urn:oid:stamp'" - ).fetchone() - assert row2["first_broadcast_at"] == 100.0 # unchanged - assert row2["last_broadcast_at"] == 4 * 3600.0 diff --git a/work/tests/test_nws_handler.py b/work/tests/test_nws_handler.py deleted file mode 100644 index 26ce80d..0000000 --- a/work/tests/test_nws_handler.py +++ /dev/null @@ -1,543 +0,0 @@ -"""Tests for v0.5.10 NWS handler.""" -import pytest - -from meshai.central.nws_handler import handle_nws, _emoji_for_event, _render -from meshai.persistence import close_thread_connection, init_db -from meshai.persistence import db as persistence_db - - -@pytest.fixture -def mem_db(monkeypatch, tmp_path): - db_path = str(tmp_path / "nws-test.sqlite") - monkeypatch.setenv("MESHAI_DB_PATH", db_path) - persistence_db._initialised.clear() - close_thread_connection() - conn = init_db() - yield conn - close_thread_connection() - persistence_db._initialised.discard(db_path) - - -def _nws_env(*, cap_id="urn:oid:test.001", - event="Severe Thunderstorm Warning", - severity_str="Severe", - area_desc="Twin Falls County", - county="Twin Falls", state="ID", - expires="2026-06-05T03:00:00Z", - msg_type=None, - lat=42.500, lon=-114.460, - geocoder_city=None, - category="wx.alert.severe_thunderstorm_warning"): - return { - "id": cap_id, "subject": "central.wx.alert.us.id", - "data": { - "id": cap_id, "adapter": "nws", "category": category, - "severity": 2, - "geo": {"centroid": [lon, lat], "primary_region": "US-ID"}, - "data": { - "id": cap_id, "@type": "wx:Alert", - "event": event, "severity": severity_str, - "areaDesc": area_desc, "msgType": msg_type or "Alert", - "headline": f"{event} for {area_desc}", - "description": "Storm details.", - "expires": expires, - "_enriched": {"geocoder": {"city": geocoder_city, - "county": county, "state": state}}, - }, - }, - } - - -def _commit(data, t): - data["_on_broadcast_committed"](float(t)) - - -# ---- severity gate ---- - - -def test_severe_thunderstorm_warning_broadcasts(mem_db): - env = _nws_env(severity_str="Severe", event="Severe Thunderstorm Warning") - data = {} - wire = handle_nws(env, env["subject"], data=data, now=1_000_000) - assert wire is not None - assert wire.startswith("🌩️") - assert "Severe Thunderstorm Warning" in wire - - -def test_extreme_emergency_broadcasts(mem_db): - env = _nws_env(severity_str="Extreme", event="Tornado Warning", - category="wx.alert.tornado_warning") - data = {} - wire = handle_nws(env, env["subject"], data=data, now=1_000_000) - assert wire is not None - assert wire.startswith("🌪️") - - -def test_special_weather_statement_passes_through(mem_db): - # GATE A removed: Minor/SWS is no longer dropped on CAP severity alone. - env = _nws_env(severity_str="Minor", event="Special Weather Statement", - category="wx.alert.special_weather_statement") - data = {} - wire = handle_nws(env, env["subject"], data=data, now=1_000_000) - assert wire is not None, "SWS should now pass through (GATE A removed)" - assert "Special Weather Statement" in wire - # Row inserted in nws_alerts (not a warning category → no override). - n_rows = mem_db.execute("SELECT COUNT(*) AS n FROM nws_alerts").fetchone()["n"] - assert n_rows == 1 - # _severity_override should NOT be set for a non-warning category. - assert data.get("_severity_override") is None - - -def test_watch_severity_moderate_passes_through(mem_db): - # GATE A removed: Moderate watches now pass through; dispatcher threshold governs. - env = _nws_env(severity_str="Moderate", event="Severe Thunderstorm Watch", - category="wx.alert.severe_thunderstorm_watch") - data = {} - wire = handle_nws(env, env["subject"], data=data, now=1_000_000) - assert wire is not None, "Moderate watch should now pass through (GATE A removed)" - assert "Severe Thunderstorm Watch" in wire - # Watches end in _watch, not _warning — no severity override. - assert data.get("_severity_override") is None - - -# ---- emoji map ---- - - -@pytest.mark.parametrize("event_type, expected_emoji", [ - ("Severe Thunderstorm Warning", "🌩️"), - ("Tornado Warning", "🌪️"), - ("Flash Flood Warning", "🌊"), - ("Flood Warning", "🌊"), - ("Winter Storm Warning", "❄️"), - ("Blizzard Warning", "❄️"), - ("Excessive Heat Warning", "🌡️"), - ("High Wind Warning", "🌬️"), - ("Red Flag Warning", "🔥"), - ("Fire Weather Watch", "🔥"), - ("Air Quality Alert", "😷"), - ("Freeze Warning", "🥶"), - ("Coastal Flood Warning", "🌊"), - ("(some other warning)", "⚠️"), -]) -def test_emoji_map(event_type, expected_emoji): - assert _emoji_for_event(event_type) == expected_emoji - - -# ---- tombstone ---- - - -def test_cancel_msgType_tombstone_skipped(mem_db): - env = _nws_env(severity_str="Severe", event="Severe Thunderstorm Warning", - msg_type="Cancel") - data = {} - wire = handle_nws(env, env["subject"], data=data, now=1_000_000) - assert wire is None - n_log = mem_db.execute( - "SELECT COUNT(*) AS n FROM event_log WHERE source='nws' AND handled=0" - ).fetchone()["n"] - assert n_log == 1 - - -def test_expire_msgType_tombstone_skipped(mem_db): - env = _nws_env(severity_str="Severe", event="Tornado Warning", - msg_type="Expire") - wire = handle_nws(env, env["subject"], data={}, now=1_000_000) - assert wire is None - - -# ---- per-CAP-id dedup ---- - - -def test_per_cap_id_dedup_no_reissue(mem_db): - env = _nws_env(severity_str="Severe") - data1 = {} - wire1 = handle_nws(env, env["subject"], data=data1, now=1_000_000) - assert wire1 is not None - _commit(data1, 1_000_001) - - # Same CAP id republishes (e.g. headline update). Should NOT re-broadcast. - data2 = {} - wire2 = handle_nws(env, env["subject"], data=data2, now=1_000_300) - assert wire2 is None - - -# ---- area_desc fallback ---- - - -def test_area_desc_used_when_geocoder_city_missing(mem_db): - env = _nws_env(severity_str="Severe", area_desc="Twin Falls County", - geocoder_city=None) - wire = handle_nws(env, env["subject"], data={}, now=1_000_000) - assert "Twin Falls" in wire - - -def test_geocoder_city_preferred_over_area_desc(mem_db): - env = _nws_env(severity_str="Severe", area_desc="Twin Falls County", - geocoder_city="Twin Falls") - wire = handle_nws(env, env["subject"], data={}, now=1_000_000) - assert "Twin Falls" in wire # either source serves the same anchor - - -# ---- commit callback ---- - - -def test_commit_callback_updates_last_broadcast(mem_db): - env = _nws_env(severity_str="Severe") - data = {} - handle_nws(env, env["subject"], data=data, now=1_000_000) - fr_pre = mem_db.execute( - "SELECT last_broadcast_at FROM nws_alerts").fetchone() - assert fr_pre["last_broadcast_at"] is None - _commit(data, 1_000_001) - fr_post = mem_db.execute( - "SELECT last_broadcast_at FROM nws_alerts").fetchone() - assert fr_post["last_broadcast_at"] == 1_000_001 - # event_log row flipped to handled=1. - el = mem_db.execute( - "SELECT handled FROM event_log WHERE source='nws' ORDER BY id DESC LIMIT 1" - ).fetchone() - assert el["handled"] == 1 - - -def test_wire_includes_event_and_headline(mem_db): - env = _nws_env(severity_str="Severe", lat=42.500, lon=-114.460) - wire = handle_nws(env, env["subject"], data={}, now=1_000_000) - assert "Severe Thunderstorm Warning" in wire - assert "Twin Falls County" in wire - -# ---- warning → immediate promotion (Step 2) ---- - - -def test_warning_category_sets_severity_override_immediate(mem_db): - """A *_warning category sets data[_severity_override]='immediate'.""" - env = _nws_env(severity_str="Severe", event="Severe Thunderstorm Warning", - category="wx.alert.severe_thunderstorm_warning") - data = {} - wire = handle_nws(env, env["subject"], data=data, now=1_000_000) - assert wire is not None - assert data.get("_severity_override") == "immediate" - - -def test_tornado_warning_dotted_category_sets_severity_override(mem_db): - """A category ending in .warning also sets _severity_override='immediate'.""" - env = _nws_env(severity_str="Extreme", event="Tornado Warning", - category="wx.alert.tornado_warning") - # Override the data.data.severity to use dotted-style category check - env["data"]["category"] = "wx.alert.tornado.warning" - env["data"]["data"]["severity"] = "Extreme" - data = {} - wire = handle_nws(env, env["subject"], data=data, now=2_000_000) - assert wire is not None - assert data.get("_severity_override") == "immediate" - - -def test_non_warning_category_no_severity_override(mem_db): - """A non-warning category (watch, advisory, statement) leaves no override.""" - env = _nws_env(severity_str="Severe", event="Severe Thunderstorm Watch", - category="wx.alert.severe_thunderstorm_watch") - data = {} - wire = handle_nws(env, env["subject"], data=data, now=3_000_000) - assert wire is not None - assert "_severity_override" not in data - - -# ---- packet-budget enforcement ---- - - -def test_svr_long_locations_path_sampled(mem_db): - """SVR with a long town list: render must fit in 200 chars, and the town - list must be represented as a PATH SAMPLE (first -> middle -> last) rather - than a tail-drop. The old bug dropped the final town ('Shoshone').""" - # Long list; first town "Buhl", last town "and Shoshone" (exercises the - # leading-"and " strip on the tail element). - long_locations = ( - "Buhl, Eden, Hazelton, Murtaugh, Richfield, Dietrich, " - "Gooding, Hagerman, Wendell, and Shoshone" - ) - description = ( - "HAZARD...Damaging winds to 60 mph and quarter-size hail.\n\n" - f"Locations impacted include...{long_locations}" - ) - d = { - "eventCode": {"SAME": ["SVR"]}, - "certainty": "Observed", - "parameters": { - "maxWindGust": ["60 MPH"], - "maxHailSize": ["1.00"], - # 254 DEG 35 KT -> "Moving W 40 mph" - "eventMotionDescription": ["2200000T254DEG...35KT 42.5,-114.5"], - }, - "description": description, - } - rendered = _render( - event_type="Severe Thunderstorm Warning", - area_desc="Twin Falls County", - geocoder_city=None, - county="Twin Falls", - state="ID", - expires_epoch=1_751_400_000, - lat=42.5, - lon=-114.46, - now=1_751_400_000, - d=d, - ) - - # (a) fits in one mesh packet (budget is now the 140-char LoRa max) - assert len(rendered) <= 140, ( - f"rendered is {len(rendered)} chars (expected <= 140):\n{rendered!r}" - ) - - # (b) all data-point categories present, hazard wording TIGHTENED - assert "Severe Thunderstorm Warning" in rendered, "event type missing" - assert "Until" in rendered, "expiry time segment missing" - assert "Twin Falls County" in rendered, "area missing" - assert "60mph winds" in rendered, "wind hazard not tightened to '60mph winds'" - assert '1" hail' in rendered, "hail hazard not rendered as numeric inches" - assert "radar" in rendered, "certainty not collapsed to 'radar'" - assert "Moving" in rendered, "motion segment missing" - - # (c) path-sampling applied (arrow) with the soonest-impact town retained. - # At the 140 budget the farthest-along town may be trimmed by the final - # backstop; the hard cap wins over endpoint preservation. - assert "→" in rendered, "no arrow -> not path-sampled" - assert "Buhl" in rendered, "first (soonest-impact) town missing" - - -def test_svr_short_locations_shown_in_full(mem_db): - """Short town list that fits in one packet: show the FULL comma-joined - list, and never emit the path-sample arrow.""" - short_locations = "Buhl, Eden, and Hazelton" - description = ( - "HAZARD...Damaging winds to 60 mph and quarter-size hail.\n\n" - f"Locations impacted include...{short_locations}" - ) - d = { - "eventCode": {"SAME": ["SVR"]}, - "certainty": "Observed", - "parameters": { - "maxWindGust": ["60 MPH"], - "maxHailSize": ["1.00"], - "eventMotionDescription": ["2200000T254DEG...35KT 42.5,-114.5"], - }, - "description": description, - } - rendered = _render( - event_type="Severe Thunderstorm Warning", - area_desc="Twin Falls County", - geocoder_city=None, - county="Twin Falls", - state="ID", - expires_epoch=1_751_400_000, - lat=42.5, - lon=-114.46, - now=1_751_400_000, - d=d, - ) - - assert len(rendered) <= 140 - assert "→" not in rendered, "short list should not be path-sampled" - assert "Buhl, Eden, Hazelton" in rendered, "full comma-joined list expected" - # Hazard wording is tightened even on the short-list path. - assert "60mph winds" in rendered - assert '1" hail' in rendered - assert "radar" in rendered - - -def test_svr_worst_case_fits_140(mem_db): - """Pathologically long SVR payload: the final wire MUST fit 140 chars while - still carrying event name, area, time, tightened hazard, and >=1 town.""" - long_locations = ( - "Buhl, Eden, Hazelton, Murtaugh, Richfield, Dietrich, Gooding, " - "Hagerman, Wendell, Jerome, Kimberly, Hansen, Filer, and Shoshone" - ) - description = ( - "HAZARD...Damaging winds to 70 mph and golf ball size hail.\n\n" - f"Locations impacted include...{long_locations}" - ) - d = { - "eventCode": {"SAME": ["SVR"]}, - "certainty": "Observed", - "parameters": { - "maxWindGust": ["70 MPH"], - "maxHailSize": ["1.75"], - "eventMotionDescription": ["2200000T254DEG...35KT 42.5,-114.5"], - }, - "description": description, - } - rendered = _render( - event_type="Severe Thunderstorm Warning", - area_desc="Twin Falls County", - geocoder_city=None, county="Twin Falls", state="ID", - expires_epoch=1_751_400_000, lat=42.5, lon=-114.46, - now=1_751_400_000, d=d, - ) - assert len(rendered) <= 140, f"{len(rendered)} chars:\n{rendered!r}" - assert "Severe Thunderstorm Warning" in rendered # event name - assert "Twin Falls County" in rendered # area - assert "Until" in rendered # time - assert "70mph winds" in rendered # tightened hazard (wind) - assert '1.75" hail' in rendered # golf ball -> 1.75" - assert "Buhl" in rendered # >=1 town present - - -# ---- no dangling "— …" across ALL product types ---- - - -def _assert_no_dangling_separator(rendered: str): - """The L4 motion/locations line must never end in a stray separator: - no trailing '—…', '— …', or a bare '—'. Either a real location list - follows the em-dash, or the em-dash (and its locations) are absent.""" - for line in rendered.splitlines(): - stripped = line.rstrip() - assert not stripped.endswith("—…"), f"dangling '—…': {line!r}" - assert not stripped.endswith("— …"), f"dangling '— …': {line!r}" - assert not stripped.endswith("—"), f"bare trailing '—': {line!r}" - # And the "— …" fragment must not appear mid-line either. - assert "—…" not in stripped, f"'—…' fragment: {line!r}" - assert "— …" not in stripped, f"'— …' fragment: {line!r}" - - -def test_sps_worst_case_tightened_and_no_dangling(mem_db): - """The reported live-log bug: a Special Weather Statement (SPS) with wind - gusts + motion + a long town list previously collapsed L4 to - 'Moving SW 24 mph —…' (all towns lost, dangling separator). After the fix: - hazard is tightened, output fits 140, and L4 is either - 'Moving … — ' or 'Moving …' — never a trailing '—…'.""" - long_locations = ( - "Twin Falls, Kimberly, Filer, Buhl, Hansen, Murtaugh, Hollister, " - "Eden, Hazelton, and Rogerson" - ) - description = ( - "HAZARD...Wind gusts in excess of 45 mph and pea size hail.\n\n" - "SOURCE...Radar indicated.\n\n" - f"Locations impacted include...{long_locations}" - ) - d = { - "eventCode": {"SAME": ["SPS"]}, - "certainty": "Observed", - "parameters": { - # 225 DEG 21 KT -> "Moving SW 24 mph" - "eventMotionDescription": ["2200000T225DEG...21KT 42.5,-114.5"], - }, - "description": description, - } - rendered = _render( - event_type="Special Weather Statement", area_desc="Twin Falls County", - geocoder_city=None, county="Twin Falls", state="ID", - expires_epoch=1_751_400_000, lat=42.5, lon=-114.46, - now=1_751_400_000, d=d, - ) - assert len(rendered) <= 140, f"{len(rendered)} chars:\n{rendered!r}" - assert "Special Weather Statement" in rendered - # (a) hazard tightened: "Wind gusts in excess of 45 mph" -> "45mph gusts", - # "pea size hail" -> '0.25" hail'; filler dropped. - assert "45mph gusts" in rendered, f"wind not tightened:\n{rendered!r}" - assert '0.25" hail' in rendered, f"hail not numeric:\n{rendered!r}" - assert "in excess of" not in rendered, "filler 'in excess of' survived" - assert "· observed" in rendered, "certainty not collapsed" - # (b) NEVER a dangling separator. - _assert_no_dangling_separator(rendered) - # (c) the motion line, when present, either carries a town or stands alone. - last = rendered.splitlines()[-1] - if last.startswith("Moving"): - assert last == "Moving SW 24 mph" or " — " in last, ( - f"L4 neither motion-only nor motion+towns:\n{last!r}") - if " — " in last: - # A real town must follow the em-dash. - tail = last.split(" — ", 1)[1].strip() - assert tail and tail != "…", f"empty tail after em-dash:\n{last!r}" - - -def test_wsw_hazard_tightened_and_no_dangling(mem_db): - """Winter Weather product (WSW SAME code): wind-gust hazard is tightened and - no dangling '—…' can appear.""" - long_locations = ( - "Sun Valley, Ketchum, Hailey, Bellevue, Carey, Picabo, Fairfield, " - "and Gooding" - ) - description = ( - "HAZARD...Wind gusts up to 45 mph and heavy snow.\n\n" - f"Locations impacted include...{long_locations}" - ) - d = { - "eventCode": {"SAME": ["WSW"]}, - "certainty": "Observed", - "parameters": { - "eventMotionDescription": ["2200000T225DEG...21KT 42.5,-114.5"], - }, - "description": description, - } - rendered = _render( - event_type="Winter Weather Advisory", area_desc="Blaine County", - geocoder_city=None, county="Blaine", state="ID", - expires_epoch=1_751_400_000, lat=43.5, lon=-114.3, - now=1_751_400_000, d=d, - ) - assert len(rendered) <= 140, f"{len(rendered)} chars:\n{rendered!r}" - assert "45mph gusts" in rendered, f"WSW wind not tightened:\n{rendered!r}" - assert "heavy snow" in rendered - assert "up to" not in rendered, "filler 'up to' survived" - _assert_no_dangling_separator(rendered) - - -def test_sps_pathological_towns_degrade_to_motion_only(mem_db): - """When even a single sampled town cannot fit the remaining budget, L4 must - degrade to motion-only ('Moving …') with NO trailing separator — never - 'Moving … —…'.""" - # One absurdly long town name that cannot coexist with the em-dash + motion - # in the leftover budget. - long_town = "Averyverylongimpossibletownnamethatwillnotfitthebudgetatall" * 2 - description = ( - "HAZARD...Wind gusts in excess of 45 mph.\n\n" - f"Locations impacted include...{long_town}" - ) - d = { - "eventCode": {"SAME": ["SPS"]}, - "certainty": "Observed", - "parameters": { - "eventMotionDescription": ["2200000T225DEG...21KT 42.5,-114.5"], - }, - "description": description, - } - rendered = _render( - event_type="Special Weather Statement", area_desc="Twin Falls County", - geocoder_city=None, county="Twin Falls", state="ID", - expires_epoch=1_751_400_000, lat=42.5, lon=-114.46, - now=1_751_400_000, d=d, - ) - assert len(rendered) <= 140 - _assert_no_dangling_separator(rendered) - last = rendered.splitlines()[-1] - # The town can't fit, so L4 (if present) is bare motion. - if last.startswith("Moving"): - assert " — " not in last, f"expected motion-only, got:\n{last!r}" - - -def test_svr_no_dangling_separator(mem_db): - """Re-verify SVR (the branch tightened earlier) still never dangles.""" - long_locations = ( - "Buhl, Eden, Hazelton, Murtaugh, Richfield, Dietrich, Gooding, " - "Hagerman, Wendell, Jerome, Kimberly, Hansen, Filer, and Shoshone" - ) - description = ( - "HAZARD...Damaging winds to 70 mph and golf ball size hail.\n\n" - f"Locations impacted include...{long_locations}" - ) - d = { - "eventCode": {"SAME": ["SVR"]}, - "certainty": "Observed", - "parameters": { - "maxWindGust": ["70 MPH"], "maxHailSize": ["1.75"], - "eventMotionDescription": ["2200000T254DEG...35KT 42.5,-114.5"], - }, - "description": description, - } - rendered = _render( - event_type="Severe Thunderstorm Warning", area_desc="Twin Falls County", - geocoder_city=None, county="Twin Falls", state="ID", - expires_epoch=1_751_400_000, lat=42.5, lon=-114.46, - now=1_751_400_000, d=d, - ) - assert len(rendered) <= 140 - assert "70mph winds" in rendered - _assert_no_dangling_separator(rendered) diff --git a/work/tests/test_nws_refactor.py b/work/tests/test_nws_refactor.py index 4a856c2..8621613 100644 --- a/work/tests/test_nws_refactor.py +++ b/work/tests/test_nws_refactor.py @@ -1,57 +1,49 @@ -"""Phase-2 NWS refactor tests — formatter+gater architecture verification. +"""NWS refactor tests — formatter+gater architecture verification. -Four test groups: +Originally four test groups; groups 1-3 below were golden-parity tests +against the now-deleted Central NATS-consumer bridge +(meshai.central.nws_handler / handle_nws / _render). That bridge is dead — +production runs the native formatter+gater path exclusively — so byte-parity +and old-vs-new comparisons against it no longer have anything to compare +against and were deleted (git history preserves the original handler and +the parity tests that proved the rewrite matched it). What remains exercises +the LIVE native path only: -1. Golden byte-parity (tier-a): for each NWS fixture, run the OLD _render() - under pinned_time+pinned_tz, then run the NEW format() from canonical - data built by the Central bridge, and assert_byte_identical. This MUST - be exactly equal — any difference is a regression. +1. Formatter golden: formatters.nws.format() renders the expected wire text + for real fixtures and pathological synthetic cases (SVR path-sampling, + dangling-separator regression, TOR/FFW branches). These goldens are + hardcoded literals, NOT computed by importing the deleted handler. They + were derived by temporarily restoring the pre-excision + meshai.central.nws_handler._render() from git history (ca751fb5^) in a + throwaway script, confirming it produced byte-identical output to the + current native format() for every case below, and pinning the resulting + string as the literal. That verification script/module was never + committed; only the confirmed-matching literals live here. See "golden + verified against pre-excision _render()" comments below. -2. Cross-source identity: the native adapter's to_event() canonical dict - (for a synthetic fixture) produces the same formatter output as the - Central-bridge canonical dict built from the same alert data. +2. Gate-sequence: replay a synthetic 4-step lifecycle (first→dup<3h→ + dup>3h→Cancel) through gating.nws.decide(), and a reference-triggered + "Update" prefix case — both against native code only. -3. Gate-sequence: replay a synthetic 4-step lifecycle (first→dup<3h→ - dup>3h→Cancel) through the OLD handle_nws gating and the NEW - gating.nws.decide(), and assert broadcast/suppress match at every step. - -4. Schema-conformance: env/nws.py _fetch() emits all canonical schema keys; +3. Schema-conformance: env/nws.py _fetch() emits all canonical schema keys; description is not truncated; to_event() produces a canonical event.data. + +4. Formatter/gater registration: formatters/__init__ and gating/__init__ + register the NWS categories against the native format()/decide(). """ from __future__ import annotations -import json -import os -import pathlib import time +from datetime import datetime import pytest -from meshai.central.nws_handler import _render, handle_nws from meshai.central.budget import budget_for from meshai.notifications.formatters.nws import format as nws_format from meshai.notifications.gating.nws import decide as nws_decide -from meshai.notifications.gating.base import GateResult -from meshai.persistence import close_thread_connection, get_db, init_db +from meshai.persistence import close_thread_connection, init_db from meshai.persistence import db as persistence_db -from tests.harness.goldens import ( - assert_byte_identical, - load_fixtures, - pinned_time, - pinned_tz, - run_gate_sequence, -) - -# ── Shared epoch for deterministic renders ──────────────────────────────────── -_AT = 1_783_206_513.0 # captured_epoch for fixture 0000 - - -# ── Minimal fake Event for calling formatter without full pipeline ──────────── - -class _FakeEvent: - def __init__(self, data: dict): - self.data = data - +from tests.harness.goldens import assert_byte_identical, load_fixtures, pinned_tz # ── DB fixture ──────────────────────────────────────────────────────────────── @@ -67,357 +59,350 @@ def mem_db(monkeypatch, tmp_path): persistence_db._initialised.discard(db_path) -# ── Helper: build canonical data from a Central fixture ────────────────────── +class _FakeEvent: + """Minimal fake Event for calling the formatter without the full pipeline.""" + def __init__(self, data: dict): + self.data = data -def _canonical_from_fixture(fix: dict) -> dict: - """Extract canonical event.data dict from a Central NWS fixture. - - Mirrors exactly what handle_nws (cutover path) writes into data dict. - Used in formatter golden tests without going through the full handler. - """ - envelope = fix["envelope"] - inner = envelope.get("data") or {} - d = inner.get("data") or {} - geo = inner.get("geo") or {} - ge = (d.get("_enriched") or {}).get("geocoder") or {} - category_raw = inner.get("category") or "" - - from meshai.central.nws_handler import _category_to_event_type, _parse_iso - - cap_id = d.get("id") or inner.get("id") - event_type = d.get("event") or _category_to_event_type(category_raw) - area_desc = d.get("areaDesc") - headline = d.get("headline") - description = d.get("description") - cap_severity = d.get("severity") - county = d.get("areaDesc") or ge.get("county") - state = ge.get("state") or d.get("state") - expires_epoch = _parse_iso(d.get("expires")) - same_code = ((d.get("eventCode") or {}).get("SAME") or [""])[0] - certainty = d.get("certainty") or "" - references = d.get("references") or [] - parameters = d.get("parameters") or {} - msg_type = d.get("msgType") +def _canonical(event_type, *, same_code="", area_desc="Twin Falls County", + county="Twin Falls", state="ID", expires_epoch=1_751_400_000, + certainty="Observed", parameters=None, description="", + prefix="") -> dict: + """Build a canonical event.data dict as the native adapter's to_event() + (or the decider's data_patch) would produce it, for feeding directly to + nws_format().""" return { - "cap_id": cap_id, + "cap_id": "test", "event": event_type, "same_code": same_code, - "cap_severity": cap_severity, + "cap_severity": None, "certainty": certainty, "expires_at": expires_epoch, "area_desc": area_desc, - "geocoder": { - "city": ge.get("city"), - "county": county, - "state": state, - }, + "geocoder": {"city": None, "county": county, "state": state}, "description": description, - "parameters": parameters, - "msgType": msg_type, - "references": references, - "category": category_raw, - "headline": headline, - # prefix injected by gater: "" for first sighting (no references) - "_nws_prefix": "", + "parameters": parameters or {}, + "msgType": "Alert", + "references": [], + "category": "", + "headline": None, + "_nws_prefix": prefix, } -def _old_render_from_fixture(fix: dict) -> str: - """Call old _render() from a Central NWS fixture with pinned clock+tz. - - Returns the wire string. - """ - envelope = fix["envelope"] - inner = envelope.get("data") or {} - d = inner.get("data") or {} - geo = inner.get("geo") or {} - ge = (d.get("_enriched") or {}).get("geocoder") or {} - category_raw = inner.get("category") or "" - - from meshai.central.nws_handler import _category_to_event_type, _parse_iso - - event_type = d.get("event") or _category_to_event_type(category_raw) - area_desc = d.get("areaDesc") - cap_severity = d.get("severity") - county = d.get("areaDesc") or ge.get("county") - state = ge.get("state") or d.get("state") - expires_epoch = _parse_iso(d.get("expires")) - - lat = lon = None - cent = geo.get("centroid") or [] - if isinstance(cent, list) and len(cent) >= 2: - lon, lat = cent[0], cent[1] - - epoch = float(fix.get("captured_epoch", _AT)) - return _render( - event_type=event_type, area_desc=area_desc, - geocoder_city=ge.get("city"), county=county, state=state, - expires_epoch=expires_epoch, lat=lat, lon=lon, - now=epoch, prefix="", d=d, - ) - - # ============================================================================= -# 1. Golden byte-parity (tier-a) +# 1. Formatter golden — native format() wire text # ============================================================================= -class TestGoldenByteParity: - """formatters/nws.format() is byte-identical to _render() for all fixtures. +class TestFormatterGolden: + """formatters.nws.format() renders the expected wire text. - Both the nws/ fixtures (first-sighting, no prefix) and nws_last/ fixtures - (may have references → "Update" prefix) are tested. + Fixture-driven cases (golden verified against pre-excision _render(), + see module docstring) plus hand-built pathological cases that pin + known-tricky behavior: SVR path-sampling, the "no dangling separator" + regression, and the TOR/FFW hazard branches. """ - def _render_and_format(self, fix: dict, prefix: str = ""): - """Run old _render and new format() under identical pinned clock+tz. + def _canonical_from_fixture(self, fix: dict) -> dict: + """Extract canonical event.data from a Central-style NWS fixture. - Returns (golden, new_output). + Standalone re-implementation of the field extraction that used to + live in the deleted meshai.central.nws_handler (event-type fallback + via category, ISO-to-epoch parsing) — kept here only as test + scaffolding to turn a raw fixture into a canonical dict. """ - epoch = float(fix.get("captured_epoch", _AT)) - - canonical = _canonical_from_fixture(fix) - canonical["_nws_prefix"] = prefix - - budget = budget_for("nws") - - with pinned_tz("America/Boise"): - with pinned_time(epoch): - golden = _old_render_from_fixture(fix) - # Override prefix in _render for parity (handler uses "" for first-sight) - golden = _render( - **{k: canonical.get(k) for k in - ("event_type",)}, # we'll call _render directly below - ) - # Actually call _render directly with same params as _old_render_from_fixture - golden = _old_render_from_fixture(fix) - new_out = nws_format(_FakeEvent(canonical), now=epoch, budget=budget) - - return golden, new_out - - @pytest.mark.parametrize("n", list(range(27))) - def test_fixture_nws_byte_identical(self, n): - """All 27 nws/ fixtures render byte-identically old vs new.""" - fixes = load_fixtures("nws") - if n >= len(fixes): - pytest.skip(f"fixture {n} not found (only {len(fixes)} fixtures)") - fix = fixes[n] - epoch = float(fix.get("captured_epoch", _AT)) - canonical = _canonical_from_fixture(fix) - budget = budget_for("nws") - - with pinned_tz("America/Boise"): - golden = _old_render_from_fixture(fix) - new_out = nws_format(_FakeEvent(canonical), now=epoch, budget=budget) - - assert_byte_identical(new_out, golden) - - @pytest.mark.parametrize("n", list(range(10))) - def test_fixture_nws_last_byte_identical(self, n): - """All 10 nws_last/ fixtures render byte-identically old vs new.""" - fixes = load_fixtures("nws_last") - if n >= len(fixes): - pytest.skip(f"fixture {n} not found (only {len(fixes)} fixtures)") - fix = fixes[n] - epoch = float(fix.get("captured_epoch", _AT)) - canonical = _canonical_from_fixture(fix) - budget = budget_for("nws") - - with pinned_tz("America/Boise"): - golden = _old_render_from_fixture(fix) - new_out = nws_format(_FakeEvent(canonical), now=epoch, budget=budget) - - assert_byte_identical(new_out, golden) - - def test_update_prefix_byte_identical(self): - """'Update:' prefix variant is byte-identical.""" - fixes = load_fixtures("nws_last") - if not fixes: - pytest.skip("no nws_last fixtures") - fix = fixes[0] - epoch = float(fix.get("captured_epoch", _AT)) - canonical = _canonical_from_fixture(fix) - canonical["_nws_prefix"] = "Update" - budget = budget_for("nws") - envelope = fix["envelope"] inner = envelope.get("data") or {} d = inner.get("data") or {} - geo = inner.get("geo") or {} ge = (d.get("_enriched") or {}).get("geocoder") or {} - from meshai.central.nws_handler import _category_to_event_type, _parse_iso category_raw = inner.get("category") or "" - event_type = d.get("event") or _category_to_event_type(category_raw) + + event_type = d.get("event") or "Weather Alert" area_desc = d.get("areaDesc") county = d.get("areaDesc") or ge.get("county") state = ge.get("state") or d.get("state") expires_epoch = _parse_iso(d.get("expires")) - lat = lon = None - cent = geo.get("centroid") or [] - if isinstance(cent, list) and len(cent) >= 2: - lon, lat = cent[0], cent[1] + same_code = ((d.get("eventCode") or {}).get("SAME") or [""])[0] - with pinned_tz("America/Boise"): - golden = _render(event_type=event_type, area_desc=area_desc, - geocoder_city=ge.get("city"), county=county, state=state, - expires_epoch=expires_epoch, lat=lat, lon=lon, - now=epoch, prefix="Update", d=d) - new_out = nws_format(_FakeEvent(canonical), now=epoch, budget=budget) - - assert_byte_identical(new_out, golden) - - def test_active_prefix_byte_identical(self): - """'Active:' prefix variant is byte-identical.""" - fixes = load_fixtures("nws") - if not fixes: - pytest.skip("no nws fixtures") - fix = fixes[0] - epoch = float(fix.get("captured_epoch", _AT)) - canonical = _canonical_from_fixture(fix) - canonical["_nws_prefix"] = "Active" - budget = budget_for("nws") - - envelope = fix["envelope"] - inner = envelope.get("data") or {} - d = inner.get("data") or {} - geo = inner.get("geo") or {} - ge = (d.get("_enriched") or {}).get("geocoder") or {} - from meshai.central.nws_handler import _category_to_event_type, _parse_iso - category_raw = inner.get("category") or "" - event_type = d.get("event") or _category_to_event_type(category_raw) - area_desc = d.get("areaDesc") - county = d.get("areaDesc") or ge.get("county") - state = ge.get("state") or d.get("state") - expires_epoch = _parse_iso(d.get("expires")) - lat = lon = None - cent = geo.get("centroid") or [] - if isinstance(cent, list) and len(cent) >= 2: - lon, lat = cent[0], cent[1] - - with pinned_tz("America/Boise"): - golden = _render(event_type=event_type, area_desc=area_desc, - geocoder_city=ge.get("city"), county=county, state=state, - expires_epoch=expires_epoch, lat=lat, lon=lon, - now=epoch, prefix="Active", d=d) - new_out = nws_format(_FakeEvent(canonical), now=epoch, budget=budget) - - assert_byte_identical(new_out, golden) - - -# ============================================================================= -# 2. Cross-source identity: native canonical == Central-sourced render -# ============================================================================= - -class TestCrossSourceIdentity: - """Native adapter to_event() canonical == Central-bridge canonical for same alert.""" - - def _make_native_raw(self, props: dict, onset: float, expires: float) -> dict: - """Simulate what _fetch() builds for a single NWS API feature.""" return { - "source": "nws", - "event_id": props.get("id", ""), - "event_type": props.get("event", "Unknown"), - "severity": (props.get("severity") or "Unknown").lower(), - "headline": props.get("headline", ""), - "description": props.get("description") or "", - "onset": onset, - "expires": expires, - "expires_at": expires, - "areas": (props.get("geocode") or {}).get("UGC", []), - "area_desc": props.get("areaDesc", ""), - "fetched_at": time.time(), - "cap_id": props.get("id", ""), - "same_code": ((props.get("eventCode") or {}).get("SAME") or [""])[0], - "cap_severity": props.get("severity", "Unknown"), - "certainty": props.get("certainty", "Unknown"), - "parameters": props.get("parameters") or {}, - "msgType": props.get("messageType", "Alert"), - "references": props.get("references") or [], - } - - def test_native_and_central_render_identically(self): - """For a synthetic SVR alert, native and Central canonical render the same wire. - - Both paths must produce byte-identical output when given the same underlying - alert data. The key equality constraints are: - - same expires_at epoch - - same same_code - - same area_desc / geocoder.county - - same parameters (wind/hail) - - same _nws_prefix (both "") - """ - from unittest.mock import MagicMock - from meshai.env.nws import NWSAlertsAdapter - from meshai.central.nws_handler import _parse_iso - - expires_iso = "2026-07-04T01:00:00-06:00" - # Derive epoch from the ISO string so both paths use the same value. - expires_epoch = _parse_iso(expires_iso) # int - - props = { - "id": "urn:oid:test.svr.001", - "event": "Severe Thunderstorm Warning", - "severity": "Severe", - "certainty": "Observed", - "areaDesc": "Twin Falls County", - "headline": "SVR Warning Twin Falls County", - "description": "HAZARD...60 MPH winds and 1.00 inch hail.", - "expires": expires_iso, - "messageType": "Alert", - "references": [], - "parameters": { - "maxWindGust": ["60 MPH"], - "maxHailSize": ["1.00"], - }, - "eventCode": {"SAME": ["SVR"]}, - "geocode": {"UGC": ["IDZ016"]}, - } - - # Native path: build raw → to_event() → event.data - mock_cfg = MagicMock() - mock_cfg.areas = ["ID"] - mock_cfg.user_agent = "(test)" - mock_cfg.severity_min = "moderate" - mock_cfg.tick_seconds = 60 - adapter = NWSAlertsAdapter(mock_cfg) - - raw = self._make_native_raw(props, onset=expires_epoch - 7200, expires=expires_epoch) - native_event = adapter.to_event(raw) - native_canonical = native_event.data - - # Central path: build canonical manually (same logic as bridge) - central_canonical = { - "cap_id": props["id"], - "event": "Severe Thunderstorm Warning", - "same_code": "SVR", - "cap_severity": "Severe", - "certainty": "Observed", + "cap_id": d.get("id") or inner.get("id"), + "event": event_type, + "same_code": same_code, + "cap_severity": d.get("severity"), + "certainty": d.get("certainty") or "", "expires_at": expires_epoch, - "area_desc": "Twin Falls County", - "geocoder": {"city": None, "county": "Twin Falls County", "state": None}, - "description": props["description"], - "parameters": props["parameters"], - "msgType": "Alert", - "references": [], - "category": "weather_warning", - "headline": props["headline"], + "area_desc": area_desc, + "geocoder": {"city": ge.get("city"), "county": county, "state": state}, + "description": d.get("description"), + "parameters": d.get("parameters") or {}, + "msgType": d.get("msgType"), + "references": d.get("references") or [], + "category": category_raw, + "headline": d.get("headline"), "_nws_prefix": "", } + @pytest.mark.parametrize("n,expected", [ + (0, "🌬️ Special Weather Statement\nUntil 5:45 PM MDT — Northern Elko County" + "\nLandspouts, 40mph gusts, and half inch hail · observed" + "\nMoving W 23 mph"), + (8, "⛈️ Severe Thunderstorm Warning\nUntil 4:30 PM MDT — Cassia, ID" + "\nup to 50mph winds, 1\" hail · radar" + "\nMoving SW 20 mph"), + (9, "🌩️ Severe Thunderstorm Warning\nUntil 4:30 PM MDT — Cassia, ID" + "\n1\" hail · observed" + "\nMoving SW 20 mph — Oakley Reservoir and Oakley"), + ]) + def test_fixture_golden(self, n, expected): + """Real nws/ fixtures render to the pinned wire text. + + golden verified against pre-excision _render() (see module docstring). + """ + fixes = load_fixtures("nws") + fix = fixes[n] + epoch = float(fix.get("captured_epoch", 1_783_206_513.0)) + canonical = self._canonical_from_fixture(fix) budget = budget_for("nws") + with pinned_tz("America/Boise"): - native_wire = nws_format(_FakeEvent(native_canonical), now=expires_epoch - 100, budget=budget) - central_wire = nws_format(_FakeEvent(central_canonical), now=expires_epoch - 100, budget=budget) + result = nws_format(_FakeEvent(canonical), now=epoch, budget=budget) - assert_byte_identical(native_wire, central_wire) + assert_byte_identical(result, expected) + + def test_svr_long_locations_path_sampled(self): + """SVR with a long town list renders a PATH SAMPLE (first → last), + never a tail-drop that silently loses the final town. + + golden verified against pre-excision _render() (see module docstring). + """ + long_locations = ( + "Buhl, Eden, Hazelton, Murtaugh, Richfield, Dietrich, " + "Gooding, Hagerman, Wendell, and Shoshone" + ) + description = ( + "HAZARD...Damaging winds to 60 mph and quarter-size hail.\n\n" + f"Locations impacted include...{long_locations}" + ) + canonical = _canonical( + "Severe Thunderstorm Warning", same_code="SVR", + certainty="Observed", description=description, + parameters={ + "maxWindGust": ["60 MPH"], "maxHailSize": ["1.00"], + "eventMotionDescription": ["2200000T254DEG...35KT 42.5,-114.5"], + }, + ) + expected = ( + "⛈️ Severe Thunderstorm Warning\nUntil 2:00 PM MDT — Twin Falls County" + "\n60mph winds, 1\" hail · radar" + "\nMoving W 40 mph — Buhl → Shoshone" + ) + + with pinned_tz("America/Boise"): + result = nws_format(_FakeEvent(canonical), now=1_751_400_000, + budget=budget_for("nws")) + + assert len(result) <= 140 + assert "→" in result, "long town list must be path-sampled" + assert "Buhl" in result and "Shoshone" in result + assert_byte_identical(result, expected) + + def test_svr_short_locations_shown_in_full(self): + """SVR with a short town list shows the FULL comma-joined list — + never the path-sample arrow. + + golden verified against pre-excision _render() (see module docstring). + """ + description = ( + "HAZARD...Damaging winds to 60 mph and quarter-size hail.\n\n" + "Locations impacted include...Buhl, Eden, and Hazelton" + ) + canonical = _canonical( + "Severe Thunderstorm Warning", same_code="SVR", + certainty="Observed", description=description, + parameters={ + "maxWindGust": ["60 MPH"], "maxHailSize": ["1.00"], + "eventMotionDescription": ["2200000T254DEG...35KT 42.5,-114.5"], + }, + ) + expected = ( + "⛈️ Severe Thunderstorm Warning\nUntil 2:00 PM MDT — Twin Falls County" + "\n60mph winds, 1\" hail · radar" + "\nMoving W 40 mph — Buhl, Eden, Hazelton" + ) + + with pinned_tz("America/Boise"): + result = nws_format(_FakeEvent(canonical), now=1_751_400_000, + budget=budget_for("nws")) + + assert "→" not in result, "short list must not be path-sampled" + assert_byte_identical(result, expected) + + def test_sps_no_dangling_separator(self): + """Regression: an SPS with wind+motion+long town list must never + collapse to a trailing bare em-dash ('Moving SW 24 mph —…'). + + golden verified against pre-excision _render() (see module docstring). + """ + long_locations = ( + "Twin Falls, Kimberly, Filer, Buhl, Hansen, Murtaugh, Hollister, " + "Eden, Hazelton, and Rogerson" + ) + description = ( + "HAZARD...Wind gusts in excess of 45 mph and pea size hail.\n\n" + "SOURCE...Radar indicated.\n\n" + f"Locations impacted include...{long_locations}" + ) + canonical = _canonical( + "Special Weather Statement", same_code="SPS", + certainty="Observed", description=description, + parameters={"eventMotionDescription": ["2200000T225DEG...21KT 42.5,-114.5"]}, + ) + expected = ( + "🌬️ Special Weather Statement\nUntil 2:00 PM MDT — Twin Falls County" + "\n45mph gusts and 0.25\" hail · observed" + "\nMoving SW 24 mph — Twin Falls" + ) + + with pinned_tz("America/Boise"): + result = nws_format(_FakeEvent(canonical), now=1_751_400_000, + budget=budget_for("nws")) + + assert len(result) <= 140 + for line in result.splitlines(): + stripped = line.rstrip() + assert not stripped.endswith("—"), f"bare trailing em-dash: {line!r}" + assert "—…" not in stripped and "— …" not in stripped + assert "45mph gusts" in result, "wind hazard not tightened" + assert '0.25" hail' in result, "hail not rendered numerically ('pea' -> 0.25\")" + assert "in excess of" not in result, "filler phrase survived tightening" + assert_byte_identical(result, expected) + + def test_sps_pathological_towns_degrade_to_motion_only(self): + """When not even one sampled town fits the remaining budget, line 4 + degrades to motion-only — never a dangling separator. + + golden verified against pre-excision _render() (see module docstring). + """ + long_town = "Averyverylongimpossibletownnamethatwillnotfitthebudgetatall" * 2 + description = ( + "HAZARD...Wind gusts in excess of 45 mph.\n\n" + f"Locations impacted include...{long_town}" + ) + canonical = _canonical( + "Special Weather Statement", same_code="SPS", + certainty="Observed", description=description, + parameters={"eventMotionDescription": ["2200000T225DEG...21KT 42.5,-114.5"]}, + ) + expected = ( + "🌬️ Special Weather Statement\nUntil 2:00 PM MDT — Twin Falls County" + "\n45mph gusts · observed" + "\nMoving SW 24 mph" + ) + + with pinned_tz("America/Boise"): + result = nws_format(_FakeEvent(canonical), now=1_751_400_000, + budget=budget_for("nws")) + + assert len(result) <= 140 + last = result.splitlines()[-1] + if last.startswith("Moving"): + assert " — " not in last, f"expected motion-only, got: {last!r}" + assert_byte_identical(result, expected) + + def test_tor_observed_on_ground_with_damage_threat(self): + """TOR branch: OBSERVED detection -> 'on ground'; damage threat appended. + + golden verified against pre-excision _render() (see module docstring). + """ + canonical = _canonical( + "Tornado Warning", same_code="TOR", certainty="Observed", + description="TORNADO...OBSERVED\n\nLocations impacted include...Twin Falls.", + parameters={"tornadoDetection": ["OBSERVED"], + "tornadoDamageThreat": ["Considerable"]}, + ) + expected = ( + "🌪️ Tornado Warning\nUntil 2:00 PM MDT — Twin Falls County" + "\ntornado on ground · considerable damage" + "\nTwin Falls" + ) + + with pinned_tz("America/Boise"): + result = nws_format(_FakeEvent(canonical), now=1_751_400_000, + budget=budget_for("nws")) + + assert_byte_identical(result, expected) + + def test_tor_radar_indicated_no_threat(self): + """TOR branch: non-OBSERVED detection -> 'radar'; no threat segment + when tornadoDamageThreat is empty. + + golden verified against pre-excision _render() (see module docstring). + """ + canonical = _canonical( + "Tornado Warning", same_code="TOR", certainty="Possible", + description="TORNADO...RADAR INDICATED\n\nLocations impacted include...Buhl.", + parameters={"tornadoDetection": ["RADAR INDICATED"], "tornadoDamageThreat": []}, + ) + expected = ( + "🌪️ Tornado Warning\nUntil 2:00 PM MDT — Twin Falls County" + "\ntornado radar" + "\nBuhl" + ) + + with pinned_tz("America/Boise"): + result = nws_format(_FakeEvent(canonical), now=1_751_400_000, + budget=budget_for("nws")) + + assert_byte_identical(result, expected) + + def test_ffw_thunderstorm_flood_cause(self): + """FFW/FLW branch: flood-cause keyword ('thunderstorm') is appended + as a ' · thunderstorms' segment. + + golden verified against pre-excision _render() (see module docstring). + """ + canonical = _canonical( + "Flash Flood Warning", same_code="FFW", certainty="Observed", + description=("HAZARD...Flash flooding caused by thunderstorms. Excessive " + "runoff will result in flooding of small creeks.\n\n" + "Locations impacted include...Twin Falls."), + parameters={}, + ) + expected = ( + "🌊 Flash Flood Warning\nUntil 2:00 PM MDT — Twin Falls County" + "\nFlash flooding caused by thunderstorms · thunderstorms" + "\nTwin Falls" + ) + + with pinned_tz("America/Boise"): + result = nws_format(_FakeEvent(canonical), now=1_751_400_000, + budget=budget_for("nws")) + + assert_byte_identical(result, expected) # ============================================================================= -# 3. Gate-sequence: old handle_nws vs new gating/nws.decide() +# 1. Gate-sequence: native gating/nws.decide() only # ============================================================================= +def _parse_iso(s): + """Parse a CAP ISO datetime string to an epoch int (or None). + + Standalone equivalent of the now-deleted meshai.central.nws_handler + ._parse_iso, kept here only as test scaffolding for building canonical + dicts to feed nws_decide(). + """ + if not s: + return None + try: + return int(datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp()) + except Exception: + return None + + class TestGateSequence: - """Replay a 4-step lifecycle and assert old/new gating decisions match. + """Replay a 4-step lifecycle through the native gating.nws.decide(). Steps: 1. First sighting → broadcast @@ -456,15 +441,20 @@ class TestGateSequence: } def _make_canonical(self, fixture: dict) -> dict: - """Build canonical dict from a fixture for nws_decide().""" + """Build canonical dict from a fixture for nws_decide(). + + `_make_envelope` always sets an explicit "event" field, so the + deleted central _category_to_event_type() fallback is never + actually exercised here; "Weather Alert" documents that fallback + without depending on the deleted module. + """ env = fixture["envelope"] inner = env.get("data") or {} d = inner.get("data") or {} category_raw = inner.get("category") or "" - from meshai.central.nws_handler import _category_to_event_type, _parse_iso return { "cap_id": d.get("id"), - "event": d.get("event") or _category_to_event_type(category_raw), + "event": d.get("event") or "Weather Alert", "same_code": ((d.get("eventCode") or {}).get("SAME") or [""])[0], "cap_severity": d.get("severity"), "certainty": d.get("certainty") or "", @@ -479,27 +469,6 @@ class TestGateSequence: "headline": d.get("headline"), } - def test_old_gate_sequence(self, mem_db): - """4-step lifecycle through OLD handle_nws: first→dup<3h→dup>3h→Cancel.""" - cap_id = "urn:oid:old.gate.001" - t0 = 1_783_200_000 - t1 = t0 + 1000 # <3h - t2 = t0 + 11000 # >3h (10800s window) - t3 = t0 + 12000 - - def go(msg_type="Alert", now=t0): - env = self._make_envelope(cap_id, msg_type=msg_type)["envelope"] - data = {} - wire = handle_nws(env, "central.wx.alert.us.id", data=data, now=int(now)) - if wire is not None and "_on_broadcast_committed" in data: - data["_on_broadcast_committed"](float(now)) - return wire is not None - - assert go(now=t0) is True, "step1: first sighting should broadcast" - assert go(now=t1) is False, "step2: dup within 3h should suppress" - assert go(now=t2) is True, "step3: after 3h should rebroadcast" - assert go("Cancel", now=t3) is False, "step4: Cancel tombstone should suppress" - def test_new_gate_sequence(self, mem_db): """4-step lifecycle through NEW nws_decide(): first→dup<3h→dup>3h→Cancel.""" cap_id = "urn:oid:new.gate.001" @@ -539,13 +508,14 @@ class TestGateSequence: t0 = 1_783_200_000.0 t1 = t0 + 500 - # Broadcast parent first + # Broadcast parent first, through the same native nws_decide() path + # used everywhere else in this class (mirrors test_new_gate_sequence). fix_parent = self._make_envelope(parent_id) - data_p = {} - wire_p = handle_nws(fix_parent["envelope"], fix_parent["subject"], data=data_p, now=int(t0)) - assert wire_p is not None, "parent should broadcast" - if "_on_broadcast_committed" in data_p: - data_p["_on_broadcast_committed"](t0) + canon_parent = self._make_canonical(fix_parent) + gate_parent = nws_decide(canon_parent, source="nws", now=t0) + assert gate_parent.broadcast is True, "parent should broadcast" + if gate_parent.commit: + gate_parent.commit(t0) # Child references parent fix_child = self._make_envelope( @@ -553,27 +523,7 @@ class TestGateSequence: references=[{"identifier": parent_id, "sent": "2026-07-04T00:00:00Z", "effective": "2026-07-04T00:00:00Z"}], ) - - from meshai.central.nws_handler import _parse_iso, _category_to_event_type - inner = fix_child["envelope"].get("data") or {} - d = inner.get("data") or {} - category_raw = inner.get("category") or "" - canonical = { - "cap_id": child_id, - "event": d.get("event") or _category_to_event_type(category_raw), - "same_code": ((d.get("eventCode") or {}).get("SAME") or [""])[0], - "cap_severity": d.get("severity"), - "certainty": d.get("certainty") or "", - "expires_at": _parse_iso(d.get("expires")), - "area_desc": d.get("areaDesc"), - "geocoder": {"city": None, "county": d.get("areaDesc"), "state": None}, - "description": d.get("description"), - "parameters": d.get("parameters") or {}, - "msgType": d.get("msgType"), - "references": d.get("references") or [], - "category": category_raw, - "headline": d.get("headline"), - } + canonical = self._make_canonical(fix_child) gate_child = nws_decide(canonical, source="nws", now=t1) assert gate_child.broadcast is True, f"child should broadcast: {gate_child.reason}" @@ -584,7 +534,7 @@ class TestGateSequence: # ============================================================================= -# 4. Schema-conformance: native env/nws.py emits canonical schema +# 2. Schema-conformance: native env/nws.py emits canonical schema # ============================================================================= class TestSchemaConformance: @@ -688,7 +638,7 @@ class TestSchemaConformance: # ============================================================================= -# 5. Formatter registration: weather_warning + weather_statement registered +# 3. Formatter registration: weather_warning + weather_statement registered # ============================================================================= class TestFormatterRegistration: diff --git a/work/tests/test_quake_handler.py b/work/tests/test_quake_handler.py deleted file mode 100644 index 65df6dd..0000000 --- a/work/tests/test_quake_handler.py +++ /dev/null @@ -1,193 +0,0 @@ -"""Tests for v0.5.10 USGS earthquakes handler.""" -import pytest - -from meshai.central.quake_handler import ( - handle_quake, - within_250mi_of_idaho, -) -from meshai.persistence import close_thread_connection, init_db -from meshai.persistence import db as persistence_db - - -@pytest.fixture -def mem_db(monkeypatch, tmp_path): - db_path = str(tmp_path / "quake-test.sqlite") - monkeypatch.setenv("MESHAI_DB_PATH", db_path) - persistence_db._initialised.clear() - close_thread_connection() - conn = init_db() - yield conn - close_thread_connection() - persistence_db._initialised.discard(db_path) - - -def _quake_env(*, event_id="uu80141266", mag=3.5, depth_km=9.0, - place="9 km SW of Stanley, Idaho", - lat=44.094, lon=-115.962, - tsunami=0, alert=None, - time_ms=1780006952030, - category="quake.event.minor"): - return { - "id": event_id, "subject": "central.quake.event.minor.unknown", - "data": { - "id": event_id, "adapter": "usgs_quake", "category": category, - "severity": 0, - "geo": {"centroid": [lon, lat], "primary_region": None}, - "data": { - "id": event_id, "magnitude": mag, "place": place, - "depth_km": depth_km, "time_ms": time_ms, - "tsunami": tsunami, "alert": alert, "status": "reviewed", - }, - }, - } - - -def _commit(data, t): - data["_on_broadcast_committed"](float(t)) - - -# ---- magnitude floor ---- - - -def test_m3_anywhere_broadcasts(mem_db): - env = _quake_env(mag=3.5, lat=37.0, lon=-122.0) # SF Bay area, outside Idaho - wire = handle_quake(env, env["subject"], data={}, now=1_000_000) - assert wire is not None - assert "M3.5" in wire - - -def test_m25_inside_idaho_broadcasts(mem_db): - env = _quake_env(mag=2.7, lat=44.094, lon=-115.962, event_id="uu1") - wire = handle_quake(env, env["subject"], data={}, now=1_000_000) - assert wire is not None - assert "M2.7" in wire - - -def test_m25_outside_idaho_skipped(mem_db): - # San Francisco -- well outside 250mi of Idaho centroid. - env = _quake_env(mag=2.7, lat=37.0, lon=-122.0, event_id="uu2") - wire = handle_quake(env, env["subject"], data={}, now=1_000_000) - assert wire is None - - -def test_below_25_skipped(mem_db): - env = _quake_env(mag=1.01, lat=44.0, lon=-114.0, event_id="uu3") - wire = handle_quake(env, env["subject"], data={}, now=1_000_000) - assert wire is None - - -# ---- tsunami special ---- - - -def test_tsunami_any_magnitude_broadcasts(mem_db): - env = _quake_env(mag=4.5, lat=10.0, lon=140.0, tsunami=1, event_id="japan1") - wire = handle_quake(env, env["subject"], data={}, now=1_000_000) - assert wire is not None - assert "TSUNAMI WARNING" in wire - assert wire.startswith("🚨") - - -# ---- PAGER alert ---- - - -def test_pager_orange_broadcasts(mem_db): - env = _quake_env(mag=2.0, lat=37.0, lon=-122.0, alert="orange", - event_id="pager1") - wire = handle_quake(env, env["subject"], data={}, now=1_000_000) - assert wire is not None - - -def test_pager_red_broadcasts(mem_db): - env = _quake_env(mag=2.0, lat=37.0, lon=-122.0, alert="red", - event_id="pager2") - wire = handle_quake(env, env["subject"], data={}, now=1_000_000) - assert wire is not None - - -# ---- wire format ---- - - -def test_uses_usgs_place_string(mem_db): - env = _quake_env(mag=4.1, place="9 km SW of Stanley, Idaho", - event_id="usgs1") - wire = handle_quake(env, env["subject"], data={}, now=1_000_000) - assert "9 km SW of Stanley, Idaho" in wire - - -def test_m5_uses_warning_emoji(mem_db): - env = _quake_env(mag=5.2, lat=44.0, lon=-114.0, event_id="big1") - wire = handle_quake(env, env["subject"], data={}, now=1_000_000) - assert wire.startswith("⚠️") - - -def test_wire_includes_depth_and_coords(mem_db): - env = _quake_env(mag=4.1, depth_km=9.0, lat=44.094, lon=-115.962, - event_id="d1") - wire = handle_quake(env, env["subject"], data={}, now=1_000_000) - assert "Depth: 9 km" in wire - assert "@ 44.094, -115.962" in wire - - -# ---- per-event dedup ---- - - -def test_per_event_id_dedup_no_reissue(mem_db): - env = _quake_env(mag=4.0, event_id="dedup1") - data1 = {} - handle_quake(env, env["subject"], data=data1, now=1_000_000) - _commit(data1, 1_000_001) - - # Same event_id republishes (magnitude revision). Should NOT re-broadcast. - env_rev = _quake_env(mag=4.2, event_id="dedup1") # higher mag, same id - wire2 = handle_quake(env_rev, env_rev["subject"], data={}, now=1_000_300) - assert wire2 is None - - -# ---- distance helper ---- - - -def test_within_250mi_of_idaho_boundary(): - # Boise, ID -- inside - assert within_250mi_of_idaho(43.6, -116.2) is True - # San Francisco -- outside - assert within_250mi_of_idaho(37.0, -122.0) is False - # Seattle -- inside (250mi from Idaho centroid; verify the boundary) - assert within_250mi_of_idaho(47.6, -122.3) is False - # Boundary edge case (Idaho center) - assert within_250mi_of_idaho(44.36, -114.61) is True - - -# ---- commit callback ---- - - -def test_commit_callback_updates_last_broadcast(mem_db): - env = _quake_env(mag=4.0, event_id="cb1") - data = {} - handle_quake(env, env["subject"], data=data, now=1_000_000) - pre = mem_db.execute( - "SELECT last_broadcast_at FROM quake_events WHERE event_id='cb1'" - ).fetchone() - assert pre["last_broadcast_at"] is None - _commit(data, 1_000_001) - post = mem_db.execute( - "SELECT last_broadcast_at FROM quake_events WHERE event_id='cb1'" - ).fetchone() - assert post["last_broadcast_at"] == 1_000_001 - - -# ============================================================================ -# Budget-fit SAFETY CAP: a freak-long USGS place string must still fit 140. -# ============================================================================ - -from meshai.central.quake_handler import _render as _quake_render - - -def test_quake_render_worst_case_fits_140(): - place = ("293 km SSW of a pathologically long place description island " - "region in the remote northern pacific ocean near absolutely nowhere " - "at all off the coast of the far edge of the map") - wire = _quake_render(mag=7.9, place=place, depth_km=12, lat=44.123, - lon=-114.987, tsunami=True, is_update=False) - assert len(wire) <= 140, f"{len(wire)} chars:\n{wire!r}" - # magnitude survives on the (critical) first line - assert "M7.9" in wire diff --git a/work/tests/test_quake_refactor.py b/work/tests/test_quake_refactor.py index b3ac555..c162e28 100644 --- a/work/tests/test_quake_refactor.py +++ b/work/tests/test_quake_refactor.py @@ -1,18 +1,24 @@ """Phase-1 quake refactor tests — reference implementation verification. -Four test groups: +The Central `quake_handler` module (`_render()`, `handle_quake()`) has been +deleted — the native path is the only production path now. Pure old-vs-new +parity assertions and tests that only replayed decisions through the +deleted `handle_quake` have been removed; original diffs are preserved in +git history. What remains exercises native code directly (hand-written +expected strings are kept as regression pins on the current wire format). + +Three test groups: 1. Parity (tier-b): fixture 0002 → canonical data → formatter. - Expected string is hand-written (the new correct format). - The OLD _render() output for the same fixture is captured in a comment so - the intended tier-b diff is explicit and reviewable. - Two synthetic cases show the PAGER + update-prefix diffs explicitly. + Expected string is hand-written (the current correct format). + Two synthetic cases show the PAGER + update-prefix rendering explicitly. -2. Cross-source identity: native adapter builds the same canonical data as - the Central path for fixture 0002. Both render byte-identically. +2. Cross-source identity: native adapter builds the same canonical data + shape the formatter reads. -3. Gate-sequence: replay four synthetic events through the OLD handle_quake - gating and the NEW gating.quake.decide(); assert broadcast/suppress match. +3. Gate-sequence: exercise gating.quake.decide() directly across a + synthetic event sequence; assert broadcast/suppress thresholds and the + commit → suppress-on-replay lifecycle. 4. Schema-conformance: env/usgs_quake.py to_event() emits all canonical keys. """ @@ -26,7 +32,6 @@ from tests.harness.goldens import ( assert_byte_identical, load_fixtures, pinned_time, - run_gate_sequence, ) # ── Shared clock epoch for deterministic renders ───────────────────────────── @@ -62,26 +67,13 @@ def _make_fake_event(data: dict): class TestFormatterParity: """formatter/quake.format() renders correct output from canonical data.""" - def _render_old(self, *, mag, place, depth_km, lat, lon, tsunami, is_update=False): - """Capture OLD _render() output for diff comments.""" - from meshai.central.quake_handler import _render - from meshai.central.budget import budget_for - return _render(mag=mag, place=place, depth_km=depth_km, lat=lat, - lon=lon, tsunami=tsunami, is_update=is_update) - def test_fixture_0002_new_format(self): - """Fixture 0002 (M3.3 Lima Montana) → NEW formatter output matches hand-written expected. + """Fixture 0002 (M3.3 Lima Montana) → formatter output matches hand-written expected. Fixture 0002 has alert=null and no tsunami so the tier-b additions - (PAGER line, update-prefix) are not visible. The old and new outputs - are IDENTICAL for this fixture — which is correct. The hand-written + (PAGER line, update-prefix) are not visible. The hand-written expected below documents the canonical format; synthetic tests below show the tier-b additions. - - OLD _render() output (captured for diff transparency): - "🌐 New: M3.3 — 19 km S of Lima, Montana\\nDepth: 11 km · @ 44.460, -112.611" - NEW formatter output (same — no tier-b changes triggered): - "🌐 New: M3.3 — 19 km S of Lima, Montana\\nDepth: 11 km · @ 44.460, -112.611" """ from meshai.notifications.formatters.quake import format as qfmt @@ -119,27 +111,8 @@ class TestFormatterParity: assert_byte_identical(result, expected) - # Verify old _render matches new for this fixture (no tier-b diff) - old_wire = self._render_old( - mag=canonical["magnitude"], place=canonical["place"], - depth_km=canonical["depth_km"], lat=canonical["lat"], - lon=canonical["lon"], tsunami=canonical["tsunami"], - is_update=False, - ) - assert_byte_identical(result, old_wire), ( - "For fixture 0002 (null PAGER, is_update=False) old and new " - "outputs must be identical — the tier-b diff only appears when " - "PAGER or is_update are set." - ) - def test_tier_b_pager_orange_rendered(self): - """Tier-b ①: PAGER=orange is NOW rendered on a 4th line. - - OLD _render() output (captured): - "🌐 New: M2.0 — Off the coast of Oregon\\nDepth: 10 km · @ 44.000, -125.000" - NEW formatter output (tier-b change — PAGER line added): - "🌐 New: M2.0 — Off the coast of Oregon\\nDepth: 10 km · @ 44.000, -125.000\\n⚠️ PAGER: orange" - """ + """Tier-b ①: PAGER=orange is rendered on a 4th line.""" from meshai.notifications.formatters.quake import format as qfmt canonical = { @@ -158,12 +131,6 @@ class TestFormatterParity: "distance_km": 500.0, } - # OLD _render() output (PAGER not rendered) - old_wire = self._render_old( - mag=2.0, place="Off the coast of Oregon", depth_km=10.0, - lat=44.0, lon=-125.0, tsunami=False, is_update=False, - ) - # NEW formatter output (PAGER rendered as 4th line) expected_new = ( "\U0001f310 New: M2.0 — Off the coast of Oregon" "\nDepth: 10 km · @ 44.000, -125.000" @@ -174,19 +141,9 @@ class TestFormatterParity: result = qfmt(_make_fake_event(canonical), now=_AT, budget=140) assert_byte_identical(result, expected_new) - # Confirm the old wire does NOT have the PAGER line - assert "PAGER" not in old_wire, ( - f"OLD _render() must not contain PAGER line; got: {old_wire!r}" - ) def test_tier_b_update_prefix_rendered(self): - """Tier-b ②: is_update=True produces 'Update:' prefix (was hard-coded 'New:'). - - OLD _render() output (is_update always False): - "🌐 New: M3.0 — 5 km NE of Stanley, Idaho\\nDepth: 8 km · @ 44.200, -114.900" - NEW formatter output (is_update=True): - "🌐 Update: M3.0 — 5 km NE of Stanley, Idaho\\nDepth: 8 km · @ 44.200, -114.900" - """ + """Tier-b ②: is_update=True produces 'Update:' prefix.""" from meshai.notifications.formatters.quake import format as qfmt canonical = { @@ -205,30 +162,20 @@ class TestFormatterParity: "distance_km": 10.0, } - # OLD _render() always uses is_update=False - old_wire = self._render_old( - mag=3.0, place="5 km NE of Stanley, Idaho", depth_km=8.0, - lat=44.2, lon=-114.9, tsunami=False, is_update=False, - ) expected_new = ( "\U0001f310 Update: M3.0 — 5 km NE of Stanley, Idaho" "\nDepth: 8 km · @ 44.200, -114.900" ) - expected_old = ( - "\U0001f310 New: M3.0 — 5 km NE of Stanley, Idaho" - "\nDepth: 8 km · @ 44.200, -114.900" - ) with pinned_time(_AT): result = qfmt(_make_fake_event(canonical), now=_AT, budget=140) assert_byte_identical(result, expected_new) - assert_byte_identical(old_wire, expected_old) assert "Update:" in result assert "New:" not in result def test_tsunami_escalation_preserved(self): - """Tsunami escalation (🚨 emoji + TSUNAMI WARNING line) unchanged from _render.""" + """Tsunami escalation renders the 🚨 emoji + TSUNAMI WARNING line.""" from meshai.notifications.formatters.quake import format as qfmt canonical = { @@ -247,16 +194,18 @@ class TestFormatterParity: "distance_km": 8000.0, } + expected = ( + "\U0001f6a8 New: M4.5 — off the coast of Japan" + "\nDepth: 5 km · @ 35.000, 141.000" + "\n\U0001f6a8 TSUNAMI WARNING" + ) + with pinned_time(_AT): result = qfmt(_make_fake_event(canonical), now=_AT, budget=140) - old_wire = self._render_old( - mag=4.5, place="off the coast of Japan", depth_km=5.0, - lat=35.0, lon=141.0, tsunami=True, - ) assert result.startswith("\U0001f6a8"), "Tsunami emoji must be 🚨" assert "\U0001f6a8 TSUNAMI WARNING" in result - assert_byte_identical(result, old_wire) + assert_byte_identical(result, expected) def test_m5_escalation_emoji_preserved(self): """M5+ uses ⚠️ emoji — unchanged from _render.""" @@ -433,22 +382,15 @@ def _make_envelope(*, event_id, mag, lat, lon, depth_km=10.0, place=None, class TestGateSequence: - """Gate parity: old handle_quake decisions match new gating.quake.decide().""" + """gating.quake.decide() gate thresholds + commit/suppress lifecycle.""" @pytest.fixture(autouse=True) def _db(self, mem_db): """All tests in this class share the same mem_db.""" self.db = mem_db - def _old_gate(self, fixture, *, now): - """Old path: handle_quake returning non-None = broadcast.""" - from meshai.central.quake_handler import handle_quake - env = fixture["envelope"] - wire = handle_quake(env, fixture["subject"], data={}, now=int(now)) - return wire is not None - - def _new_gate(self, fixture, *, now): - """New path: gating.quake.decide().""" + def _decide(self, fixture, *, now): + """Build canonical data from a Central-style fixture and call decide().""" from meshai.notifications.gating.quake import decide env = fixture["envelope"] inner = env.get("data") or {} @@ -475,18 +417,12 @@ class TestGateSequence: return decide(canonical, source="usgs_quake", now=float(now)) def test_gate_sequence_matches(self): - """Four-event sequence: old and new gates make identical broadcast/suppress decisions. + """Four-event sequence exercises all of decide()'s broadcast thresholds. - Sequence (each event has a DISTINCT event_id — the commit/suppress - cycle is tested separately in test_suppress_after_commit): [0] M2.0, far (below all thresholds) → suppress [1] M2.7, within Idaho (regional gate) → broadcast [2] M3.5, anywhere (global floor) → broadcast [3] M6.0 + tsunami (any-magnitude tsunami gate) → broadcast - - Gate decisions (broadcast True/False) must match between old and new. - NOTE: PAGER/update-prefix are formatter-only tier-b changes; they do - NOT affect gate decisions — any divergence here is a regression. """ t_base = 1_780_000_000.0 @@ -503,84 +439,35 @@ class TestGateSequence: fx3 = _make_envelope(event_id="gs_seq_3", mag=6.0, lat=35.0, lon=141.0, tsunami=1, time_ms=int((t_base + 300) * 1000)) - ordered = [fx0, fx1, fx2, fx3] - timeline = [t_base, t_base + 100, t_base + 200, t_base + 300] + r0 = self._decide(fx0, now=t_base) + r1 = self._decide(fx1, now=t_base + 100) + r2 = self._decide(fx2, now=t_base + 200) + r3 = self._decide(fx3, now=t_base + 300) - results = run_gate_sequence( - self._old_gate, - self._new_gate, - ordered, - timeline=timeline, - ) - - mismatches = [r for r in results if not r["match"]] - assert not mismatches, ( - "Gate sequence mismatch between old handle_quake and new decide():\n" - + "\n".join( - f" step {r['fixture_n']}: old={r['old_broadcast']} " - f"new={r['new_broadcast']} diffs={r['diffs']}" - for r in mismatches - ) - ) - - # Verify expected pattern - assert results[0]["old_broadcast"] is False, "M2.0 far must be suppressed" - assert results[1]["old_broadcast"] is True, "M2.7 Idaho must broadcast" - assert results[2]["old_broadcast"] is True, "M3.5 global must broadcast" - assert results[3]["old_broadcast"] is True, "M6.0+tsunami must broadcast" + assert r0.broadcast is False, "M2.0 far must be suppressed" + assert r1.broadcast is True, "M2.7 Idaho must broadcast" + assert r2.broadcast is True, "M3.5 global must broadcast" + assert r3.broadcast is True, "M6.0+tsunami must broadcast" def test_suppress_after_commit(self): - """After commit, the same event_id is suppressed by both old and new gates. - - The run_gate_sequence harness does not call commits between steps, so - the commit+suppress lifecycle is tested here separately by manual - sequencing. - """ - from meshai.central.quake_handler import handle_quake - from meshai.notifications.gating.quake import decide - + """After commit, a replay of the same event_id is suppressed by decide().""" t0 = 1_780_000_000.0 event_id = "suppress_after_commit_test" fx = _make_envelope(event_id=event_id, mag=3.5, lat=44.09, lon=-115.96, time_ms=int(t0 * 1000)) - env = fx["envelope"] - # First arrival: both old and new broadcast - data1 = {} - old_wire1 = handle_quake(env, fx["subject"], data=data1, now=int(t0)) - assert old_wire1 is not None, "First arrival must broadcast (old)" + # First arrival: broadcast. + result1 = self._decide(fx, now=t0) + assert result1.broadcast is True, "First arrival must broadcast" + assert result1.commit is not None, "commit callback must be attached" - # Build canonical from fixture for new gate - inner = env["data"] - d = inner["data"] - geo = inner["geo"] - cent = geo["centroid"] - canonical = { - "magnitude": d["magnitude"], - "depth_km": d.get("depth_km") or d.get("depth"), - "lat": cent[1], "lon": cent[0], - "place": d.get("place"), - "tsunami": bool(d.get("tsunami")), - "pager": d.get("alert"), - "occurred_at": int(d["time_ms"] / 1000), - "event_id": event_id, - } - # Since old gate already wrote the row (INSERT), new gate sees the - # same DB state. Both should broadcast on first arrival. - # (We test new gate's second call AFTER commit below) + # Commit (simulates confirmed delivery). + result1.commit(t0 + 1.0) - # Call commit (simulates confirmed delivery) - assert "_on_broadcast_committed" in data1, "commit callback must be attached" - data1["_on_broadcast_committed"](t0 + 1.0) - - # Second arrival with same event_id — old gate must suppress - old_wire2 = handle_quake(env, fx["subject"], data={}, now=int(t0 + 60)) - assert old_wire2 is None, "Old gate must suppress after commit" - - # New gate must also suppress - new_result2 = decide(canonical, source="usgs_quake", now=t0 + 60) - assert new_result2.broadcast is False, "New gate must suppress after commit" + # Second arrival with same event_id — must suppress. + result2 = self._decide(fx, now=t0 + 60) + assert result2.broadcast is False, "Gate must suppress after commit" def test_severity_override_from_decide(self): """decide() sets _severity_override=immediate for tsunami/PAGER.""" diff --git a/work/tests/test_rf_v057.py b/work/tests/test_rf_v057.py deleted file mode 100644 index 9b22752..0000000 --- a/work/tests/test_rf_v057.py +++ /dev/null @@ -1,245 +0,0 @@ -"""v0.5.7-rf: SWPC subject validation + protons severity=0 docs + categories audit. - -Covers three things shipped in v0.5.7-rf: - -1. SWPC subscription subject -- verifies the existing `central.space.>` - tail-only-`>` form (per Central v0.10.0 guide §swpc_*: planetary, no - region in subject; one umbrella subscription covers swpc_alerts, - swpc_kindex, swpc_protons). The pattern was already correct from v0.5.4 - work; this phase pins it explicitly so future "add a region tail" - refactors fail loudly. -2. swpc_protons severity=0 routing -- per guide §swpc_protons live sample - the adapter always publishes severity=0. Verifies map_severity(0) -> - "routine" and the NotificationToggle.severity_channels string-keyed - dict accepts "routine" with no IndexError. The "silently dropped" - failure mode the prompt described does not exist; this test is a - regression guard against a future refactor introducing it. -3. ALERT_CATEGORIES RF-family audit -- adds four missing entries that - meshai emits but the rule editor couldn't target: - - rf_anomalous_propagation (ducting.py super_refraction tier) - - rf_ducting_enhancement (ducting.py duct + surface_duct tiers) - - rf_propagation_alert (central swpc_alerts -> space.alert) - - solar_radiation_storm (central swpc_protons -> space.proton_flux) - Verifies geomagnetic_storm (central swpc_kindex -> space.kindex) - stays mapped. Legacy hf_blackout and tropospheric_ducting are kept as - selectable forward-compat targets even though no current emitter - produces them; flagged in the commit body for follow-up. -""" - -import inspect -import json -import re - -import pytest - -from meshai.central.consumer import ( - CentralConsumer, - _SUBJECTS_BARE, - _subjects_for, - map_category, - map_severity, -) -from meshai.config import EnvironmentalConfig, NotificationToggle -from meshai.notifications.categories import ALERT_CATEGORIES -from meshai.notifications.pipeline.bus import EventBus - - -def _assert_legal_nats(subject: str) -> None: - tokens = subject.split(".") - if ">" in tokens: - assert tokens[-1] == ">", f"`>` not at tail in {subject!r}" - assert tokens.count(">") == 1, f"multiple `>` in {subject!r}" - for tok in tokens: - assert tok, f"empty token in {subject!r}" - if tok not in {"*", ">"}: - assert "*" not in tok and ">" not in tok, f"mixed wildcard in token {tok!r}" - - -# ---------- FIX 1: SWPC subject pattern ----------------------------------- - - -def test_swpc_subject_is_global_umbrella(): - """Per Central v0.10.0 guide §space stream, all SWPC adapters publish - under `central.space.>`. Single tail-only-`>` subscription catches - all three (swpc_alerts / swpc_kindex / swpc_protons).""" - subs = _subjects_for("swpc", "us.id") - assert subs == ["central.space.>"] - for s in subs: - _assert_legal_nats(s) - - -def test_swpc_subject_ignores_region(): - """Space weather is planetary; region argument MUST be a no-op.""" - assert _subjects_for("swpc", "us.id") == ["central.space.>"] - assert _subjects_for("swpc", "us.mt") == ["central.space.>"] - assert _subjects_for("swpc", "") == ["central.space.>"] - assert _subjects_for("swpc", None) == ["central.space.>"] - - -def test_swpc_subject_covers_all_three_adapter_subjects(): - """The umbrella `central.space.>` matches every per-adapter subject - documented in the guide.""" - sub = _subjects_for("swpc", "us.id")[0] - # `>` matches one or more tokens at the tail. - assert sub.endswith(".>") - prefix = sub[:-2] # strip the .> - for published in ( - "central.space.alert.a20f", # swpc_alerts (4 tokens, product_id tail) - "central.space.kindex", # swpc_kindex (3 tokens, fixed) - "central.space.proton_flux", # swpc_protons (3 tokens, fixed) - ): - assert published.startswith(prefix), f"{published!r} not covered by {sub!r}" - - -# ---------- FIX 2: severity=0 routing ------------------------------------- - - -def test_map_severity_zero_routes_to_routine(): - """All three SWPC adapters publish severity=0 by default. The boundary - contract: 0 -> 'routine' (not dropped, not error).""" - assert map_severity(0) == "routine" - - -def test_severity_channels_dict_accepts_routine_key(): - """NotificationToggle.severity_channels is dict-keyed by severity STRING - -- so "routine" is a valid key with no IndexError vector. Pins the - contract so a refactor to an int-indexed list would break this test.""" - t = NotificationToggle(name="rf_propagation") - assert isinstance(t.severity_channels, dict) - # dict.get returns the default for unknown keys; no exception possible. - assert t.severity_channels.get("routine", ["mesh_broadcast"]) == ["mesh_broadcast"] - - -@pytest.mark.skip(reason="v0.5.13 default-deny: sub-threshold SWPC envelopes intentionally do NOT route through consumer to produce broadcasts. This is the architectural fix.") -def test_swpc_protons_severity_zero_routes_through_consumer(): - """Synthetic swpc_protons envelope (severity=0 per guide §swpc_protons) - -- verify it normalizes to ev.severity='routine' and emits on the bus - with no exception.""" - rec = [] - bus = EventBus(); bus.subscribe(rec.append) - c = CentralConsumer(EnvironmentalConfig(), bus) - env = {"id": "2026-05-18T05:55:00Z|>=100 MeV", "data": { - "id": "2026-05-18T05:55:00Z|>=100 MeV", "adapter": "swpc_protons", - "category": "space.proton_flux", - "time": "2026-05-18T05:55:00Z", "severity": 0, - "geo": {"centroid": None, "primary_region": None, "regions": []}, - "data": {"flux": 0.16, "energy": ">=100 MeV", - "time_tag": "2026-05-18T05:55:00Z", "satellite": 19}}} - ev = c._handle("central.space.proton_flux", json.dumps(env).encode()) - assert ev is not None - assert ev.severity == "routine" - assert ev.category == "solar_radiation_storm" - assert ev.source == "swpc" - assert len(rec) == 1 - - -@pytest.mark.skip(reason="v0.5.13 default-deny: sub-threshold SWPC envelopes intentionally do NOT route through consumer to produce broadcasts. This is the architectural fix.") -def test_swpc_kindex_severity_zero_routes_through_consumer(): - """Synthetic swpc_kindex envelope -- verifies central path mapping for - a second SWPC adapter (severity=0 -> 'routine', space.kindex -> - geomagnetic_storm).""" - rec = [] - bus = EventBus(); bus.subscribe(rec.append) - c = CentralConsumer(EnvironmentalConfig(), bus) - env = {"id": "2026-05-12T00:00:00", "data": { - "id": "2026-05-12T00:00:00", "adapter": "swpc_kindex", - "category": "space.kindex", - "time": "2026-05-12T00:00:00Z", "severity": 0, - "geo": {"centroid": None, "primary_region": None, "regions": []}, - "data": {"Kp": 0.67, "time_tag": "2026-05-12T00:00:00", - "a_running": 3, "station_count": 8}}} - ev = c._handle("central.space.kindex", json.dumps(env).encode()) - assert ev is not None - assert ev.severity == "routine" - assert ev.category == "geomagnetic_storm" - - -# ---------- FIX 3: ALERT_CATEGORIES RF-family audit ---------------------- - - -@pytest.mark.parametrize("cat", [ - "rf_anomalous_propagation", - "rf_ducting_enhancement", - "rf_propagation_alert", - "solar_radiation_storm", -]) -def test_v057_rf_added_categories_present(cat): - """v0.5.7-rf: four new rf_propagation categories must be registry-present - so the Advanced Rules editor can target them.""" - assert cat in ALERT_CATEGORIES - info = ALERT_CATEGORIES[cat] - assert info["toggle"] == "rf_propagation" - assert info["name"] - assert info["description"] - assert info["default_severity"] in {"routine", "priority", "immediate"} - assert info["example_message"] - - -def test_geomagnetic_storm_still_in_registry(): - """swpc_kindex -> space.kindex -> geomagnetic_storm: registry entry - survives the v0.5.7-rf edit.""" - assert "geomagnetic_storm" in ALERT_CATEGORIES - assert ALERT_CATEGORIES["geomagnetic_storm"]["toggle"] == "rf_propagation" - - -@pytest.mark.parametrize( - "central_cat,expected", - [ - ("space.alert.a20f", "rf_propagation_alert"), - ("space.alert", "rf_propagation_alert"), - ("space.kindex", "geomagnetic_storm"), - ("space.proton_flux", "solar_radiation_storm"), - ("space.unknown_sub", "geomagnetic_storm"), # catchall - ], -) -def test_map_category_swpc_routings(central_cat, expected): - """Pin the central -> meshai category map for each SWPC adapter.""" - assert map_category(central_cat) == expected - - -def _native_emitted_rf_categories() -> set[str]: - """Walk ducting.py for _TIER_CATEGORY values mapping to toggle=rf_propagation.""" - from meshai.env import ducting as ducting_mod - src = inspect.getsource(ducting_mod) - # _TIER_CATEGORY entries are `"": "",` literals. - emitted = set(re.findall( - r'_TIER_CATEGORY\s*=\s*\{([^}]+)\}', src, re.DOTALL)) - cats: set[str] = set() - for block in emitted: - cats |= set(re.findall(r':\s*"([a-z_]+)"', block)) - return {c for c in cats if c in ALERT_CATEGORIES - and ALERT_CATEGORIES[c].get("toggle") == "rf_propagation"} - - -def _central_path_rf_categories() -> set[str]: - central_inputs = [ - "space.alert.a20f", "space.alert", - "space.kindex", - "space.proton_flux", - "space.unknown", - ] - return {map_category(c) for c in central_inputs} - - -def test_alert_categories_rf_complete(): - """Native + central-path emit set must be a SUBSET of registry rf - entries (i.e., everything we emit is selectable). Legacy entries - without an emitter are allowed as forward-compat targets and - documented in the commit body.""" - registry_rf = { - cid for cid, info in ALERT_CATEGORIES.items() - if info.get("toggle") == "rf_propagation" - } - native = _native_emitted_rf_categories() - central = _central_path_rf_categories() - emitted = native | central - missing = emitted - registry_rf - assert not missing, f"rf emit set missing from ALERT_CATEGORIES: {missing}" - # Sanity: at minimum the four v0.5.7-rf additions + geomagnetic_storm - # must be in the emit set. - for required in ( - "rf_anomalous_propagation", "rf_ducting_enhancement", - "rf_propagation_alert", "solar_radiation_storm", - "geomagnetic_storm", - ): - assert required in emitted, f"{required!r} not emitted by native or central path" diff --git a/work/tests/test_satpass_event_path.py b/work/tests/test_satpass_event_path.py index 71c25c3..78f8fdc 100644 --- a/work/tests/test_satpass_event_path.py +++ b/work/tests/test_satpass_event_path.py @@ -91,32 +91,6 @@ def _enable_satpass(): invalidate_cache() -# ── CENTRAL_ADAPTER_TO_SOURCE mapping ─────────────────────────────── - -def test_adapter_to_source_celestrak_tle(): - from meshai.central.consumer import CENTRAL_ADAPTER_TO_SOURCE - assert CENTRAL_ADAPTER_TO_SOURCE["celestrak_tle"] == "satpass" - - -def test_adapter_to_source_n2yo_visualpasses(): - from meshai.central.consumer import CENTRAL_ADAPTER_TO_SOURCE - assert CENTRAL_ADAPTER_TO_SOURCE["n2yo_visualpasses"] == "satpass" - - -def test_adapter_to_source_satpass_predict(): - from meshai.central.consumer import CENTRAL_ADAPTER_TO_SOURCE - assert CENTRAL_ADAPTER_TO_SOURCE["satpass_predict"] == "satpass" - - -def test_no_stale_sat_names_in_adapter_map(): - """Wire names sat_passes / sat_tles / sat_tle must NOT appear.""" - from meshai.central.consumer import CENTRAL_ADAPTER_TO_SOURCE - for stale in ("sat_passes", "sat_tles", "sat_tle"): - assert stale not in CENTRAL_ADAPTER_TO_SOURCE, ( - f"stale adapter name {stale!r} still in CENTRAL_ADAPTER_TO_SOURCE" - ) - - # ── TLE handler route ────────────────────────────────────────────── def test_tle_handler_inserts_sat_tles_row(): @@ -216,25 +190,3 @@ def test_handler_reads_min_elevation(): src = inspect.getsource(satpass_handler) assert "min_elevation" in src assert "min_elevation_deg" not in src - - -# ── Dispatch routing in consumer._normalize ───────────────────────── - -def test_consumer_dispatch_celestrak_tle(): - """consumer._normalize dispatch must route celestrak_tle to tle_handler.""" - import inspect - from meshai.central import consumer - src = inspect.getsource(consumer) - assert '"celestrak_tle"' in src - assert '"sat_tles"' not in src - assert '"sat_tle"' not in src - - -def test_consumer_dispatch_n2yo_and_satpass_predict(): - """consumer._normalize dispatch must route n2yo/satpass_predict to satpass_handler.""" - import inspect - from meshai.central import consumer - src = inspect.getsource(consumer) - assert '"n2yo_visualpasses"' in src - assert '"satpass_predict"' in src - assert '"sat_passes"' not in src diff --git a/work/tests/test_satpass_persisted_timer.py b/work/tests/test_satpass_persisted_timer.py index adb84a8..d38d2bf 100644 --- a/work/tests/test_satpass_persisted_timer.py +++ b/work/tests/test_satpass_persisted_timer.py @@ -1,26 +1,26 @@ -"""Tests for the satpass persisted-timer reboot-recovery fix. +"""Tests for the satpass persisted-timer `due_at` column. Pending satellite-pass consolidations used to be scheduled only as in-memory asyncio TimerHandles, so a restart orphaned any satpass_pending rows: the row -survived but its timer did not, and it was never consolidated/broadcast. +survived but its timer did not, and it was never consolidated/broadcast. The +fix persisted a durable `due_at` on each pending row and added a startup +sweep (`CentralConsumer._sweep_pending_satpass`) that reconstructed a timer +for every pending consolidated_id off its persisted due_at. -The fix persists a durable `due_at` on each pending row and adds a startup -sweep (`CentralConsumer._sweep_pending_satpass`) that reconstructs a timer for -every pending consolidated_id off its persisted due_at, reusing the existing -`_satpass_consolidation_fire` emit path. - -These tests cover: - - a PAST-due orphan is recovered (its timer fires -> consolidation invoked) - - a FUTURE-due row is scheduled, NOT fired immediately - - `due_at` is persisted on the normal ingest path - - SCHEMA_VERSION == 22 and the v22 migration applies cleanly on a fresh DB +The Central NATS consumer (and `_sweep_pending_satpass` with it) was retired +2026-07 -- the sweep's tests are gone with it. The native satpass path +(env/satpass.py) never used the satpass_pending buffer or this sweep in the +first place (it consolidates in-memory within a single tick), so nothing +live is affected. What remains here: + - `due_at` is persisted on the normal ingest path (satpass_handler.py, + still live -- shared by both paths historically, now native-only) + - SCHEMA_VERSION == 26 and the v22 migration (which added the due_at + column) still applies cleanly on a fresh DB """ from __future__ import annotations -import asyncio import json import time -import types import pytest @@ -44,33 +44,6 @@ def _enable_satpass_db(norad_ids=(25544,), dry_run=True): invalidate_cache() -def _insert_pending(consolidated_id, *, due_at, observer="Boise", - norad_id=25544, received_at=None): - """Write a single satpass_pending row with an explicit due_at.""" - conn = get_db() - now = int(time.time()) if received_at is None else received_at - aos = now + 600 - los = aos + 360 - conn.execute( - "INSERT OR REPLACE INTO satpass_pending(" - "consolidated_id, observer, sat_name, norad_id, max_elevation, " - "aos_at, los_at, aos_compass, los_compass, peak_compass, received_at, " - "due_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", - (consolidated_id, observer, "ISS", norad_id, 72.5, - aos, los, "SW", "NE", "S", now, due_at)) - - -def _make_consumer(bus=None): - """Construct a CentralConsumer with minimal fakes (no NATS needed).""" - from meshai.central.consumer import CentralConsumer - env = types.SimpleNamespace(central=None) - return CentralConsumer(env, bus) - - -def _run(coro): - return asyncio.new_event_loop().run_until_complete(coro) - - def _ingest_envelope(norad_id=25544, observer="Boise", max_el=72.5, aos="2026-06-12T03:32:00Z", los="2026-06-12T03:38:00Z"): return { @@ -147,100 +120,3 @@ def test_due_at_persisted_on_normal_ingest(): assert row["due_at"] is not None assert row["due_at"] == row["received_at"] + CONSOLIDATION_DELAY assert row["due_at"] == now + CONSOLIDATION_DELAY - - -# ── startup sweep: past-due orphan is recovered ────────────────────── - -def test_sweep_recovers_past_due_orphan(monkeypatch): - """A pending row with due_at in the PAST fires consolidation via the sweep.""" - _enable_satpass_db(norad_ids=[25544], dry_run=True) - now = int(time.time()) - cid = "25544:ORPHAN" - _insert_pending(cid, due_at=now - 100, received_at=now - 105) - - fired = [] - import meshai.central.satpass_handler as sh - real = sh.consolidate_satpass_pending - - def _spy(consolidated_id): - fired.append(consolidated_id) - return real(consolidated_id) # exercise the real path (dry-run -> None) - - monkeypatch.setattr(sh, "consolidate_satpass_pending", _spy) - - consumer = _make_consumer(bus=None) - - async def _main(): - consumer._sweep_pending_satpass(now=now) - # overdue orphan is armed at ~0.5s; give the loop time to fire it. - await asyncio.sleep(1.0) - - _run(_main()) - - assert cid in fired, "sweep did not fire consolidation for the orphaned cid" - # Orphan recovered: consolidation (dry-run) drained its pending rows. - conn = get_db() - remaining = conn.execute( - "SELECT COUNT(*) AS n FROM satpass_pending WHERE consolidated_id=?", - (cid,)).fetchone()["n"] - assert remaining == 0 - - -# ── startup sweep: future row scheduled, not fired now ─────────────── - -def test_sweep_schedules_future_row_without_firing(monkeypatch): - """A pending row with due_at in the FUTURE is armed but does not fire yet.""" - _enable_satpass_db(norad_ids=[25544], dry_run=True) - now = int(time.time()) - cid = "25544:FUTURE" - _insert_pending(cid, due_at=now + 3600, received_at=now) - - fired = [] - import meshai.central.satpass_handler as sh - monkeypatch.setattr(sh, "consolidate_satpass_pending", - lambda c: fired.append(c)) - - consumer = _make_consumer(bus=None) - - async def _main(): - consumer._sweep_pending_satpass(now=now) - await asyncio.sleep(0.3) - - _run(_main()) - - assert cid not in fired, "future row fired immediately" - assert cid in consumer._pending_satpass_timers, "future row was not armed" - # Pending row untouched (still awaiting its future fire). - conn = get_db() - remaining = conn.execute( - "SELECT COUNT(*) AS n FROM satpass_pending WHERE consolidated_id=?", - (cid,)).fetchone()["n"] - assert remaining == 1 - - -# ── sweep does not double-schedule an already-armed cid ────────────── - -def test_sweep_does_not_double_schedule(monkeypatch): - """A cid already armed by the live path is skipped by the sweep.""" - _enable_satpass_db(norad_ids=[25544], dry_run=True) - now = int(time.time()) - cid = "25544:ARMED" - _insert_pending(cid, due_at=now - 10, received_at=now - 15) - - consumer = _make_consumer(bus=None) - - fired = [] - import meshai.central.satpass_handler as sh - monkeypatch.setattr(sh, "consolidate_satpass_pending", - lambda c: fired.append(c)) - - async def _main(): - sentinel = object() - consumer._pending_satpass_timers[cid] = sentinel # live path owns it - consumer._sweep_pending_satpass(now=now) - # The sweep must not have replaced the live handle. - assert consumer._pending_satpass_timers[cid] is sentinel - await asyncio.sleep(0.1) - - _run(_main()) - assert cid not in fired, "sweep double-scheduled an already-armed cid" diff --git a/work/tests/test_satpass_registration.py b/work/tests/test_satpass_registration.py index e652ed6..1d3464d 100644 --- a/work/tests/test_satpass_registration.py +++ b/work/tests/test_satpass_registration.py @@ -43,28 +43,6 @@ def test_environmental_satpass_default_central(): assert env.satpass.feed_source == "central" -# -- _subject_owned() integration --------------------------------------------- - -def test_subject_owned_includes_satpass_subjects(): - """When EnvironmentalConfig has satpass with feed_source='central', - _subject_owned() must return subjects containing 'central.sat.'.""" - from meshai.config import EnvironmentalConfig - from meshai.central.consumer import _SUBJECTS_BARE - - env = EnvironmentalConfig() - - # Simulate what _subject_owned does for the satpass attr - cfg = getattr(env, "satpass", None) - assert cfg is not None, "satpass attr missing from EnvironmentalConfig" - assert getattr(cfg, "feed_source", "native") == "central" - - # Verify _SUBJECTS_BARE has satpass entry - assert "satpass" in _SUBJECTS_BARE, "satpass missing from _SUBJECTS_BARE" - subjects = _SUBJECTS_BARE["satpass"] - assert any("central.sat.pass" in s for s in subjects) - assert any("central.sat.tle" in s for s in subjects) - - # -- adapter_config REGISTRY -------------------------------------------------- def test_registry_has_satpass_enabled(): @@ -116,32 +94,3 @@ def test_yaml_parsing_satpass(): assert env.satpass.enabled is True assert env.satpass.feed_source == "central" - - -# -- _subjects_for() region rewrite table ------------------------------------ - -def test_subjects_for_satpass_with_region(): - """_subjects_for('satpass', 'us.id') must return region-scoped pass - subjects and global TLE subject.""" - from meshai.central.consumer import _subjects_for - result = _subjects_for('satpass', 'us.id') - assert len(result) == 2, f'Expected 2 subjects, got {len(result)}: {result}' - assert result[0] == 'central.sat.pass.us.id.>' - assert result[1] == 'central.sat.tle.>' - - -def test_subjects_for_satpass_no_region_falls_back(): - """_subjects_for('satpass', None) must return bare-wildcard forms - from _SUBJECTS_BARE.""" - from meshai.central.consumer import _subjects_for - result = _subjects_for('satpass', None) - assert len(result) == 2, f'Expected 2 subjects, got {len(result)}: {result}' - assert result[0] == 'central.sat.pass.>' - assert result[1] == 'central.sat.tle.>' - - -def test_subjects_for_satpass_empty_region_falls_back(): - """_subjects_for('satpass', '') must behave like None — bare wildcards.""" - from meshai.central.consumer import _subjects_for - result = _subjects_for('satpass', '') - assert result == _subjects_for('satpass', None) diff --git a/work/tests/test_satpass_wire_fields.py b/work/tests/test_satpass_wire_fields.py index 9467fcb..5bc5fd0 100644 --- a/work/tests/test_satpass_wire_fields.py +++ b/work/tests/test_satpass_wire_fields.py @@ -260,30 +260,3 @@ def test_observer_fallback_to_slug(): ).fetchone() assert row is not None assert row["observer"] == "filer" - - - -# ── Consumer category mapping ────────────────────────────────────── - -def test_category_map_pass_prefix(): - """pass.n2yo_visualpasses must map to sat_pass, not other.""" - from meshai.central.consumer import map_category - assert map_category("pass.n2yo_visualpasses") == "sat_pass" - - -def test_category_map_pass_satpass_predict(): - """pass.satpass_predict must map to sat_pass.""" - from meshai.central.consumer import map_category - assert map_category("pass.satpass_predict") == "sat_pass" - - -def test_category_map_sat_prefix_still_works(): - """sat.pass must still map to sat_pass (backward compat).""" - from meshai.central.consumer import map_category - assert map_category("sat.pass") == "sat_pass" - - -def test_subject_domain_sat_fallback(): - """Subject central.sat.pass.* must map to sat_pass via domain fallback.""" - from meshai.central.consumer import category_from_subject - assert category_from_subject("central.sat.pass.us.id.filer") == "sat_pass" diff --git a/work/tests/test_seismic_v057.py b/work/tests/test_seismic_v057.py deleted file mode 100644 index e6f5acd..0000000 --- a/work/tests/test_seismic_v057.py +++ /dev/null @@ -1,199 +0,0 @@ -"""v0.5.7-seismic: USGS quake NATS pattern + severity clamp + categories audit. - -Covers three things shipped in v0.5.7-seismic: - -1. USGS quake subject pattern -- per Central v0.10.0 guide §usgs_quake the - pattern is `central.quake.event.` (4 tokens, NO region). Pre-v0.5.7 - we shipped `central.quake.event.>.us.id` which is invalid NATS (`>` - mid-subject) AND wouldn't have matched anything Central publishes. -2. Severity clamp -- documents/regression-tests the existing `map_severity` - behavior. The v0.5.7-seismic prompt described a "severity=5 great-quake - IndexError / drop" bug; investigation confirmed that bug does NOT exist: - - map_severity already clamps any int >= 3 to "immediate" - (so severity=5, 99, etc. all map safely). - - NotificationToggle.severity_channels is dict-keyed by severity STRING - ({"routine","priority","immediate"}), not int -- IndexError is - structurally impossible from this boundary. - - Per the guide §5b severity vocabulary is documented as 0-4 only; - severity=5 is not in Central's contract. The clamp is defensive - padding against contract drift. - These tests pin the clamp so a future regression doesn't introduce the - bug Matt was guarding against. -3. ALERT_CATEGORIES seismic-family audit -- earthquake_event was MISSING - from the registry. Native usgs_quake.py emits it and the central path - maps every quake.event. to it via map_category, but the - Advanced Rules editor couldn't select it (it fell through to - get_category's mesh_health default). Added in v0.5.7-seismic. The - hydro entries (stream_flood_warning / stream_high_water under - toggle='seismic' from v0.5.2) are out of scope; this audit only adds - the quake side and verifies hydro toggles are unchanged. -""" - -import inspect -import json -import re - -import pytest - -from meshai.central.consumer import ( - CentralConsumer, - _SUBJECTS_BARE, - _subjects_for, - map_category, - map_severity, -) -from meshai.config import EnvironmentalConfig -from meshai.notifications.categories import ALERT_CATEGORIES -from meshai.notifications.pipeline.bus import EventBus - - -def _assert_legal_nats(subject: str) -> None: - tokens = subject.split(".") - if ">" in tokens: - assert tokens[-1] == ">", f"`>` not at tail in {subject!r}" - assert tokens.count(">") == 1, f"multiple `>` in {subject!r}" - for tok in tokens: - assert tok, f"empty token in {subject!r}" - if tok not in {"*", ">"}: - assert "*" not in tok and ">" not in tok, f"mixed wildcard in token {tok!r}" - - -# ---------- FIX 1: USGS quake subject pattern ----------------------------- - - -def test_usgs_quake_subject_uses_tail_only_wildcard(): - """Per Central v0.10.0 guide §usgs_quake: `central.quake.event.`, - 4 tokens, no region. Tail-only `>` is the legal wildcard form.""" - subs = _subjects_for("usgs_quake", "us.id") - assert subs == ["central.quake.event.>"] - for s in subs: - _assert_legal_nats(s) - - -def test_usgs_quake_subject_has_no_mid_subject_wildcard(): - """Belt-and-braces NATS-syntax check.""" - for s in _subjects_for("usgs_quake", "us.id"): - tokens = s.split(".") - for tok in tokens[:-1]: - assert tok != ">", f"`>` mid-subject in {s!r}" - - -def test_usgs_quake_bare_form_unchanged(): - """Empty region falls back to the broader bare wildcard for backward compat.""" - assert _subjects_for("usgs_quake", "") == ["central.quake.>"] - - -# ---------- FIX 2: severity clamp regression guard ------------------------ - - -@pytest.mark.parametrize("sev,expected", [ - (0, "routine"), - (1, "routine"), - (2, "priority"), - (3, "immediate"), - (4, "immediate"), - # v0.5.7-seismic regression guard: hypothetical "great quake" severity=5 - # (not in the Central v0.10.0 contract, but defensible if it ever appears) - # MUST clamp to "immediate", not raise / not drop. - (5, "immediate"), - (10, "immediate"), - (99, "immediate"), - # Edge cases that previously degraded to "routine". - (None, "routine"), - ("nonsense", "routine"), - (-1, "routine"), -]) -def test_map_severity_handles_full_range(sev, expected): - assert map_severity(sev) == expected - - -def test_severity_5_quake_routes_through_consumer_without_crashing(): - """Inject a synthetic Central quake envelope with severity=5 (out-of- - contract great-quake hypothetical) and verify it normalizes cleanly - into an Event with severity='immediate' -- no IndexError, no drop.""" - rec = [] - bus = EventBus(); bus.subscribe(rec.append) - c = CentralConsumer(EnvironmentalConfig(), bus) - env = {"id": "us8000mc12", "data": { - "id": "us8000mc12", "adapter": "usgs_quake", - "category": "quake.event.great", - "time": "2026-05-19T02:50:39+00:00", - "severity": 5, # the out-of-contract value - "geo": {"centroid": [-148.93, 61.32], "primary_region": "US-AK", "regions": ["US-AK"]}, - "data": {"title": "M 8.2 - 23 km ESE of Anchorage, AK", - "magnitude": 8.2, "depth": 32.0, "magType": "mw", - "alert": "red", "tsunami": 1, "type": "earthquake"}}} - ev = c._handle("central.quake.event.great", json.dumps(env).encode()) - assert ev is not None - assert ev.severity == "immediate" - assert ev.category == "earthquake_event" - assert ev.source == "usgs_quake" - assert len(rec) == 1 - - -def test_severity_channels_is_string_keyed_no_int_indexerror_risk(): - """The shape that would make severity=5 dangerous is an int-indexed - list; ours is a dict keyed by severity STRING. This pins that contract - so a refactor can't quietly introduce the IndexError vector.""" - from meshai.config import NotificationToggle - t = NotificationToggle(name="seismic") - assert isinstance(t.severity_channels, dict) - # dict.get with an unknown key returns None / default, never raises. - assert t.severity_channels.get("any_string", []) == [] - - -# ---------- FIX 3: seismic-family categories audit ------------------------ - - -def test_earthquake_event_in_registry(): - """v0.5.7-seismic: registry now has earthquake_event so the Advanced - Rules editor can target it. Pre-v0.5.7-seismic it was missing entirely - and fell through to the mesh_health default via get_category().""" - assert "earthquake_event" in ALERT_CATEGORIES - assert ALERT_CATEGORIES["earthquake_event"]["toggle"] == "seismic" - - -def test_hydro_entries_still_seismic_toggle(): - """The v0.5.2 USGS-water migration to toggle='seismic' (geohazards - family in the GUI) must survive the v0.5.7-seismic edit. Out of scope - for THIS phase to modify; in scope to verify-unchanged.""" - assert ALERT_CATEGORIES["stream_flood_warning"]["toggle"] == "seismic" - assert ALERT_CATEGORIES["stream_high_water"]["toggle"] == "seismic" - - -def _native_emitted_quake_categories() -> set[str]: - """Walk usgs_quake.py for category= literals routing to toggle=seismic.""" - from meshai.env import usgs_quake as quake_mod - src = inspect.getsource(quake_mod) - emitted = set(re.findall(r'category="([a-z_]+)"', src)) - return {c for c in emitted if c in ALERT_CATEGORIES - and ALERT_CATEGORIES[c].get("toggle") == "seismic"} - - -def _central_path_quake_categories() -> set[str]: - central_inputs = [ - "quake.event.minor", "quake.event.light", "quake.event.moderate", - "quake.event.strong", "quake.event.major", "quake.event.great", - ] - return {map_category(c) for c in central_inputs} - - -def test_alert_categories_quake_complete(): - """Every quake-side category that meshai emits (native or central path) - must have an ALERT_CATEGORIES entry under toggle='seismic'. Hydro - entries are out of scope for this audit but kept as a control.""" - native = _native_emitted_quake_categories() - central = _central_path_quake_categories() - emitted = native | central - # All six tiers should fold to earthquake_event via the central path. - assert emitted == {"earthquake_event"}, f"unexpected quake emit set: {emitted}" - assert "earthquake_event" in ALERT_CATEGORIES - - -def test_seismic_family_required_fields(): - info = ALERT_CATEGORIES["earthquake_event"] - assert info["toggle"] == "seismic" - assert info["name"] - assert info["description"] - assert info["default_severity"] in {"routine", "priority", "immediate"} - assert info["example_message"] diff --git a/work/tests/test_swpc_handler.py b/work/tests/test_swpc_handler.py deleted file mode 100644 index 5ea0d65..0000000 --- a/work/tests/test_swpc_handler.py +++ /dev/null @@ -1,228 +0,0 @@ -"""Tests for v0.5.10 SWPC space-weather handler.""" -import pytest - -from meshai.central.swpc_handler import handle_swpc -from meshai.persistence import close_thread_connection, init_db -from meshai.persistence import db as persistence_db - - -@pytest.fixture -def mem_db(monkeypatch, tmp_path): - db_path = str(tmp_path / "swpc-test.sqlite") - monkeypatch.setenv("MESHAI_DB_PATH", db_path) - persistence_db._initialised.clear() - close_thread_connection() - conn = init_db() - # Clear module-level geomag dedup caches between tests. - # Phase-1: _geomag_recent moved to gating.swpc._geomag_window. - from meshai.central import swpc_handler as _swpc_mod - if hasattr(_swpc_mod, '_geomag_recent'): - _swpc_mod._geomag_recent.clear() - from meshai.notifications.gating import swpc as _swpc_gate - _swpc_gate._geomag_window.clear() - yield conn - close_thread_connection() - persistence_db._initialised.discard(db_path) - - -def _kindex_env(*, kp=3.0, event_id="kp_2026_06_05_15Z"): - return { - "id": event_id, "subject": "central.space.kindex", - "data": { - "id": event_id, "adapter": "swpc_kindex", - "category": "space.kindex", "severity": 0, - "geo": {}, - "data": {"id": event_id, "kp_index": kp, - "time": "2026-06-05T15:00:00Z"}, - }, - } - - -def _protons_env(*, flux=1.0, event_id="p_2026_06_05_15Z"): - return { - "id": event_id, "subject": "central.space.proton_flux", - "data": { - "id": event_id, "adapter": "swpc_protons", - "category": "space.proton_flux", "severity": 0, - "geo": {}, - "data": {"id": event_id, "p10mev": flux, - "time": "2026-06-05T15:00:00Z"}, - }, - } - - -def _alert_env(*, flare_class=None, kp=None, pfu=None, - event_id="alert_001", product_id="ALTPRO"): - d = {"id": event_id, "product_id": product_id, - "time": "2026-06-05T15:00:00Z"} - if flare_class: d["flare_class"] = flare_class - if kp: d["kp_index"] = kp - if pfu: d["p10mev"] = pfu - return { - "id": event_id, "subject": "central.space.alert.xrayflare", - "data": { - "id": event_id, "adapter": "swpc_alerts", - "category": "space.alert", "severity": 1, - "geo": {}, "data": d, - }, - } - - -def _commit(data, t): - cb = data.get("_on_broadcast_committed") - if cb is not None: - cb(float(t)) - - -# ---- geomagnetic storm ---- - - -def test_kp_below_7_skipped(mem_db): - env = _kindex_env(kp=4.0, event_id="kp_low") - wire = handle_swpc(env, env["subject"], data={}, now=1_000_000) - assert wire is None - # Row persisted for trending, not broadcast. - row = mem_db.execute( - "SELECT last_broadcast_at FROM swpc_events WHERE event_id='kp_low'" - ).fetchone() - assert row is not None - assert row["last_broadcast_at"] is None - - -def test_kp7_g3_broadcasts(mem_db): - env = _kindex_env(kp=7.0, event_id="kp_g3") - wire = handle_swpc(env, env["subject"], data={}, now=1_000_000) - assert wire is not None - assert wire.startswith("🧲") - assert "G3" in wire - assert "Kp7" in wire - assert "Geomagnetic Storm" in wire - - -def test_kp9_g5_broadcasts_with_extreme_label(mem_db): - env = _kindex_env(kp=9.0, event_id="kp_g5") - wire = handle_swpc(env, env["subject"], data={}, now=1_000_000) - assert wire is not None - assert "G5" in wire - assert "Kp9" in wire - - -# ---- solar flares ---- - - -def test_m_class_flare_skipped(mem_db): - env = _alert_env(flare_class="M5.5", event_id="m55_flare") - wire = handle_swpc(env, env["subject"], data={}, now=1_000_000) - assert wire is None - - -def test_x1_flare_r3_broadcasts(mem_db): - env = _alert_env(flare_class="X1.2", event_id="x1_flare") - wire = handle_swpc(env, env["subject"], data={}, now=1_000_000) - assert wire is not None - assert wire.startswith("☀️") - assert "R3" in wire - assert "X1.2" in wire - - -def test_x10_flare_r4_broadcasts(mem_db): - env = _alert_env(flare_class="X10", event_id="x10_flare") - wire = handle_swpc(env, env["subject"], data={}, now=1_000_000) - assert wire is not None - assert "R4" in wire or "R5" in wire - - -def test_flare_class_in_product_id(mem_db): - """Some swpc_alerts encode the class in product_id rather than flare_class.""" - env = _alert_env(event_id="prod_id_flare", product_id="X2.1 FLARE EVENT") - wire = handle_swpc(env, env["subject"], data={}, now=1_000_000) - assert wire is not None - assert "R3" in wire - - -# ---- proton events ---- - - -def test_proton_below_threshold_skipped(mem_db): - env = _protons_env(flux=0.5, event_id="p_low") - wire = handle_swpc(env, env["subject"], data={}, now=1_000_000) - assert wire is None - row = mem_db.execute( - "SELECT last_broadcast_at FROM swpc_events WHERE event_id='p_low'" - ).fetchone() - assert row is not None - assert row["last_broadcast_at"] is None - - -def test_proton_s1_threshold_broadcasts(mem_db): - env = _protons_env(flux=15, event_id="p_s1") - wire = handle_swpc(env, env["subject"], data={}, now=1_000_000) - assert wire is not None - assert wire.startswith("☢️") - assert "S1" in wire - - -def test_proton_s2_broadcasts(mem_db): - env = _protons_env(flux=200, event_id="p_s2") - wire = handle_swpc(env, env["subject"], data={}, now=1_000_000) - assert wire is not None - assert "S2" in wire - - -# ---- wire format ---- - - -def test_wire_has_scale_code_and_scalar_tail(mem_db): - env = _kindex_env(kp=7.0, event_id="fmt1") - wire = handle_swpc(env, env["subject"], data={}, now=1_000_000) - # Wire format: "🧲 New: G3 Geomagnetic Storm — Kp7\nHF degraded, ..." - assert "G3" in wire - assert "Kp7" in wire - assert "\n" in wire - - -# ---- per-event dedup ---- - - -def test_per_event_dedup_no_reissue(mem_db): - env = _kindex_env(kp=7.0, event_id="dedup_kp") - data1 = {} - handle_swpc(env, env["subject"], data=data1, now=1_000_000) - _commit(data1, 1_000_001) - # Re-publish with same id and same Kp -- should not re-broadcast. - wire2 = handle_swpc(env, env["subject"], data={}, now=1_000_300) - assert wire2 is None - - -# ---- commit callback ---- - - -def test_commit_callback_updates_last_broadcast(mem_db): - env = _kindex_env(kp=7.0, event_id="cb_swpc") - data = {} - handle_swpc(env, env["subject"], data=data, now=1_000_000) - pre = mem_db.execute( - "SELECT last_broadcast_at FROM swpc_events WHERE event_id='cb_swpc'" - ).fetchone() - assert pre["last_broadcast_at"] is None - _commit(data, 1_000_001) - post = mem_db.execute( - "SELECT last_broadcast_at FROM swpc_events WHERE event_id='cb_swpc'" - ).fetchone() - assert post["last_broadcast_at"] == 1_000_001 - - -# ---- routine readings persist but never broadcast ---- - - -def test_routine_kp_reading_persists_no_broadcast(mem_db): - """Sub-G3 Kp must still be saved for trending queries.""" - env = _kindex_env(kp=4.5, event_id="routine_kp") - wire = handle_swpc(env, env["subject"], data={}, now=1_000_000) - assert wire is None - row = mem_db.execute( - "SELECT event_type, payload_json FROM swpc_events " - "WHERE event_id='routine_kp'").fetchone() - assert row is not None - assert row["event_type"] == "swpc_kindex" - assert "kp_index" in row["payload_json"] diff --git a/work/tests/test_swpc_refactor.py b/work/tests/test_swpc_refactor.py index e59b0a0..0d21f3c 100644 --- a/work/tests/test_swpc_refactor.py +++ b/work/tests/test_swpc_refactor.py @@ -1,9 +1,20 @@ """Phase-1 SWPC refactor tests. -Six test groups: +The Central `swpc_handler` module (`_render()`, `handle_swpc()`) has been +deleted — the native path is the only production path now. Pure old-vs-new +parity assertions, the flare_class-string-parsing tests that only existed +to exercise the deleted handler's Central-only mapping, and the +`solar_radiation_storm`/proton "legacy path" test (which imported the now +also-deleted `swpc_handler` -- proton events have no broadcast path at all +post-excision, native or otherwise; flagged for Matt, not fixed here) have +been removed. Original diffs are preserved in git history. What remains +exercises native code directly (hand-written expected strings are kept as +regression pins on the current wire format). -1. Parity — for a kindex-style fixture and a flare fixture, the new formatter - produces output equivalent to old _render() (noting tier-b severity fix). +Five test groups: + +1. Parity — for a kindex-style fixture and a flare fixture, the formatter + produces the expected wire text (noting tier-b severity fix). 2. Cross-source identity — same Kp from swpc_kindex and swpc_alerts shares the 600s geomag dedup window (committed broadcast suppresses the second). @@ -108,20 +119,13 @@ def _commit(data: dict, t: float) -> None: # ───────────────────────────────────────────────────────────────────────────── class TestFormatterParity: - """formatters/swpc.format() renders equivalent output to swpc_handler._render().""" - - def _render_old(self, event_kind: str, scale_code: str, label: str, - scalar_str: str, *, detail: str = "", time_tag: str = "") -> str: - from meshai.central.swpc_handler import _render - return _render(event_kind, scale_code, label, scalar_str, - is_update=False, detail=detail, time_tag=time_tag) + """formatters/swpc.format() renders the expected wire text from canonical data.""" def test_kindex_g3_parity(self, mem_db): - """Kp=7 (G3) kindex envelope → new formatter ≈ old _render. + """Kp=7 (G3) kindex envelope → formatter output matches hand-written expected. - Tier-b note: the only intentional delta is _severity_override (now - "priority" instead of missing/routine), which does NOT affect the - wire text — parity is exact for the text body. + Tier-b note: _severity_override (now "priority" instead of + missing/routine) does NOT affect the wire text. """ from meshai.notifications.formatters.swpc import format as sfmt @@ -135,29 +139,24 @@ class TestFormatterParity: "issued_at": "2026-07-04T05:00:00Z", } - old_wire = self._render_old( - "geomag", "G3", "strong", "Kp7", - detail="HF degraded, aurora possible", - time_tag="2026-07-04 05:00", + expected = ( + "🧲 New: G3 Geomagnetic Storm — Kp7" + "\nHF degraded, aurora possible" + "\nSWPC · 2026-07-04 05:00" ) with pinned_time(_AT): new_wire = sfmt(_make_fake_event(canonical), now=_AT, budget=140) - # Content must match: same line 1 and line 2. assert "G3" in new_wire, f"scale_code missing from wire: {new_wire!r}" assert "Kp7" in new_wire, f"scalar 'Kp7' missing from wire: {new_wire!r}" assert "Geomagnetic Storm" in new_wire - - # Old wire content also present - assert "G3" in old_wire - assert "Kp7" in old_wire - assert new_wire == old_wire, ( - f"Parity failure for G3/Kp7:\n old: {old_wire!r}\n new: {new_wire!r}" + assert new_wire == expected, ( + f"Wire mismatch for G3/Kp7:\n expected: {expected!r}\n got: {new_wire!r}" ) def test_flare_x1_r3_parity(self, mem_db): - """X1.0 flare (R3) alert → new formatter ≈ old _render. + """X1.0 flare (R3) alert → formatter output matches hand-written expected. Fixture mirrors swpc_last/0003.json (XX0S, X1.0 flare, R3 Strong). """ @@ -172,10 +171,10 @@ class TestFormatterParity: "issued_at": "2026-06-03T11:59:00Z", } - old_wire = self._render_old( - "flare", "R3", "strong", "X1.0", - detail="HF radio fading, GPS may glitch", - time_tag="2026-06-03 11:59", + expected = ( + "☀️ New: X1.0 Solar Flare — R3" + "\nHF radio fading, GPS may glitch" + "\nSWPC · 2026-06-03 11:59" ) with pinned_time(_AT): @@ -184,8 +183,8 @@ class TestFormatterParity: assert "R3" in new_wire assert "X1.0" in new_wire assert "Solar Flare" in new_wire - assert new_wire == old_wire, ( - f"Parity failure for X1.0/R3:\n old: {old_wire!r}\n new: {new_wire!r}" + assert new_wire == expected, ( + f"Wire mismatch for X1.0/R3:\n expected: {expected!r}\n got: {new_wire!r}" ) def test_g5_kp9_parity(self, mem_db): @@ -201,10 +200,10 @@ class TestFormatterParity: "issued_at": "2026-07-04T08:00:00Z", } - old_wire = self._render_old( - "geomag", "G5", "extreme", "Kp9", - detail="Widespread power disruptions possible", - time_tag="2026-07-04 08:00", + expected = ( + "🧲 New: G5 Geomagnetic Storm — Kp9" + "\nWidespread power disruptions possible" + "\nSWPC · 2026-07-04 08:00" ) with pinned_time(_AT): @@ -212,8 +211,8 @@ class TestFormatterParity: assert "G5" in new_wire assert "Kp9" in new_wire - assert new_wire == old_wire, ( - f"G5 parity failure:\n old: {old_wire!r}\n new: {new_wire!r}" + assert new_wire == expected, ( + f"G5 wire mismatch:\n expected: {expected!r}\n got: {new_wire!r}" ) def test_null_scalar_renders_without_dash_tail(self, mem_db): @@ -444,48 +443,6 @@ class TestFlareRScaleFloor: assert gate.broadcast, "R5 must broadcast" assert gate.data_patch.get("_severity_override") == "immediate" - def test_m5_flare_suppressed_via_handler(self, mem_db): - """M5.5 flare maps to R2 via old path → new arch suppresses at R2 floor.""" - from meshai.central.swpc_handler import handle_swpc - - env = { - "id": "m55_new_arch", - "subject": "central.space.alert.m55", - "data": { - "id": "m55_new_arch", - "adapter": "swpc_alerts", - "category": "space.alert", - "severity": 0, - "geo": {}, - "data": {"id": "m55_new_arch", "flare_class": "M5.5", - "time": "2026-07-04T06:00:00Z"}, - }, - } - wire = handle_swpc(env, env["subject"], data={}, now=int(_AT)) - assert wire is None, "M5.5 (R2) must be suppressed" - - def test_x1_flare_broadcasts_via_handler(self, mem_db): - """X1.0 flare maps to R3 → broadcasts via new arch.""" - from meshai.central.swpc_handler import handle_swpc - - env = { - "id": "x10_new_arch", - "subject": "central.space.alert.x10", - "data": { - "id": "x10_new_arch", - "adapter": "swpc_alerts", - "category": "space.alert", - "severity": 0, - "geo": {}, - "data": {"id": "x10_new_arch", "flare_class": "X1.0", - "time": "2026-07-04T06:00:00Z"}, - }, - } - wire = handle_swpc(env, env["subject"], data={}, now=int(_AT)) - assert wire is not None, "X1.0 (R3) must broadcast" - assert "R3" in wire - assert "X1.0" in wire - # ───────────────────────────────────────────────────────────────────────────── # 5. Schema conformance — to_event() emits required canonical fields @@ -646,52 +603,3 @@ class TestProtonNotRegistered: assert "rf_propagation_alert" in DECIDERS, ( "rf_propagation_alert must be in DECIDERS" ) - - def test_proton_stays_on_legacy_path(self): - """Proton events (S1+) still broadcast via legacy path in swpc_handler. - - Uses swpc_protons adapter with 15 pfu (S1 threshold). The legacy path - must still work — no regression from the new arch changes. - """ - import pytest - pytest.importorskip("meshai.central.swpc_handler") - - # This test needs a DB fixture — create one inline - import tempfile, os - from meshai.persistence import close_thread_connection, init_db - from meshai.persistence import db as persistence_db - - with tempfile.TemporaryDirectory() as tmp: - db_path = os.path.join(tmp, "proton-test.sqlite") - old_env = os.environ.get("MESHAI_DB_PATH") - os.environ["MESHAI_DB_PATH"] = db_path - persistence_db._initialised.clear() - close_thread_connection() - try: - init_db() - from meshai.central.swpc_handler import handle_swpc - - env = { - "id": "p_s1_legacy", - "subject": "central.space.proton_flux", - "data": { - "id": "p_s1_legacy", - "adapter": "swpc_protons", - "category": "space.proton_flux", - "severity": 0, - "geo": {}, - "data": {"id": "p_s1_legacy", "p10mev": 15.0, - "time": "2026-07-04T06:00:00Z"}, - }, - } - wire = handle_swpc(env, env["subject"], data={}, now=int(_AT)) - assert wire is not None, "S1 proton must still broadcast via legacy path" - assert "S1" in wire - assert "☢️" in wire - finally: - close_thread_connection() - persistence_db._initialised.discard(db_path) - if old_env is None: - os.environ.pop("MESHAI_DB_PATH", None) - else: - os.environ["MESHAI_DB_PATH"] = old_env diff --git a/work/tests/test_tail_followups.py b/work/tests/test_tail_followups.py index 86f64bd..974bd71 100644 --- a/work/tests/test_tail_followups.py +++ b/work/tests/test_tail_followups.py @@ -334,14 +334,3 @@ def test_reminder_fires_when_fire_not_tombstoned(): import asyncio fired = asyncio.run(sch.tick_once()) assert fired == 1 - - -# ============================================================================ -# Item 5 -- dead-code removal -# ============================================================================ - - -def test_incident_broadcast_heartbeat_constant_gone(): - """The dead constant is not importable anymore.""" - from meshai.central import incident_handler - assert not hasattr(incident_handler, "INCIDENT_BROADCAST_HEARTBEAT_S") diff --git a/work/tests/test_tracking_v057.py b/work/tests/test_tracking_v057.py deleted file mode 100644 index 148a450..0000000 --- a/work/tests/test_tracking_v057.py +++ /dev/null @@ -1,189 +0,0 @@ -"""v0.5.7-tracking: Central tracking adapter check + categories audit. - -The tracking family is a PLACEHOLDER for Phase 7 (per -meshai/notifications/categories.py:19 header comment: "tracking - ADS-B, -AIS, satellite passes (Phase 7)"). As of v0.5.7-tracking it has: - - - "tracking" in VALID_TOGGLES (reserved toggle name) - - dashboard-frontend/src/pages/Environment.tsx FAMILIES list entry with - label="Tracking", icon=Satellite, adapters=['satpass'] (a pre-existing - UI-only grouping of the native satpass adapter under the "Tracking" - display section; satpass has its own "satpass" backend toggle and is - NOT itself a Phase-7 tracking-family adapter -- the guard below allows - only this one known entry and fails on anything else) - - ZERO native adapter files in meshai/env/ - - ZERO ALERT_CATEGORIES entries with toggle="tracking" - - ZERO Central wires (no central.tracking.* / central.aprs.* / etc.; - no entries in _SUBJECTS_BARE; no entries in CENTRAL_ADAPTER_TO_SOURCE) - -This file pins all of those invariants as regression guards. The intent -is that a future Phase 7 commit that flips any of these (e.g. adds an -APRS adapter, wires a Central tracking subject) will fail these tests -and FORCE the implementer to come back and complete the family-audit -shape (registry entries with required fields, composer emoji/labels, -test file refresh) the same way every other family in v0.5.7 was done. - -Central v0.10.0 cross-check ---------------------------- -The Central v0.10.0 guide (docs/CONSUMER-INTEGRATION.md at v0.10.0-itd-511) -documents 22 per-adapter sections covering wx / fire / quake / space / -disaster / traffic / hydro -- NONE are tracking-related. The producer -source tree src/central/adapters/ contains 24 adapter files; NONE are -named for any tracking concept (aprs / adsb / opensky / satellite / -position). Subject prefixes used: central.{disaster,fire,fires,hydro, -meta,models,quake,space,traffic,traffic_cameras,traffic_flow,wx}.> -- -no central.tracking.* / central.aprs.*. - -Same shape as v0.5.7-avalanche: no Central counterpart, native-only -(currently native-empty). -""" - -import os -from pathlib import Path - -import pytest - -from meshai.central.consumer import ( - CENTRAL_ADAPTER_TO_SOURCE, - CentralConsumer, - _SUBJECTS_BARE, - _subjects_for, -) -from meshai.config import EnvironmentalConfig -from meshai.notifications.categories import ALERT_CATEGORIES, VALID_TOGGLES - - -# ---------- FIX 1: Central has no tracking adapter ----------------------- - - -def test_central_has_no_tracking_subject_prefix(): - """No Central stream/subject namespace uses a tracking-style prefix.""" - for adapter, subs in _SUBJECTS_BARE.items(): - for s in subs: - assert "tracking" not in s.lower(), \ - f"unexpected tracking subject for adapter {adapter}: {s!r}" - for needle in ("aprs", "adsb", "opensky", "ads_b"): - assert needle not in s.lower(), \ - f"unexpected {needle!r} subject for adapter {adapter}: {s!r}" - - -def test_central_adapter_remap_has_no_tracking_entries(): - """No Central adapter name remaps to a tracking source on either side.""" - for src_name, mesh_name in CENTRAL_ADAPTER_TO_SOURCE.items(): - for needle in ("tracking", "aprs", "adsb", "opensky"): - assert needle not in src_name.lower(), \ - f"unexpected Central adapter name: {src_name}" - assert needle not in mesh_name.lower(), \ - f"unexpected meshai source name: {mesh_name}" - - -def test_tracking_source_is_unknown_to_subjects_for(): - """Asking the consumer for tracking-source subjects returns empty for - every region -- the source isn't in the table at all.""" - for region in ("us.id", "us.mt", "", None): - assert _subjects_for("tracking", region) == [], \ - f"_subjects_for('tracking', {region!r}) should be []" - assert _subjects_for("aprs", region) == [] - assert _subjects_for("adsb", region) == [] - - -# ---------- meshai-side placeholder invariants --------------------------- - - -def test_tracking_toggle_is_reserved_in_valid_toggles(): - """The toggle name 'tracking' is reserved (placeholder for Phase 7) - even though no categories use it yet.""" - assert "tracking" in VALID_TOGGLES - - -def test_alert_categories_has_zero_tracking_entries(): - """v0.5.7-tracking placeholder check: registry has no toggle='tracking' - entries. If Phase 7 lands, this test should be updated alongside the - new entries -- not silently deleted.""" - tracking_entries = [ - cid for cid, info in ALERT_CATEGORIES.items() - if info.get("toggle") == "tracking" - ] - assert tracking_entries == [], \ - f"unexpected tracking-family entries: {tracking_entries}" - - -def test_no_native_tracking_adapter_files(): - """meshai/env/ has no tracking-related adapter files. Phase 7 will - add at least one (likely aprs.py / adsb.py / opensky.py); when it - does, this test should be updated to point at the new adapter.""" - env_dir = Path("meshai/env") - if not env_dir.is_dir(): - pytest.skip("meshai/env not present in this working tree") - files = {p.name for p in env_dir.iterdir() if p.suffix == ".py"} - for needle in ("aprs", "adsb", "ads_b", "opensky", "tracking", "satellite"): - for fname in files: - assert needle not in fname.lower(), \ - f"unexpected env adapter file {fname!r} hints at a tracking adapter" - - -# ---------- frontend placeholder invariant ------------------------------- - - -def test_environment_tsx_tracking_family_has_only_the_satpass_preview(): - """Environment.tsx FAMILIES entry for 'tracking' must have adapters - limited to the pre-existing ['satpass'] preview grouping -- it must - NOT gain any actual Phase-7 tracking adapter (aprs/adsb/opensky/etc). - - This guard originally required adapters=[] outright, but Environment.tsx - has grouped the native satpass (SGP4) adapter under the "Tracking" UI - section (icon: Satellite) since before this test's earliest visible - history -- satellite-pass tracking is thematically "tracking" for - display purposes, even though satpass has always had its OWN dedicated - backend registry toggle ("satpass", see - meshai/notifications/categories.py) rather than "tracking". All 7 other - guards in this file (zero Central subjects, zero ALERT_CATEGORIES - entries, zero native adapter files, etc.) confirm the backend-side - Phase-7 tracking family genuinely has not landed; only this test's - stricter-than-reality assumption about the frontend grouping was wrong. - If Phase 7 lands for real, update this test together with the new - ALERT_CATEGORIES entries + adapter files + composer glyphs. - """ - tsx = Path("dashboard-frontend/src/pages/Environment.tsx") - if not tsx.is_file(): - pytest.skip("Environment.tsx not present in this working tree") - text = tsx.read_text() - # Look for the FAMILIES line for tracking; the adapter list must be - # limited to the known satpass preview entry. - assert "key: 'tracking'" in text or 'key: "tracking"' in text, \ - "Environment.tsx FAMILIES is missing the tracking placeholder entry" - # Pattern: `key: 'tracking', label: 'Tracking', icon: Satellite, adapters: [...]` - # Accept any quote style + minor formatting tolerance. - import re - m = re.search( - r"""key:\s*['"]tracking['"]\s*,\s*""" - r"""label:\s*['"]Tracking['"]\s*,\s*""" - r"""icon:\s*\w+\s*,\s*""" - r"""adapters:\s*\[([^\]]*)\]""", - text, re.DOTALL, - ) - assert m, ( - "Environment.tsx FAMILIES tracking entry not found in the expected shape" - ) - adapters = {a.strip().strip("'\"") for a in m.group(1).split(",") if a.strip()} - assert adapters == {"satpass"}, ( - f"Environment.tsx tracking-family adapter list is {sorted(adapters)!r}, " - "expected only the pre-existing {'satpass'} preview entry. " - "If you're landing Phase 7, update this test together with the " - "new ALERT_CATEGORIES entries + adapter files + composer glyphs." - ) - - -# ---------- safety: no orphan routing into tracking ---------------------- - - -def test_no_prefix_fallback_routes_to_tracking(): - """The _TOGGLE_PREFIX_FALLBACK chain in categories.py has no rule that - silently routes unknown categories to toggle='tracking'. Adding such - a rule without paired registry entries would create orphan tracking - events -- the v0.5.7 audit purity rule (every emitted = selectable) - forbids this.""" - from meshai.notifications.categories import _TOGGLE_PREFIX_FALLBACK - for prefix, toggle in _TOGGLE_PREFIX_FALLBACK: - assert toggle != "tracking", \ - f"prefix-fallback {prefix!r} -> tracking without registry entries" diff --git a/work/tests/test_traffic_v057.py b/work/tests/test_traffic_v057.py deleted file mode 100644 index ae1e1d3..0000000 --- a/work/tests/test_traffic_v057.py +++ /dev/null @@ -1,177 +0,0 @@ -"""v0.5.7-traffic: NATS pattern fix + itd_511 sub-adapter routing + categories audit. - -Covers four things shipped in v0.5.7-traffic: - -1. NATS pattern syntax — `>` is legal only at the tail. Pre-v0.5.7-traffic - we shipped `central.traffic.>.` (mid-subject `>`), invalid per - NATS rules. Now: `central.traffic.*.` (Convention B, bare state) - for traffic; roads511 dual-subscribes both Convention B and - `central.traffic.*.us.` (Convention A, itd_511 form). -2. roads511 dual subscription — owns both shared bare-state and us. - subjects so itd_511 events route to the roads511 source in meshai. -3. CENTRAL_ADAPTER_TO_SOURCE['itd_511'] == 'roads511'. -4. ALERT_CATEGORIES roads-family parity — every category we can emit - (native + central path post-map_category) has a registry entry. -""" - -import inspect - -import pytest - -from meshai.central.consumer import ( - CENTRAL_ADAPTER_TO_SOURCE, - _SUBJECTS_BARE, - _subjects_for, - map_category, -) -from meshai.notifications.categories import ALERT_CATEGORIES - - -# ---------- NATS pattern validation (Convention A / B) --------------------- - - -def _assert_legal_nats(subject: str) -> None: - """Assert NATS multi-level wildcard `>` only appears at the tail token.""" - tokens = subject.split(".") - if ">" in tokens: - assert tokens[-1] == ">", f"`>` not at tail in {subject!r}" - assert tokens.count(">") == 1, f"multiple `>` in {subject!r}" - for tok in tokens: - # `*` and `>` are wildcards; everything else must be a non-empty - # token without further wildcard characters mixed in. - assert tok, f"empty token in {subject!r}" - if tok not in {"*", ">"}: - assert "*" not in tok and ">" not in tok, f"mixed wildcard in token {tok!r}" - - -def test_subjects_for_traffic_uses_convention_b(): - """traffic adapter -> bare-state Convention B; no `>` anywhere.""" - subs = _subjects_for("traffic", "us.id") - assert subs == ["central.traffic.*.id"] - for s in subs: - _assert_legal_nats(s) - assert ">" not in s, f"`>` in {s!r}" - - -def test_subjects_for_roads511_dual_subscribes(): - """roads511 owns bare-state (shared with traffic) AND us. (itd_511).""" - subs = _subjects_for("roads511", "us.id") - assert subs == ["central.traffic.*.id", "central.traffic.*.us.id"] - for s in subs: - _assert_legal_nats(s) - assert ">" not in s, f"`>` in {s!r}" - - -def test_traffic_and_roads511_share_convention_b_subject(): - """The bare-state subject is shared so sub-adapter routing kicks in.""" - traffic_subs = set(_subjects_for("traffic", "us.id")) - roads511_subs = set(_subjects_for("roads511", "us.id")) - shared = traffic_subs & roads511_subs - assert shared == {"central.traffic.*.id"} - - -def test_no_invalid_mid_subject_wildcards_in_traffic_family(): - """Sanity sweep, scoped to this phase: traffic + roads511 region-aware - subjects are NATS-legal (no `>` mid-subject). Other adapters (firms, - usgs, usgs_quake, fires, nws) carry the v0.5.4 mid-`>` patterns and - are intentionally OUT OF SCOPE for v0.5.7-traffic -- they'll be fixed - per-family later in the v0.5.7 campaign.""" - for adapter in ("traffic", "roads511"): - for s in _subjects_for(adapter, "us.id"): - _assert_legal_nats(s) - assert ">" not in s, f"`>` still present in {adapter} subject {s!r}" - - -def test_bare_form_unchanged_when_region_empty(): - """Empty region returns _SUBJECTS_BARE for backward compat.""" - assert _subjects_for("traffic", "") == ["central.traffic.>"] - assert _subjects_for("roads511", None) == ["central.traffic.>"] - - -# ---------- itd_511 -> roads511 remap -------------------------------------- - - -def test_itd_511_remaps_to_roads511(): - assert CENTRAL_ADAPTER_TO_SOURCE.get("itd_511") == "roads511" - - -def test_state_511_atis_still_remaps_to_roads511(): - """v0.5.3 mapping must survive the v0.5.7-traffic edit.""" - assert CENTRAL_ADAPTER_TO_SOURCE.get("state_511_atis") == "roads511" - - -# ---------- map_category preserves event_type distinctions ----------------- - - -@pytest.mark.parametrize("central_cat,expected", [ - ("work_zone.wzdx", "work_zone"), - ("work_zone", "work_zone"), - ("incident.tomtom_incidents", "road_incident"), - ("incident", "road_incident"), - ("closure.itd_511", "road_closure"), - ("closure", "road_closure"), - # The catchall still flattens unknown traffic.* shapes. - ("traffic.unknown_thing", "traffic_congestion"), -]) -def test_map_category_traffic_event_types(central_cat, expected): - assert map_category(central_cat) == expected - - -# ---------- ALERT_CATEGORIES roads-family parity --------------------------- - - -def _native_emitted_roads_categories() -> set[str]: - """Walk traffic.py and roads511.py for category= literals.""" - import re - from meshai.env import traffic as traffic_mod - from meshai.env import roads511 as roads511_mod - emitted: set[str] = set() - for mod in (traffic_mod, roads511_mod): - src = inspect.getsource(mod) - emitted |= set(re.findall(r'category="([a-z_]+)"', src)) - return emitted - - -def _central_path_roads_categories() -> set[str]: - """Categories the central path can deliver into the roads family. - - Drives off map_category() so the test breaks if the routing changes. - """ - central_inputs = [ - "work_zone.wzdx", - "incident.tomtom_incidents", - "closure.itd_511", - "closure", - "incident", - "traffic.flow_slow", - ] - return {map_category(c) for c in central_inputs} - - -def test_alert_categories_roads_complete(): - """Every category emitted by native traffic/roads511 OR delivered via - the central path (post-map_category) must have an ALERT_CATEGORIES - entry with toggle='roads'. No orphans. - """ - registry_roads = { - cid for cid, info in ALERT_CATEGORIES.items() - if info.get("toggle") == "roads" - } - emitted = _native_emitted_roads_categories() | _central_path_roads_categories() - missing = emitted - registry_roads - orphans = registry_roads - emitted - assert not missing, f"emit set has roads categories missing from ALERT_CATEGORIES: {missing}" - assert not orphans, f"ALERT_CATEGORIES has orphan roads entries: {orphans}" - - -@pytest.mark.parametrize( - "cat", - ["road_closure", "traffic_congestion", "work_zone", "road_incident"], -) -def test_roads_categories_have_required_fields(cat): - info = ALERT_CATEGORIES[cat] - assert info["toggle"] == "roads" - assert info["name"] - assert info["description"] - assert info["default_severity"] in {"routine", "priority", "immediate"} - assert info["example_message"] diff --git a/work/tests/test_water_v057.py b/work/tests/test_water_v057.py deleted file mode 100644 index 899e255..0000000 --- a/work/tests/test_water_v057.py +++ /dev/null @@ -1,197 +0,0 @@ -"""v0.5.7-water: USGS NWIS hydro NATS pattern + water/hydro categories audit. - -Covers two things shipped in v0.5.7-water: - -1. USGS NWIS hydro subject pattern -- per Central v0.10.0-itd-511 nwis.py - producer subject_for() body, the actual published subject is - `central.hydro....` where is - `us.` (7 tokens) or `unknown` (6 tokens). The pre-v0.5.7-water - `central.hydro.>.us.id` was invalid NATS (`>` mid-subject) -- replaced - with three single-token `*` wildcards in the param/agency/site slots - plus the bare region tail. - - Note on guide vs code: the Central guide §nwis text shows only the - 4-token category-shape stem `central.hydro... - ` without the regional suffix. That doc text is stale - w.r.t. the producer code. The producer code is the ground truth (it's - what NATS actually delivers); we follow the code. - -2. ALERT_CATEGORIES water/hydro audit -- pre-v0.5.7-water the registry had - `stream_flood_warning` and `stream_high_water` (both toggle=seismic from - the v0.5.2 geohazards migration). The central path's - `("hydro.", "stream_flow")` _CATEGORY_MAP entry produced a category - `stream_flow` that had no registry entry -- the rule editor couldn't - target it. Added `stream_flow` (toggle=seismic) so central-delivered - raw gauge readings are UI-selectable. The native usgs.py threshold- - classified categories are unchanged. -""" - -import inspect -import re - -import pytest - -from meshai.central.consumer import ( - _SUBJECTS_BARE, - _subjects_for, - map_category, - map_severity, -) -from meshai.notifications.categories import ALERT_CATEGORIES - - -def _assert_legal_nats(subject: str) -> None: - tokens = subject.split(".") - if ">" in tokens: - assert tokens[-1] == ">", f"`>` not at tail in {subject!r}" - assert tokens.count(">") == 1, f"multiple `>` in {subject!r}" - for tok in tokens: - assert tok, f"empty token in {subject!r}" - if tok not in {"*", ">"}: - assert "*" not in tok and ">" not in tok, f"mixed wildcard in token {tok!r}" - - -# ---------- FIX 1: USGS NWIS hydro subject pattern ------------------------ - - -def test_usgs_subjects_are_nats_legal(): - """No `>` mid-subject; all wildcards are single-token `*`.""" - subs = _subjects_for("usgs", "us.id") - assert subs == [ - "central.hydro.*.*.*.us.id", - "central.hydro.*.*.*.unknown", - ] - for s in subs: - _assert_legal_nats(s) - # Per-state filter has 7 tokens; .unknown has 6. - assert ">" not in s, f"`>` should not appear in fixed-token form: {s!r}" - - -def test_usgs_subjects_match_producer_published_shape(): - """Sanity: the subscription patterns match what nwis.py actually - publishes. Producer publishes: - central.hydro.... - where is us. (2 tokens) or unknown (1 token). - """ - sub_state, sub_unknown = _subjects_for("usgs", "us.id") - # Per-state form: matches a 7-token published subject. - sample_published_state = "central.hydro.00060.usgs.06898000.us.id" - sample_published_unknown = "central.hydro.00060.usgs.06898000.unknown" - # Token-count check (NATS `*` matches exactly one token). - assert len(sub_state.split(".")) == len(sample_published_state.split(".")) - assert len(sub_unknown.split(".")) == len(sample_published_unknown.split(".")) - # Per-state must end with the requested region, .unknown with literal. - assert sub_state.endswith(".us.id") - assert sub_unknown.endswith(".unknown") - - -def test_usgs_bare_form_unchanged(): - """Empty region falls back to the bare wildcard (backward compat).""" - assert _subjects_for("usgs", "") == ["central.hydro.>"] - assert _subjects_for("usgs", None) == ["central.hydro.>"] - - -def test_usgs_per_state_filter_does_not_match_wrong_state(): - """Sanity: a Montana-region subscription wouldn't match an Idaho subject. - (Just verifies the substitution flows through cleanly per region.)""" - mt_subs = _subjects_for("usgs", "us.mt") - assert mt_subs == [ - "central.hydro.*.*.*.us.mt", - "central.hydro.*.*.*.unknown", - ] - - -# ---------- FIX 2: ALERT_CATEGORIES water/hydro audit --------------------- - - -def test_stream_flow_in_registry(): - """v0.5.7-water: central path's `hydro.* -> stream_flow` mapping now has - a corresponding ALERT_CATEGORIES entry under toggle='seismic'.""" - assert "stream_flow" in ALERT_CATEGORIES - assert ALERT_CATEGORIES["stream_flow"]["toggle"] == "seismic" - assert ALERT_CATEGORIES["stream_flow"]["default_severity"] == "routine" - - -def test_existing_hydro_entries_unchanged(): - """v0.5.2 USGS-water -> toggle='seismic' migration must survive.""" - for cat in ("stream_flood_warning", "stream_high_water"): - assert cat in ALERT_CATEGORIES - assert ALERT_CATEGORIES[cat]["toggle"] == "seismic" - - -def _native_emitted_water_categories() -> set[str]: - """Walk usgs.py for category= literals routing to toggle=seismic.""" - from meshai.env import usgs as usgs_mod - src = inspect.getsource(usgs_mod) - emitted = set(re.findall(r'category\s*=\s*"([a-z_]+)"', src)) - return {c for c in emitted if c in ALERT_CATEGORIES - and ALERT_CATEGORIES[c].get("toggle") == "seismic"} - - -def _central_path_water_categories() -> set[str]: - """Map a representative set of central hydro category strings through - map_category() to see what meshai categories we'd emit downstream. - Per the guide §nwis, every NWIS event has category - `hydro...`.""" - central_inputs = [ - "hydro.00060.usgs.06898000", # discharge - "hydro.00065.usgs.06898000", # gage height - "hydro.00010.usgs.06898000", # water temperature - "hydro.00060.mo005.0000123", # cooperator agency - ] - return {map_category(c) for c in central_inputs} - - -def test_alert_categories_water_complete(): - """Native + central-path water emit must equal registry's water-side - subset of toggle='seismic'. (The quake-side earthquake_event added in - v0.5.7-seismic is also under toggle='seismic' but emitted by a - different adapter — exclude it from this water-only audit.)""" - registry_water = { - cid for cid, info in ALERT_CATEGORIES.items() - if info.get("toggle") == "seismic" - and (cid.startswith("stream_") or cid == "stream_flow") - } - native = _native_emitted_water_categories() - central = _central_path_water_categories() - emitted = native | central - missing = emitted - registry_water - orphans = registry_water - emitted - assert not missing, f"water emit set missing from ALERT_CATEGORIES: {missing}" - assert not orphans, f"ALERT_CATEGORIES has orphan water entries: {orphans}" - - -def test_native_threshold_categories_still_emitted(): - """Spot-check that usgs.py still has the two threshold-classified - categories (regression guard against accidental removal).""" - native = _native_emitted_water_categories() - assert "stream_flood_warning" in native - assert "stream_high_water" in native - - -def test_central_hydro_pcode_strings_all_map_to_stream_flow(): - """Every realistic central hydro category collapses to stream_flow - via the catchall `("hydro.", "stream_flow")` _CATEGORY_MAP entry.""" - for pcode in ("00060", "00065", "00010", "00045", "00095"): - assert map_category(f"hydro.{pcode}.usgs.12345678") == "stream_flow" - - -@pytest.mark.parametrize( - "cat", ["stream_flow", "stream_flood_warning", "stream_high_water"], -) -def test_water_categories_have_required_fields(cat): - info = ALERT_CATEGORIES[cat] - assert info["toggle"] == "seismic" - assert info["name"] - assert info["description"] - assert info["default_severity"] in {"routine", "priority", "immediate"} - assert info["example_message"] - - -# ---------- Severity sanity for central NWIS events ----------------------- - - -def test_central_nwis_severity_zero_routes_to_routine(): - """Central NWIS publishes severity=0 (no threshold classification). - Confirm that becomes 'routine' in meshai's three-level scale.""" - assert map_severity(0) == "routine"