mirror of
https://github.com/zvx-echo6/meshai.git
synced 2026-06-11 01:14:45 +02:00
feat(v0.5.8b): persistence foundation + WFIGS handler + universal cold-start grace
Three integrated pieces that ship together because they were designed as one safety story: (1) PERSISTENCE FOUNDATION -- new meshai/persistence/ module with SQLite db.py, schema migration framework (v1), 13 tables covering all adapter event shapes (traffic_events, fires, firms_pixels, quake_events, nws_alerts, gauge_readings, swpc_events) + mesh state (mesh_nodes, mesh_telemetry, mesh_positions, mesh_messages_in, mesh_broadcasts_out, mesh_health_events) + cross-cutting event_log + schema_meta. WAL mode for reader concurrency, single-writer pattern, MESHAI_DB_PATH env var, mounted at /data/meshai.sqlite via existing docker-compose meshai_data volume. .gitignore updated. (2) WFIGS HANDLER -- meshai/central/wfigs_handler.py implements the first per-adapter handler that uses the persistence layer. Format: MEDIUM style with town/landclass/county fallback chain, lat/lon at 3-decimal precision, New:/Update: prefix. 8h-rate-limited change-detection per IRWIN via fires.last_broadcast_at. Skips tombstones and perimeters silently (logged to event_log with handled=0). Acres fallback chain DailyAcres -> IncidentSize -> raw.DiscoveryAcres -> raw.FinalAcres -> N/A. Pass-through Initial Attack auto-numbered names (IA 1, IA 2). (3) UNIVERSAL COLD-START GRACE -- meshai/notifications/pipeline/dispatcher.py grows a configurable grace window (cold_start_grace_seconds, default 60s, GUI-editable per Rule 17). Anchored to first-event-seen (not container boot), so the grace activates the moment broadcasts could fire. Suppresses mesh delivery during the window; handler-side persistence (fires UPSERT, event_log) still happens normally. New _cold_start_dropped counter exposed in dispatch_stats(). Designed to protect against JetStream backlog spam at toggle-flip time, applies universally to ALL adapters. (4) WFIGS HANDLER CALLBACK REFACTOR -- New:/Update: prefix now keys on fires.last_broadcast_at IS NULL (not row-missing), and last_broadcast_* field updates moved to a post-broadcast commit callback that the dispatcher invokes ONLY on successful delivery. This means: cold-start-suppressed events leave fires.last_broadcast_at NULL, so when they eventually broadcast post-grace, they correctly render as New: (first ACTUAL delivery for that IRWIN), not Update:. event_log.handled and mesh_broadcasts_out audit row also gated on the same callback -- decoupling persistence rows from broadcast rows for an honest audit trail. New tests: 15 in test_wfigs_handler.py, 15 in test_persistence.py, additional cold-start grace tests in test_dispatcher.py (+4 WFIGS callback scenarios). Synthetic probes wfigs-cleaned-samples.md (initial) and wfigs-cleaned-samples-v2.md (cold-start verification) generated against isolated temp SQLite databases. CT108 /data/meshai.sqlite untouched during build. Master stays off. No live toggle flips. Test count: was 535 (v0.5.7 baseline) -> 566 (persistence) -> 581 (wfigs handler) -> 589 expected (cold-start grace). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
7751a40c6c
commit
053d67db6e
16 changed files with 2652 additions and 1 deletions
|
|
@ -392,3 +392,284 @@ def test_geocoder_name_is_never_used_as_town_fallback(monkeypatch):
|
|||
n = normalize(env)
|
||||
# Must NOT pick up "Cache Nf Road 444" from geocoder.name.
|
||||
assert n["town"] is None
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# v0.5.8-wzdx federal parser tests
|
||||
# ============================================================================
|
||||
|
||||
# --- representative envelopes (flat shape, as Central actually publishes) ---
|
||||
|
||||
_WZDX_ID_FULL = {
|
||||
"data": {
|
||||
"adapter": "wzdx",
|
||||
"category": "work_zone.wzdx",
|
||||
"time": "2026-06-01T13:00:00Z",
|
||||
"severity": 3,
|
||||
"geo": {"centroid": [-112.408309608311, 43.0208066348276],
|
||||
"primary_region": "US-ID", "regions": ["US-ID"]},
|
||||
"data": {
|
||||
"road_names": ["Exit 80 On Ramp"],
|
||||
"direction": "southbound",
|
||||
"description": " Road construction on Exit 80 On Ramp Southbound near MM (80)."
|
||||
" All lanes closed. 6/1/2026 7:00 AM to 6/10/2026 6:00 PM Mon, Tue ...",
|
||||
"vehicle_impact": "all-lanes-closed",
|
||||
"event_status": None,
|
||||
"start_date": "2026-06-01T13:00:00Z",
|
||||
"end_date": "2026-06-11T00:00:00Z",
|
||||
"data_source_id": "ERS",
|
||||
"feed_name": "iddot",
|
||||
"feed_state_code": "ID",
|
||||
"latitude": 43.0208066348276,
|
||||
"longitude": -112.408309608311,
|
||||
"_enriched": {"geocoder": {"city": None, "name": "Ross Fork Creek",
|
||||
"county": "Bannock", "state": "Idaho"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
_WZDX_WA = {
|
||||
"data": {
|
||||
"adapter": "wzdx",
|
||||
"category": "work_zone.wzdx",
|
||||
"time": "2026-06-01T00:00:00+00:00",
|
||||
"severity": 1,
|
||||
"geo": {"centroid": [-117.33633, 46.433365], "primary_region": "US-WA"},
|
||||
"data": {
|
||||
"road_names": ["012"],
|
||||
"direction": "westbound",
|
||||
"description": "Contract - XE3608 SR 12",
|
||||
"vehicle_impact": "unknown",
|
||||
"event_status": "pending",
|
||||
"start_date": "2026-06-01T00:00:00+00:00",
|
||||
"end_date": "2026-06-05T00:00:00+00:00",
|
||||
"data_source_id": "WSDOT-WZDB",
|
||||
"feed_name": "wsdot",
|
||||
"feed_state_code": "WA",
|
||||
"latitude": 46.433365,
|
||||
"longitude": -117.33633,
|
||||
"_enriched": {"geocoder": {"city": None, "name": "US Highway 12",
|
||||
"county": "Garfield", "state": "Washington"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
_WZDX_MCCALL = {
|
||||
"data": {
|
||||
"adapter": "wzdx", "category": "work_zone.wzdx",
|
||||
"time": "2026-05-28T23:00:00Z", "severity": 1,
|
||||
"geo": {"centroid": [-116.09759, 44.9065083834611], "primary_region": "US-ID"},
|
||||
"data": {
|
||||
"road_names": ["SH-55"],
|
||||
"direction": "unknown",
|
||||
"description": " Emergency repairs on SH-55 Both Directions near Washington St."
|
||||
" 5/28/2026 5:00 PM to 5/29/2026 8:00 AM Thu, Fri: ...",
|
||||
"vehicle_impact": "all-lanes-open",
|
||||
"start_date": "2026-05-28T23:00:00Z",
|
||||
"end_date": "2026-05-29T14:00:00Z",
|
||||
"feed_state_code": "ID",
|
||||
"latitude": 44.9065083834611,
|
||||
"longitude": -116.09759,
|
||||
"_enriched": {"geocoder": {"city": "McCall", "county": "Valley", "state": "ID"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _normalize_wzdx(env):
|
||||
n = normalize(env)
|
||||
assert n is not None
|
||||
assert n["source"] == "wzdx"
|
||||
return n
|
||||
|
||||
|
||||
# --- (a) Idaho wzdx full-field parse ---------------------------------------
|
||||
|
||||
def test_wzdx_idaho_full_fields_normalized(monkeypatch):
|
||||
_clear_h3_cache()
|
||||
from meshai import central_normalizer as cn
|
||||
# Mock Photon for the SECONDARY town path (city is null in this envelope).
|
||||
monkeypatch.setattr(cn, "_photon_reverse_places",
|
||||
lambda lat, lon: [
|
||||
{"geometry": {"coordinates": [-112.4373, 43.0299]},
|
||||
"properties": {"name": "Fort Hall",
|
||||
"osm_key": "place", "osm_value": "village"}},
|
||||
])
|
||||
n = _normalize_wzdx(_WZDX_ID_FULL)
|
||||
assert n["road"] is None # Exit-ramp pattern → uninformative-road drop
|
||||
assert n["direction"] == "southbound"
|
||||
# sub_type combines impact-phrase (suppressed under full-closure) + work_type
|
||||
# (None here — types_of_work absent). With full-closure, sub_type stays None
|
||||
# and the renderer prepends "all lanes closed".
|
||||
assert n["sub_type"] is None
|
||||
assert n["impact"] == "full_closure"
|
||||
assert n["mile_start"] == 80 and n["mile_end"] is None
|
||||
assert n["town"] == "Fort Hall" # via Photon nearest_town
|
||||
assert isinstance(n["ends_at"], datetime)
|
||||
assert n["ends_at"].year == 2026 and n["ends_at"].month == 6 and n["ends_at"].day == 11
|
||||
|
||||
|
||||
def test_wzdx_wa_road_passes_through_verbatim(monkeypatch):
|
||||
_clear_h3_cache()
|
||||
from meshai import central_normalizer as cn
|
||||
monkeypatch.setattr(cn, "_photon_reverse_places", lambda lat, lon: [])
|
||||
n = _normalize_wzdx(_WZDX_WA)
|
||||
# Per spec: "honor upstream verbatim, no expansion" -- raw '012' passes through.
|
||||
assert n["road"] == "012"
|
||||
assert n["direction"] == "westbound"
|
||||
# vehicle_impact='unknown' → impact_phrase=None; sub_type stays None.
|
||||
assert n["sub_type"] is None
|
||||
assert n["impact"] == "partial"
|
||||
# No MM in WA descriptions; mile_start stays None.
|
||||
assert n["mile_start"] is None
|
||||
assert isinstance(n["ends_at"], datetime)
|
||||
assert n["town"] is None # city null + Photon returned no places
|
||||
|
||||
|
||||
# --- (c) vehicle_impact mapping for each main value ------------------------
|
||||
|
||||
@pytest.mark.parametrize("vi_raw,expected_sub_type,expected_impact", [
|
||||
("all-lanes-closed", None, "full_closure"),
|
||||
("some-lanes-closed", "lanes reduced", "partial"),
|
||||
("alternating-one-way", "one-way alternating", "partial"),
|
||||
("unknown", None, "partial"),
|
||||
("all-lanes-open", None, "partial"),
|
||||
("totally-made-up", None, "partial"),
|
||||
])
|
||||
def test_wzdx_vehicle_impact_mapping(vi_raw, expected_sub_type, expected_impact, monkeypatch):
|
||||
_clear_h3_cache()
|
||||
from meshai import central_normalizer as cn
|
||||
monkeypatch.setattr(cn, "_photon_reverse_places", lambda lat, lon: [])
|
||||
env = {"data": {"adapter": "wzdx", "category": "work_zone.wzdx", "time": "2026-06-01T00:00:00Z",
|
||||
"geo": {"centroid": [-116.0, 44.0]},
|
||||
"data": {"road_names": ["SH-1"], "direction": "northbound",
|
||||
"description": "X", "vehicle_impact": vi_raw,
|
||||
"end_date": "2026-06-05T17:00:00Z",
|
||||
"latitude": 44.0, "longitude": -116.0,
|
||||
"_enriched": {"geocoder": {"city": "Boise"}}}}}
|
||||
n = normalize(env)
|
||||
assert n["sub_type"] == expected_sub_type
|
||||
assert n["impact"] == expected_impact
|
||||
|
||||
|
||||
# --- (d) structured end_date parses to friendly format --------------------
|
||||
|
||||
def test_wzdx_end_date_iso_parsed_to_datetime(monkeypatch):
|
||||
_clear_h3_cache()
|
||||
from meshai import central_normalizer as cn
|
||||
monkeypatch.setattr(cn, "_photon_reverse_places", lambda lat, lon: [])
|
||||
env = {"data": {"adapter": "wzdx", "category": "work_zone.wzdx",
|
||||
"geo": {"centroid": [-116.0, 44.0]},
|
||||
"data": {"road_names": ["SH-1"], "direction": "northbound",
|
||||
"description": "x", "vehicle_impact": "unknown",
|
||||
"end_date": "2026-06-15T18:30:00+00:00",
|
||||
"latitude": 44.0, "longitude": -116.0,
|
||||
"_enriched": {"geocoder": {"city": "Boise"}}}}}
|
||||
n = normalize(env)
|
||||
assert isinstance(n["ends_at"], datetime)
|
||||
assert n["ends_at"].month == 6 and n["ends_at"].day == 15
|
||||
assert n["ends_at"].hour in (18, 11, 12) # depending on local-tz coercion
|
||||
|
||||
|
||||
# --- (e) MM regex extraction on ID description ----------------------------
|
||||
|
||||
def test_wzdx_mile_post_regex_from_description(monkeypatch):
|
||||
_clear_h3_cache()
|
||||
from meshai import central_normalizer as cn
|
||||
monkeypatch.setattr(cn, "_photon_reverse_places", lambda lat, lon: [])
|
||||
env = {"data": {"adapter": "wzdx", "category": "work_zone.wzdx",
|
||||
"geo": {"centroid": [-116.0, 44.0]},
|
||||
"data": {"road_names": ["I-15"], "direction": "southbound",
|
||||
"description": "Bridge work on I-15 SB from MM (89) to MM (93). 6/1/2026 7:00 AM to 6/3/2026 5:00 PM",
|
||||
"vehicle_impact": "some-lanes-closed",
|
||||
"end_date": "2026-06-03T22:00:00Z",
|
||||
"latitude": 44.0, "longitude": -116.0,
|
||||
"_enriched": {"geocoder": {"city": "Blackfoot"}}}}}
|
||||
n = normalize(env)
|
||||
assert n["mile_start"] == 89
|
||||
assert n["mile_end"] == 93
|
||||
|
||||
|
||||
# --- (f) WA event without MM yields mile_start=None -----------------------
|
||||
|
||||
def test_wzdx_wa_no_mm_in_description(monkeypatch):
|
||||
_clear_h3_cache()
|
||||
from meshai import central_normalizer as cn
|
||||
monkeypatch.setattr(cn, "_photon_reverse_places", lambda lat, lon: [])
|
||||
n = _normalize_wzdx(_WZDX_WA)
|
||||
assert n["mile_start"] is None
|
||||
assert n["mile_end"] is None
|
||||
|
||||
|
||||
# --- (g) town fallback chain ----------------------------------------------
|
||||
|
||||
def test_wzdx_town_uses_geocoder_city_when_present(monkeypatch):
|
||||
_clear_h3_cache()
|
||||
from meshai import central_normalizer as cn
|
||||
calls = []
|
||||
monkeypatch.setattr(cn, "_photon_reverse_places",
|
||||
lambda lat, lon: calls.append("called") or [])
|
||||
n = _normalize_wzdx(_WZDX_MCCALL)
|
||||
assert n["town"] == "McCall"
|
||||
assert calls == [] # city present → Photon NOT called
|
||||
|
||||
|
||||
def test_wzdx_town_falls_back_to_nearest_town_when_city_null(monkeypatch):
|
||||
_clear_h3_cache()
|
||||
from meshai import central_normalizer as cn
|
||||
monkeypatch.setattr(cn, "_photon_reverse_places",
|
||||
lambda lat, lon: [
|
||||
{"geometry": {"coordinates": [-117.293, 46.475]},
|
||||
"properties": {"name": "Pomeroy",
|
||||
"osm_key": "place", "osm_value": "city"}},
|
||||
])
|
||||
n = _normalize_wzdx(_WZDX_WA)
|
||||
assert n["town"] == "Pomeroy"
|
||||
|
||||
|
||||
# --- adapter dispatch routes wzdx → _parse_wzdx_federal -------------------
|
||||
|
||||
def test_wzdx_adapter_routes_to_wzdx_parser(monkeypatch):
|
||||
_clear_h3_cache()
|
||||
from meshai import central_normalizer as cn
|
||||
monkeypatch.setattr(cn, "_photon_reverse_places", lambda lat, lon: [])
|
||||
n = normalize(_WZDX_WA)
|
||||
assert n is not None
|
||||
assert n["source"] == "wzdx"
|
||||
|
||||
|
||||
# --- work_type from types_of_work or event_type ---------------------------
|
||||
|
||||
def test_wzdx_sub_type_from_types_of_work(monkeypatch):
|
||||
_clear_h3_cache()
|
||||
from meshai import central_normalizer as cn
|
||||
monkeypatch.setattr(cn, "_photon_reverse_places", lambda lat, lon: [])
|
||||
env = {"data": {"adapter": "wzdx", "category": "work_zone.wzdx",
|
||||
"geo": {"centroid": [-116.0, 44.0]},
|
||||
"data": {"road_names": ["SH-1"], "direction": "both",
|
||||
"description": "x",
|
||||
"types_of_work": [{"type_name": "paving"}],
|
||||
"vehicle_impact": "some-lanes-closed",
|
||||
"end_date": "2026-06-05T17:00:00Z",
|
||||
"latitude": 44.0, "longitude": -116.0,
|
||||
"_enriched": {"geocoder": {"city": "Boise"}}}}}
|
||||
n = normalize(env)
|
||||
# Folded form: impact_phrase + work_type (paving)
|
||||
assert n["sub_type"] == "lanes reduced, paving"
|
||||
|
||||
|
||||
def test_wzdx_sub_type_unknown_vocab_is_lowercased_with_spaces(monkeypatch):
|
||||
_clear_h3_cache()
|
||||
from meshai import central_normalizer as cn
|
||||
monkeypatch.setattr(cn, "_photon_reverse_places", lambda lat, lon: [])
|
||||
env = {"data": {"adapter": "wzdx", "category": "work_zone.wzdx",
|
||||
"geo": {"centroid": [-116.0, 44.0]},
|
||||
"data": {"road_names": ["SH-1"], "direction": "northbound",
|
||||
"description": "x",
|
||||
"types_of_work": [{"type_name": "Some-Custom-Work"}],
|
||||
"vehicle_impact": "all-lanes-open",
|
||||
"end_date": "2026-06-05T17:00:00Z",
|
||||
"latitude": 44.0, "longitude": -116.0,
|
||||
"_enriched": {"geocoder": {"city": "Boise"}}}}}
|
||||
n = normalize(env)
|
||||
assert n["sub_type"] == "some custom work" # lowercased + hyphens→spaces
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue