mirror of
https://github.com/zvx-echo6/meshai.git
synced 2026-08-26 17:31:34 +00:00
fix(usgs_quake): stop dropping first-sighted quakes at the freshness gate
The USGS feed is a rolling PAST-DAY feed (2.5_day.geojson), so a genuinely first-seen quake's age-at-ingest routinely exceeds the generic per-toggle freshness_seconds (600s) window, silently dropping it downstream of decide()/DB-insert (last_broadcast_at stayed NULL on every sampled first-sighting row). - dispatcher.py: earthquake_event gets its own adapter_config-backed freshness override (adapter_config.usgs_quake.freshness_seconds, default 3600s), scoped to the CATEGORY (not the "seismic" toggle, which also carries hydro stream_flood_warning/stream_high_water and must keep the generic per-toggle freshness). - usgs_quake.py: to_event() no longer passes the adapter's fixed freshness in, deferring to the dispatcher override. - adapter_config/defaults.py: adds the freshness_seconds field/default for usgs_quake.
This commit is contained in:
parent
50553b26f5
commit
738ffc71c3
6 changed files with 261 additions and 3 deletions
|
|
@ -85,7 +85,8 @@ REGISTRY: dict[tuple[str, str], dict[str, Any]] = {
|
|||
},
|
||||
|
||||
# =================================================================
|
||||
# USGS_QUAKE -- 6 settings (regional geography + 3 mag floors + PAGER set)
|
||||
# USGS_QUAKE -- 7 settings (regional geography + 3 mag floors + PAGER set
|
||||
# + freshness gate)
|
||||
# =================================================================
|
||||
("usgs_quake", "regional_centroid"): {
|
||||
"default": [44.36, -114.61], # quake_handler.py:36-37 (Idaho centroid)
|
||||
|
|
@ -117,6 +118,11 @@ REGISTRY: dict[tuple[str, str], dict[str, Any]] = {
|
|||
"type": "float",
|
||||
"description": "Magnitude floor for the visual escalation emoji.",
|
||||
},
|
||||
("usgs_quake", "freshness_seconds"): {
|
||||
"default": 3600,
|
||||
"type": "int",
|
||||
"description": "Staleness gate for earthquake_event broadcasts (age = now - quake origin time), read by the dispatcher instead of the generic per-toggle freshness. The upstream feed is a rolling PAST-DAY feed, so quakes are routinely detected 10min-20h after origin; 1 hour keeps quakes broadcast-worthy (still situationally relevant) while excluding the day-feed's stale backlog. 0 = disabled (not recommended -- would readmit day-old quakes).",
|
||||
},
|
||||
|
||||
# =================================================================
|
||||
# SWPC -- 3 settings (three storm-tier broadcast floors)
|
||||
|
|
|
|||
9
work/meshai/env/usgs_quake.py
vendored
9
work/meshai/env/usgs_quake.py
vendored
|
|
@ -246,7 +246,14 @@ class USGSQuakeAdapter:
|
|||
expires=evt.get("expires"),
|
||||
lat=lat,
|
||||
lon=lon,
|
||||
region=evt.get("region"),
|
||||
# Deliberately NOT passing region=evt.get("region") here: the
|
||||
# raw dict's region is a fixed config default (self._region,
|
||||
# "magic_valley"), which doesn't match any region_routes cell
|
||||
# key. Event.region is documented as "set by region tagger"
|
||||
# (notifications/events.py) -- leaving it unset lets
|
||||
# CoverageFilter's geometry-based tagging (event.lat/lon vs
|
||||
# the named coverage areas) populate the real region name
|
||||
# from the quake's actual coordinates instead.
|
||||
group_key=event_id,
|
||||
inhibit_keys=[event_id],
|
||||
data=canonical_data,
|
||||
|
|
|
|||
|
|
@ -359,6 +359,18 @@ class Dispatcher:
|
|||
# v0.6-3b: fire toggle uses wfigs adapter_config freshness (0 = disabled)
|
||||
if fam == "fire":
|
||||
freshness_s = int(adapter_config.wfigs.freshness_seconds)
|
||||
elif event.category == "earthquake_event":
|
||||
# The upstream USGS feed is a rolling PAST-DAY feed
|
||||
# (2.5_day.geojson), so a quake's age-at-ingest routinely exceeds
|
||||
# the generic 600s toggle window even for a genuinely first-seen
|
||||
# event -- every native quake was being silently dropped here
|
||||
# (measured age-at-ingest 548s-72169s across 12 sampled quakes,
|
||||
# confirmed against quake_events: 12/12 first-sighted rows with
|
||||
# last_broadcast_at=NULL). Scoped to the earthquake_event
|
||||
# CATEGORY, not the "seismic" family/toggle, because that family
|
||||
# also carries stream_flood_warning/stream_high_water (hydro),
|
||||
# which must keep using the generic per-toggle freshness_seconds.
|
||||
freshness_s = int(adapter_config.usgs_quake.freshness_seconds)
|
||||
else:
|
||||
freshness_s = int(getattr(tog, "freshness_seconds", 600) or 600)
|
||||
if event.timestamp and freshness_s > 0:
|
||||
|
|
|
|||
|
|
@ -86,6 +86,7 @@ def test_per_adapter_list(client):
|
|||
"regional_centroid", "regional_radius_mi",
|
||||
"broadcast_pager_alerts", "global_mag_floor",
|
||||
"regional_mag_floor", "escalate_mag_floor",
|
||||
"freshness_seconds",
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -200,12 +200,32 @@ def test_populates_core_fields(adapter):
|
|||
assert event.source == "usgs_quake"
|
||||
assert event.lat == 42.61
|
||||
assert event.lon == -114.48
|
||||
assert event.region == "magic_valley"
|
||||
# to_event() deliberately does NOT propagate the raw dict's "region"
|
||||
# (a fixed config default that never matches region_routes cell keys)
|
||||
# onto the Event -- Event.region is left unset so CoverageFilter's
|
||||
# geometry-based tagger (lat/lon vs named coverage areas) can set the
|
||||
# real region name downstream. See test_region_left_unset_for_tagger
|
||||
# and tests/test_coverage_area.py for the tagging behavior itself.
|
||||
assert event.region is None
|
||||
assert event.expires == evt["expires"]
|
||||
assert event.timestamp == evt["quake_time"]
|
||||
assert event.id
|
||||
|
||||
|
||||
def test_region_left_unset_for_tagger(adapter):
|
||||
"""Regression: to_event() must never propagate the adapter's fixed
|
||||
config region (e.g. "magic_valley") onto the Event. That value never
|
||||
matches a region_routes cell key ("East Idaho"/"SC Idaho"/"SW Idaho"),
|
||||
so pre-setting it would silently break region-routed delivery. Leaving
|
||||
event.region/regions unset lets CoverageFilter's geometry tagger (which
|
||||
only fires when `not event.regions`) stamp the real area name from the
|
||||
quake's actual lat/lon."""
|
||||
evt = make_quake_event(lat=44.2, lon=-114.0, region="magic_valley")
|
||||
event = adapter.to_event(evt)
|
||||
assert event.region is None
|
||||
assert event.regions == []
|
||||
|
||||
|
||||
# ============================================================
|
||||
# DEFENSIVE TESTS
|
||||
# ============================================================
|
||||
|
|
|
|||
212
work/tests/test_quake_freshness_and_region_tagging.py
Normal file
212
work/tests/test_quake_freshness_and_region_tagging.py
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
"""Regression tests for the usgs_quake dispatcher-freshness + region-tagging fix.
|
||||
|
||||
Problem (see meshai/notifications/pipeline/dispatcher.py:359-374 and
|
||||
meshai/env/usgs_quake.py): the usgs_quake adapter polls a rolling PAST-DAY
|
||||
USGS feed (2.5_day.geojson), so a genuinely first-seen quake is routinely
|
||||
already 10min-20h old by the time it's detected. The generic per-toggle
|
||||
freshness_seconds (600s) silently dropped essentially every native quake
|
||||
before broadcast -- confirmed against production: quake_events had 12
|
||||
first-sighting rows (decide() ran, magnitude/region gate passed, DB row
|
||||
inserted) with last_broadcast_at=NULL on every single one (commit() never
|
||||
reached because the staleness filter dropped the event downstream first).
|
||||
|
||||
Fix 1 (dispatcher.py): earthquake_event gets its own adapter_config-backed
|
||||
freshness override (adapter_config.usgs_quake.freshness_seconds, default
|
||||
3600s), mirroring the existing wfigs/"fire" override -- but scoped to the
|
||||
CATEGORY, not the "seismic" family/toggle, because stream_flood_warning /
|
||||
stream_high_water (hydro) also live under toggle="seismic" and must keep
|
||||
using the generic per-toggle freshness unchanged.
|
||||
|
||||
Fix 2 (env/usgs_quake.py): to_event() no longer passes the adapter's fixed
|
||||
config region ("magic_valley", which never matches a region_routes cell
|
||||
key) onto the Event. Event.region is left unset so CoverageFilter's
|
||||
geometry-based tagger (lat/lon vs named coverage areas) can stamp the real
|
||||
region name.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from meshai.config import Config
|
||||
from meshai.coverage_area import MonitoringArea
|
||||
from meshai.env.usgs_quake import USGSQuakeAdapter
|
||||
from meshai.notifications.events import make_event
|
||||
from meshai.notifications.pipeline.coverage_filter import CoverageFilter
|
||||
from meshai.notifications.pipeline.dispatcher import Dispatcher
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Shared dispatcher-test helpers (mirror tests/test_v052_dispatcher.py)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
class RecChannel:
|
||||
def __init__(self, rec):
|
||||
self.rec = rec
|
||||
|
||||
async def deliver(self, payload, rule):
|
||||
self.rec.append({"name": rule.name, "message": payload.message})
|
||||
return True
|
||||
|
||||
|
||||
def _make_dispatcher(cfg):
|
||||
rec: list = []
|
||||
d = Dispatcher(cfg, lambda rule, conn: RecChannel(rec), connector=None)
|
||||
return d, rec
|
||||
|
||||
|
||||
def _dispatch_one(cfg, event):
|
||||
d, rec = _make_dispatcher(cfg)
|
||||
asyncio.run(d.dispatch(event))
|
||||
return d, rec
|
||||
|
||||
|
||||
def _quake_cfg():
|
||||
"""Config with the seismic toggle enabled, cold-start grace disabled."""
|
||||
cfg = Config()
|
||||
cfg.notifications.rules = []
|
||||
cfg.notifications.cold_start_grace_seconds = 0
|
||||
t = cfg.notifications.toggles["seismic"]
|
||||
t.enabled = True
|
||||
t.min_severity = "routine"
|
||||
t.regions = []
|
||||
t.severity_channels = {
|
||||
"routine": ["mesh_broadcast"],
|
||||
"priority": ["mesh_broadcast"],
|
||||
"immediate": ["mesh_broadcast"],
|
||||
}
|
||||
t.cooldown_seconds = 0
|
||||
# Generic per-toggle freshness -- deliberately tight (600s, the old
|
||||
# default) so these tests prove the earthquake_event category is NOT
|
||||
# using this value anymore.
|
||||
t.freshness_seconds = 600
|
||||
return cfg
|
||||
|
||||
|
||||
def _quake_event(age_seconds: float, event_id="us6000abcd", lat=42.6, lon=-114.5):
|
||||
return make_event(
|
||||
source="usgs_quake",
|
||||
category="earthquake_event",
|
||||
severity="routine",
|
||||
title=f"M3.0 -- {event_id}",
|
||||
timestamp=time.time() - age_seconds,
|
||||
lat=lat,
|
||||
lon=lon,
|
||||
group_key=event_id,
|
||||
inhibit_keys=[event_id],
|
||||
)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# (a) fresh quake passes under the new per-adapter override
|
||||
# ============================================================
|
||||
|
||||
def test_fresh_quake_passes_staleness_gate_with_override():
|
||||
"""A 30-minute-old quake (1800s) exceeds the generic 600s toggle window
|
||||
but must pass under the adapter_config.usgs_quake.freshness_seconds
|
||||
override (default 3600s)."""
|
||||
cfg = _quake_cfg()
|
||||
event = _quake_event(age_seconds=1800)
|
||||
d, rec = _dispatch_one(cfg, event)
|
||||
assert len(rec) == 1, "30-min-old quake must broadcast under the 3600s override"
|
||||
assert d.dispatch_stats()["stale_dropped"] == 0
|
||||
|
||||
|
||||
# ============================================================
|
||||
# (b) old quake (12h) is still dropped
|
||||
# ============================================================
|
||||
|
||||
def test_old_quake_still_dropped_by_override():
|
||||
"""A 12-hour-old quake (43200s) must still be dropped -- the override
|
||||
widens the window, it does not disable the staleness check."""
|
||||
cfg = _quake_cfg()
|
||||
event = _quake_event(age_seconds=12 * 3600)
|
||||
d, rec = _dispatch_one(cfg, event)
|
||||
assert rec == [], "12h-old quake must still be dropped as stale"
|
||||
assert d.dispatch_stats()["stale_dropped"] == 1
|
||||
|
||||
|
||||
def test_hydro_seismic_sibling_unaffected_by_quake_override():
|
||||
"""Guardrail: stream_flood_warning shares toggle='seismic' with
|
||||
earthquake_event but must keep using the GENERIC per-toggle freshness
|
||||
(600s here), not the quake-specific 3600s override. A 1800s-old hydro
|
||||
event must still be dropped."""
|
||||
cfg = _quake_cfg()
|
||||
event = make_event(
|
||||
source="usgs", category="stream_flood_warning", severity="priority",
|
||||
title="Snake River nr Twin Falls 12.8 ft",
|
||||
timestamp=time.time() - 1800,
|
||||
)
|
||||
d, rec = _dispatch_one(cfg, event)
|
||||
assert rec == [], "hydro sibling must NOT inherit the quake freshness override"
|
||||
assert d.dispatch_stats()["stale_dropped"] == 1
|
||||
|
||||
|
||||
# ============================================================
|
||||
# (c) region-tagging from real lat/lon
|
||||
# ============================================================
|
||||
|
||||
SW_IDAHO = MonitoringArea(name="SW Idaho", west=-117.993408, south=42.05,
|
||||
east=-115.389404, north=44.331707)
|
||||
SC_IDAHO = MonitoringArea(name="SC Idaho", west=-115.389404, south=42.05,
|
||||
east=-112.8, north=44.331707)
|
||||
EAST_IDAHO = MonitoringArea(name="East Idaho", west=-112.8, south=41.9,
|
||||
east=-110.9, north=45.331707)
|
||||
PROD_AREAS = [SW_IDAHO, SC_IDAHO, EAST_IDAHO]
|
||||
|
||||
|
||||
def _quake_adapter():
|
||||
cfg = MagicMock()
|
||||
cfg.feed_url = "https://example.test/feed.geojson"
|
||||
cfg.min_magnitude = 2.5
|
||||
cfg.bbox = [-115.5, 42.0, -110.0, 45.2]
|
||||
cfg.region = "magic_valley"
|
||||
cfg.tick_seconds = 300
|
||||
return USGSQuakeAdapter(cfg)
|
||||
|
||||
|
||||
def test_quake_event_region_tags_to_matching_coverage_area():
|
||||
"""A quake with real Idaho lat/lon, run through the actual adapter's
|
||||
to_event() and then the real CoverageFilter (named exactly like prod's
|
||||
SW/SC/East Idaho areas), must end up tagged with a region name that
|
||||
matches a region_routes cell key -- NOT the adapter's stale
|
||||
"magic_valley" default."""
|
||||
adapter = _quake_adapter()
|
||||
raw = {
|
||||
"source": "usgs_quake",
|
||||
"event_id": "us7000example",
|
||||
"event_type": "Earthquake",
|
||||
"severity": "routine",
|
||||
"headline": "M2.7 -- 10 km N of Twin Falls, ID",
|
||||
"magnitude": 2.7,
|
||||
"place": "10 km N of Twin Falls, ID",
|
||||
"depth_km": 8.0,
|
||||
"sig": 80,
|
||||
"url": "https://earthquake.usgs.gov/x",
|
||||
"region": "magic_valley",
|
||||
"lat": 42.61, # Twin Falls -- falls inside SC Idaho
|
||||
"lon": -114.48,
|
||||
"quake_time": time.time(),
|
||||
"expires": time.time() + 86400,
|
||||
"fetched_at": time.time(),
|
||||
}
|
||||
event = adapter.to_event(raw)
|
||||
assert event is not None
|
||||
assert event.region is None # adapter no longer pre-stamps a region
|
||||
|
||||
received: list = []
|
||||
flt = CoverageFilter(next_handler=received.append, areas=PROD_AREAS, enabled=True)
|
||||
flt.handle(event)
|
||||
|
||||
assert len(received) == 1, "in-region quake must pass the coverage gate"
|
||||
tagged = received[0]
|
||||
assert tagged.region == "SC Idaho", (
|
||||
f"expected geometry tagging to set 'SC Idaho', got {tagged.region!r} "
|
||||
"(pre-fix this would have stayed 'magic_valley' and never matched a "
|
||||
"region_routes cell key)"
|
||||
)
|
||||
assert tagged.regions == ["SC Idaho"]
|
||||
Loading…
Add table
Add a link
Reference in a new issue