mirror of
https://github.com/zvx-echo6/meshai.git
synced 2026-08-26 17:31:34 +00:00
fix(usgs_quake): stop dropping first-sighted quakes at the freshness gate
The USGS feed is a rolling PAST-DAY feed (2.5_day.geojson), so a genuinely first-seen quake's age-at-ingest routinely exceeds the generic per-toggle freshness_seconds (600s) window, silently dropping it downstream of decide()/DB-insert (last_broadcast_at stayed NULL on every sampled first-sighting row). - dispatcher.py: earthquake_event gets its own adapter_config-backed freshness override (adapter_config.usgs_quake.freshness_seconds, default 3600s), scoped to the CATEGORY (not the "seismic" toggle, which also carries hydro stream_flood_warning/stream_high_water and must keep the generic per-toggle freshness). - usgs_quake.py: to_event() no longer passes the adapter's fixed freshness in, deferring to the dispatcher override. - adapter_config/defaults.py: adds the freshness_seconds field/default for usgs_quake.
This commit is contained in:
parent
50553b26f5
commit
738ffc71c3
6 changed files with 261 additions and 3 deletions
|
|
@ -85,7 +85,8 @@ REGISTRY: dict[tuple[str, str], dict[str, Any]] = {
|
||||||
},
|
},
|
||||||
|
|
||||||
# =================================================================
|
# =================================================================
|
||||||
# USGS_QUAKE -- 6 settings (regional geography + 3 mag floors + PAGER set)
|
# USGS_QUAKE -- 7 settings (regional geography + 3 mag floors + PAGER set
|
||||||
|
# + freshness gate)
|
||||||
# =================================================================
|
# =================================================================
|
||||||
("usgs_quake", "regional_centroid"): {
|
("usgs_quake", "regional_centroid"): {
|
||||||
"default": [44.36, -114.61], # quake_handler.py:36-37 (Idaho centroid)
|
"default": [44.36, -114.61], # quake_handler.py:36-37 (Idaho centroid)
|
||||||
|
|
@ -117,6 +118,11 @@ REGISTRY: dict[tuple[str, str], dict[str, Any]] = {
|
||||||
"type": "float",
|
"type": "float",
|
||||||
"description": "Magnitude floor for the visual escalation emoji.",
|
"description": "Magnitude floor for the visual escalation emoji.",
|
||||||
},
|
},
|
||||||
|
("usgs_quake", "freshness_seconds"): {
|
||||||
|
"default": 3600,
|
||||||
|
"type": "int",
|
||||||
|
"description": "Staleness gate for earthquake_event broadcasts (age = now - quake origin time), read by the dispatcher instead of the generic per-toggle freshness. The upstream feed is a rolling PAST-DAY feed, so quakes are routinely detected 10min-20h after origin; 1 hour keeps quakes broadcast-worthy (still situationally relevant) while excluding the day-feed's stale backlog. 0 = disabled (not recommended -- would readmit day-old quakes).",
|
||||||
|
},
|
||||||
|
|
||||||
# =================================================================
|
# =================================================================
|
||||||
# SWPC -- 3 settings (three storm-tier broadcast floors)
|
# SWPC -- 3 settings (three storm-tier broadcast floors)
|
||||||
|
|
|
||||||
9
work/meshai/env/usgs_quake.py
vendored
9
work/meshai/env/usgs_quake.py
vendored
|
|
@ -246,7 +246,14 @@ class USGSQuakeAdapter:
|
||||||
expires=evt.get("expires"),
|
expires=evt.get("expires"),
|
||||||
lat=lat,
|
lat=lat,
|
||||||
lon=lon,
|
lon=lon,
|
||||||
region=evt.get("region"),
|
# Deliberately NOT passing region=evt.get("region") here: the
|
||||||
|
# raw dict's region is a fixed config default (self._region,
|
||||||
|
# "magic_valley"), which doesn't match any region_routes cell
|
||||||
|
# key. Event.region is documented as "set by region tagger"
|
||||||
|
# (notifications/events.py) -- leaving it unset lets
|
||||||
|
# CoverageFilter's geometry-based tagging (event.lat/lon vs
|
||||||
|
# the named coverage areas) populate the real region name
|
||||||
|
# from the quake's actual coordinates instead.
|
||||||
group_key=event_id,
|
group_key=event_id,
|
||||||
inhibit_keys=[event_id],
|
inhibit_keys=[event_id],
|
||||||
data=canonical_data,
|
data=canonical_data,
|
||||||
|
|
|
||||||
|
|
@ -359,6 +359,18 @@ class Dispatcher:
|
||||||
# v0.6-3b: fire toggle uses wfigs adapter_config freshness (0 = disabled)
|
# v0.6-3b: fire toggle uses wfigs adapter_config freshness (0 = disabled)
|
||||||
if fam == "fire":
|
if fam == "fire":
|
||||||
freshness_s = int(adapter_config.wfigs.freshness_seconds)
|
freshness_s = int(adapter_config.wfigs.freshness_seconds)
|
||||||
|
elif event.category == "earthquake_event":
|
||||||
|
# The upstream USGS feed is a rolling PAST-DAY feed
|
||||||
|
# (2.5_day.geojson), so a quake's age-at-ingest routinely exceeds
|
||||||
|
# the generic 600s toggle window even for a genuinely first-seen
|
||||||
|
# event -- every native quake was being silently dropped here
|
||||||
|
# (measured age-at-ingest 548s-72169s across 12 sampled quakes,
|
||||||
|
# confirmed against quake_events: 12/12 first-sighted rows with
|
||||||
|
# last_broadcast_at=NULL). Scoped to the earthquake_event
|
||||||
|
# CATEGORY, not the "seismic" family/toggle, because that family
|
||||||
|
# also carries stream_flood_warning/stream_high_water (hydro),
|
||||||
|
# which must keep using the generic per-toggle freshness_seconds.
|
||||||
|
freshness_s = int(adapter_config.usgs_quake.freshness_seconds)
|
||||||
else:
|
else:
|
||||||
freshness_s = int(getattr(tog, "freshness_seconds", 600) or 600)
|
freshness_s = int(getattr(tog, "freshness_seconds", 600) or 600)
|
||||||
if event.timestamp and freshness_s > 0:
|
if event.timestamp and freshness_s > 0:
|
||||||
|
|
|
||||||
|
|
@ -86,6 +86,7 @@ def test_per_adapter_list(client):
|
||||||
"regional_centroid", "regional_radius_mi",
|
"regional_centroid", "regional_radius_mi",
|
||||||
"broadcast_pager_alerts", "global_mag_floor",
|
"broadcast_pager_alerts", "global_mag_floor",
|
||||||
"regional_mag_floor", "escalate_mag_floor",
|
"regional_mag_floor", "escalate_mag_floor",
|
||||||
|
"freshness_seconds",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -200,12 +200,32 @@ def test_populates_core_fields(adapter):
|
||||||
assert event.source == "usgs_quake"
|
assert event.source == "usgs_quake"
|
||||||
assert event.lat == 42.61
|
assert event.lat == 42.61
|
||||||
assert event.lon == -114.48
|
assert event.lon == -114.48
|
||||||
assert event.region == "magic_valley"
|
# to_event() deliberately does NOT propagate the raw dict's "region"
|
||||||
|
# (a fixed config default that never matches region_routes cell keys)
|
||||||
|
# onto the Event -- Event.region is left unset so CoverageFilter's
|
||||||
|
# geometry-based tagger (lat/lon vs named coverage areas) can set the
|
||||||
|
# real region name downstream. See test_region_left_unset_for_tagger
|
||||||
|
# and tests/test_coverage_area.py for the tagging behavior itself.
|
||||||
|
assert event.region is None
|
||||||
assert event.expires == evt["expires"]
|
assert event.expires == evt["expires"]
|
||||||
assert event.timestamp == evt["quake_time"]
|
assert event.timestamp == evt["quake_time"]
|
||||||
assert event.id
|
assert event.id
|
||||||
|
|
||||||
|
|
||||||
|
def test_region_left_unset_for_tagger(adapter):
|
||||||
|
"""Regression: to_event() must never propagate the adapter's fixed
|
||||||
|
config region (e.g. "magic_valley") onto the Event. That value never
|
||||||
|
matches a region_routes cell key ("East Idaho"/"SC Idaho"/"SW Idaho"),
|
||||||
|
so pre-setting it would silently break region-routed delivery. Leaving
|
||||||
|
event.region/regions unset lets CoverageFilter's geometry tagger (which
|
||||||
|
only fires when `not event.regions`) stamp the real area name from the
|
||||||
|
quake's actual lat/lon."""
|
||||||
|
evt = make_quake_event(lat=44.2, lon=-114.0, region="magic_valley")
|
||||||
|
event = adapter.to_event(evt)
|
||||||
|
assert event.region is None
|
||||||
|
assert event.regions == []
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
# ============================================================
|
||||||
# DEFENSIVE TESTS
|
# DEFENSIVE TESTS
|
||||||
# ============================================================
|
# ============================================================
|
||||||
|
|
|
||||||
212
work/tests/test_quake_freshness_and_region_tagging.py
Normal file
212
work/tests/test_quake_freshness_and_region_tagging.py
Normal file
|
|
@ -0,0 +1,212 @@
|
||||||
|
"""Regression tests for the usgs_quake dispatcher-freshness + region-tagging fix.
|
||||||
|
|
||||||
|
Problem (see meshai/notifications/pipeline/dispatcher.py:359-374 and
|
||||||
|
meshai/env/usgs_quake.py): the usgs_quake adapter polls a rolling PAST-DAY
|
||||||
|
USGS feed (2.5_day.geojson), so a genuinely first-seen quake is routinely
|
||||||
|
already 10min-20h old by the time it's detected. The generic per-toggle
|
||||||
|
freshness_seconds (600s) silently dropped essentially every native quake
|
||||||
|
before broadcast -- confirmed against production: quake_events had 12
|
||||||
|
first-sighting rows (decide() ran, magnitude/region gate passed, DB row
|
||||||
|
inserted) with last_broadcast_at=NULL on every single one (commit() never
|
||||||
|
reached because the staleness filter dropped the event downstream first).
|
||||||
|
|
||||||
|
Fix 1 (dispatcher.py): earthquake_event gets its own adapter_config-backed
|
||||||
|
freshness override (adapter_config.usgs_quake.freshness_seconds, default
|
||||||
|
3600s), mirroring the existing wfigs/"fire" override -- but scoped to the
|
||||||
|
CATEGORY, not the "seismic" family/toggle, because stream_flood_warning /
|
||||||
|
stream_high_water (hydro) also live under toggle="seismic" and must keep
|
||||||
|
using the generic per-toggle freshness unchanged.
|
||||||
|
|
||||||
|
Fix 2 (env/usgs_quake.py): to_event() no longer passes the adapter's fixed
|
||||||
|
config region ("magic_valley", which never matches a region_routes cell
|
||||||
|
key) onto the Event. Event.region is left unset so CoverageFilter's
|
||||||
|
geometry-based tagger (lat/lon vs named coverage areas) can stamp the real
|
||||||
|
region name.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import time
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from meshai.config import Config
|
||||||
|
from meshai.coverage_area import MonitoringArea
|
||||||
|
from meshai.env.usgs_quake import USGSQuakeAdapter
|
||||||
|
from meshai.notifications.events import make_event
|
||||||
|
from meshai.notifications.pipeline.coverage_filter import CoverageFilter
|
||||||
|
from meshai.notifications.pipeline.dispatcher import Dispatcher
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
# Shared dispatcher-test helpers (mirror tests/test_v052_dispatcher.py)
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class RecChannel:
|
||||||
|
def __init__(self, rec):
|
||||||
|
self.rec = rec
|
||||||
|
|
||||||
|
async def deliver(self, payload, rule):
|
||||||
|
self.rec.append({"name": rule.name, "message": payload.message})
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _make_dispatcher(cfg):
|
||||||
|
rec: list = []
|
||||||
|
d = Dispatcher(cfg, lambda rule, conn: RecChannel(rec), connector=None)
|
||||||
|
return d, rec
|
||||||
|
|
||||||
|
|
||||||
|
def _dispatch_one(cfg, event):
|
||||||
|
d, rec = _make_dispatcher(cfg)
|
||||||
|
asyncio.run(d.dispatch(event))
|
||||||
|
return d, rec
|
||||||
|
|
||||||
|
|
||||||
|
def _quake_cfg():
|
||||||
|
"""Config with the seismic toggle enabled, cold-start grace disabled."""
|
||||||
|
cfg = Config()
|
||||||
|
cfg.notifications.rules = []
|
||||||
|
cfg.notifications.cold_start_grace_seconds = 0
|
||||||
|
t = cfg.notifications.toggles["seismic"]
|
||||||
|
t.enabled = True
|
||||||
|
t.min_severity = "routine"
|
||||||
|
t.regions = []
|
||||||
|
t.severity_channels = {
|
||||||
|
"routine": ["mesh_broadcast"],
|
||||||
|
"priority": ["mesh_broadcast"],
|
||||||
|
"immediate": ["mesh_broadcast"],
|
||||||
|
}
|
||||||
|
t.cooldown_seconds = 0
|
||||||
|
# Generic per-toggle freshness -- deliberately tight (600s, the old
|
||||||
|
# default) so these tests prove the earthquake_event category is NOT
|
||||||
|
# using this value anymore.
|
||||||
|
t.freshness_seconds = 600
|
||||||
|
return cfg
|
||||||
|
|
||||||
|
|
||||||
|
def _quake_event(age_seconds: float, event_id="us6000abcd", lat=42.6, lon=-114.5):
|
||||||
|
return make_event(
|
||||||
|
source="usgs_quake",
|
||||||
|
category="earthquake_event",
|
||||||
|
severity="routine",
|
||||||
|
title=f"M3.0 -- {event_id}",
|
||||||
|
timestamp=time.time() - age_seconds,
|
||||||
|
lat=lat,
|
||||||
|
lon=lon,
|
||||||
|
group_key=event_id,
|
||||||
|
inhibit_keys=[event_id],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# (a) fresh quake passes under the new per-adapter override
|
||||||
|
# ============================================================
|
||||||
|
|
||||||
|
def test_fresh_quake_passes_staleness_gate_with_override():
|
||||||
|
"""A 30-minute-old quake (1800s) exceeds the generic 600s toggle window
|
||||||
|
but must pass under the adapter_config.usgs_quake.freshness_seconds
|
||||||
|
override (default 3600s)."""
|
||||||
|
cfg = _quake_cfg()
|
||||||
|
event = _quake_event(age_seconds=1800)
|
||||||
|
d, rec = _dispatch_one(cfg, event)
|
||||||
|
assert len(rec) == 1, "30-min-old quake must broadcast under the 3600s override"
|
||||||
|
assert d.dispatch_stats()["stale_dropped"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# (b) old quake (12h) is still dropped
|
||||||
|
# ============================================================
|
||||||
|
|
||||||
|
def test_old_quake_still_dropped_by_override():
|
||||||
|
"""A 12-hour-old quake (43200s) must still be dropped -- the override
|
||||||
|
widens the window, it does not disable the staleness check."""
|
||||||
|
cfg = _quake_cfg()
|
||||||
|
event = _quake_event(age_seconds=12 * 3600)
|
||||||
|
d, rec = _dispatch_one(cfg, event)
|
||||||
|
assert rec == [], "12h-old quake must still be dropped as stale"
|
||||||
|
assert d.dispatch_stats()["stale_dropped"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_hydro_seismic_sibling_unaffected_by_quake_override():
|
||||||
|
"""Guardrail: stream_flood_warning shares toggle='seismic' with
|
||||||
|
earthquake_event but must keep using the GENERIC per-toggle freshness
|
||||||
|
(600s here), not the quake-specific 3600s override. A 1800s-old hydro
|
||||||
|
event must still be dropped."""
|
||||||
|
cfg = _quake_cfg()
|
||||||
|
event = make_event(
|
||||||
|
source="usgs", category="stream_flood_warning", severity="priority",
|
||||||
|
title="Snake River nr Twin Falls 12.8 ft",
|
||||||
|
timestamp=time.time() - 1800,
|
||||||
|
)
|
||||||
|
d, rec = _dispatch_one(cfg, event)
|
||||||
|
assert rec == [], "hydro sibling must NOT inherit the quake freshness override"
|
||||||
|
assert d.dispatch_stats()["stale_dropped"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# (c) region-tagging from real lat/lon
|
||||||
|
# ============================================================
|
||||||
|
|
||||||
|
SW_IDAHO = MonitoringArea(name="SW Idaho", west=-117.993408, south=42.05,
|
||||||
|
east=-115.389404, north=44.331707)
|
||||||
|
SC_IDAHO = MonitoringArea(name="SC Idaho", west=-115.389404, south=42.05,
|
||||||
|
east=-112.8, north=44.331707)
|
||||||
|
EAST_IDAHO = MonitoringArea(name="East Idaho", west=-112.8, south=41.9,
|
||||||
|
east=-110.9, north=45.331707)
|
||||||
|
PROD_AREAS = [SW_IDAHO, SC_IDAHO, EAST_IDAHO]
|
||||||
|
|
||||||
|
|
||||||
|
def _quake_adapter():
|
||||||
|
cfg = MagicMock()
|
||||||
|
cfg.feed_url = "https://example.test/feed.geojson"
|
||||||
|
cfg.min_magnitude = 2.5
|
||||||
|
cfg.bbox = [-115.5, 42.0, -110.0, 45.2]
|
||||||
|
cfg.region = "magic_valley"
|
||||||
|
cfg.tick_seconds = 300
|
||||||
|
return USGSQuakeAdapter(cfg)
|
||||||
|
|
||||||
|
|
||||||
|
def test_quake_event_region_tags_to_matching_coverage_area():
|
||||||
|
"""A quake with real Idaho lat/lon, run through the actual adapter's
|
||||||
|
to_event() and then the real CoverageFilter (named exactly like prod's
|
||||||
|
SW/SC/East Idaho areas), must end up tagged with a region name that
|
||||||
|
matches a region_routes cell key -- NOT the adapter's stale
|
||||||
|
"magic_valley" default."""
|
||||||
|
adapter = _quake_adapter()
|
||||||
|
raw = {
|
||||||
|
"source": "usgs_quake",
|
||||||
|
"event_id": "us7000example",
|
||||||
|
"event_type": "Earthquake",
|
||||||
|
"severity": "routine",
|
||||||
|
"headline": "M2.7 -- 10 km N of Twin Falls, ID",
|
||||||
|
"magnitude": 2.7,
|
||||||
|
"place": "10 km N of Twin Falls, ID",
|
||||||
|
"depth_km": 8.0,
|
||||||
|
"sig": 80,
|
||||||
|
"url": "https://earthquake.usgs.gov/x",
|
||||||
|
"region": "magic_valley",
|
||||||
|
"lat": 42.61, # Twin Falls -- falls inside SC Idaho
|
||||||
|
"lon": -114.48,
|
||||||
|
"quake_time": time.time(),
|
||||||
|
"expires": time.time() + 86400,
|
||||||
|
"fetched_at": time.time(),
|
||||||
|
}
|
||||||
|
event = adapter.to_event(raw)
|
||||||
|
assert event is not None
|
||||||
|
assert event.region is None # adapter no longer pre-stamps a region
|
||||||
|
|
||||||
|
received: list = []
|
||||||
|
flt = CoverageFilter(next_handler=received.append, areas=PROD_AREAS, enabled=True)
|
||||||
|
flt.handle(event)
|
||||||
|
|
||||||
|
assert len(received) == 1, "in-region quake must pass the coverage gate"
|
||||||
|
tagged = received[0]
|
||||||
|
assert tagged.region == "SC Idaho", (
|
||||||
|
f"expected geometry tagging to set 'SC Idaho', got {tagged.region!r} "
|
||||||
|
"(pre-fix this would have stayed 'magic_valley' and never matched a "
|
||||||
|
"region_routes cell key)"
|
||||||
|
)
|
||||||
|
assert tagged.regions == ["SC Idaho"]
|
||||||
Loading…
Add table
Add a link
Reference in a new issue