mirror of
https://github.com/zvx-echo6/meshai.git
synced 2026-08-26 17:31:34 +00:00
feat(env): native adapters broadcast only newly-RECEIVED items (no backlog) (#43)
Replace the native path's "scan accumulated state + suppress what we've already broadcast" model with "broadcast only what newly arrived from the API this poll." Storage is unchanged (self._events + firms_pixels etc. are populated for EVERY received item, so the LLM/get_active backlog is intact); only the BROADCAST decision changes. - env/store.py: per-adapter in-memory seen-set (_seen) + _seeded. First data-bearing poll for an adapter seeds keys and emits NOTHING (that batch is pre-existing backlog); later polls emit only keys not seen before. Restart => empty sets => next poll re-seeds silently. Structurally impossible to broadcast backlog on cold start / restart / re-enable. Key = external_id -> event_id -> content hash, namespaced per adapter. self._events[key]=evt still runs unconditionally (storage preserved). - Fixes the ~175 (roads511) / ~782 (wzdx) cold-start bursts AND the latent quake/nws version (they only looked safe because Central pre-populated their broadcast tables). - env/satpass.py: broadcast on AOS IMMINENCE (now < aos <= now+lead, broadcast_lead_seconds default 3600), future-only; window_hours still governs prediction depth. Strict norad_ids post-filter + fixed _parse_norad_ids char-iteration bug (cause of GOES/METEOR leak). - Central path + broadcast-state tables untouched (native-only gate). 13 new tests; full suite at 10-failure baseline (1697 passed). Co-authored-by: Matt Johnson <mj@k7zvx.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
63245b8fba
commit
0c76a806b8
6 changed files with 421 additions and 24 deletions
|
|
@ -513,6 +513,11 @@ class SatpassConfig(_SourcedFeed):
|
||||||
# Predictor pass filters (used by the next task):
|
# Predictor pass filters (used by the next task):
|
||||||
min_elevation_deg: float = 10.0
|
min_elevation_deg: float = 10.0
|
||||||
window_hours: int = 24
|
window_hours: int = 24
|
||||||
|
# BROADCAST imminence lead (seconds): a predicted pass is broadcast only
|
||||||
|
# once its AOS is within this near-term window (and still in the future).
|
||||||
|
# `window_hours` governs how far ahead we PREDICT; this governs when a
|
||||||
|
# predicted pass is actually announced. Default 60 min.
|
||||||
|
broadcast_lead_seconds: int = 3600
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
|
|
|
||||||
49
work/meshai/env/satpass.py
vendored
49
work/meshai/env/satpass.py
vendored
|
|
@ -60,6 +60,18 @@ logger = logging.getLogger(__name__)
|
||||||
# pointless). Overridable via SatpassConfig.pass_refresh_seconds if present.
|
# pointless). Overridable via SatpassConfig.pass_refresh_seconds if present.
|
||||||
DEFAULT_POLL_SECONDS = 900
|
DEFAULT_POLL_SECONDS = 900
|
||||||
|
|
||||||
|
# Broadcast imminence lead. Satpass is NOT an API-delta feed — it computes the
|
||||||
|
# SAME future passes every poll, so a pure "new since last poll" gate would
|
||||||
|
# seed them all as backlog and then never broadcast. A pass's "newly received"
|
||||||
|
# analog is IMMINENCE: it becomes broadcast-worthy only once its AOS first
|
||||||
|
# enters this near-term lead window (and is still in the future). Combined with
|
||||||
|
# the store's received-delta seen-set (keyed on the pass canonical id), each
|
||||||
|
# pass then emits exactly once as it crosses into imminence, and the
|
||||||
|
# first-poll-seed suppresses any startup burst. `window_hours` still governs
|
||||||
|
# how far ahead we PREDICT (the schedule); this only governs the BROADCAST
|
||||||
|
# trigger. Overridable via SatpassConfig.broadcast_lead_seconds.
|
||||||
|
DEFAULT_BROADCAST_LEAD_SECONDS = 3600
|
||||||
|
|
||||||
|
|
||||||
class SatpassAdapter:
|
class SatpassAdapter:
|
||||||
"""Native SGP4 pass predictor — consolidates in-memory, gates synchronously."""
|
"""Native SGP4 pass predictor — consolidates in-memory, gates synchronously."""
|
||||||
|
|
@ -71,6 +83,9 @@ class SatpassAdapter:
|
||||||
or DEFAULT_POLL_SECONDS)
|
or DEFAULT_POLL_SECONDS)
|
||||||
self._window_h = int(getattr(config, "window_hours", 24) or 24)
|
self._window_h = int(getattr(config, "window_hours", 24) or 24)
|
||||||
self._min_el = float(getattr(config, "min_elevation_deg", 10.0))
|
self._min_el = float(getattr(config, "min_elevation_deg", 10.0))
|
||||||
|
self._lead_s = int(
|
||||||
|
getattr(config, "broadcast_lead_seconds", DEFAULT_BROADCAST_LEAD_SECONDS)
|
||||||
|
or DEFAULT_BROADCAST_LEAD_SECONDS)
|
||||||
self._norad_ids = self._parse_norad_ids(
|
self._norad_ids = self._parse_norad_ids(
|
||||||
getattr(config, "norad_ids", None) or [])
|
getattr(config, "norad_ids", None) or [])
|
||||||
|
|
||||||
|
|
@ -84,21 +99,39 @@ class SatpassAdapter:
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _parse_norad_ids(raw) -> list[int]:
|
def _parse_norad_ids(raw) -> list[int]:
|
||||||
"""Coerce a config norad_ids list (ints or GUI strings) to int list."""
|
"""Coerce a config norad_ids value to a sorted unique int list.
|
||||||
|
|
||||||
|
Accepts a real list (ints or GUI strings) OR a single comma/space
|
||||||
|
separated string (as some GUI paths persist it). The string case
|
||||||
|
MUST be split first — iterating a bare string would walk its
|
||||||
|
CHARACTERS, turning "25544,33591" into garbage single-digit ids and
|
||||||
|
silently defeating the norad filter (a cause of non-listed sats
|
||||||
|
leaking into predictions).
|
||||||
|
"""
|
||||||
|
if isinstance(raw, str):
|
||||||
|
raw = raw.replace(",", " ").split()
|
||||||
return sorted({int(x) for x in raw if str(x).strip().isdigit()})
|
return sorted({int(x) for x in raw if str(x).strip().isdigit()})
|
||||||
|
|
||||||
def _resolve_tles(self) -> list[dict]:
|
def _resolve_tles(self) -> list[dict]:
|
||||||
"""Resolve the target TLE set from `sat_tles` (fresh only).
|
"""Resolve the target TLE set from `sat_tles` (fresh only).
|
||||||
|
|
||||||
`norad_ids` configured -> exactly those; empty -> all fresh.
|
`norad_ids` configured -> exactly those; empty -> all fresh.
|
||||||
|
|
||||||
|
When `norad_ids` is configured the result is ALSO post-filtered to
|
||||||
|
that set: this is belt-and-suspenders so that only configured NORADs
|
||||||
|
are ever predicted, regardless of which lookup path ran. Without a
|
||||||
|
configured list, the fetcher's `tle_groups` (e.g. "weather") stock
|
||||||
|
sat_tles with GOES/METEOR/FENGYUN etc.; the strict post-filter
|
||||||
|
guarantees those never leak into predictions once a list is set.
|
||||||
"""
|
"""
|
||||||
from meshai.central.tle_handler import get_fresh_tles, get_tle_by_norad
|
from meshai.central.tle_handler import get_fresh_tles, get_tle_by_norad
|
||||||
|
|
||||||
if self._norad_ids:
|
if self._norad_ids:
|
||||||
|
allowed = set(self._norad_ids)
|
||||||
out: list[dict] = []
|
out: list[dict] = []
|
||||||
for nid in self._norad_ids:
|
for nid in self._norad_ids:
|
||||||
tle = get_tle_by_norad(nid)
|
tle = get_tle_by_norad(nid)
|
||||||
if tle is not None:
|
if tle is not None and int(tle["norad_id"]) in allowed:
|
||||||
out.append(tle)
|
out.append(tle)
|
||||||
return out
|
return out
|
||||||
return get_fresh_tles()
|
return get_fresh_tles()
|
||||||
|
|
@ -186,6 +219,18 @@ class SatpassAdapter:
|
||||||
staged: list[dict] = []
|
staged: list[dict] = []
|
||||||
for cid, recs in groups.items():
|
for cid, recs in groups.items():
|
||||||
consolidated = self._consolidate(cid, recs)
|
consolidated = self._consolidate(cid, recs)
|
||||||
|
# IMMINENCE gate — applied on the CONSOLIDATED pass (earliest AOS
|
||||||
|
# across observers), so multi-observer consolidation is never
|
||||||
|
# fragmented. A pass is broadcast-worthy only once its AOS is both
|
||||||
|
# in the FUTURE and within the near-term lead window. Far-future
|
||||||
|
# passes are predicted (on the schedule) but not staged until they
|
||||||
|
# become imminent; a past AOS is never staged. This is satpass's
|
||||||
|
# "just received" signal — the store's received-delta seen-set then
|
||||||
|
# emits each pass exactly once as it crosses into the window, and
|
||||||
|
# the first-poll-seed prevents any startup burst.
|
||||||
|
aos = consolidated["aos_epoch"]
|
||||||
|
if aos <= now_epoch or aos > now_epoch + self._lead_s:
|
||||||
|
continue
|
||||||
try:
|
try:
|
||||||
result = sh.gate_consolidated_pass(consolidated, now=now_epoch)
|
result = sh.gate_consolidated_pass(consolidated, now=now_epoch)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|
|
||||||
113
work/meshai/env/store.py
vendored
113
work/meshai/env/store.py
vendored
|
|
@ -1,5 +1,7 @@
|
||||||
"""Environmental data store with tick-based adapter polling."""
|
"""Environmental data store with tick-based adapter polling."""
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
import logging
|
import logging
|
||||||
import time
|
import time
|
||||||
from typing import TYPE_CHECKING, Optional
|
from typing import TYPE_CHECKING, Optional
|
||||||
|
|
@ -29,6 +31,21 @@ class EnvironmentalStore:
|
||||||
self._mesh_zones = config.nws_zones or []
|
self._mesh_zones = config.nws_zones or []
|
||||||
self._region_anchors = region_anchors or []
|
self._region_anchors = region_anchors or []
|
||||||
|
|
||||||
|
# ── Received-delta gate (NATIVE-only) ────────────────────────────
|
||||||
|
# Per-adapter set of item keys seen in PRIOR polls, plus the set of
|
||||||
|
# adapters whose first poll has completed. The model the operator
|
||||||
|
# demanded: a native adapter broadcasts an item ONLY when it was newly
|
||||||
|
# RECEIVED from the API this poll — never by scanning an accumulated
|
||||||
|
# backlog. So the FIRST poll for an adapter records every current key
|
||||||
|
# as "seen" and emits NOTHING (that batch is pre-existing backlog, not
|
||||||
|
# ours to announce); every later poll emits only keys not already
|
||||||
|
# seen. In-memory only (per process) — a restart empties the sets, so
|
||||||
|
# the next poll is a fresh "first poll" that re-seeds silently. This
|
||||||
|
# makes broadcasting a backlog item structurally impossible on cold
|
||||||
|
# start, restart, or after the adapter was disabled for days.
|
||||||
|
self._seen: dict[str, set] = {} # adapter name -> set of item keys
|
||||||
|
self._seeded: set[str] = set() # adapters past their first poll
|
||||||
|
|
||||||
# Create adapter instances with error isolation
|
# Create adapter instances with error isolation
|
||||||
self._register_adapter("nws", config.nws, ".nws", "NWSAlertsAdapter",
|
self._register_adapter("nws", config.nws, ".nws", "NWSAlertsAdapter",
|
||||||
lambda cfg: (cfg,))
|
lambda cfg: (cfg,))
|
||||||
|
|
@ -112,46 +129,106 @@ class EnvironmentalStore:
|
||||||
self._purge_expired()
|
self._purge_expired()
|
||||||
return changed
|
return changed
|
||||||
|
|
||||||
|
def _seen_key(self, name: str, raw_evt: dict) -> str:
|
||||||
|
"""Derive a STABLE per-item key for the received-delta gate.
|
||||||
|
|
||||||
|
Stability across polls is the whole point: the SAME real-world item
|
||||||
|
must produce the SAME key every poll, or it would look "newly
|
||||||
|
received" forever and re-broadcast on each tick. Preference order,
|
||||||
|
most→least explicit id:
|
||||||
|
1. external_id — the upstream feed's own stable id (e.g. WZDx)
|
||||||
|
2. event_id — the adapter's stable per-item id; every native raw
|
||||||
|
event carries one (the store's own dedup already keys on it, and
|
||||||
|
the adapters build it from natural ids: USGS quake id, 511 event
|
||||||
|
id, NWS alert id, ``swpc_<scale><level>``, ``ducting_<tier>_<loc>``,
|
||||||
|
``avy_<center>_<zone>``, etc. — all stable across polls).
|
||||||
|
3. content hash — last-resort fallback if an item somehow carries no
|
||||||
|
id at all.
|
||||||
|
Namespaced by adapter so two feeds never cross-contaminate.
|
||||||
|
"""
|
||||||
|
ext = raw_evt.get("external_id")
|
||||||
|
if ext:
|
||||||
|
return f"{name}\x1eext:{ext}"
|
||||||
|
eid = raw_evt.get("event_id")
|
||||||
|
if eid:
|
||||||
|
return f"{name}\x1eeid:{eid}"
|
||||||
|
blob = json.dumps(raw_evt, sort_keys=True, default=str)
|
||||||
|
return f"{name}\x1ehash:" + hashlib.sha1(blob.encode()).hexdigest()[:16]
|
||||||
|
|
||||||
|
def _delta_emit(self, name: str, adapter, raw_evt: dict, force: bool = False):
|
||||||
|
"""Received-delta gate — emit ONLY items newly received THIS poll.
|
||||||
|
|
||||||
|
- First poll for ``name`` (name not yet in ``self._seeded``): record
|
||||||
|
the item's key as seen and emit NOTHING. That batch is the
|
||||||
|
pre-existing backlog.
|
||||||
|
- Later polls: emit only when the key is not already in the seen-set
|
||||||
|
(it just appeared upstream = "just received"); then record it.
|
||||||
|
- ``force=True`` (avalanche danger-level rise) re-emits an already-seen
|
||||||
|
item on a legitimate CONTENT change — but is IGNORED on the first
|
||||||
|
poll, so a restart still can never replay the backlog.
|
||||||
|
|
||||||
|
This replaces the native path's reliance on the deciders'
|
||||||
|
broadcast-state tables for the "is this new" decision. Legitimate
|
||||||
|
content filtering (severity/threshold/impact, decider gates) still runs
|
||||||
|
downstream in ``_emit_event``.
|
||||||
|
"""
|
||||||
|
seen = self._seen.setdefault(name, set())
|
||||||
|
key = self._seen_key(name, raw_evt)
|
||||||
|
first_poll = name not in self._seeded
|
||||||
|
|
||||||
|
if first_poll:
|
||||||
|
seen.add(key) # seed silently — this is backlog
|
||||||
|
return
|
||||||
|
if key in seen and not force:
|
||||||
|
return # already received in a prior poll
|
||||||
|
seen.add(key)
|
||||||
|
|
||||||
|
if self._event_bus is not None and hasattr(adapter, "to_event"):
|
||||||
|
self._emit_event(adapter, raw_evt)
|
||||||
|
|
||||||
def _ingest(self, name: str, adapter):
|
def _ingest(self, name: str, adapter):
|
||||||
"""Ingest data from an adapter after it ticks."""
|
"""Ingest data from an adapter after it ticks.
|
||||||
|
|
||||||
|
Emission goes through the received-delta gate (``_delta_emit``): the
|
||||||
|
adapter's FIRST data-bearing poll seeds the seen-set and broadcasts
|
||||||
|
nothing; only items that newly appear on later polls are broadcast.
|
||||||
|
``self._events`` is still maintained for state/queries as before.
|
||||||
|
"""
|
||||||
if name == "swpc":
|
if name == "swpc":
|
||||||
self._swpc_status = adapter.get_status()
|
self._swpc_status = adapter.get_status()
|
||||||
# Also ingest any alert events (R-scale >= 3)
|
# Also ingest any alert events (R-scale >= 3)
|
||||||
for evt in adapter.get_events():
|
for evt in adapter.get_events():
|
||||||
key = (evt["source"], evt["event_id"])
|
key = (evt["source"], evt["event_id"])
|
||||||
is_new = key not in self._events
|
|
||||||
self._events[key] = evt
|
self._events[key] = evt
|
||||||
if is_new and self._event_bus and hasattr(adapter, "to_event"):
|
self._delta_emit(name, adapter, evt)
|
||||||
self._emit_event(adapter, evt)
|
|
||||||
elif name == "ducting":
|
elif name == "ducting":
|
||||||
self._ducting_status = adapter.get_status()
|
self._ducting_status = adapter.get_status()
|
||||||
for evt in adapter.get_events():
|
for evt in adapter.get_events():
|
||||||
key = (evt["source"], evt["event_id"])
|
key = (evt["source"], evt["event_id"])
|
||||||
is_new = key not in self._events
|
|
||||||
self._events[key] = evt
|
self._events[key] = evt
|
||||||
if is_new and self._event_bus and hasattr(adapter, "to_event"):
|
self._delta_emit(name, adapter, evt)
|
||||||
self._emit_event(adapter, evt)
|
|
||||||
elif name == "avalanche":
|
elif name == "avalanche":
|
||||||
# Avalanche: re-emit on danger_level rise (Update:) not just new events.
|
# Avalanche: re-emit on danger_level rise (Update:) not just new
|
||||||
|
# events. The rise is a legitimate CONTENT change, so it passes
|
||||||
|
# `force=True` — but the received-delta gate still suppresses it on
|
||||||
|
# the first poll (backlog stays silent).
|
||||||
for evt in adapter.get_events():
|
for evt in adapter.get_events():
|
||||||
key = (evt["source"], evt["event_id"])
|
key = (evt["source"], evt["event_id"])
|
||||||
prior = self._events.get(key)
|
prior = self._events.get(key)
|
||||||
is_new = prior is None
|
|
||||||
prior_level = prior.get("danger_level", -1) if prior else -1
|
prior_level = prior.get("danger_level", -1) if prior else -1
|
||||||
level_rose = (not is_new) and (evt.get("danger_level", -1) > prior_level)
|
level_rose = (prior is not None) and (
|
||||||
|
evt.get("danger_level", -1) > prior_level)
|
||||||
if (is_new or level_rose) and self._event_bus and hasattr(adapter, "to_event"):
|
evt["_is_update"] = level_rose # signal to to_event()
|
||||||
evt["_is_update"] = level_rose # signal to to_event()
|
self._delta_emit(name, adapter, evt, force=level_rose)
|
||||||
self._emit_event(adapter, evt)
|
|
||||||
|
|
||||||
self._events[key] = evt # always update stored state
|
self._events[key] = evt # always update stored state
|
||||||
else:
|
else:
|
||||||
for evt in adapter.get_events():
|
for evt in adapter.get_events():
|
||||||
key = (evt["source"], evt["event_id"])
|
key = (evt["source"], evt["event_id"])
|
||||||
is_new = key not in self._events
|
|
||||||
self._events[key] = evt
|
self._events[key] = evt
|
||||||
if is_new and self._event_bus and hasattr(adapter, "to_event"):
|
self._delta_emit(name, adapter, evt)
|
||||||
self._emit_event(adapter, evt)
|
|
||||||
|
# First poll for this adapter is now complete: later polls may emit.
|
||||||
|
self._seeded.add(name)
|
||||||
|
|
||||||
def _emit_event(self, adapter, raw_evt: dict):
|
def _emit_event(self, adapter, raw_evt: dict):
|
||||||
"""Convert raw event to pipeline Event and emit to bus.
|
"""Convert raw event to pipeline Event and emit to bus.
|
||||||
|
|
|
||||||
|
|
@ -330,6 +330,13 @@ class TestAdapterTickFusion:
|
||||||
store = EnvironmentalStore.__new__(EnvironmentalStore)
|
store = EnvironmentalStore.__new__(EnvironmentalStore)
|
||||||
store._events = {}
|
store._events = {}
|
||||||
store._event_bus = bus
|
store._event_bus = bus
|
||||||
|
# Received-delta gate state. This test exercises STEADY-STATE fusion
|
||||||
|
# routing (fusion reaches the bus, raw hotspots never do), so mark firms
|
||||||
|
# as already past its first-poll seed — otherwise the store's
|
||||||
|
# received-delta gate would (correctly) seed this first batch silently.
|
||||||
|
# First-poll seed-silence is covered by test_store_received_delta.
|
||||||
|
store._seen = {}
|
||||||
|
store._seeded = {"firms"}
|
||||||
assert a.tick() is True
|
assert a.tick() is True
|
||||||
store._ingest("firms", a)
|
store._ingest("firms", a)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -162,8 +162,10 @@ def test_second_tick_does_not_rebroadcast_after_commit(monkeypatch):
|
||||||
commit = staged[0]["data"]["_on_broadcast_committed"]
|
commit = staged[0]["data"]["_on_broadcast_committed"]
|
||||||
commit(float(T0)) # marks satpass_events.last_broadcast_at
|
commit(float(T0)) # marks satpass_events.last_broadcast_at
|
||||||
|
|
||||||
# Tick 2 (interval elapsed): same canonical pass must be suppressed.
|
# Tick 2 (interval elapsed): the pass is STILL imminent (AOS 30 min out,
|
||||||
assert adapter.tick(now=T0 + 2000) is False
|
# inside the 60-min lead) so the imminence gate would stage it — but the
|
||||||
|
# satpass_events commit from tick 1 must suppress the re-broadcast.
|
||||||
|
assert adapter.tick(now=T0 - 1800) is False
|
||||||
assert adapter.get_events() == []
|
assert adapter.get_events() == []
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -176,8 +178,9 @@ def test_second_tick_without_commit_is_not_deduped(monkeypatch):
|
||||||
|
|
||||||
adapter = _adapter()
|
adapter = _adapter()
|
||||||
assert adapter.tick(now=T0 - 3600) is True
|
assert adapter.tick(now=T0 - 3600) is True
|
||||||
# No commit fired -> last_broadcast_at still NULL -> re-stages next tick.
|
# No commit fired -> last_broadcast_at still NULL. The pass is still
|
||||||
assert adapter.tick(now=T0 + 2000) is True
|
# imminent at T0-1800 (AOS 30 min out) so it re-stages next tick.
|
||||||
|
assert adapter.tick(now=T0 - 1800) is True
|
||||||
|
|
||||||
|
|
||||||
# ══════════════════════════════════════════════════════════════════════
|
# ══════════════════════════════════════════════════════════════════════
|
||||||
|
|
@ -305,3 +308,107 @@ def test_central_consolidate_feeds_shared_gate_merged(monkeypatch):
|
||||||
"SELECT COUNT(*) AS n FROM satpass_pending WHERE consolidated_id=?",
|
"SELECT COUNT(*) AS n FROM satpass_pending WHERE consolidated_id=?",
|
||||||
(cid,)).fetchone()
|
(cid,)).fetchone()
|
||||||
assert left["n"] == 0
|
assert left["n"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
# ══════════════════════════════════════════════════════════════════════
|
||||||
|
# 6. IMMINENCE BROADCAST TRIGGER (satpass's "just received" analog)
|
||||||
|
# ══════════════════════════════════════════════════════════════════════
|
||||||
|
#
|
||||||
|
# Satpass is NOT an API-delta feed — it recomputes the SAME future passes
|
||||||
|
# every poll. Its "newly received" signal is IMMINENCE: a pass is staged for
|
||||||
|
# broadcast only once its AOS is both in the FUTURE and within the near-term
|
||||||
|
# lead window (default 60 min). A far-future pass is predicted (on schedule)
|
||||||
|
# but not staged; a past AOS is never staged. Combined with the store's
|
||||||
|
# received-delta seen-set, each pass then emits exactly once as it crosses in.
|
||||||
|
|
||||||
|
# The fixed mock pass sits at AOS=T0 (both observers consolidate to one).
|
||||||
|
|
||||||
|
def test_far_future_pass_not_staged(monkeypatch):
|
||||||
|
_enable_satpass_db(dry_run=False)
|
||||||
|
_seed_iss_tle()
|
||||||
|
_patch_observers(monkeypatch, [_BOISE, _TWIN])
|
||||||
|
_patch_predictor(monkeypatch, _two_observer_pass)
|
||||||
|
|
||||||
|
adapter = _adapter() # default broadcast_lead_seconds = 3600 (60 min)
|
||||||
|
# AOS=T0 is 4 hours ahead of now -> well outside the 60-min lead window.
|
||||||
|
assert adapter.tick(now=T0 - 4 * 3600) is False
|
||||||
|
assert adapter.get_events() == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_imminent_pass_is_staged(monkeypatch):
|
||||||
|
_enable_satpass_db(dry_run=False)
|
||||||
|
_seed_iss_tle()
|
||||||
|
_patch_observers(monkeypatch, [_BOISE, _TWIN])
|
||||||
|
_patch_predictor(monkeypatch, _two_observer_pass)
|
||||||
|
|
||||||
|
adapter = _adapter()
|
||||||
|
# AOS=T0 is 30 min ahead of now -> inside the 60-min lead window.
|
||||||
|
assert adapter.tick(now=T0 - 1800) is True
|
||||||
|
staged = adapter.get_events()
|
||||||
|
assert len(staged) == 1
|
||||||
|
assert staged[0]["event_id"] == f"25544:{T0 // 3600}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_past_aos_never_staged(monkeypatch):
|
||||||
|
_enable_satpass_db(dry_run=False)
|
||||||
|
_seed_iss_tle()
|
||||||
|
_patch_observers(monkeypatch, [_BOISE, _TWIN])
|
||||||
|
_patch_predictor(monkeypatch, _two_observer_pass)
|
||||||
|
|
||||||
|
adapter = _adapter()
|
||||||
|
# now is 10 min AFTER AOS=T0 -> the pass is in the past, never staged.
|
||||||
|
assert adapter.tick(now=T0 + 600) is False
|
||||||
|
assert adapter.get_events() == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_configurable_lead_window(monkeypatch):
|
||||||
|
_enable_satpass_db(dry_run=False)
|
||||||
|
_seed_iss_tle()
|
||||||
|
_patch_observers(monkeypatch, [_BOISE, _TWIN])
|
||||||
|
_patch_predictor(monkeypatch, _two_observer_pass)
|
||||||
|
|
||||||
|
# With a 3-hour lead, an AOS 2 hours out is now imminent.
|
||||||
|
adapter = _adapter(broadcast_lead_seconds=3 * 3600)
|
||||||
|
assert adapter.tick(now=T0 - 2 * 3600) is True
|
||||||
|
assert len(adapter.get_events()) == 1
|
||||||
|
|
||||||
|
|
||||||
|
# ══════════════════════════════════════════════════════════════════════
|
||||||
|
# 7. NORAD_IDS FILTER IS STRICT (non-listed sats never predicted)
|
||||||
|
# ══════════════════════════════════════════════════════════════════════
|
||||||
|
|
||||||
|
# A non-ISS TLE (a stand-in for a GOES/METEOR/FENGYUN weather sat the fetcher
|
||||||
|
# stocks via tle_groups). Different NORAD so a filter break would surface as a
|
||||||
|
# second consolidated pass with a different canonical id.
|
||||||
|
OTHER_L1 = "1 43226U 18022A 26182.50000000 .00000000 00000-0 00000-0 0 9999"
|
||||||
|
OTHER_L2 = "2 43226 0.0300 100.0000 0001000 90.0000 270.0000 1.00270000 12345"
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_other_tle():
|
||||||
|
conn = get_db()
|
||||||
|
fresh = datetime.now(timezone.utc).isoformat()
|
||||||
|
upsert_tle(conn, 43226, "GOES-17", OTHER_L1, OTHER_L2, fresh)
|
||||||
|
|
||||||
|
|
||||||
|
def test_norad_ids_filter_excludes_non_listed_sats(monkeypatch):
|
||||||
|
_enable_satpass_db(dry_run=False)
|
||||||
|
_seed_iss_tle() # 25544 — configured
|
||||||
|
_seed_other_tle() # 43226 — fresh but NOT in norad_ids
|
||||||
|
_patch_observers(monkeypatch, [_BOISE, _TWIN])
|
||||||
|
# Predictor would return a pass for ANY satellite (keyed on observer lat),
|
||||||
|
# so if 43226 leaked through it would produce a second staged pass.
|
||||||
|
_patch_predictor(monkeypatch, _two_observer_pass)
|
||||||
|
|
||||||
|
adapter = _adapter() # norad_ids=[25544]
|
||||||
|
assert adapter.tick(now=T0 - 1800) is True
|
||||||
|
staged = adapter.get_events()
|
||||||
|
# Only the configured NORAD is predicted, despite 43226 being fresh.
|
||||||
|
assert len(staged) == 1
|
||||||
|
assert staged[0]["norad_id"] == 25544
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_norad_ids_handles_comma_string():
|
||||||
|
# A GUI-persisted comma string must NOT be char-iterated into garbage ids.
|
||||||
|
assert SatpassAdapter._parse_norad_ids("25544, 33591") == [25544, 33591]
|
||||||
|
assert SatpassAdapter._parse_norad_ids([25544, "33591"]) == [25544, 33591]
|
||||||
|
assert SatpassAdapter._parse_norad_ids([]) == []
|
||||||
|
|
|
||||||
156
work/tests/test_store_received_delta.py
Normal file
156
work/tests/test_store_received_delta.py
Normal file
|
|
@ -0,0 +1,156 @@
|
||||||
|
"""Received-delta gate tests for the native EnvironmentalStore path.
|
||||||
|
|
||||||
|
The operator's required model: a native adapter broadcasts an item ONLY when
|
||||||
|
it was newly RECEIVED from the API this poll — never by scanning an accumulated
|
||||||
|
backlog. The store enforces this with an in-memory, per-adapter seen-set:
|
||||||
|
|
||||||
|
* the FIRST data-bearing poll for an adapter records every current item key
|
||||||
|
as "seen" and broadcasts NOTHING (that batch is pre-existing backlog);
|
||||||
|
* every later poll broadcasts only items whose key is not already seen;
|
||||||
|
* a fresh store (process restart) has empty sets, so its next poll is again a
|
||||||
|
"first poll" that re-seeds silently — backlog is never re-broadcast.
|
||||||
|
|
||||||
|
These tests drive the real EnvironmentalStore + EventBus with a fake adapter
|
||||||
|
whose per-poll batch we control, and assert exactly which events reach the bus.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from meshai.env.store import EnvironmentalStore
|
||||||
|
from meshai.config import EnvironmentalConfig
|
||||||
|
from meshai.notifications.pipeline.bus import EventBus
|
||||||
|
from meshai.notifications.events import make_event
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeAdapter:
|
||||||
|
"""Native-adapter stand-in: returns a controllable batch of raw events.
|
||||||
|
|
||||||
|
Raw events carry the (source, event_id) shape every native adapter emits,
|
||||||
|
which is exactly what the store's seen-set keys on. Each poll's batch is
|
||||||
|
set by the test via `set_batch`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, source: str = "roads511"):
|
||||||
|
self._source = source
|
||||||
|
self._batch: list[dict] = []
|
||||||
|
|
||||||
|
def set_batch(self, ids: list[str]) -> None:
|
||||||
|
self._batch = [
|
||||||
|
{"source": self._source, "event_id": eid, "fetched_at": 0}
|
||||||
|
for eid in ids
|
||||||
|
]
|
||||||
|
|
||||||
|
def tick(self) -> bool:
|
||||||
|
# Data is "fetched" every poll; the store decides what is new.
|
||||||
|
return True
|
||||||
|
|
||||||
|
def get_events(self) -> list:
|
||||||
|
return list(self._batch)
|
||||||
|
|
||||||
|
def to_event(self, raw_evt: dict):
|
||||||
|
eid = raw_evt["event_id"]
|
||||||
|
return make_event(
|
||||||
|
source=raw_evt["source"],
|
||||||
|
category="test_delta", # no decider registered -> emits directly
|
||||||
|
severity="routine",
|
||||||
|
title=eid,
|
||||||
|
summary=eid,
|
||||||
|
group_key=eid,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_store(adapter_name: str = "roads511"):
|
||||||
|
"""Build a store with NO real adapters, then inject one fake adapter."""
|
||||||
|
bus = EventBus()
|
||||||
|
captured: list = []
|
||||||
|
bus.subscribe(lambda e: captured.append(e))
|
||||||
|
|
||||||
|
# All feeds default disabled -> zero native adapters register.
|
||||||
|
store = EnvironmentalStore(EnvironmentalConfig(), event_bus=bus)
|
||||||
|
adapter = _FakeAdapter(source="511")
|
||||||
|
store._adapters[adapter_name] = adapter
|
||||||
|
return store, adapter, captured
|
||||||
|
|
||||||
|
|
||||||
|
def _emitted_ids(captured) -> list[str]:
|
||||||
|
return [e.title for e in captured]
|
||||||
|
|
||||||
|
|
||||||
|
def test_first_poll_seeds_and_broadcasts_nothing():
|
||||||
|
store, adapter, captured = _make_store()
|
||||||
|
adapter.set_batch(["A", "B", "C"])
|
||||||
|
|
||||||
|
store.refresh() # poll 1 — the backlog
|
||||||
|
|
||||||
|
assert captured == [], "first poll must broadcast NOTHING (backlog seed)"
|
||||||
|
|
||||||
|
|
||||||
|
def test_second_poll_emits_only_newly_received():
|
||||||
|
store, adapter, captured = _make_store()
|
||||||
|
|
||||||
|
adapter.set_batch(["A", "B", "C"])
|
||||||
|
store.refresh() # poll 1: seed
|
||||||
|
assert _emitted_ids(captured) == []
|
||||||
|
|
||||||
|
adapter.set_batch(["A", "B", "C", "D"])
|
||||||
|
store.refresh() # poll 2: only D is new
|
||||||
|
assert _emitted_ids(captured) == ["D"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_unchanged_poll_emits_nothing():
|
||||||
|
store, adapter, captured = _make_store()
|
||||||
|
|
||||||
|
adapter.set_batch(["A", "B", "C"])
|
||||||
|
store.refresh() # poll 1: seed
|
||||||
|
adapter.set_batch(["A", "B", "C", "D"])
|
||||||
|
store.refresh() # poll 2: D
|
||||||
|
adapter.set_batch(["A", "B", "C", "D"])
|
||||||
|
store.refresh() # poll 3: nothing new
|
||||||
|
|
||||||
|
assert _emitted_ids(captured) == ["D"], "poll 3 has no new items"
|
||||||
|
|
||||||
|
|
||||||
|
def test_restart_reseeds_and_never_rebroadcasts_backlog():
|
||||||
|
# Process 1 sees A,B,C,D and broadcasts D.
|
||||||
|
store1, adapter1, cap1 = _make_store()
|
||||||
|
adapter1.set_batch(["A", "B", "C"])
|
||||||
|
store1.refresh()
|
||||||
|
adapter1.set_batch(["A", "B", "C", "D"])
|
||||||
|
store1.refresh()
|
||||||
|
assert _emitted_ids(cap1) == ["D"]
|
||||||
|
|
||||||
|
# RESTART: a fresh store has an empty seen-set. The SAME backlog [A,B,C,D]
|
||||||
|
# arriving on its first poll must be re-seeded silently, not re-broadcast.
|
||||||
|
store2, adapter2, cap2 = _make_store()
|
||||||
|
adapter2.set_batch(["A", "B", "C", "D"])
|
||||||
|
store2.refresh()
|
||||||
|
assert cap2 == [], "restart must NEVER re-broadcast the existing backlog"
|
||||||
|
|
||||||
|
# And a genuinely new item after the restart still broadcasts once.
|
||||||
|
adapter2.set_batch(["A", "B", "C", "D", "E"])
|
||||||
|
store2.refresh()
|
||||||
|
assert _emitted_ids(cap2) == ["E"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_stable_key_prevents_reemit_when_batch_reorders():
|
||||||
|
# The same real-world items in a different order are NOT "newly received".
|
||||||
|
store, adapter, captured = _make_store()
|
||||||
|
adapter.set_batch(["A", "B", "C"])
|
||||||
|
store.refresh() # seed
|
||||||
|
adapter.set_batch(["C", "A", "B"]) # reordered, same items
|
||||||
|
store.refresh()
|
||||||
|
assert captured == [], "reordering the same items emits nothing"
|
||||||
|
|
||||||
|
|
||||||
|
def test_disabled_for_days_then_backlog_is_not_broadcast():
|
||||||
|
# Simulate an adapter that was off for days: its first poll after coming
|
||||||
|
# back returns a large accumulated backlog. None of it may broadcast.
|
||||||
|
store, adapter, captured = _make_store()
|
||||||
|
backlog = [f"evt{i}" for i in range(200)]
|
||||||
|
adapter.set_batch(backlog)
|
||||||
|
store.refresh() # first poll after re-enable
|
||||||
|
assert captured == [], "a days-old backlog is seeded silently, never sent"
|
||||||
|
|
||||||
|
# Only a truly new arrival afterward is announced.
|
||||||
|
adapter.set_batch(backlog + ["fresh"])
|
||||||
|
store.refresh()
|
||||||
|
assert _emitted_ids(captured) == ["fresh"]
|
||||||
Loading…
Add table
Add a link
Reference in a new issue