feat(env): native adapters broadcast only newly-RECEIVED items (no backlog) (#43)

Replace the native path's "scan accumulated state + suppress what we've
already broadcast" model with "broadcast only what newly arrived from the
API this poll." Storage is unchanged (self._events + firms_pixels etc. are
populated for EVERY received item, so the LLM/get_active backlog is intact);
only the BROADCAST decision changes.

- env/store.py: per-adapter in-memory seen-set (_seen) + _seeded. First
  data-bearing poll for an adapter seeds keys and emits NOTHING (that batch
  is pre-existing backlog); later polls emit only keys not seen before.
  Restart => empty sets => next poll re-seeds silently. Structurally
  impossible to broadcast backlog on cold start / restart / re-enable.
  Key = external_id -> event_id -> content hash, namespaced per adapter.
  self._events[key]=evt still runs unconditionally (storage preserved).
- Fixes the ~175 (roads511) / ~782 (wzdx) cold-start bursts AND the latent
  quake/nws version (they only looked safe because Central pre-populated
  their broadcast tables).
- env/satpass.py: broadcast on AOS IMMINENCE (now < aos <= now+lead,
  broadcast_lead_seconds default 3600), future-only; window_hours still
  governs prediction depth. Strict norad_ids post-filter + fixed
  _parse_norad_ids char-iteration bug (cause of GOES/METEOR leak).
- Central path + broadcast-state tables untouched (native-only gate).

13 new tests; full suite at 10-failure baseline (1697 passed).

Co-authored-by: Matt Johnson <mj@k7zvx.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
malice 2026-07-05 14:38:40 -06:00 committed by GitHub
commit 0c76a806b8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 421 additions and 24 deletions

View file

@ -0,0 +1,156 @@
"""Received-delta gate tests for the native EnvironmentalStore path.
The operator's required model: a native adapter broadcasts an item ONLY when
it was newly RECEIVED from the API this poll never by scanning an accumulated
backlog. The store enforces this with an in-memory, per-adapter seen-set:
* the FIRST data-bearing poll for an adapter records every current item key
as "seen" and broadcasts NOTHING (that batch is pre-existing backlog);
* every later poll broadcasts only items whose key is not already seen;
* a fresh store (process restart) has empty sets, so its next poll is again a
"first poll" that re-seeds silently backlog is never re-broadcast.
These tests drive the real EnvironmentalStore + EventBus with a fake adapter
whose per-poll batch we control, and assert exactly which events reach the bus.
"""
from __future__ import annotations
from meshai.env.store import EnvironmentalStore
from meshai.config import EnvironmentalConfig
from meshai.notifications.pipeline.bus import EventBus
from meshai.notifications.events import make_event
class _FakeAdapter:
"""Native-adapter stand-in: returns a controllable batch of raw events.
Raw events carry the (source, event_id) shape every native adapter emits,
which is exactly what the store's seen-set keys on. Each poll's batch is
set by the test via `set_batch`.
"""
def __init__(self, source: str = "roads511"):
self._source = source
self._batch: list[dict] = []
def set_batch(self, ids: list[str]) -> None:
self._batch = [
{"source": self._source, "event_id": eid, "fetched_at": 0}
for eid in ids
]
def tick(self) -> bool:
# Data is "fetched" every poll; the store decides what is new.
return True
def get_events(self) -> list:
return list(self._batch)
def to_event(self, raw_evt: dict):
eid = raw_evt["event_id"]
return make_event(
source=raw_evt["source"],
category="test_delta", # no decider registered -> emits directly
severity="routine",
title=eid,
summary=eid,
group_key=eid,
)
def _make_store(adapter_name: str = "roads511"):
"""Build a store with NO real adapters, then inject one fake adapter."""
bus = EventBus()
captured: list = []
bus.subscribe(lambda e: captured.append(e))
# All feeds default disabled -> zero native adapters register.
store = EnvironmentalStore(EnvironmentalConfig(), event_bus=bus)
adapter = _FakeAdapter(source="511")
store._adapters[adapter_name] = adapter
return store, adapter, captured
def _emitted_ids(captured) -> list[str]:
return [e.title for e in captured]
def test_first_poll_seeds_and_broadcasts_nothing():
store, adapter, captured = _make_store()
adapter.set_batch(["A", "B", "C"])
store.refresh() # poll 1 — the backlog
assert captured == [], "first poll must broadcast NOTHING (backlog seed)"
def test_second_poll_emits_only_newly_received():
store, adapter, captured = _make_store()
adapter.set_batch(["A", "B", "C"])
store.refresh() # poll 1: seed
assert _emitted_ids(captured) == []
adapter.set_batch(["A", "B", "C", "D"])
store.refresh() # poll 2: only D is new
assert _emitted_ids(captured) == ["D"]
def test_unchanged_poll_emits_nothing():
store, adapter, captured = _make_store()
adapter.set_batch(["A", "B", "C"])
store.refresh() # poll 1: seed
adapter.set_batch(["A", "B", "C", "D"])
store.refresh() # poll 2: D
adapter.set_batch(["A", "B", "C", "D"])
store.refresh() # poll 3: nothing new
assert _emitted_ids(captured) == ["D"], "poll 3 has no new items"
def test_restart_reseeds_and_never_rebroadcasts_backlog():
# Process 1 sees A,B,C,D and broadcasts D.
store1, adapter1, cap1 = _make_store()
adapter1.set_batch(["A", "B", "C"])
store1.refresh()
adapter1.set_batch(["A", "B", "C", "D"])
store1.refresh()
assert _emitted_ids(cap1) == ["D"]
# RESTART: a fresh store has an empty seen-set. The SAME backlog [A,B,C,D]
# arriving on its first poll must be re-seeded silently, not re-broadcast.
store2, adapter2, cap2 = _make_store()
adapter2.set_batch(["A", "B", "C", "D"])
store2.refresh()
assert cap2 == [], "restart must NEVER re-broadcast the existing backlog"
# And a genuinely new item after the restart still broadcasts once.
adapter2.set_batch(["A", "B", "C", "D", "E"])
store2.refresh()
assert _emitted_ids(cap2) == ["E"]
def test_stable_key_prevents_reemit_when_batch_reorders():
# The same real-world items in a different order are NOT "newly received".
store, adapter, captured = _make_store()
adapter.set_batch(["A", "B", "C"])
store.refresh() # seed
adapter.set_batch(["C", "A", "B"]) # reordered, same items
store.refresh()
assert captured == [], "reordering the same items emits nothing"
def test_disabled_for_days_then_backlog_is_not_broadcast():
# Simulate an adapter that was off for days: its first poll after coming
# back returns a large accumulated backlog. None of it may broadcast.
store, adapter, captured = _make_store()
backlog = [f"evt{i}" for i in range(200)]
adapter.set_batch(backlog)
store.refresh() # first poll after re-enable
assert captured == [], "a days-old backlog is seeded silently, never sent"
# Only a truly new arrival afterward is announced.
adapter.set_batch(backlog + ["fresh"])
store.refresh()
assert _emitted_ids(captured) == ["fresh"]