mirror of
https://github.com/zvx-echo6/meshai.git
synced 2026-06-11 01:14:45 +02:00
Fixes the two real bugs C.3 surfaced when flipping usgs_quake to central. BUG #1 -- GUI save dropped ${VAR} secret refs (config_loader.save_section). before: A GUI PUT round-trips the *interpolated* secret value (GET returns the resolved key string, e.g. the real TomTom key). save_section's check_secrets saw a literal string at a SECRET_FIELDS path, didn't recognize it as a ref, and DROPPED it -- losing the on-disk ${TOMTOM_API_KEY} placeholder. C.3's flip PUT stripped TomTom's key. after: check_secrets now reads the raw on-disk value (pre-interpolation) for each secret field and decides three ways: on-disk ${VAR} and new == resolved(VAR) -> keep the ${VAR} ref on-disk ${VAR} and new != resolved(VAR) -> intentional change, store it no on-disk ${VAR} ref -> reject (never write a raw secret to a domain file) ${VAR} resolution mirrors load: os.environ first, then /data/secrets/.env. The common case (GUI re-saves unchanged config) now preserves the placeholder instead of dropping it. BUG #2 -- CentralConsumer replayed the entire retained backlog on first flip. before: js.subscribe(...) with no config -> default deliver_policy=all. Fine for quake (682 msgs) but would flood the bus with ~330k traffic_flow messages on first flip. after: consumer_config() -> ConsumerConfig(deliver_policy=DeliverPolicy.NEW): only messages published AFTER consumer creation. meshai won't see the backlog on first flip -- acceptable, Central is a live firehose for current events. (NOT geo-filtering -- that's a Central-side issue filed separately for the Central project.) Files: meshai/config_loader.py (save_section secret preservation), meshai/central/consumer.py (consumer_config() + deliver_policy=NEW), tests/test_save_section_secret_preserve.py (new), tests/test_central_consumer.py (deliver_policy assertion). Verification: - (A) py_compile clean on config_loader.py + consumer.py. - (C) pytest -q: 276 passed (272 + 4 new -- preserve-unchanged-ref, changed-value-written, no-placeholder-still-rejects, deliver_policy=NEW). The C.2.1 strip test still passes (no placeholder -> reject). - (D) In-prod (rebuilt): GET+PUT /api/config/environmental round-trip -> {"saved":true}; on-disk traffic.api_key stayed '${TOMTOM_API_KEY}' (SECRET_REF_PRESERVED: True), not the literal key; disk restored to baseline. consumer_config().deliver_policy == DeliverPolicy.NEW in the built image. Follow-up for D rollout: the durable 'meshai-v04-central_quake_' created during C.3 was made with deliver_policy=all; re-flipping a domain may need that stale durable deleted on the Central NATS server first (config mismatch on re-subscribe). D rollout (remaining domains) is now safe: GUI flips preserve secret refs and new subscriptions don't replay huge backlogs. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
146 lines
5.1 KiB
Python
146 lines
5.1 KiB
Python
"""v0.4 C.1: Central connector backend — normalization, lifecycle, source gate."""
|
|
|
|
import asyncio
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from meshai.config import EnvironmentalConfig
|
|
from meshai.central.consumer import CentralConsumer, map_category, map_severity
|
|
from meshai.notifications.pipeline.bus import EventBus
|
|
|
|
|
|
def make_consumer():
|
|
env = EnvironmentalConfig()
|
|
bus = EventBus()
|
|
rec = []
|
|
bus.subscribe(rec.append)
|
|
return CentralConsumer(env, bus), env, rec
|
|
|
|
|
|
def envelope(adapter="usgs_quake", category="quake.event", severity=2,
|
|
eid="us6000abcd", centroid=(-114.5, 42.6), upstream=None,
|
|
time="2026-05-27T12:00:00Z", expires=None):
|
|
return {
|
|
"id": eid, "source": "central.echo6.co",
|
|
"type": f"central.{category}.v1", "time": time,
|
|
"centralcategory": category, "centralseverity": severity,
|
|
"specversion": "1.0", "datacontenttype": "application/json",
|
|
"data": {
|
|
"id": eid, "adapter": adapter, "category": category,
|
|
"time": time, "expires": expires, "severity": severity,
|
|
"geo": {"centroid": list(centroid), "bbox": None,
|
|
"regions": ["US-ID"], "primary_region": "US-ID"},
|
|
"data": upstream if upstream is not None else {"magnitude": 4.2, "place": "near Twin Falls"},
|
|
},
|
|
}
|
|
|
|
|
|
class FakeMsg:
|
|
def __init__(self, subject, env):
|
|
self.subject = subject
|
|
self.data = json.dumps(env).encode()
|
|
self.acked = False
|
|
|
|
async def ack(self):
|
|
self.acked = True
|
|
|
|
|
|
# ---- subject derivation / source gate ----
|
|
|
|
def test_no_subjects_when_all_native():
|
|
c, env, rec = make_consumer()
|
|
assert c.subjects() == []
|
|
|
|
|
|
def test_subjects_when_central():
|
|
c, env, rec = make_consumer()
|
|
env.usgs_quake.feed_source = "central"
|
|
assert "central.quake.>" in c.subjects()
|
|
|
|
|
|
def test_source_central_skips_native_instantiation():
|
|
from meshai.env.store import EnvironmentalStore
|
|
env = EnvironmentalConfig()
|
|
env.enabled = True
|
|
env.usgs_quake.enabled = True
|
|
env.usgs_quake.feed_source = "central" # should be skipped natively
|
|
env.nws.enabled = True # native -> present
|
|
store = EnvironmentalStore(config=env, region_anchors=[], event_bus=None)
|
|
assert "usgs_quake" not in store._adapters
|
|
assert "nws" in store._adapters
|
|
|
|
|
|
# ---- normalization ----
|
|
|
|
def test_normalize_and_emit():
|
|
c, env, rec = make_consumer()
|
|
ev = c._handle("central.quake.event.moderate", json.dumps(envelope()).encode())
|
|
assert ev is not None
|
|
assert len(rec) == 1
|
|
e = rec[0]
|
|
assert e.source == "usgs_quake"
|
|
assert e.category == "earthquake_event"
|
|
assert e.severity == "priority" # central severity 2
|
|
assert e.lat == 42.6 and e.lon == -114.5 # [lon,lat] -> (lat,lon)
|
|
assert e.group_key == "us6000abcd"
|
|
assert e.region == "US-ID"
|
|
assert e.data.get("magnitude") == 4.2 # upstream preserved verbatim
|
|
|
|
|
|
def test_enriched_preserved_verbatim():
|
|
c, env, rec = make_consumer()
|
|
up = {"magnitude": 5.1, "_enriched": {"geocoder": {"state": "Idaho"}, "usgs_stats": {"x": 1}}}
|
|
ev = c._handle("central.quake.event.strong", json.dumps(envelope(severity=4, upstream=up)).encode())
|
|
assert ev.severity == "immediate"
|
|
assert ev.data["_enriched"]["geocoder"]["state"] == "Idaho"
|
|
assert ev.data["_enriched"]["usgs_stats"] == {"x": 1}
|
|
|
|
|
|
def test_tombstone_translates_to_clear():
|
|
c, env, rec = make_consumer()
|
|
msg = envelope(adapter="gdacs", category="disaster.fl.removed", severity=0, eid="FL1103885:removed")
|
|
ev = c._handle("central.disaster.fl.removed.austria", json.dumps(msg).encode())
|
|
assert ev is not None
|
|
assert ev.group_key == "FL1103885" # ':removed' stripped -> matches original
|
|
assert ev.data.get("_central_tombstone") is True
|
|
|
|
|
|
def test_severity_mapping():
|
|
assert map_severity(0) == "routine"
|
|
assert map_severity(1) == "routine"
|
|
assert map_severity(2) == "priority"
|
|
assert map_severity(3) == "immediate"
|
|
assert map_severity(4) == "immediate"
|
|
assert map_severity(None) == "routine"
|
|
|
|
|
|
def test_category_mapping():
|
|
assert map_category("wx.alert.severe_thunderstorm_warning") == "weather_warning"
|
|
assert map_category("quake.event") == "earthquake_event"
|
|
assert map_category("fire.hotspot.viirs_noaa20.high") == "wildfire_hotspot"
|
|
assert map_category("hydro.00060.usgs.06901250") == "stream_flow"
|
|
|
|
|
|
# ---- async callback path ----
|
|
|
|
def test_on_message_emits_and_acks():
|
|
c, env, rec = make_consumer()
|
|
msg = FakeMsg("central.quake.event.moderate", envelope())
|
|
asyncio.run(c._on_message(msg))
|
|
assert msg.acked is True
|
|
assert len(rec) == 1
|
|
|
|
|
|
def test_start_no_op_when_all_native():
|
|
"""start() is a no-op (no NATS connect) when no adapter is central."""
|
|
c, env, rec = make_consumer()
|
|
asyncio.run(c.start()) # must not raise / must not require NATS
|
|
assert c._nc is None
|
|
|
|
|
|
def test_consumer_config_uses_deliver_policy_new():
|
|
"""C.3.1: Central subscriptions use deliver_policy=NEW (no full-backlog replay)."""
|
|
from meshai.central.consumer import consumer_config
|
|
from nats.js.api import DeliverPolicy
|
|
assert consumer_config().deliver_policy == DeliverPolicy.NEW
|