meshai/tests/test_central_consumer.py
K7ZVX a491684861 fix(central): v0.4 C.3.1 -- preserve secret refs in save_section + deliver_policy=NEW (no backlog flood)
Fixes the two real bugs C.3 surfaced when flipping usgs_quake to central.

BUG #1 -- GUI save dropped ${VAR} secret refs (config_loader.save_section).
  before: A GUI PUT round-trips the *interpolated* secret value (GET returns the
          resolved key string, e.g. the real TomTom key). save_section's
          check_secrets saw a literal string at a SECRET_FIELDS path, didn't
          recognize it as a ref, and DROPPED it -- losing the on-disk
          ${TOMTOM_API_KEY} placeholder. C.3's flip PUT stripped TomTom's key.
  after:  check_secrets now reads the raw on-disk value (pre-interpolation) for
          each secret field and decides three ways:
            on-disk ${VAR} and new == resolved(VAR)  -> keep the ${VAR} ref
            on-disk ${VAR} and new != resolved(VAR)  -> intentional change, store it
            no on-disk ${VAR} ref                    -> reject (never write a raw
                                                        secret to a domain file)
          ${VAR} resolution mirrors load: os.environ first, then /data/secrets/.env.
          The common case (GUI re-saves unchanged config) now preserves the
          placeholder instead of dropping it.

BUG #2 -- CentralConsumer replayed the entire retained backlog on first flip.
  before: js.subscribe(...) with no config -> default deliver_policy=all. Fine
          for quake (682 msgs) but would flood the bus with ~330k traffic_flow
          messages on first flip.
  after:  consumer_config() -> ConsumerConfig(deliver_policy=DeliverPolicy.NEW):
          only messages published AFTER consumer creation. meshai won't see the
          backlog on first flip -- acceptable, Central is a live firehose for
          current events. (NOT geo-filtering -- that's a Central-side issue filed
          separately for the Central project.)

Files: meshai/config_loader.py (save_section secret preservation),
meshai/central/consumer.py (consumer_config() + deliver_policy=NEW),
tests/test_save_section_secret_preserve.py (new),
tests/test_central_consumer.py (deliver_policy assertion).

Verification:
- (A) py_compile clean on config_loader.py + consumer.py.
- (C) pytest -q: 276 passed (272 + 4 new -- preserve-unchanged-ref,
  changed-value-written, no-placeholder-still-rejects, deliver_policy=NEW).
  The C.2.1 strip test still passes (no placeholder -> reject).
- (D) In-prod (rebuilt): GET+PUT /api/config/environmental round-trip ->
  {"saved":true}; on-disk traffic.api_key stayed '${TOMTOM_API_KEY}'
  (SECRET_REF_PRESERVED: True), not the literal key; disk restored to baseline.
  consumer_config().deliver_policy == DeliverPolicy.NEW in the built image.

Follow-up for D rollout: the durable 'meshai-v04-central_quake_' created during
C.3 was made with deliver_policy=all; re-flipping a domain may need that stale
durable deleted on the Central NATS server first (config mismatch on re-subscribe).

D rollout (remaining domains) is now safe: GUI flips preserve secret refs and
new subscriptions don't replay huge backlogs.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-28 04:55:20 +00:00

146 lines
5.1 KiB
Python

"""v0.4 C.1: Central connector backend — normalization, lifecycle, source gate."""
import asyncio
import json
import pytest
from meshai.config import EnvironmentalConfig
from meshai.central.consumer import CentralConsumer, map_category, map_severity
from meshai.notifications.pipeline.bus import EventBus
def make_consumer():
env = EnvironmentalConfig()
bus = EventBus()
rec = []
bus.subscribe(rec.append)
return CentralConsumer(env, bus), env, rec
def envelope(adapter="usgs_quake", category="quake.event", severity=2,
eid="us6000abcd", centroid=(-114.5, 42.6), upstream=None,
time="2026-05-27T12:00:00Z", expires=None):
return {
"id": eid, "source": "central.echo6.co",
"type": f"central.{category}.v1", "time": time,
"centralcategory": category, "centralseverity": severity,
"specversion": "1.0", "datacontenttype": "application/json",
"data": {
"id": eid, "adapter": adapter, "category": category,
"time": time, "expires": expires, "severity": severity,
"geo": {"centroid": list(centroid), "bbox": None,
"regions": ["US-ID"], "primary_region": "US-ID"},
"data": upstream if upstream is not None else {"magnitude": 4.2, "place": "near Twin Falls"},
},
}
class FakeMsg:
def __init__(self, subject, env):
self.subject = subject
self.data = json.dumps(env).encode()
self.acked = False
async def ack(self):
self.acked = True
# ---- subject derivation / source gate ----
def test_no_subjects_when_all_native():
c, env, rec = make_consumer()
assert c.subjects() == []
def test_subjects_when_central():
c, env, rec = make_consumer()
env.usgs_quake.feed_source = "central"
assert "central.quake.>" in c.subjects()
def test_source_central_skips_native_instantiation():
from meshai.env.store import EnvironmentalStore
env = EnvironmentalConfig()
env.enabled = True
env.usgs_quake.enabled = True
env.usgs_quake.feed_source = "central" # should be skipped natively
env.nws.enabled = True # native -> present
store = EnvironmentalStore(config=env, region_anchors=[], event_bus=None)
assert "usgs_quake" not in store._adapters
assert "nws" in store._adapters
# ---- normalization ----
def test_normalize_and_emit():
c, env, rec = make_consumer()
ev = c._handle("central.quake.event.moderate", json.dumps(envelope()).encode())
assert ev is not None
assert len(rec) == 1
e = rec[0]
assert e.source == "usgs_quake"
assert e.category == "earthquake_event"
assert e.severity == "priority" # central severity 2
assert e.lat == 42.6 and e.lon == -114.5 # [lon,lat] -> (lat,lon)
assert e.group_key == "us6000abcd"
assert e.region == "US-ID"
assert e.data.get("magnitude") == 4.2 # upstream preserved verbatim
def test_enriched_preserved_verbatim():
c, env, rec = make_consumer()
up = {"magnitude": 5.1, "_enriched": {"geocoder": {"state": "Idaho"}, "usgs_stats": {"x": 1}}}
ev = c._handle("central.quake.event.strong", json.dumps(envelope(severity=4, upstream=up)).encode())
assert ev.severity == "immediate"
assert ev.data["_enriched"]["geocoder"]["state"] == "Idaho"
assert ev.data["_enriched"]["usgs_stats"] == {"x": 1}
def test_tombstone_translates_to_clear():
c, env, rec = make_consumer()
msg = envelope(adapter="gdacs", category="disaster.fl.removed", severity=0, eid="FL1103885:removed")
ev = c._handle("central.disaster.fl.removed.austria", json.dumps(msg).encode())
assert ev is not None
assert ev.group_key == "FL1103885" # ':removed' stripped -> matches original
assert ev.data.get("_central_tombstone") is True
def test_severity_mapping():
assert map_severity(0) == "routine"
assert map_severity(1) == "routine"
assert map_severity(2) == "priority"
assert map_severity(3) == "immediate"
assert map_severity(4) == "immediate"
assert map_severity(None) == "routine"
def test_category_mapping():
assert map_category("wx.alert.severe_thunderstorm_warning") == "weather_warning"
assert map_category("quake.event") == "earthquake_event"
assert map_category("fire.hotspot.viirs_noaa20.high") == "wildfire_hotspot"
assert map_category("hydro.00060.usgs.06901250") == "stream_flow"
# ---- async callback path ----
def test_on_message_emits_and_acks():
c, env, rec = make_consumer()
msg = FakeMsg("central.quake.event.moderate", envelope())
asyncio.run(c._on_message(msg))
assert msg.acked is True
assert len(rec) == 1
def test_start_no_op_when_all_native():
"""start() is a no-op (no NATS connect) when no adapter is central."""
c, env, rec = make_consumer()
asyncio.run(c.start()) # must not raise / must not require NATS
assert c._nc is None
def test_consumer_config_uses_deliver_policy_new():
"""C.3.1: Central subscriptions use deliver_policy=NEW (no full-backlog replay)."""
from meshai.central.consumer import consumer_config
from nats.js.api import DeliverPolicy
assert consumer_config().deliver_policy == DeliverPolicy.NEW