fix(central): v0.4 C.3.1 -- preserve secret refs in save_section + deliver_policy=NEW (no backlog flood)
Fixes the two real bugs C.3 surfaced when flipping usgs_quake to central.
BUG #1 -- GUI save dropped ${VAR} secret refs (config_loader.save_section).
before: A GUI PUT round-trips the *interpolated* secret value (GET returns the
resolved key string, e.g. the real TomTom key). save_section's
check_secrets saw a literal string at a SECRET_FIELDS path, didn't
recognize it as a ref, and DROPPED it -- losing the on-disk
${TOMTOM_API_KEY} placeholder. C.3's flip PUT stripped TomTom's key.
after: check_secrets now reads the raw on-disk value (pre-interpolation) for
each secret field and decides three ways:
on-disk ${VAR} and new == resolved(VAR) -> keep the ${VAR} ref
on-disk ${VAR} and new != resolved(VAR) -> intentional change, store it
no on-disk ${VAR} ref -> reject (never write a raw
secret to a domain file)
${VAR} resolution mirrors load: os.environ first, then /data/secrets/.env.
The common case (GUI re-saves unchanged config) now preserves the
placeholder instead of dropping it.
BUG #2 -- CentralConsumer replayed the entire retained backlog on first flip.
before: js.subscribe(...) with no config -> default deliver_policy=all. Fine
for quake (682 msgs) but would flood the bus with ~330k traffic_flow
messages on first flip.
after: consumer_config() -> ConsumerConfig(deliver_policy=DeliverPolicy.NEW):
only messages published AFTER consumer creation. meshai won't see the
backlog on first flip -- acceptable, Central is a live firehose for
current events. (NOT geo-filtering -- that's a Central-side issue filed
separately for the Central project.)
Files: meshai/config_loader.py (save_section secret preservation),
meshai/central/consumer.py (consumer_config() + deliver_policy=NEW),
tests/test_save_section_secret_preserve.py (new),
tests/test_central_consumer.py (deliver_policy assertion).
Verification:
- (A) py_compile clean on config_loader.py + consumer.py.
- (C) pytest -q: 276 passed (272 + 4 new -- preserve-unchanged-ref,
changed-value-written, no-placeholder-still-rejects, deliver_policy=NEW).
The C.2.1 strip test still passes (no placeholder -> reject).
- (D) In-prod (rebuilt): GET+PUT /api/config/environmental round-trip ->
{"saved":true}; on-disk traffic.api_key stayed '${TOMTOM_API_KEY}'
(SECRET_REF_PRESERVED: True), not the literal key; disk restored to baseline.
consumer_config().deliver_policy == DeliverPolicy.NEW in the built image.
Follow-up for D rollout: the durable 'meshai-v04-central_quake_' created during
C.3 was made with deliver_policy=all; re-flipping a domain may need that stale
durable deleted on the Central NATS server first (config mismatch on re-subscribe).
D rollout (remaining domains) is now safe: GUI flips preserve secret refs and
new subscriptions don't replay huge backlogs.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-28 04:55:20 +00:00
|
|
|
"""v0.4 C.3.1: save_section preserves on-disk ${VAR} secret refs instead of
|
|
|
|
|
dropping them when a GUI save round-trips the interpolated value."""
|
|
|
|
|
|
|
|
|
|
import yaml
|
|
|
|
|
|
|
|
|
|
from meshai.config_loader import save_section
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _setup(tmp_path, env_yaml, dotenv):
|
|
|
|
|
cfg = tmp_path / "config"
|
|
|
|
|
cfg.mkdir()
|
|
|
|
|
sec = tmp_path / "secrets"
|
|
|
|
|
sec.mkdir()
|
|
|
|
|
(cfg / "env_feeds.yaml").write_text(env_yaml)
|
|
|
|
|
(sec / ".env").write_text(dotenv)
|
|
|
|
|
return cfg
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_preserves_unchanged_secret_ref(tmp_path):
|
|
|
|
|
# on-disk has ${C31_TEST_KEY}; GUI submits the resolved value -> keep the ref
|
|
|
|
|
cfg = _setup(
|
|
|
|
|
tmp_path,
|
|
|
|
|
"enabled: true\ntraffic:\n enabled: true\n api_key: ${C31_TEST_KEY}\n",
|
|
|
|
|
"C31_TEST_KEY=realkey123\n",
|
|
|
|
|
)
|
|
|
|
|
res = save_section("environmental",
|
|
|
|
|
{"enabled": True, "traffic": {"enabled": True, "api_key": "realkey123"}},
|
|
|
|
|
cfg)
|
|
|
|
|
written = yaml.safe_load((cfg / "env_feeds.yaml").read_text())
|
|
|
|
|
assert written["traffic"]["api_key"] == "${C31_TEST_KEY}" # placeholder preserved
|
|
|
|
|
assert "traffic.api_key" not in res["rejected_secrets"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_changed_secret_value_is_written(tmp_path):
|
|
|
|
|
# on-disk ${C31_TEST_KEY}; GUI submits a DIFFERENT value -> intentional change stored
|
|
|
|
|
cfg = _setup(
|
|
|
|
|
tmp_path,
|
|
|
|
|
"enabled: true\ntraffic:\n enabled: true\n api_key: ${C31_TEST_KEY}\n",
|
|
|
|
|
"C31_TEST_KEY=oldkey\n",
|
|
|
|
|
)
|
|
|
|
|
save_section("environmental",
|
|
|
|
|
{"enabled": True, "traffic": {"enabled": True, "api_key": "NEWKEY999"}},
|
|
|
|
|
cfg)
|
|
|
|
|
written = yaml.safe_load((cfg / "env_feeds.yaml").read_text())
|
|
|
|
|
assert written["traffic"]["api_key"] == "NEWKEY999"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_no_placeholder_still_rejects(tmp_path):
|
|
|
|
|
# no on-disk ${VAR} ref -> a raw secret must be rejected, never written
|
|
|
|
|
cfg = tmp_path / "config"
|
|
|
|
|
cfg.mkdir()
|
|
|
|
|
res = save_section("environmental",
|
|
|
|
|
{"enabled": True, "traffic": {"enabled": True, "api_key": "RAWSECRET"}},
|
|
|
|
|
cfg)
|
|
|
|
|
written = yaml.safe_load((cfg / "env_feeds.yaml").read_text())
|
|
|
|
|
assert "api_key" not in written.get("traffic", {})
|
|
|
|
|
assert "traffic.api_key" in res["rejected_secrets"]
|
chore(meshai): v0.5.5 -- cleanup bundle (gitignore env anchor, ducting health event_count, mesh_sources secret stripping, delete unused SeverityRouter)
Four independent low-risk fixes from the deferred list. Bundled in a single
commit because none are large enough to warrant their own tag and none
touch the safe-mode-sensitive paths (dispatcher / consumer / toggle config).
1) .gitignore: change bare `env/` to `/env/` so the rule anchors at the
repo root only. The unanchored form was matching `meshai/env/` (the
adapter package directory) and forced `git add -f` workarounds during
2.14 / 2.16.1. Verified post-fix: `git check-ignore -vn meshai/env/test.py`
reports no pattern match; `git check-ignore -v env/foo` still matches
the new `/env/` rule.
2) meshai/env/ducting.py: health_status.event_count was hardcoded `0`
from before Phase 2.13 added real event emission. Replaced with
`len(self._events)`, which is the pattern every other env adapter
already uses (fires/firms/nws/swpc/traffic/roads511/usgs/usgs_quake/
avalanche). Flows through env.store.health_status → /api/env/status
so the dashboard counter starts reflecting reality.
3) meshai/config_loader.py save_section: list-section secret stripping.
The path landed in C.2.1 fed list items into check_secrets() with
path="" or with `<field>[<i>]` syntax, neither of which matched the
`mesh_sources.*.api_token` / `notifications.rules.*.smtp_password`
regexes in SECRET_FIELDS (where `*` matches a single dotted token).
Result: a raw secret submitted on a list-section save could slip
through to the YAML file. Fix uses dotted-index form `<field>.<i>.<key>`
for both nested-list (notifications.rules) and top-level-list
(mesh_sources) paths. Also extended _raw_section construction +
_ondisk_ref to walk list-shaped on-disk YAML by integer index so
the C.3.1 ${VAR}-placeholder preservation now works for list sections
too. Three new tests round-trip the mesh_sources placeholder case,
the mesh_sources raw-secret rejection, and the nested-list
notifications.rules placeholder case.
4) meshai/notifications/pipeline/severity_router.py: deleted.
The fork-by-severity routing it implemented was never wired in
production -- _tee in build_pipeline does the dispatcher+digest
fanout directly. The class had two test references in
tests/test_pipeline_skeleton.py that exercised "no matching rule"
and "unknown severity" paths; those guarantees are now covered by
tests/test_v052_dispatcher.py (stats counters) and the existing
Dispatcher-class tests. Removed the file, the __init__.py imports
and __all__ entries (SeverityRouter + StubDigestQueue both), the
two test methods, and the docstring mention.
Verification:
- py_compile clean on all four touched modules.
- `grep -rn SeverityRouter meshai/ tests/` returns zero.
- pytest 328 passed (was 327 at v0.5.4; net: -2 SeverityRouter tests,
+3 secret-preservation tests = +1).
- .gitignore anchor diagnosed via `git check-ignore -vn`.
Safe-mode preserved -- no toggle enabled, no master enabled, no central
enabled, no adapter feed_source flipped.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-06-04 02:50:45 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
# v0.5.5: secret stripping for LIST-shaped sections.
|
|
|
|
|
# C.2.1 added save_section's list-section branch but field paths inside list
|
|
|
|
|
# items used `<key>` instead of `<index>.<key>`, so SECRET_FIELDS patterns
|
|
|
|
|
# like `mesh_sources.*.api_token` never matched on the list-section save
|
|
|
|
|
# path. The fix lands a dotted-index form (mesh_sources.0.api_token).
|
|
|
|
|
|
|
|
|
|
def test_mesh_sources_list_preserves_secret_ref(tmp_path):
|
|
|
|
|
"""List section (mesh_sources): an item's ${VAR} api_token must survive a
|
|
|
|
|
GUI round-trip that submits the resolved value."""
|
|
|
|
|
cfg = _setup(
|
|
|
|
|
tmp_path,
|
|
|
|
|
"", # env_feeds.yaml unused; we write mesh_sources.yaml directly below
|
|
|
|
|
"MS_TEST_TOKEN=realtoken-xyz\n",
|
|
|
|
|
)
|
|
|
|
|
(cfg / "mesh_sources.yaml").write_text(
|
|
|
|
|
"- name: src-a\n url: http://a.local\n api_token: ${MS_TEST_TOKEN}\n"
|
|
|
|
|
)
|
|
|
|
|
res = save_section(
|
|
|
|
|
"mesh_sources",
|
|
|
|
|
[{"name": "src-a", "url": "http://a.local", "api_token": "realtoken-xyz"}],
|
|
|
|
|
cfg,
|
|
|
|
|
)
|
|
|
|
|
written = yaml.safe_load((cfg / "mesh_sources.yaml").read_text())
|
|
|
|
|
assert written[0]["api_token"] == "${MS_TEST_TOKEN}" # placeholder preserved
|
|
|
|
|
# rejected_secrets stores section-relative paths (no section prefix),
|
|
|
|
|
# matching the convention asserted by test_no_placeholder_still_rejects.
|
|
|
|
|
assert "0.api_token" not in res["rejected_secrets"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_mesh_sources_list_rejects_raw_when_no_placeholder(tmp_path):
|
|
|
|
|
"""List section: raw secret without an on-disk placeholder must be rejected
|
|
|
|
|
(same negative case the object-section path already enforces)."""
|
|
|
|
|
cfg = tmp_path / "config"
|
|
|
|
|
cfg.mkdir()
|
|
|
|
|
res = save_section(
|
|
|
|
|
"mesh_sources",
|
|
|
|
|
[{"name": "src-a", "url": "http://a.local", "api_token": "RAW_LEAK"}],
|
|
|
|
|
cfg,
|
|
|
|
|
)
|
|
|
|
|
written = yaml.safe_load((cfg / "mesh_sources.yaml").read_text())
|
|
|
|
|
assert "api_token" not in (written[0] if written else {})
|
|
|
|
|
assert "0.api_token" in res["rejected_secrets"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_notifications_rules_nested_list_preserves_secret_ref(tmp_path):
|
|
|
|
|
"""Object section with a nested list (notifications.rules) -- same dotted-
|
|
|
|
|
index path semantics; smtp_password placeholder must survive round-trip."""
|
|
|
|
|
cfg = _setup(
|
|
|
|
|
tmp_path,
|
|
|
|
|
"",
|
|
|
|
|
"C55_SMTP_PASS=letmein\n",
|
|
|
|
|
)
|
|
|
|
|
# notifications lives in its own file per SECTION_TO_FILE.
|
|
|
|
|
(cfg / "notifications.yaml").write_text(
|
|
|
|
|
"enabled: true\n"
|
|
|
|
|
"rules:\n"
|
|
|
|
|
" - name: r0\n enabled: true\n smtp_password: ${C55_SMTP_PASS}\n"
|
|
|
|
|
)
|
|
|
|
|
res = save_section(
|
|
|
|
|
"notifications",
|
|
|
|
|
{
|
|
|
|
|
"enabled": True,
|
|
|
|
|
"rules": [{"name": "r0", "enabled": True, "smtp_password": "letmein"}],
|
|
|
|
|
},
|
|
|
|
|
cfg,
|
|
|
|
|
)
|
|
|
|
|
written = yaml.safe_load((cfg / "notifications.yaml").read_text())
|
|
|
|
|
assert written["rules"][0]["smtp_password"] == "${C55_SMTP_PASS}"
|
|
|
|
|
assert "rules.0.smtp_password" not in res["rejected_secrets"]
|