mirror of
https://github.com/zvx-echo6/central.git
synced 2026-05-21 18:14:44 +02:00
feat(3-K.5): operator-settable EnrichmentConfig (config plumbing)
Bridge PR for v0.5.0. PR J wired the supervisor with a hardcoded EnrichmentConfig() default; PR K added real backends to the registry but left no operator path to select one. K.5 closes that gap by mirroring the config.adapters storage + LISTEN/NOTIFY hot-reload pattern. config.enrichment (migration 024): single-row table (id BOOLEAN PK CHECK (id = true), mirroring config.system). Columns enricher_class, backend_class, backend_settings JSONB, cache_ttl_s, updated_at. Reuses the existing config.set_updated_at + config.notify_config_change triggers (the NOTIFY function's ELSE branch emits 'enrichment:' for this keyless single-row table). Seeds framework DEFAULTS ONLY — GeocoderEnricher + NoOpBackend, empty backend_settings, 24h TTL. NO URLs/IPs/auth in the seed; a fresh deploy runs NoOp out of the box. Idempotent (CREATE IF NOT EXISTS / DROP TRIGGER IF EXISTS / INSERT ON CONFLICT DO NOTHING). Supervisor: - Reads config.enrichment at startup (start() -> config_source .get_enrichment_config()), overriding the constructor default. - Hot-reloads via _on_config_change(table == "enrichment"): re-reads the row, rebuilds the enricher set, and invalidates the enrichment cache when the enricher/backend/settings changed (a new backend must not keep serving the old backend's cached bundles until TTL). TTL-only changes retain the cache. - build_enrichers now takes an explicit EnrichmentCache (the supervisor owns it so it can invalidate); cache no longer built inside build_enrichers. ConfigStore / ConfigSource: get_enrichment_config() (falls back to defaults if the row is somehow absent) + upsert_enrichment_config(). Mirrors the adapter accessors. cache.py: EnrichmentCache.invalidate(enricher_name=None) — DELETE all or enricher-scoped; returns rows deleted. GUI /enrichment: GET renders the EnrichmentConfig form via the generic describe_fields machinery (no enrichment-specific Jinja); POST validates via Pydantic, writes config.enrichment, and lets the NOTIFY trigger propagate the hot-reload. New enrichment.html + a nav link. backend_settings (a dict field) needed a generic "json" widget in describe_fields + the template — usable by any dict-typed settings field, not enrichment-specific. Necessary deviation (surfaced): PR K shipped a deployment-specific default DEFAULT_BASE_URL = "http://192.168.1.130:8440" in navi.py. Bar (b) forbids deployer IPs in src, and operator-settable base_url is exactly K.5's purpose, so the default is changed to http://localhost:8440 (matching Photon/Nominatim defaults). The live integration smoke (tests/, env-gated, skipped) now reads the endpoint from NAVI_BASE_URL — no IP anywhere in src. Tests (test_enrichment_config_plumbing.py, 10): ConfigStore read / default fallback / upsert-passes-dict; cache invalidate all + scoped; supervisor builds NaviBackend from config; hot-reload rebuilds + invalidates on backend change; no-invalidate on TTL-only change; describe_fields json widget; /enrichment GET render. test_firms updated for the build_enrichers signature change. Hot-reload mechanism mirrored: Postgres LISTEN/NOTIFY on channel 'config_changed' (payload 'table:key'), same path adapters/streams use; the supervisor's existing _on_config_change dispatch gains an "enrichment" branch. Verification: full pytest 535 passed, 1 skipped (was 525; +10). Migration applied cleanly on the live prod schema; SELECT * FROM config.enrichment returns the NoOp default row. grep subject_for_event/_ADAPTER_REGISTRY and grep 100.64.0./192.168.1. in src both empty. Does NOT activate NaviBackend (ships NoOp default; operator action) and does NOT declare enrichment_locations on other adapters (PR L scope). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
54238093a5
commit
04c1d07b3f
13 changed files with 604 additions and 15 deletions
208
tests/test_enrichment_config_plumbing.py
Normal file
208
tests/test_enrichment_config_plumbing.py
Normal file
|
|
@ -0,0 +1,208 @@
|
|||
"""Tests for operator-settable EnrichmentConfig plumbing (PR K.5).
|
||||
|
||||
Covers: ConfigStore DB read/upsert, supervisor startup read + hot-reload
|
||||
rebuild, cache invalidation on backend change (but not on TTL-only change),
|
||||
EnrichmentCache.invalidate, the generic json widget for backend_settings, and
|
||||
the /enrichment GUI render. No real DB / NATS — pool, config_source, and the
|
||||
EnrichmentCache class are mocked.
|
||||
"""
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from central.config_models import EnrichmentConfig
|
||||
from central.enrichment.cache import EnrichmentCache
|
||||
from central.enrichment.backends.navi import NaviBackend
|
||||
from central.enrichment.backends.no_op import NoOpBackend
|
||||
from central.gui.form_descriptors import describe_fields
|
||||
|
||||
|
||||
# --- mock pool/conn helpers -------------------------------------------------
|
||||
|
||||
def _mock_pool(conn: MagicMock) -> MagicMock:
|
||||
pool = MagicMock()
|
||||
acquire_cm = MagicMock()
|
||||
acquire_cm.__aenter__ = AsyncMock(return_value=conn)
|
||||
acquire_cm.__aexit__ = AsyncMock(return_value=None)
|
||||
pool.acquire = MagicMock(return_value=acquire_cm)
|
||||
return pool
|
||||
|
||||
|
||||
# --- ConfigStore --------------------------------------------------------------
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_config_store_reads_enrichment_row():
|
||||
from central.config_store import ConfigStore
|
||||
|
||||
conn = MagicMock()
|
||||
conn.fetchrow = AsyncMock(return_value={
|
||||
"enricher_class": "GeocoderEnricher",
|
||||
"backend_class": "NaviBackend",
|
||||
"backend_settings": {"base_url": "http://example.test:8440"},
|
||||
"cache_ttl_s": 3600,
|
||||
})
|
||||
store = ConfigStore(_mock_pool(conn))
|
||||
cfg = await store.get_enrichment_config()
|
||||
assert isinstance(cfg, EnrichmentConfig)
|
||||
assert cfg.backend_class == "NaviBackend"
|
||||
assert cfg.backend_settings == {"base_url": "http://example.test:8440"}
|
||||
assert cfg.cache_ttl_s == 3600
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_config_store_falls_back_to_defaults_when_row_absent():
|
||||
from central.config_store import ConfigStore
|
||||
|
||||
conn = MagicMock()
|
||||
conn.fetchrow = AsyncMock(return_value=None)
|
||||
store = ConfigStore(_mock_pool(conn))
|
||||
cfg = await store.get_enrichment_config()
|
||||
assert cfg == EnrichmentConfig() # framework defaults
|
||||
assert cfg.backend_class == "NoOpBackend"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_config_store_upsert_passes_dict_settings():
|
||||
from central.config_store import ConfigStore
|
||||
|
||||
conn = MagicMock()
|
||||
conn.execute = AsyncMock()
|
||||
store = ConfigStore(_mock_pool(conn))
|
||||
cfg = EnrichmentConfig(backend_class="NaviBackend", backend_settings={"base_url": "x"})
|
||||
await store.upsert_enrichment_config(cfg)
|
||||
args = conn.execute.call_args.args
|
||||
assert "INSERT INTO config.enrichment" in args[0]
|
||||
# backend_settings passed as a dict (pool codec encodes to jsonb), not a str.
|
||||
assert {"base_url": "x"} in args
|
||||
|
||||
|
||||
# --- EnrichmentCache.invalidate ----------------------------------------------
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cache_invalidate_all(tmp_path):
|
||||
cache = EnrichmentCache(tmp_path / "c.db", ttl_s=3600)
|
||||
await cache.set("geocoder", 1.0, 2.0, {"name": "x"})
|
||||
await cache.set("geocoder", 3.0, 4.0, {"name": "y"})
|
||||
deleted = await cache.invalidate()
|
||||
assert deleted == 2
|
||||
assert await cache.get("geocoder", 1.0, 2.0) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cache_invalidate_scoped_to_enricher(tmp_path):
|
||||
cache = EnrichmentCache(tmp_path / "c.db", ttl_s=3600)
|
||||
await cache.set("geocoder", 1.0, 2.0, {"name": "x"})
|
||||
await cache.set("other", 1.0, 2.0, {"name": "z"})
|
||||
deleted = await cache.invalidate("geocoder")
|
||||
assert deleted == 1
|
||||
assert await cache.get("geocoder", 1.0, 2.0) is None
|
||||
assert await cache.get("other", 1.0, 2.0) == {"name": "z"}
|
||||
|
||||
|
||||
# --- Supervisor startup read + hot-reload ------------------------------------
|
||||
|
||||
def _supervisor_with(enrichment_cfg: EnrichmentConfig):
|
||||
"""Build a Supervisor with mocked deps and a mocked EnrichmentCache class
|
||||
(so no real /var/lib cache file is touched)."""
|
||||
from central import supervisor as sup_mod
|
||||
|
||||
config_source = MagicMock()
|
||||
config_source.get_enrichment_config = AsyncMock(return_value=enrichment_cfg)
|
||||
config_store = MagicMock()
|
||||
sup = sup_mod.Supervisor(
|
||||
config_source=config_source,
|
||||
config_store=config_store,
|
||||
nats_url="nats://localhost:4222",
|
||||
)
|
||||
return sup
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_supervisor_builds_navi_from_config():
|
||||
"""Given a config naming NaviBackend, the supervisor's enricher set wraps a
|
||||
NaviBackend — proves the registry resolution end-to-end."""
|
||||
with patch("central.supervisor.EnrichmentCache") as cache_cls:
|
||||
cache_cls.return_value = MagicMock(invalidate=AsyncMock(return_value=0))
|
||||
sup = _supervisor_with(
|
||||
EnrichmentConfig(backend_class="NaviBackend",
|
||||
backend_settings={"base_url": "http://x:8440", "warmup": False})
|
||||
)
|
||||
cfg = await sup._config_source.get_enrichment_config()
|
||||
sup._rebuild_enrichers(cfg)
|
||||
assert isinstance(sup._enrichers[0]._backend, NaviBackend)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hot_reload_rebuilds_and_invalidates_on_backend_change():
|
||||
from central import supervisor as sup_mod
|
||||
|
||||
with patch("central.supervisor.EnrichmentCache") as cache_cls:
|
||||
invalidate = AsyncMock(return_value=5)
|
||||
cache_cls.return_value = MagicMock(invalidate=invalidate)
|
||||
# Start at NoOp.
|
||||
sup = _supervisor_with(EnrichmentConfig())
|
||||
sup._rebuild_enrichers(EnrichmentConfig())
|
||||
assert isinstance(sup._enrichers[0]._backend, NoOpBackend)
|
||||
# Config flips to Navi.
|
||||
sup._config_source.get_enrichment_config = AsyncMock(
|
||||
return_value=EnrichmentConfig(
|
||||
backend_class="NaviBackend",
|
||||
backend_settings={"base_url": "http://x:8440", "warmup": False},
|
||||
)
|
||||
)
|
||||
await sup._handle_enrichment_change()
|
||||
assert isinstance(sup._enrichers[0]._backend, NaviBackend)
|
||||
invalidate.assert_awaited() # backend changed -> cache wiped
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hot_reload_does_not_invalidate_on_ttl_only_change():
|
||||
with patch("central.supervisor.EnrichmentCache") as cache_cls:
|
||||
invalidate = AsyncMock(return_value=0)
|
||||
cache_cls.return_value = MagicMock(invalidate=invalidate)
|
||||
sup = _supervisor_with(EnrichmentConfig())
|
||||
sup._rebuild_enrichers(EnrichmentConfig())
|
||||
# Same backend, only TTL changes.
|
||||
sup._config_source.get_enrichment_config = AsyncMock(
|
||||
return_value=EnrichmentConfig(cache_ttl_s=3600)
|
||||
)
|
||||
await sup._handle_enrichment_change()
|
||||
invalidate.assert_not_awaited()
|
||||
|
||||
|
||||
# --- generic json widget + GUI render ----------------------------------------
|
||||
|
||||
def test_describe_fields_renders_dict_as_json_widget():
|
||||
fields = {f.name: f.widget for f in describe_fields(EnrichmentConfig, {})}
|
||||
assert fields["backend_settings"] == "json"
|
||||
assert fields["enricher_class"] == "text"
|
||||
assert fields["cache_ttl_s"] == "number"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_enrichment_form_renders():
|
||||
from central.gui.routes import enrichment_form
|
||||
|
||||
request = MagicMock()
|
||||
request.state.operator = MagicMock(username="op")
|
||||
request.state.csrf_token = "tok"
|
||||
|
||||
conn = MagicMock()
|
||||
conn.fetchrow = AsyncMock(return_value={
|
||||
"enricher_class": "GeocoderEnricher",
|
||||
"backend_class": "NoOpBackend",
|
||||
"backend_settings": {},
|
||||
"cache_ttl_s": 86400,
|
||||
})
|
||||
templates = MagicMock()
|
||||
templates.TemplateResponse.return_value = MagicMock()
|
||||
|
||||
with patch("central.gui.routes._get_templates", return_value=templates), \
|
||||
patch("central.gui.routes.get_pool", return_value=_mock_pool(conn)):
|
||||
await enrichment_form(request)
|
||||
|
||||
ctx = templates.TemplateResponse.call_args.kwargs["context"]
|
||||
field_widgets = {f.name: f.widget for f in ctx["fields"]}
|
||||
assert field_widgets["backend_settings"] == "json"
|
||||
assert ctx["csrf_token"] == "tok"
|
||||
|
|
@ -456,6 +456,7 @@ class TestEnrichmentIntegration:
|
|||
"""A FIRMS event run through the supervisor's enrichment stage emerges
|
||||
with data._enriched.geocoder populated (all-null under NoOpBackend)."""
|
||||
from central.config_models import EnrichmentConfig
|
||||
from central.enrichment.cache import EnrichmentCache
|
||||
from central.enrichment.geocoder import all_null_bundle
|
||||
from central.supervisor import apply_enrichment, build_enrichers
|
||||
|
||||
|
|
@ -469,9 +470,8 @@ class TestEnrichmentIntegration:
|
|||
event = adapter._row_to_event(rows[0], "VIIRS_SNPP_NRT")
|
||||
assert "_enriched" not in event.data
|
||||
|
||||
enrichers = build_enrichers(
|
||||
EnrichmentConfig(), cache_db_path=tmp_path / "enrichment_cache.db"
|
||||
)
|
||||
cache = EnrichmentCache(tmp_path / "enrichment_cache.db")
|
||||
enrichers = build_enrichers(EnrichmentConfig(), cache)
|
||||
await apply_enrichment(event, adapter.enrichment_locations, enrichers)
|
||||
|
||||
assert "_enriched" in event.data
|
||||
|
|
|
|||
|
|
@ -110,8 +110,13 @@ async def test_headers_passed_through_config():
|
|||
reason="set NAVI_INTEGRATION_TEST=1 to hit the live Navi endpoint",
|
||||
)
|
||||
async def test_live_navi_boise():
|
||||
"""Integration smoke against the real endpoint (default skipped)."""
|
||||
b = NaviBackend(warmup=False) # default base_url
|
||||
"""Integration smoke against the real endpoint (default skipped).
|
||||
|
||||
The endpoint host is supplied via NAVI_BASE_URL so no deployment-specific
|
||||
address lives in source; defaults to localhost when unset.
|
||||
"""
|
||||
base_url = os.environ.get("NAVI_BASE_URL", "http://localhost:8440")
|
||||
b = NaviBackend(base_url=base_url, warmup=False)
|
||||
result = await b.reverse(43.6150, -116.2023)
|
||||
assert result["name"] == "Where you are"
|
||||
assert result["city"] == "Boise"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue