From 89640f624da70c1797b01e21f61745dfcdd65d10 Mon Sep 17 00:00:00 2001 From: "Matt Johnson (via Claude)" Date: Sat, 6 Jun 2026 07:33:11 +0000 Subject: [PATCH] fix(v0.7-fire-tracker-4-revised): rip ?status; LLM DM 7-path verification 3 of 7 pass (NOT verified) Matt review caught a scope error: ?status was a hypothetical sketch in the design doc ("a node could ping ?status cache peak") treated as authorization without asking. Ripping the structured-command path entirely. The LLM DM path with env_reporter injection is the natural- language interface; ?status was redundant infrastructure parallel to the path the design depends on. What landed: - router.py: _maybe_rewrite_status_query + _lookup_fire_fuzzy + _build_fire_status_context removed. route() restored to: bang -> IGNORE-empty -> LLM with verbatim query. - tests/test_fire_tracker_phase4.py: 5 ?status tests removed; replaced with two regression guards: test_natural_language_fire_question_routes_to_llm -- "how's the cache peak fire?" returns RouteType.LLM with the verbatim query (no in-router rewriting). test_status_helpers_removed_from_router -- hard-block on _maybe_rewrite_status_query / _lookup_fire_fuzzy / "?status" appearing anywhere in router.py source. If anyone adds a structured-command path for fires, this test fails and the author has to talk to Matt first. - 56 passed in 3.80s across phase1+phase2+phase3+phase4+or-arch+ include-roundtrip. What stays (NOT ripped): - Daily fire digest -- scheduled broadcaster, not a command. Its 4 adapter_config rows (fires.digest_enabled / digest_schedule / digest_timezone / digest_max_chars) stay GUI-editable. - Bug A fix (UnboundLocalError at router.py:745) -- independent of ?status. Confirmed still in effect. LLM DM 7-path verification result -- 3 of 7 pass, INCOMPLETE: | # | query | env_reporter | verdict | |---|-----------------------------------------------|----------------------|---------| | 1 | "are there any fires near me?" | build_fires_detail | PASS | | 2 | "any weather alerts?" | build_alerts_detail | FAIL | | 3 | "any earthquakes nearby?" | build_quakes_detail | FAIL | | 4 | "how's traffic on I-84?" | build_traffic_detail | FAIL | | 5 | "what's the snake river level?" | build_gauges_detail | PASS | | 6 | "what are the band conditions?" | build_swpc_detail | PASS | | 7 | "why didn't I hear about anything today?" | build_drop_audit | FAIL | Two distinct failure classes: Class A -- routing miss (#4 traffic, #7 drop): _ENV_KEYWORDS_TO_SUBTYPE lacks "traffic" (only road/jam/crash/ closure/511/incident map to "traffic"), so a query literally mentioning "traffic" never triggers env scope -> build_traffic_detail never runs even though traffic_events has 9 rows on disk. The LLM fell back to training data and hallucinated I-84 conditions. build_drop_audit has no natural-language trigger phrase at all; "why didn't I hear about anything today?" has no env keyword. Class B -- empty data + LLM hallucination (#2 alerts, #3 quakes): Env scope IS detected, build_alerts_detail and build_quakes_detail DO run, but return empty because nws_alerts has 0 rows and quake_events 24h-window has 0 rows (legitimate empty state). The LLM has no env block to ground on and hallucinated "144 earthquakes worldwide" -- sounds authoritative, is fabricated. Not fixed in this commit -- needs Matt's call on: (a) keyword additions to _ENV_KEYWORDS_TO_SUBTYPE for traffic + drop_audit triggers (risk: false-positive env-scope triggers for unrelated phrases). (b) anti-hallucination prompt clamp: "If a topic's env block is missing/empty, say you don't have live data instead of answering from general knowledge." (risk: bot apologizes every other message.) Per the "STOP if any path fails" instruction, this commit does NOT claim verification done; the report at v0.7-firetracker-phase4.md has the full table + per-row mesh-receiver wire + per-failure root cause analysis. Co-Authored-By: Claude Opus 4.7 (1M context) --- meshai/router.py | 157 +----------------------------- tests/test_fire_tracker_phase4.py | 104 ++++++++++---------- 2 files changed, 58 insertions(+), 203 deletions(-) diff --git a/meshai/router.py b/meshai/router.py index 5990569..54a3bb6 100644 --- a/meshai/router.py +++ b/meshai/router.py @@ -373,17 +373,9 @@ class MessageRouter: if not query: return RouteResult(RouteType.IGNORE) - # v0.7-fire-tracker-4: ?status intent. - # Matches the leading "?status" sigil or a bare "status "; - # falls through to the normal LLM path on no match. We do the - # fire lookup here but return RouteType.LLM with a synthesized - # query so generate_llm_response runs the normal injection + - # chunking path with the fire's context attached. - status_query = _maybe_rewrite_status_query(query, self) - if status_query is not None: - return RouteResult(RouteType.LLM, query=status_query) - - # Route to LLM + # v0.7-fire-tracker-4-revised: the LLM DM path with env_reporter + # injection is the natural-language interface. No bolt-on + # structured-command parallel. return RouteResult(RouteType.LLM, query=query) def _is_mesh_question(self, message: str) -> bool: @@ -957,146 +949,3 @@ class MessageRouter: connector=self.connector, history=self.history, ) - - - - -# ============================================================================ -# v0.7-fire-tracker-4: ?status intent helper -# ============================================================================ - - -_STATUS_PREFIXES = ("?status ", "status ", "?status:", "status:") - - -def _maybe_rewrite_status_query(query: str, router) -> "Optional[str]": - """If `query` looks like a fire status request, rewrite it with the - fire's persisted context inlined. Return None to let the normal LLM - path handle the message verbatim. - - Triggers on the leading word patterns in _STATUS_PREFIXES OR an - interrogative referencing a known fire (e.g. "how is the X fire?"). - """ - q = query.strip() - ql = q.lower() - target_phrase = None - for prefix in _STATUS_PREFIXES: - if ql.startswith(prefix): - target_phrase = q[len(prefix):].strip() - break - - if target_phrase is None: - # Heuristic for "how is fire?" style without a sigil. - triggers = ("how is ", "tell me about ", "status of ", - "what about ", "any update on ") - for t in triggers: - if ql.startswith(t): - target_phrase = q[len(t):].rstrip("?!. ").strip() - if "fire" in target_phrase.lower(): - break - target_phrase = None - if target_phrase is None: - return None - - if not target_phrase: - return None - - fire = _lookup_fire_fuzzy(target_phrase) - if fire is None: - # No match -- leave the query alone; the LLM with env_reporter - # injection may still answer reasonably. - return None - - context = _build_fire_status_context(fire) - return ( - f"User asked for the status of {fire['incident_name']}. " - f"Reply with ONE short paragraph (<= 300 chars total) for mesh " - f"radio operators. No markdown.\n\n" - f"FIRE DATA:\n{context}\n\n" - f"Original question: {query}" - ) - - -def _lookup_fire_fuzzy(phrase: str): - """Find a fire whose incident_name fuzzy-matches phrase. Returns the - sqlite3.Row or None. - - Match priority: exact (case-insensitive) -> startswith -> - contains -> word-overlap. Active fires (tombstoned_at IS NULL) - rank above closed ones.""" - from meshai.persistence import get_db - conn = get_db() - phrase_l = phrase.lower().strip().rstrip("?!.").rstrip() - # Drop trailing " fire" so "cache peak fire" matches "Cache Peak". - if phrase_l.endswith(" fire"): - phrase_l = phrase_l[:-5].strip() - - candidates = conn.execute( - "SELECT irwin_id, incident_name, current_acres, " - "current_contained_pct, state, county, " - "tombstoned_at, last_pass_at " - "FROM fires " - "ORDER BY (tombstoned_at IS NULL) DESC, " - "COALESCE(current_acres, 0) DESC", - ).fetchall() - if not candidates: - return None - - # Tier 1: exact match. - for c in candidates: - if (c["incident_name"] or "").lower() == phrase_l: - return c - # Tier 2: startswith. - for c in candidates: - if (c["incident_name"] or "").lower().startswith(phrase_l): - return c - # Tier 3: contains. - for c in candidates: - if phrase_l in (c["incident_name"] or "").lower(): - return c - # Tier 4: word-overlap (>= 1 token). - tokens = set(phrase_l.split()) - if tokens: - best = None - best_overlap = 0 - for c in candidates: - name_tokens = set((c["incident_name"] or "").lower().split()) - overlap = len(tokens & name_tokens) - if overlap > best_overlap: - best_overlap = overlap - best = c - if best is not None and best_overlap > 0: - return best - return None - - -def _build_fire_status_context(fire) -> str: - """Compose the context block for the status query LLM prompt.""" - from meshai.persistence import get_db - conn = get_db() - passes = conn.execute( - "SELECT pass_id, drift_mi_from_prev, drift_direction, " - "drift_mi_per_hour, pixel_count, pass_ended_at " - "FROM fire_passes WHERE irwin_id=? " - "ORDER BY pass_ended_at DESC LIMIT 3", - (fire["irwin_id"],), - ).fetchall() - lines = [ - f"name: {fire['incident_name']}", - f"acres: {fire['current_acres'] or 0}", - f"contained: {fire['current_contained_pct'] or 0}%", - f"county/state: {fire['county'] or '?'}/{fire['state'] or '?'}", - f"closed: {bool(fire['tombstoned_at'])}", - ] - if passes: - lines.append("recent passes (newest first):") - for p in passes: - drift = "" - if (p["drift_mi_from_prev"] is not None - and p["drift_direction"] is not None): - drift = (f", drift {p['drift_mi_from_prev']:.1f}mi " - f"{p['drift_direction']}") - lines.append( - f" - pass {p['pass_id']}: {p['pixel_count']} pixel(s)" - f"{drift}") - return "\n".join(lines) diff --git a/tests/test_fire_tracker_phase4.py b/tests/test_fire_tracker_phase4.py index 2af1639..03a51b2 100644 --- a/tests/test_fire_tracker_phase4.py +++ b/tests/test_fire_tracker_phase4.py @@ -131,57 +131,63 @@ def test_render_digest_uses_llm_when_available(): # =========================================================================== -# Fuzzy fire lookup for ?status +# Natural-language fire DMs route to the LLM (no ?status fallback) # =========================================================================== -def test_status_lookup_exact_name(): - from meshai.router import _lookup_fire_fuzzy - _seed_fire(irwin_id="ID-A", name="Cache Peak", - lat=42.0, lon=-114.0, acres=1847) - f = _lookup_fire_fuzzy("Cache Peak") - assert f is not None - assert f["incident_name"] == "Cache Peak" +def test_natural_language_fire_question_routes_to_llm(): + """The LLM DM path is the sole interface for natural-language fire + questions. Pre-revised commit there was a `?status` intent that + rewrote the query in-router; this test confirms the rewrite is gone + and that a plain English question is forwarded verbatim.""" + import asyncio + from meshai.router import MessageRouter, RouteType + from meshai.config_loader import load_config + from meshai.history import ConversationHistory + from meshai.commands.dispatcher import create_dispatcher + + cfg = load_config() + history = ConversationHistory(cfg.history) + + async def _run(): + await history.initialize() + dispatcher = create_dispatcher( + prefix=cfg.commands.prefix, + disabled_commands=cfg.commands.disabled_commands, + custom_commands=cfg.commands.custom_commands, + ) + + class FakeConnector: + my_node_id = "!THIS_BOT" + + class FakeMessage: + text = "how's the cache peak fire?" + sender_id = "!T" + sender_name = "t" + is_dm = True + channel = 0 + + router = MessageRouter( + config=cfg, connector=FakeConnector(), + history=history, dispatcher=dispatcher, + llm_backend=None, # we only inspect the route() decision + ) + result = await router.route(FakeMessage()) + return result + + result = asyncio.run(_run()) + assert result.route_type == RouteType.LLM + # Critical: the query must be the verbatim user text, not a rewrite + # synthesized by an in-router intent helper. + assert result.query == "how's the cache peak fire?" -def test_status_lookup_trims_trailing_fire_word(): - from meshai.router import _lookup_fire_fuzzy - _seed_fire(irwin_id="ID-A", name="Cache Peak", - lat=42.0, lon=-114.0, acres=1847) - f = _lookup_fire_fuzzy("cache peak fire") - assert f is not None - assert f["incident_name"] == "Cache Peak" - - -def test_status_lookup_word_overlap_fallback(): - from meshai.router import _lookup_fire_fuzzy - _seed_fire(irwin_id="ID-A", name="Cache Peak", - lat=42.0, lon=-114.0, acres=1847) - f = _lookup_fire_fuzzy("how is peak doing") - assert f is not None - assert f["incident_name"] == "Cache Peak" - - -def test_status_lookup_returns_none_on_no_match(): - from meshai.router import _lookup_fire_fuzzy - _seed_fire(irwin_id="ID-A", name="Cache Peak", - lat=42.0, lon=-114.0, acres=1847) - assert _lookup_fire_fuzzy("nonexistent ranger station") is None - - -def test_status_query_rewrite_includes_fire_context(): - from meshai.router import _maybe_rewrite_status_query - _seed_fire(irwin_id="ID-A", name="Cache Peak", - lat=42.0, lon=-114.0, acres=1847, contained=23) - out = _maybe_rewrite_status_query("?status Cache Peak", router=None) - assert out is not None - assert "Cache Peak" in out - assert "1847" in out - # Must instruct the LLM to be terse mesh format. - assert "mesh" in out.lower() - - -def test_status_query_rewrite_returns_none_when_not_status(): - from meshai.router import _maybe_rewrite_status_query - out = _maybe_rewrite_status_query("how's the weather?", router=None) - assert out is None +def test_status_helpers_removed_from_router(): + """Hard guard against ?status helpers sneaking back in. If anyone + adds a structured-command path to router.py for fires, this test + fails and the author has to talk to Matt first.""" + from pathlib import Path + src = Path("/opt/meshai/meshai/router.py").read_text() + assert "_maybe_rewrite_status_query" not in src + assert "_lookup_fire_fuzzy" not in src + assert "?status" not in src