mirror of
https://github.com/zvx-echo6/meshai.git
synced 2026-06-10 17:04:45 +02:00
Matt review caught a scope error: ?status was a hypothetical sketch
in the design doc ("a node could ping ?status cache peak") treated as
authorization without asking. Ripping the structured-command path
entirely. The LLM DM path with env_reporter injection is the natural-
language interface; ?status was redundant infrastructure parallel to
the path the design depends on.
What landed:
- router.py: _maybe_rewrite_status_query + _lookup_fire_fuzzy +
_build_fire_status_context removed. route() restored to:
bang -> IGNORE-empty -> LLM with verbatim query.
- tests/test_fire_tracker_phase4.py: 5 ?status tests removed; replaced
with two regression guards:
test_natural_language_fire_question_routes_to_llm -- "how's the
cache peak fire?" returns RouteType.LLM with the verbatim query
(no in-router rewriting).
test_status_helpers_removed_from_router -- hard-block on
_maybe_rewrite_status_query / _lookup_fire_fuzzy / "?status"
appearing anywhere in router.py source. If anyone adds a
structured-command path for fires, this test fails and the
author has to talk to Matt first.
- 56 passed in 3.80s across phase1+phase2+phase3+phase4+or-arch+
include-roundtrip.
What stays (NOT ripped):
- Daily fire digest -- scheduled broadcaster, not a command. Its 4
adapter_config rows (fires.digest_enabled / digest_schedule /
digest_timezone / digest_max_chars) stay GUI-editable.
- Bug A fix (UnboundLocalError at router.py:745) -- independent of
?status. Confirmed still in effect.
LLM DM 7-path verification result -- 3 of 7 pass, INCOMPLETE:
| # | query | env_reporter | verdict |
|---|-----------------------------------------------|----------------------|---------|
| 1 | "are there any fires near me?" | build_fires_detail | PASS |
| 2 | "any weather alerts?" | build_alerts_detail | FAIL |
| 3 | "any earthquakes nearby?" | build_quakes_detail | FAIL |
| 4 | "how's traffic on I-84?" | build_traffic_detail | FAIL |
| 5 | "what's the snake river level?" | build_gauges_detail | PASS |
| 6 | "what are the band conditions?" | build_swpc_detail | PASS |
| 7 | "why didn't I hear about anything today?" | build_drop_audit | FAIL |
Two distinct failure classes:
Class A -- routing miss (#4 traffic, #7 drop):
_ENV_KEYWORDS_TO_SUBTYPE lacks "traffic" (only road/jam/crash/
closure/511/incident map to "traffic"), so a query literally
mentioning "traffic" never triggers env scope -> build_traffic_detail
never runs even though traffic_events has 9 rows on disk. The LLM
fell back to training data and hallucinated I-84 conditions.
build_drop_audit has no natural-language trigger phrase at all;
"why didn't I hear about anything today?" has no env keyword.
Class B -- empty data + LLM hallucination (#2 alerts, #3 quakes):
Env scope IS detected, build_alerts_detail and build_quakes_detail
DO run, but return empty because nws_alerts has 0 rows and
quake_events 24h-window has 0 rows (legitimate empty state). The
LLM has no env block to ground on and hallucinated "144 earthquakes
worldwide" -- sounds authoritative, is fabricated.
Not fixed in this commit -- needs Matt's call on:
(a) keyword additions to _ENV_KEYWORDS_TO_SUBTYPE for traffic +
drop_audit triggers (risk: false-positive env-scope triggers
for unrelated phrases).
(b) anti-hallucination prompt clamp: "If a topic's env block is
missing/empty, say you don't have live data instead of
answering from general knowledge." (risk: bot apologizes
every other message.)
Per the "STOP if any path fails" instruction, this commit does NOT
claim verification done; the report at
v0.7-firetracker-phase4.md has the full table + per-row mesh-receiver
wire + per-failure root cause analysis.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
193 lines
7.1 KiB
Python
193 lines
7.1 KiB
Python
"""v0.7-fire-tracker-4 tests."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import time
|
|
import uuid
|
|
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _isolate_db(tmp_path, monkeypatch):
|
|
db_path = str(tmp_path / f"meshai-{uuid.uuid4().hex}.sqlite")
|
|
monkeypatch.setenv("MESHAI_DB_PATH", db_path)
|
|
from meshai.persistence import db as pdb
|
|
pdb.close_thread_connection()
|
|
pdb._initialised.discard(db_path)
|
|
from meshai.persistence import init_db
|
|
init_db(db_path)
|
|
yield db_path
|
|
pdb.close_thread_connection()
|
|
pdb._initialised.discard(db_path)
|
|
|
|
|
|
def _seed_fire(*, irwin_id, name, lat, lon, acres=None,
|
|
contained=None, county="Test", state="ID"):
|
|
from meshai.persistence import get_db
|
|
get_db().execute(
|
|
"INSERT INTO fires(irwin_id, incident_name, current_acres, "
|
|
"current_contained_pct, lat, lon, county, state, last_event_at) "
|
|
"VALUES (?,?,?,?,?,?,?,?,?)",
|
|
(irwin_id, name, acres, contained, lat, lon, county, state,
|
|
int(time.time())),
|
|
)
|
|
|
|
|
|
# ===========================================================================
|
|
# Bug A regression: scope_type defined before use
|
|
# ===========================================================================
|
|
|
|
|
|
def test_router_scope_type_defined_before_env_check():
|
|
"""The env_reporter check at the top of generate_llm_response reads
|
|
scope_type. Pre-fix it was UnboundLocalError on every env query."""
|
|
import re
|
|
from pathlib import Path
|
|
src = Path("/opt/meshai/meshai/router.py").read_text()
|
|
# Find the "if should_inject_mesh and scope_type" line + the
|
|
# nearest preceding `scope_type, scope_value = ` assignment.
|
|
env_use_line = None
|
|
for i, line in enumerate(src.splitlines(), start=1):
|
|
if "should_inject_mesh and scope_type" in line:
|
|
env_use_line = i
|
|
break
|
|
assert env_use_line is not None
|
|
# There must be an assignment on or before this line.
|
|
preceding = "\n".join(src.splitlines()[: env_use_line - 1])
|
|
assert re.search(r"scope_type[, ]+scope_value\s*=", preceding) \
|
|
or "scope_type:" in preceding
|
|
|
|
|
|
# ===========================================================================
|
|
# adapter_config seed + categories registration
|
|
# ===========================================================================
|
|
|
|
|
|
def test_adapter_config_seeds_digest_keys():
|
|
from meshai.persistence import get_db
|
|
rows = {
|
|
(r["adapter"], r["key"]): r["default_json"]
|
|
for r in get_db().execute(
|
|
"SELECT adapter, key, default_json FROM adapter_config "
|
|
"WHERE adapter='fires' AND key LIKE 'digest%'"
|
|
)
|
|
}
|
|
assert rows[("fires", "digest_enabled")] == "true"
|
|
assert rows[("fires", "digest_schedule")] == '["06:00", "18:00"]'
|
|
assert rows[("fires", "digest_timezone")] == '"America/Boise"'
|
|
assert rows[("fires", "digest_max_chars")] == "200"
|
|
|
|
|
|
# ===========================================================================
|
|
# Digest renderer
|
|
# ===========================================================================
|
|
|
|
|
|
def test_render_digest_returns_no_fires_when_table_empty():
|
|
from meshai.notifications.scheduled.fire_digest import render_digest
|
|
|
|
async def _run():
|
|
return await render_digest(llm_backend=None, max_chars=200)
|
|
wire, source = asyncio.run(_run())
|
|
assert wire == ""
|
|
assert source == "no_fires"
|
|
|
|
|
|
def test_render_digest_terse_fallback_when_no_llm():
|
|
_seed_fire(irwin_id="ID-A", name="Cache Peak",
|
|
lat=42.0, lon=-114.0, acres=1847, contained=23)
|
|
_seed_fire(irwin_id="ID-B", name="Twin Peaks",
|
|
lat=43.0, lon=-115.0, acres=320, contained=5)
|
|
from meshai.notifications.scheduled.fire_digest import render_digest
|
|
|
|
async def _run():
|
|
return await render_digest(llm_backend=None, max_chars=200)
|
|
wire, source = asyncio.run(_run())
|
|
assert source == "fallback_terse"
|
|
assert wire
|
|
assert "Cache Peak" in wire
|
|
assert len(wire) <= 200
|
|
|
|
|
|
def test_render_digest_uses_llm_when_available():
|
|
"""When the LLM backend returns a string, that string IS the wire."""
|
|
_seed_fire(irwin_id="ID-A", name="Cache Peak",
|
|
lat=42.0, lon=-114.0, acres=1847)
|
|
|
|
class StubLLM:
|
|
async def generate(self, *, messages, system_prompt, max_tokens):
|
|
# The renderer must give us a single-line wire derived from
|
|
# the LLM output, with markdown stripped + cap applied.
|
|
return "Cache Peak 1847 ac stable; no spotting today."
|
|
|
|
from meshai.notifications.scheduled.fire_digest import render_digest
|
|
|
|
async def _run():
|
|
return await render_digest(llm_backend=StubLLM(), max_chars=200)
|
|
wire, source = asyncio.run(_run())
|
|
assert source == "llm"
|
|
assert wire == "Cache Peak 1847 ac stable; no spotting today."
|
|
|
|
|
|
# ===========================================================================
|
|
# Natural-language fire DMs route to the LLM (no ?status fallback)
|
|
# ===========================================================================
|
|
|
|
|
|
def test_natural_language_fire_question_routes_to_llm():
|
|
"""The LLM DM path is the sole interface for natural-language fire
|
|
questions. Pre-revised commit there was a `?status` intent that
|
|
rewrote the query in-router; this test confirms the rewrite is gone
|
|
and that a plain English question is forwarded verbatim."""
|
|
import asyncio
|
|
from meshai.router import MessageRouter, RouteType
|
|
from meshai.config_loader import load_config
|
|
from meshai.history import ConversationHistory
|
|
from meshai.commands.dispatcher import create_dispatcher
|
|
|
|
cfg = load_config()
|
|
history = ConversationHistory(cfg.history)
|
|
|
|
async def _run():
|
|
await history.initialize()
|
|
dispatcher = create_dispatcher(
|
|
prefix=cfg.commands.prefix,
|
|
disabled_commands=cfg.commands.disabled_commands,
|
|
custom_commands=cfg.commands.custom_commands,
|
|
)
|
|
|
|
class FakeConnector:
|
|
my_node_id = "!THIS_BOT"
|
|
|
|
class FakeMessage:
|
|
text = "how's the cache peak fire?"
|
|
sender_id = "!T"
|
|
sender_name = "t"
|
|
is_dm = True
|
|
channel = 0
|
|
|
|
router = MessageRouter(
|
|
config=cfg, connector=FakeConnector(),
|
|
history=history, dispatcher=dispatcher,
|
|
llm_backend=None, # we only inspect the route() decision
|
|
)
|
|
result = await router.route(FakeMessage())
|
|
return result
|
|
|
|
result = asyncio.run(_run())
|
|
assert result.route_type == RouteType.LLM
|
|
# Critical: the query must be the verbatim user text, not a rewrite
|
|
# synthesized by an in-router intent helper.
|
|
assert result.query == "how's the cache peak fire?"
|
|
|
|
|
|
def test_status_helpers_removed_from_router():
|
|
"""Hard guard against ?status helpers sneaking back in. If anyone
|
|
adds a structured-command path to router.py for fires, this test
|
|
fails and the author has to talk to Matt first."""
|
|
from pathlib import Path
|
|
src = Path("/opt/meshai/meshai/router.py").read_text()
|
|
assert "_maybe_rewrite_status_query" not in src
|
|
assert "_lookup_fire_fuzzy" not in src
|
|
assert "?status" not in src
|