chore: capture in-flight fire-spam/drain-pacer + reconnect ground truth from CT108

Preserve the live CT108 working-tree state (the running container is already
built and is unaffected by this commit).

Canonical build tree (work/ — what ships):
  - work/meshai/config.py      : ConnectionConfig watchdog reconnect knobs
  - work/meshai/connector.py   : watchdog link-state, socket-based liveness,
                                 active_probe, in-place reconnect
  - work/meshai/main.py        : connection supervisor (watchdog) task
  - work/Dockerfile            : healthcheck also asserts /tmp/meshai.link=up
  - work/docker-compose.yml    : matching healthcheck change

Root tree (stale duplicate, earlier reconnect iteration — preserved for fidelity):
  - meshai/config.py, meshai/connector.py, meshai/main.py
  - Dockerfile, docker-compose.yml

New (root only, NOT under work/, currently unimported / not in build):
  - meshai/notifications/pipeline/severity_router.py

Excluded: all *.bak / *.bak2 backup artifacts.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

work/Dockerfile

@@ -93,7 +93,7 @@ EXPOSE 7682 8080
 
 # Health check - verify bot process is alive via PID file
 HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \
-    CMD test -f /tmp/meshai.pid && kill -0 $(cat /tmp/meshai.pid) 2>/dev/null || exit 1
+    CMD test -f /tmp/meshai.pid && kill -0 $(cat /tmp/meshai.pid) 2>/dev/null && [ "$(cat /tmp/meshai.link 2>/dev/null)" = up ] || exit 1
 
 # Entrypoint handles config and ttyd
 ENTRYPOINT ["/app/docker-entrypoint.sh"]

work/docker-compose.yml

@@ -67,7 +67,7 @@ services:
           memory: 64M
 
     healthcheck:
-      test: ["CMD-SHELL", "test -f /tmp/meshai.pid && kill -0 $(cat /tmp/meshai.pid) 2>/dev/null || exit 1"]
+      test: ["CMD-SHELL", "test -f /tmp/meshai.pid && kill -0 $(cat /tmp/meshai.pid) 2>/dev/null && [ \"$(cat /tmp/meshai.link 2>/dev/null)\" = up ] || exit 1"]
       interval: 30s
       timeout: 10s
       retries: 3

work/meshai/config.py

@@ -29,6 +29,11 @@ class ConnectionConfig:
     serial_port: str = "/dev/ttyUSB0"
     tcp_host: str = "192.168.1.100"
     tcp_port: int = 4403
+    # --- app-level auto-reconnect (watchdog) ---
+    reconnect: bool = True
+    reconnect_initial_delay: float = 2.0
+    reconnect_max_delay: float = 60.0
+    reconnect_health_interval: float = 30.0
 
 
 @dataclass

work/meshai/connector.py

@@ -2,7 +2,9 @@
 
 import asyncio
 import logging
+import socket
 import threading
+import time
 from dataclasses import dataclass, field
 from typing import Callable, Optional
 
@@ -48,6 +50,12 @@ class MeshConnector:
         self._connected = False
         self._loop: Optional[asyncio.AbstractEventLoop] = None
         self._lock = threading.Lock()
+        # --- watchdog link-state (watchdog is the SINGLE source of truth) ---
+        self._last_rx: float = time.monotonic()
+        self._link_suspect: bool = False
+        self._wake: Optional[asyncio.Event] = None  # created by main once loop exists
+        self._link_path = "/tmp/meshai.link"
+        self._reconnect_lock = threading.Lock()
 
     @property
     def connected(self) -> bool:
@@ -86,6 +94,9 @@ class MeshConnector:
             # Subscribe to messages
             pub.subscribe(self._on_receive, "meshtastic.receive.text")
             pub.subscribe(self._on_node_update, "meshtastic.node.updated")
+            pub.subscribe(self._on_any_rx, "meshtastic.receive")
+            pub.subscribe(self._on_connection_lost, "meshtastic.connection.lost")
+            self._last_rx = time.monotonic()
 
             self._connected = True
 
@@ -100,6 +111,8 @@ class MeshConnector:
             try:
                 pub.unsubscribe(self._on_receive, "meshtastic.receive.text")
                 pub.unsubscribe(self._on_node_update, "meshtastic.node.updated")
+                pub.unsubscribe(self._on_any_rx, "meshtastic.receive")
+                pub.unsubscribe(self._on_connection_lost, "meshtastic.connection.lost")
             except Exception:
                 pass
 
@@ -145,6 +158,7 @@ class MeshConnector:
 
     def _on_node_update(self, node, interface) -> None:
         """Handle node info updates."""
+        self._last_rx = time.monotonic()
         node_id = f"!{node['num']:08x}"
 
         with self._lock:
@@ -162,6 +176,7 @@ class MeshConnector:
 
     def _on_receive(self, packet, interface) -> None:
         """Handle incoming text message."""
+        self._last_rx = time.monotonic()
         if not self._message_callback or not self._loop:
             return
 
@@ -207,6 +222,121 @@ class MeshConnector:
         except Exception as e:
             logger.error(f"Error processing received message: {e}")
 
+    # --- watchdog: receive ANY packet bumps last_rx (liveness) ---
+    def _on_any_rx(self, packet=None, interface=None, *args, **kwargs):
+        self._last_rx = time.monotonic()
+
+    # --- watchdog: connection.lost ONLY flags suspect + wakes watchdog.
+    # Strict pubsub arg-spec passes interface= -> signature MUST accept it.
+    # Does NOT write link state, NOT toggle _connected, NOT reconnect.
+    def _on_connection_lost(self, interface=None, *args, **kwargs):
+        self._link_suspect = True
+        logger.warning("connection.lost fired -> link suspect (waking watchdog)")
+        if self._wake is not None and self._loop is not None:
+            try:
+                self._loop.call_soon_threadsafe(self._wake.set)
+            except Exception:
+                pass
+
+    @property
+    def last_rx(self) -> float:
+        return self._last_rx
+
+    @property
+    def link_suspect(self) -> bool:
+        return self._link_suspect
+
+    def clear_suspect(self) -> None:
+        self._link_suspect = False
+
+    def write_link_status(self, status: str) -> None:
+        """SINGLE writer of the link-state file. Called ONLY by the watchdog."""
+        try:
+            with open(self._link_path, "w") as f:
+                f.write(status)
+        except Exception as e:
+            logger.warning(f"Failed to write link status: {e}")
+
+    # --- watchdog: socket-based liveness (getMyNodeInfo is NOT trusted; it
+    # returns cached data on a dead link -> false-alive). Verified meshtastic 2.7.9.
+    def socket_state(self) -> str:
+        """'open', or 'dead' from isConnected + non-blocking MSG_PEEK."""
+        iface = self._interface
+        if iface is None:
+            return "dead"
+        ev = getattr(iface, "isConnected", None)
+        if ev is not None and hasattr(ev, "is_set") and not ev.is_set():
+            return "dead"
+        sock = getattr(iface, "socket", None)
+        if sock is None:
+            return "dead"
+        try:
+            data = sock.recv(1, socket.MSG_DONTWAIT | socket.MSG_PEEK)
+            return "open" if data else "dead"
+        except BlockingIOError:
+            return "open"
+        except (OSError, AttributeError):
+            return "dead"
+
+    def interface_open(self) -> bool:
+        return self.socket_state() == "open"
+
+    def active_probe(self, probe_wait: float = 5.0) -> bool:
+        """sendHeartbeat then watch for genuine life. BLOCKING -> call via to_thread.
+        ALIVE = last_rx advances OR socket stays verifiably open. Never trusts getMyNodeInfo."""
+        if self.socket_state() == "dead":
+            logger.info("probe: DEAD (socket closed / isConnected cleared)")
+            return False
+        rx_before = self._last_rx
+        try:
+            self._interface.sendHeartbeat()
+        except Exception as e:
+            logger.info(f"probe: sendHeartbeat raised {e!r} -> link dead")
+            return False
+        deadline = time.monotonic() + probe_wait
+        while time.monotonic() < deadline:
+            if self._last_rx > rx_before:
+                logger.info("probe: ALIVE (last_rx advanced)")
+                return True
+            if self.socket_state() == "dead":
+                logger.info("probe: DEAD (socket went dead during probe)")
+                return False
+            time.sleep(0.25)
+        if self.socket_state() == "open":
+            logger.info("probe: ALIVE (socket verifiably open)")
+            return True
+        logger.info("probe: DEAD (no rx advance, socket not open)")
+        return False
+
+    def reconnect(self) -> None:
+        """Tear down old interface + build a fresh one under the reconnect lock,
+        exponential backoff forever. BLOCKING -> call via to_thread. Does NOT
+        write link state (watchdog owns that)."""
+        initial = getattr(self.config, "reconnect_initial_delay", 2.0)
+        maxd = getattr(self.config, "reconnect_max_delay", 60.0)
+        with self._reconnect_lock:
+            backoff = initial
+            attempt = 0
+            while True:
+                attempt += 1
+                logger.warning(f"reconnect attempt {attempt}...")
+                # tear down old interface (also stops the lib's internal threads)
+                try:
+                    self.disconnect()
+                except Exception as e:
+                    logger.warning(f"  teardown warn: {e!r}")
+                # build fresh
+                try:
+                    self.connect()
+                    self._last_rx = time.monotonic()
+                    self._link_suspect = False
+                    logger.info(f"reconnected as node {self._my_node_id} (attempt {attempt})")
+                    return
+                except Exception as e:
+                    logger.warning(f"  reconnect attempt {attempt} failed: {e!r}; sleeping {backoff:.0f}s")
+                    time.sleep(backoff)
+                    backoff = min(backoff * 2, maxd)
+
     def send_message(
         self,
         text: str,

work/meshai/main.py

@@ -56,6 +56,7 @@ class MeshAI:
         self.router: Optional[MessageRouter] = None
         self.responder: Optional[Responder] = None
         self._running = False
+        self._supervisor_task = None  # watchdog (connection supervisor)
         self._loop: Optional[asyncio.AbstractEventLoop] = None
         self._last_cleanup: float = 0.0
         self._last_health_compute: float = 0.0
@@ -79,6 +80,14 @@ class MeshAI:
 
         self._running = True
         self._loop = asyncio.get_event_loop()
+        # --- connection supervisor (watchdog): single source of truth for link
+        # state + the only reconnect driver. Reconnects IN-PLACE so the container
+        # never needs to restart. ---
+        if getattr(self.config.connection, "reconnect", True):
+            self.connector._wake = asyncio.Event()
+            self.connector.write_link_status("up")  # we just connected ok
+            self._supervisor_task = asyncio.create_task(self._connection_supervisor())
+            logger.info("Connection supervisor (watchdog) started")
         self._last_cleanup = time.time()
         self._last_health_compute = 0.0
 
@@ -211,6 +220,55 @@ class MeshAI:
                     self.context.prune()
                 self._last_cleanup = time.time()
 
+    async def _connection_supervisor(self) -> None:
+        """Watchdog: SINGLE source of truth for /tmp/meshai.link and the ONLY
+        reconnect driver. Woken by connector._wake (connection.lost) or a
+        health-interval timeout. Probe is socket-based (see connector.active_probe).
+        """
+        c = self.connector
+        hi = getattr(self.config.connection, "reconnect_health_interval", 30.0)
+        alive_idle = 2.0 * hi
+        probe_wait = 5.0
+        wake = c._wake
+        logger.info("watchdog loop running (health_interval=%.0fs)", hi)
+        while self._running:
+            try:
+                try:
+                    await asyncio.wait_for(wake.wait(), timeout=hi)
+                except asyncio.TimeoutError:
+                    pass
+                wake.clear()
+                if not self._running:
+                    break
+
+                idle = time.monotonic() - c.last_rx
+                alive = (not c.link_suspect) and (idle < alive_idle) and c.interface_open()
+                if alive:
+                    c.clear_suspect()
+                    c.write_link_status("up")
+                    continue
+
+                # Not trivially alive -> ACTIVE PROBE (blocking -> thread)
+                logger.info("watchdog: link check (suspect=%s, idle=%.1fs) -> probing",
+                            c.link_suspect, idle)
+                probe_alive = await asyncio.to_thread(c.active_probe, probe_wait)
+                if probe_alive:
+                    c.clear_suspect()
+                    c.write_link_status("up")
+                    continue
+
+                # DEAD -> write down, reconnect in place (blocking -> thread)
+                logger.warning("watchdog: link DOWN -> reconnecting")
+                c.write_link_status("down")
+                await asyncio.to_thread(c.reconnect)
+                c.write_link_status("up")
+            except asyncio.CancelledError:
+                break
+            except Exception:
+                logger.exception("watchdog cycle error (continuing)")
+                await asyncio.sleep(1.0)
+        logger.info("watchdog loop exited")
+
     async def stop(self) -> None:
         """Stop the bot."""
         logger.info("Stopping MeshAI...")
@@ -226,6 +284,13 @@ class MeshAI:
         if self._fire_pacer is not None:
             await self._fire_pacer.stop()
 
+        if self._supervisor_task is not None:
+            self._supervisor_task.cancel()
+            try:
+                await self._supervisor_task
+            except (asyncio.CancelledError, Exception):
+                pass
+
         if self.connector:
             self.connector.disconnect()