From e7d30b3c485706783a0eab7a019b577b6d41376d Mon Sep 17 00:00:00 2001 From: echo6-autocommit Date: Wed, 1 Jul 2026 18:00:05 +0000 Subject: [PATCH] auto: docs sync 2026-07-01T18:00:05+00:00 Files changed: engine/lint-report.md vault/.obsidian/workspace.json vault/runbooks/pymc-repeater-kiss-tnc-reenumeration.md --- engine/lint-report.md | 2 +- vault/.obsidian/workspace.json | 4 +- .../pymc-repeater-kiss-tnc-reenumeration.md | 187 ++++++++++++++++++ 3 files changed, 190 insertions(+), 3 deletions(-) create mode 100644 vault/runbooks/pymc-repeater-kiss-tnc-reenumeration.md diff --git a/engine/lint-report.md b/engine/lint-report.md index 7512513..21c472a 100644 --- a/engine/lint-report.md +++ b/engine/lint-report.md @@ -1,6 +1,6 @@ # Vault Lint Report -Generated: 2026-07-01T06:00:06Z | Docs scanned: 98 | Elapsed: 0.0s +Generated: 2026-07-01T12:00:06Z | Docs scanned: 98 | Elapsed: 0.0s ## Summary diff --git a/vault/.obsidian/workspace.json b/vault/.obsidian/workspace.json index c33b283..5dddd81 100644 --- a/vault/.obsidian/workspace.json +++ b/vault/.obsidian/workspace.json @@ -199,6 +199,8 @@ }, "active": "17bd4a6166f789d0", "lastOpenFiles": [ + "runbooks/pymc-repeater-kiss-tnc-reenumeration.md", + "runbooks/pymc-repeater-kiss-tnc-reenumeration.md.tmp.5281.e47d0cbf9c9a", "runbooks/headscale-oidc-boot-order.md", "runbooks/headscale-oidc-boot-order.md.tmp.246153.e2a5642e10dd", "runbooks/headscale-onboard-node.md.tmp.246153.9adf70459e77", @@ -209,7 +211,6 @@ "runbooks/ia-cli-reference.md.tmp.3603785.daa56d288051", "runbooks/pipeline-patterns.md.tmp.3603785.ea95256eaac1", "runbooks/headless-browser-page-verification.md.tmp.3603785.4eb1d115aefe", - "runbooks/headless-browser-page-verification.md.tmp.3603785.f017d6928577", "runbooks/headless-browser-page-verification.md", "runbooks/central-deploy-cutover.md", "runbooks/fleet-magicdns-resolved-migration.md", @@ -235,7 +236,6 @@ "concepts/ocr.md", "archive/projects/mmud/mmud-prompts/mmud-prompts/01-update-planned.md", "INDEX.md", - "glossary.md", "assets/echo6yellow_logo_422x422_square.png", "assets/echo6yellow_logo_422x81.png", "assets/echo6_logo.png", diff --git a/vault/runbooks/pymc-repeater-kiss-tnc-reenumeration.md b/vault/runbooks/pymc-repeater-kiss-tnc-reenumeration.md new file mode 100644 index 0000000..59cdecf --- /dev/null +++ b/vault/runbooks/pymc-repeater-kiss-tnc-reenumeration.md @@ -0,0 +1,187 @@ +--- +title: pymc-repeater KISS TNC Re-enumeration Fix +type: runbook +tags: + - mesh +related: + - [[meshtastic-sidecar-node]] +updated: 2026-07-01 +--- +# pymc-repeater KISS TNC Re-enumeration Fix + +`pymc-repeater.service` on **aida-nebra** (Tailscale 100.64.0.9) runs the Meshcore LoRa repeater node `EchoBase` on 910.525 MHz SF7. It talks to a RAK4631 USB KISS TNC (VID 239a / PID 8029) via the `/dev/pymc-radio` udev symlink. The Nebra SX1262 hat on the same machine runs a separate `meshtasticd` instance on its own radio — see [[meshtastic-sidecar-node]] for that hardware's setup. Do not touch `meshtasticd` or the SX1262 hat during any pymc-repeater work. + +--- + +## Background — Why This Breaks + +`KissModemWrapper` holds the serial file descriptor open but has no reopen logic. When the RAK4631 re-enumerates (power cycle, USB glitch, kernel rescan), the kernel assigns a new node: observed `ttyACM0` → `ttyACM1` around 2026-06-23. The service keeps a stale fd pointing at `/dev/ttyACMx (deleted)`. Systemd still reports `active (running)` — the service is silently deaf and mute. + +--- + +## Symptom + +Every ~30 seconds in the journal: + +``` +KissModemWrapper - ERROR - Serial write error: ... [Errno 5] Input/output error +TX frame write failed, dropping frame +``` + +Zero RF traffic in or out despite a green `systemctl status`. + +--- + +## Diagnose + +### 1. Check the journal for the I/O error spam + +```bash +ssh zvx@100.64.0.9 +journalctl -u pymc-repeater.service --since '5 min ago' --no-pager +``` + +Look for `[Errno 5] Input/output error` or `TX frame write failed` — either confirms stale fd. + +### 2. Inspect the process's open file descriptors + +```bash +sudo ls -l /proc/$(systemctl show -p MainPID --value pymc-repeater.service)/fd/ | grep ttyACM +``` + +- **Stale (broken):** symlink target contains `(deleted)` — smoking gun. +- **Healthy:** target points at a real `/dev/ttyACMx`. + +### 3. Confirm where the device landed after re-enumeration + +```bash +ls -l /dev/pymc-radio +``` + +Shows which `ttyACM*` the live RAK4631 is on now. + +--- + +## Manual Recovery + +A restart clears the stale fd and re-opens the device via the udev symlink: + +```bash +sudo systemctl restart pymc-repeater.service +``` + +### Verify recovery + +```bash +# fd should now point at a real ttyACMx (not deleted) +sudo ls -l /proc/$(systemctl show -p MainPID --value pymc-repeater.service)/fd/ | grep ttyACM + +# Journal should show reconnect and normal TX +journalctl -u pymc-repeater.service -n 20 --no-pager +``` + +Expected healthy journal lines: + +``` +KISS modem connected to /dev/pymc-radio at 115200 baud +Retransmitted packet ... +``` + +The `[Errno 5]` spam must be absent. + +--- + +## Permanent Fix (applied 2026-07-01) + +Two changes make pymc-repeater self-healing across any future re-enumeration. Original files were backed up as `.bak` alongside each. + +### 1. udev rule — `/etc/udev/rules.d/99-pymc-radio.rules` + +Added `TAG+="systemd"` and `ENV{SYSTEMD_WANTS}="pymc-repeater.service"` to the existing VID 239a / PID 8029 rule: + +```udev +SUBSYSTEM=="tty", ATTRS{idVendor}=="239a", ATTRS{idProduct}=="8029", \ + SYMLINK+="pymc-radio", \ + TAG+="systemd", \ + ENV{SYSTEMD_WANTS}="pymc-repeater.service" +``` + +On any future re-enumeration, udev creates the `dev-pymc\x2dradio.device` unit for the new node and systemd activates `pymc-repeater.service` against it automatically. + +### 2. systemd unit — `/etc/systemd/system/pymc-repeater.service` + +`[Unit]` additions: + +```ini +BindsTo=dev-pymc\x2dradio.device +After=dev-pymc\x2dradio.device +StartLimitIntervalSec=300 +StartLimitBurst=10 +``` + +`[Service]` additions: + +```ini +Restart=always +RestartSec=5 +``` + +`BindsTo` stops the service cleanly when the TNC drops (the device unit disappears) and restarts it when the device returns. `Restart=always` with `RestartSec=5` ensures fast recovery from any modem crash without the device unit cycling. + +--- + +## Verify the Resilience Wiring Is Intact + +Run these on aida-nebra after any future maintenance to confirm the permanent fix is still in place: + +```bash +# BindsTo and After wired to the device unit +systemctl show pymc-repeater.service -p BindsTo -p After | grep -i pymc + +# udev tagging correct +udevadm info -q property -n /dev/pymc-radio | grep -i SYSTEMD + +# device unit tracked by systemd +systemctl list-units --type=device --all | grep -i pymc +``` + +Expected output: + +``` +BindsTo=dev-pymc\x2dradio.device +After=... dev-pymc\x2dradio.device ... +SYSTEMD_WANTS=pymc-repeater.service +dev-pymc\x2dradio.device loaded active plugged ... +``` + +--- + +## Known Secondary Issue (non-blocking) + +RRD graphing fails with `rrdtool not available`. Metrics/graphing only — no impact on RF operation. + +--- + +## Quick Reference + +| Item | Value | +|------|-------| +| Host | aida-nebra (100.64.0.9) | +| Service | `pymc-repeater.service` | +| Node / freq | EchoBase, 910.525 MHz SF7 | +| TNC hardware | RAK4631 USB KISS TNC (VID 239a / PID 8029) | +| udev symlink | `/dev/pymc-radio` | +| udev rule | `/etc/udev/rules.d/99-pymc-radio.rules` | +| Unit file | `/etc/systemd/system/pymc-repeater.service` | +| Unrelated radio | Nebra SX1262 hat → `meshtasticd` (do not touch) | + +```bash +# Restart (manual recovery) +sudo systemctl restart pymc-repeater.service + +# Tail journal +journalctl -u pymc-repeater.service -f + +# Check fd state +sudo ls -l /proc/$(systemctl show -p MainPID --value pymc-repeater.service)/fd/ | grep ttyACM +```