From 238de7a9a178bbf270d7fdf3b8c9a19bc3171a38 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 27 Jun 2026 23:49:06 +0000 Subject: [PATCH] supervisor: shrink sat dedup window + WAL cursors.db to cut CPU/IO sat_positions and sat_orbits produce dedup IDs that are unique per second (:), so the inherited 14-day sweep window accumulates ~3.5M rows in cursors.db with no benefit. Shrink to 1 day (positions) and 2 days (orbits, to tolerate stable TLE re-emission). Open cursors.db in WAL + NORMAL sync mode in both adapters' startup(). WAL mode is file-level persistent, so this covers all adapters sharing the same cursors.db. Previously journal_mode=DELETE + synchronous=FULL forced an fsync on every mark_published() call (~190 calls/60s for sat_positions alone), sustaining ~17% CPU on the supervisor process. A redeploy is needed; the box is currently detached at v0.14.5. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/central/adapters/sat_orbits.py | 8 ++++++++ src/central/adapters/sat_positions.py | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/src/central/adapters/sat_orbits.py b/src/central/adapters/sat_orbits.py index 8b74721..a2855d5 100644 --- a/src/central/adapters/sat_orbits.py +++ b/src/central/adapters/sat_orbits.py @@ -126,6 +126,10 @@ class SatOrbitsAdapter(SourceAdapter): # Skip the publish-time/archive bbox filter (v0.14.2). Keep in sync with # archive._BYPASS_BBOX_ADAPTERS. bypass_bbox_filter = True + # Dedup IDs are ":" -- unique per second by design, + # so a 14-day window accumulates rows unnecessarily. TLEs are stable for days, + # so 2 days is sufficient to catch re-emitted orbits without bloating the table. + dedup_sweep_days = 2 def __init__( self, @@ -147,6 +151,10 @@ class SatOrbitsAdapter(SourceAdapter): async def startup(self) -> None: self._db = sqlite3.connect(self._cursor_db_path) + # WAL + NORMAL sync: eliminates per-commit fsync overhead on cursors.db. + # WAL mode is file-level persistent; applies to all adapters on this DB. + self._db.execute("PRAGMA journal_mode=WAL") + self._db.execute("PRAGMA synchronous=NORMAL") self._db.execute(_DEDUP_DDL) self._db.execute( "CREATE INDEX IF NOT EXISTS published_ids_last_seen ON published_ids (last_seen)" diff --git a/src/central/adapters/sat_positions.py b/src/central/adapters/sat_positions.py index 91d8bdb..11c2a74 100644 --- a/src/central/adapters/sat_positions.py +++ b/src/central/adapters/sat_positions.py @@ -136,6 +136,9 @@ class SatPositionsAdapter(SourceAdapter): # queries. Skip the publish-time/archive bbox filter (v0.14.2). Keep in sync # with archive._BYPASS_BBOX_ADAPTERS. bypass_bbox_filter = True + # Dedup IDs are ":" -- unique per second by design, + # so a 14-day window accumulates ~3.5M rows unnecessarily. 1 day is enough. + dedup_sweep_days = 1 def __init__( self, @@ -155,6 +158,10 @@ class SatPositionsAdapter(SourceAdapter): async def startup(self) -> None: self._db = sqlite3.connect(self._cursor_db_path) + # WAL + NORMAL sync: eliminates per-commit fsync overhead on cursors.db. + # WAL mode is file-level persistent; applies to all adapters on this DB. + self._db.execute("PRAGMA journal_mode=WAL") + self._db.execute("PRAGMA synchronous=NORMAL") self._db.execute(_DEDUP_DDL) self._db.execute( "CREATE INDEX IF NOT EXISTS published_ids_last_seen ON published_ids (last_seen)"