matt/central
1
0
Fork 0
mirror of https://github.com/zvx-echo6/central.git synced 2026-08-26 09:21:36 +00:00
Code Issues Projects Releases Packages Wiki Activity

supervisor: shrink sat dedup window + WAL cursors.db to cut CPU/IO

Browse source 
sat_positions and sat_orbits produce dedup IDs that are unique per
second (<norad_id>:<iso_timestamp>), so the inherited 14-day sweep
window accumulates ~3.5M rows in cursors.db with no benefit. Shrink
to 1 day (positions) and 2 days (orbits, to tolerate stable TLE
re-emission).

Open cursors.db in WAL + NORMAL sync mode in both adapters' startup().
WAL mode is file-level persistent, so this covers all adapters sharing
the same cursors.db. Previously journal_mode=DELETE + synchronous=FULL
forced an fsync on every mark_published() call (~190 calls/60s for
sat_positions alone), sustaining ~17% CPU on the supervisor process.

A redeploy is needed; the box is currently detached at v0.14.5.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ubuntu 2026-06-27 23:49:06 +00:00
commit 238de7a9a1
2 changed files with 15 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

8
src/central/adapters/sat_orbits.py
View file

@ -126,6 +126,10 @@ class SatOrbitsAdapter(SourceAdapter):
# Skip the publish-time/archive bbox filter (v0.14.2). Keep in sync with # Skip the publish-time/archive bbox filter (v0.14.2). Keep in sync with
# archive._BYPASS_BBOX_ADAPTERS. # archive._BYPASS_BBOX_ADAPTERS.
bypass_bbox_filter = True bypass_bbox_filter = True
# Dedup IDs are "<norad_id>:<propagation_iso>" -- unique per second by design,
# so a 14-day window accumulates rows unnecessarily. TLEs are stable for days,
# so 2 days is sufficient to catch re-emitted orbits without bloating the table.
dedup_sweep_days = 2
def __init__( def __init__(
self, self,
@ -147,6 +151,10 @@ class SatOrbitsAdapter(SourceAdapter):
async def startup(self) -> None: async def startup(self) -> None:
self._db = sqlite3.connect(self._cursor_db_path) self._db = sqlite3.connect(self._cursor_db_path)
# WAL + NORMAL sync: eliminates per-commit fsync overhead on cursors.db.
# WAL mode is file-level persistent; applies to all adapters on this DB.
self._db.execute("PRAGMA journal_mode=WAL")
self._db.execute("PRAGMA synchronous=NORMAL")
self._db.execute(_DEDUP_DDL) self._db.execute(_DEDUP_DDL)
self._db.execute( self._db.execute(
"CREATE INDEX IF NOT EXISTS published_ids_last_seen ON published_ids (last_seen)" "CREATE INDEX IF NOT EXISTS published_ids_last_seen ON published_ids (last_seen)"

7
src/central/adapters/sat_positions.py
View file

@ -136,6 +136,9 @@ class SatPositionsAdapter(SourceAdapter):
# queries. Skip the publish-time/archive bbox filter (v0.14.2). Keep in sync # queries. Skip the publish-time/archive bbox filter (v0.14.2). Keep in sync
# with archive._BYPASS_BBOX_ADAPTERS. # with archive._BYPASS_BBOX_ADAPTERS.
bypass_bbox_filter = True bypass_bbox_filter = True
# Dedup IDs are "<norad_id>:<position_iso>" -- unique per second by design,
# so a 14-day window accumulates ~3.5M rows unnecessarily. 1 day is enough.
dedup_sweep_days = 1
def __init__( def __init__(
self, self,
@ -155,6 +158,10 @@ class SatPositionsAdapter(SourceAdapter):
async def startup(self) -> None: async def startup(self) -> None:
self._db = sqlite3.connect(self._cursor_db_path) self._db = sqlite3.connect(self._cursor_db_path)
# WAL + NORMAL sync: eliminates per-commit fsync overhead on cursors.db.
# WAL mode is file-level persistent; applies to all adapters on this DB.
self._db.execute("PRAGMA journal_mode=WAL")
self._db.execute("PRAGMA synchronous=NORMAL")
self._db.execute(_DEDUP_DDL) self._db.execute(_DEDUP_DDL)
self._db.execute( self._db.execute(
"CREATE INDEX IF NOT EXISTS published_ids_last_seen ON published_ids (last_seen)" "CREATE INDEX IF NOT EXISTS published_ids_last_seen ON published_ids (last_seen)"