From 83b1e45fa89e90291580c5c17a25ac0ecea8b68e Mon Sep 17 00:00:00 2001 From: Matt Johnson Date: Sun, 17 May 2026 18:26:48 +0000 Subject: [PATCH] docs: add test database setup, restore geom to test fixture - Add docs/test-database.md with one-time setup, DSN convention, reset instructions, and explanation of why PostGIS is not in migrations - Update docs/migrations.md with "Extensions are not in migrations" section explaining superuser requirement - Restore geom GEOMETRY(Geometry, 4326) column to test fixture now that central_test has PostGIS installed - Add CREATE EXTENSION IF NOT EXISTS postgis to test fixture for self-bootstrap (central_test is superuser) - Add Testing section to README.md pointing to docs/test-database.md Co-Authored-By: Claude Opus 4.5 --- README.md | 4 ++ docs/migrations.md | 18 ++++++- docs/test-database.md | 83 +++++++++++++++++++++++++++++ tests/test_events_adapter_column.py | 10 +++- 4 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 docs/test-database.md diff --git a/README.md b/README.md index 36bcb4c..f057003 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,10 @@ Phase 0 — scaffold. Not yet operational. - One archive consumer process persisting events to TimescaleDB - Both processes systemd-managed +## Testing + +See [docs/test-database.md](docs/test-database.md) for test database setup. + ## License MIT. See LICENSE. diff --git a/docs/migrations.md b/docs/migrations.md index ddac19a..ccf2e61 100644 --- a/docs/migrations.md +++ b/docs/migrations.md @@ -26,8 +26,24 @@ Direct `psql` execution bypasses the `schema_migrations` tracker and was the cause of the v0.2.0 reconcile. If a migration needs to be applied on the live system, run: - sudo -u central /opt/central/.venv/bin/python -m scripts.migrate + sudo -u central /opt/central/.venv/bin/python -m central.migrate Never apply migration SQL directly via `psql`, even as a superuser, even "just to test." If migrate.py has a bug that's blocking you, fix migrate.py. + +## Extensions are not in migrations + +PostgreSQL extensions like PostGIS require superuser privileges to +install. The production `central` role is intentionally not a superuser. +Therefore, extensions live outside the migration system: + +- **Production bootstrap:** A DBA runs `CREATE EXTENSION postgis` once + before the first `migrate.py` run. +- **Test database:** The `central_test` role is a superuser, allowing + test fixtures to self-bootstrap extensions. + +This is documented in [docs/test-database.md](test-database.md). + +Do not add `CREATE EXTENSION` statements to migrations — they will fail +in production where migrations run as the non-superuser `central` role. diff --git a/docs/test-database.md b/docs/test-database.md new file mode 100644 index 0000000..71c71a4 --- /dev/null +++ b/docs/test-database.md @@ -0,0 +1,83 @@ +# Test Database Setup + +Central's integration tests require a PostgreSQL database. This document +covers one-time setup and maintenance of the test database. + +## DSN Convention + +Tests default to: + +``` +postgresql://central_test:testpass@localhost/central_test +``` + +Override via the `CENTRAL_TEST_DB_DSN` environment variable: + +```bash +export CENTRAL_TEST_DB_DSN="postgresql://myuser:mypass@localhost/mydb" +``` + +## One-Time Setup + +Run these commands once on a fresh PostgreSQL installation: + +```bash +# Create the test user (as postgres superuser) +sudo -u postgres createuser -s central_test +sudo -u postgres psql -c "ALTER USER central_test PASSWORD 'testpass'" + +# Create the test database +sudo -u postgres createdb -O central_test central_test + +# Install required extensions +sudo -u postgres psql central_test -c "CREATE EXTENSION IF NOT EXISTS postgis" +``` + +**Note:** The `central_test` role is created as a superuser (`-s` flag). +This allows test fixtures to self-bootstrap extensions like PostGIS via +`CREATE EXTENSION IF NOT EXISTS`. Production uses a non-superuser role. + +## Required Extensions + +| Extension | Version | Purpose | +|-----------|---------|---------| +| postgis | 3.4+ | Geometry types for geospatial event data | + +## Why PostGIS Is Not in Migrations + +PostGIS requires superuser privileges to install. The production `central` +role is intentionally not a superuser for security reasons. Therefore: + +- **Production:** A DBA must run `CREATE EXTENSION postgis` before the + first `migrate.py` run. This is a one-time bootstrap step. +- **Test:** The `central_test` role is a superuser, so test fixtures can + self-bootstrap PostGIS via `CREATE EXTENSION IF NOT EXISTS`. + +This divergence is documented rather than "fixed" because granting +superuser to production roles creates security risk, and the PostgreSQL +packaging on Ubuntu does not mark PostGIS as a trusted extension. + +## Resetting the Test Database + +If the test database gets into a bad state: + +```bash +# Drop and recreate +sudo -u postgres dropdb central_test +sudo -u postgres createdb -O central_test central_test +sudo -u postgres psql central_test -c "CREATE EXTENSION IF NOT EXISTS postgis" +``` + +Test fixtures handle their own table creation and cleanup, so this is +rarely needed. + +## Running Tests + +```bash +cd /opt/central +uv run pytest tests/ # all tests +uv run pytest tests/test_config_store.py -v # specific file +``` + +Tests that require the database will skip gracefully if the connection +fails, though most integration tests will fail without a working DB. diff --git a/tests/test_events_adapter_column.py b/tests/test_events_adapter_column.py index 7a22396..c0d630c 100644 --- a/tests/test_events_adapter_column.py +++ b/tests/test_events_adapter_column.py @@ -5,6 +5,8 @@ verifying backfill logic, FK constraints, NOT NULL enforcement, and source column removal. Requires CENTRAL_TEST_DB_DSN or uses default central_test database. +The test database must have PostGIS installed, or the central_test role +must be a superuser (which it is by default) to self-bootstrap PostGIS. """ import os @@ -75,7 +77,11 @@ async def pre_migration_events_table(db_conn: asyncpg.Connection) -> None: """Create events table with pre-migration schema (source column, no adapter). Also ensures config.adapters exists with test adapters. + Self-bootstraps PostGIS if not already installed (central_test is superuser). """ + # Self-bootstrap PostGIS extension (central_test role is superuser) + await db_conn.execute("CREATE EXTENSION IF NOT EXISTS postgis") + # Ensure config schema and adapters table exist await db_conn.execute("CREATE SCHEMA IF NOT EXISTS config") await db_conn.execute(""" @@ -100,7 +106,7 @@ async def pre_migration_events_table(db_conn: asyncpg.Connection) -> None: await db_conn.execute("DROP TABLE IF EXISTS public.events CASCADE") # Create events table with PRE-MIGRATION schema (has source, no adapter) - # Note: geom column omitted since test DB lacks PostGIS extension + # Matches production schema including geom column await db_conn.execute(""" CREATE TABLE public.events ( id TEXT NOT NULL, @@ -109,6 +115,7 @@ async def pre_migration_events_table(db_conn: asyncpg.Connection) -> None: time TIMESTAMPTZ NOT NULL, expires TIMESTAMPTZ, severity SMALLINT, + geom GEOMETRY(Geometry, 4326), regions TEXT[], primary_region TEXT, payload JSONB NOT NULL, @@ -118,6 +125,7 @@ async def pre_migration_events_table(db_conn: asyncpg.Connection) -> None: """) # Insert test rows with different source values + # geom is NULL (production schema permits this) test_rows = [ ("event-nws-1", "central/adapters/nws", "wx.alert.tornado_warning"), ("event-nws-2", "central/adapters/nws", "wx.alert.flood_warning"),