feat(2-D): add NOAA SWPC space weather adapters (alerts, kindex, protons)

Three independent adapters sharing src/central/adapters/swpc_common.py,
mirroring the WFIGS two-adapter pattern. Each adapter has its own row in
config.adapters (ships disabled), its own cadence, and its own dedup
state, so operators can independently enable/disable and so a broken
upstream endpoint does not silently mask a healthy one.

Subjects:
  swpc_alerts   -> central.space.alert.<product_id_lower>
  swpc_kindex   -> central.space.kindex
  swpc_protons  -> central.space.proton_flux

Dedup keys:
  alerts:   product_id + issue_datetime
  kindex:   time_tag
  protons:  time_tag + energy

Severity: G-scale on product_id for K0[5-9][AW] alerts (G1-G5 -> 1-4),
G-scale on Kp for kindex, 0 for protons (raw flux carried in event.data).

No geo on any SWPC events (centroid=None, regions=[], primary_region=None).
No fall-off detection for alerts -- a single 115-row sample cannot confirm
whether alerts disappear from the upstream JSON when expired; deferred to
a later pass after 24h of observation.

CENTRAL_SPACE stream seeded with 7-day retention / 1 GiB max_bytes, mirroring
CENTRAL_FIRE / CENTRAL_QUAKE. STREAM_SUBJECTS, archive STREAMS, and
DASHBOARD_STREAMS each pick up the new stream.

Tests: 16 new cases in tests/test_swpc.py using real-shape frozen JSON
fixtures (alerts product_ids EF3A/K05A/K07A; kindex Kp boundaries; protons
composite dedup). Two existing tests updated for the new stream count
(test_archive_multi_stream.test_streams_list_has_three_entries renamed to
_has_four_entries; test_dashboard expects 5 streams not 4); added a
test_streams_contains_central_space companion.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
zvx-echo6 2026-05-19 05:55:29 +00:00
commit 72ec498365
12 changed files with 1007 additions and 5 deletions

View file

@ -0,0 +1,186 @@
"""NOAA SWPC space weather alerts adapter."""
import logging
import sqlite3
from collections.abc import AsyncIterator
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import aiohttp
from tenacity import (
retry,
retry_if_exception_type,
stop_after_attempt,
wait_exponential_jitter,
)
from central.adapter import SourceAdapter
from central.adapters.swpc_common import (
SWPC_ALERTS_URL,
SWPCSettings,
parse_swpc_timestamp,
severity_from_alert_product_id,
)
from central.config_models import AdapterConfig
from central.config_store import ConfigStore
from central.models import Event, Geo
logger = logging.getLogger(__name__)
class SWPCAlertsAdapter(SourceAdapter):
"""NOAA SWPC space weather alerts adapter."""
name = "swpc_alerts"
display_name = "NOAA SWPC — Space Weather Alerts"
description = "Active NOAA SWPC space weather alerts, watches, warnings, and summaries."
settings_schema = SWPCSettings
requires_api_key = None
api_key_field = None
wizard_order = None
default_cadence_s = 300
def __init__(
self,
config: AdapterConfig,
config_store: ConfigStore,
cursor_db_path: Path,
) -> None:
self._config_store = config_store
self._cursor_db_path = cursor_db_path
self._session: aiohttp.ClientSession | None = None
self._db: sqlite3.Connection | None = None
async def startup(self) -> None:
self._session = aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=60),
)
self._db = sqlite3.connect(self._cursor_db_path)
self._db.execute("""
CREATE TABLE IF NOT EXISTS published_ids (
adapter TEXT NOT NULL,
event_id TEXT NOT NULL,
first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (adapter, event_id)
)
""")
self._db.execute("""
CREATE INDEX IF NOT EXISTS published_ids_last_seen
ON published_ids (last_seen)
""")
self._db.commit()
logger.info("SWPC alerts adapter started")
async def shutdown(self) -> None:
if self._session:
await self._session.close()
self._session = None
if self._db:
self._db.close()
self._db = None
logger.info("SWPC alerts adapter shut down")
async def apply_config(self, new_config: AdapterConfig) -> None:
logger.info("SWPC alerts config updated")
def is_published(self, event_id: str) -> bool:
if not self._db:
return False
cur = self._db.execute(
"SELECT 1 FROM published_ids WHERE adapter = ? AND event_id = ?",
(self.name, event_id),
)
return cur.fetchone() is not None
def mark_published(self, event_id: str) -> None:
if not self._db:
return
self._db.execute(
"""
INSERT INTO published_ids (adapter, event_id, first_seen, last_seen)
VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (adapter, event_id) DO UPDATE SET
last_seen = CURRENT_TIMESTAMP
""",
(self.name, event_id),
)
self._db.commit()
def sweep_old_ids(self) -> int:
if not self._db:
return 0
cur = self._db.execute(
"DELETE FROM published_ids WHERE adapter = ? AND last_seen < datetime('now', '-14 days')",
(self.name,),
)
self._db.commit()
count = cur.rowcount
if count > 0:
logger.info("SWPC alerts swept old dedup entries", extra={"count": count})
return count
def subject_for(self, event: Event) -> str:
product_id = event.data.get("product_id") or "unknown"
return f"central.space.alert.{product_id.lower()}"
@retry(
stop=stop_after_attempt(3),
wait=wait_exponential_jitter(initial=1, max=30),
retry=retry_if_exception_type((aiohttp.ClientError, TimeoutError)),
)
async def _fetch(self) -> list[dict[str, Any]]:
if not self._session:
raise RuntimeError("Session not initialized")
async with self._session.get(
SWPC_ALERTS_URL, headers={"User-Agent": "Central/0.4"}
) as resp:
resp.raise_for_status()
data = await resp.json()
logger.info("SWPC alerts fetch completed", extra={"item_count": len(data)})
return data
async def poll(self) -> AsyncIterator[Event]:
if not self._db:
raise RuntimeError("Database not initialized")
try:
items = await self._fetch()
except Exception as e:
logger.error("SWPC alerts fetch failed", extra={"error": str(e)})
raise
events_yielded = 0
for item in items:
product_id = item.get("product_id")
issue_dt_raw = item.get("issue_datetime")
if not product_id or not issue_dt_raw:
continue
event_id = f"{product_id}|{issue_dt_raw}"
if self.is_published(event_id):
continue
issue_dt = parse_swpc_timestamp(issue_dt_raw, "alerts") or datetime.now(timezone.utc)
event = Event(
id=event_id,
adapter=self.name,
category="space.alert",
time=issue_dt,
severity=severity_from_alert_product_id(product_id),
geo=Geo(),
data={
"product_id": product_id,
"issue_datetime": issue_dt_raw,
"message": item.get("message", ""),
},
)
yield event
self.mark_published(event_id)
events_yielded += 1
self.sweep_old_ids()
logger.info("SWPC alerts poll completed", extra={"events_yielded": events_yielded})

View file

@ -0,0 +1,81 @@
"""Shared utilities for NOAA SWPC space weather adapters."""
import re
from datetime import datetime, timezone
from pydantic import BaseModel
SWPC_ALERTS_URL = "https://services.swpc.noaa.gov/products/alerts.json"
SWPC_KINDEX_URL = "https://services.swpc.noaa.gov/products/noaa-planetary-k-index.json"
SWPC_PROTONS_URL = "https://services.swpc.noaa.gov/json/goes/primary/integral-protons-1-day.json"
class SWPCSettings(BaseModel):
"""Settings schema for SWPC adapters. No operator-tunable knobs today."""
def parse_swpc_timestamp(raw: str | None, endpoint_kind: str) -> datetime | None:
"""Normalize SWPC timestamp strings to UTC datetime.
endpoint_kind shapes:
alerts -> "2026-05-19 05:14:59.780" (space-separated, no TZ; UTC per message body)
kindex -> "2026-05-12T00:00:00" (ISO without TZ; UTC by convention)
protons -> "2026-05-18T05:35:00Z" (ISO with Z)
"""
if not raw:
return None
if endpoint_kind == "alerts":
try:
dt = datetime.strptime(raw, "%Y-%m-%d %H:%M:%S.%f")
except ValueError:
dt = datetime.strptime(raw, "%Y-%m-%d %H:%M:%S")
return dt.replace(tzinfo=timezone.utc)
if endpoint_kind == "kindex":
dt = datetime.fromisoformat(raw)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
return dt.astimezone(timezone.utc)
if endpoint_kind == "protons":
raw_norm = raw[:-1] + "+00:00" if raw.endswith("Z") else raw
dt = datetime.fromisoformat(raw_norm)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
return dt.astimezone(timezone.utc)
raise ValueError(f"unknown endpoint_kind: {endpoint_kind!r}")
def severity_from_kp(kp: float | int | None) -> int:
"""Map planetary K-index value (0-9) to severity 0-4 via the G-scale.
Kp 5 = G1 = severity 1, Kp 6 = G2 = severity 2, Kp 7 = G3 = severity 3,
Kp 8 = G4 = severity 4, Kp 9 = G5 = severity 4 (capped).
"""
if kp is None:
return 0
if kp < 5:
return 0
if kp < 6:
return 1
if kp < 7:
return 2
if kp < 8:
return 3
return 4
_ALERT_KP_PATTERN = re.compile(r"^K0([5-9])[AW]$")
def severity_from_alert_product_id(product_id: str | None) -> int:
"""Best-effort severity for an alert from its product_id G-scale.
Product IDs of form K0[5-9][AW] identify Kp-based geomagnetic storm
alerts and warnings (K05A=G1, K06A=G2, K07A=G3, K08A=G4, K09A=G5).
All other product IDs return 0.
"""
if not product_id:
return 0
m = _ALERT_KP_PATTERN.match(product_id.upper())
if not m:
return 0
return severity_from_kp(int(m.group(1)))

View file

@ -0,0 +1,186 @@
"""NOAA SWPC Planetary K-Index adapter."""
import logging
import sqlite3
from collections.abc import AsyncIterator
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import aiohttp
from tenacity import (
retry,
retry_if_exception_type,
stop_after_attempt,
wait_exponential_jitter,
)
from central.adapter import SourceAdapter
from central.adapters.swpc_common import (
SWPC_KINDEX_URL,
SWPCSettings,
parse_swpc_timestamp,
severity_from_kp,
)
from central.config_models import AdapterConfig
from central.config_store import ConfigStore
from central.models import Event, Geo
logger = logging.getLogger(__name__)
class SWPCKindexAdapter(SourceAdapter):
"""NOAA SWPC planetary K-index adapter."""
name = "swpc_kindex"
display_name = "NOAA SWPC — Planetary K-Index"
description = "Planetary K-index measurements at 3-hour cadence from NOAA SWPC."
settings_schema = SWPCSettings
requires_api_key = None
api_key_field = None
wizard_order = None
default_cadence_s = 600
def __init__(
self,
config: AdapterConfig,
config_store: ConfigStore,
cursor_db_path: Path,
) -> None:
self._config_store = config_store
self._cursor_db_path = cursor_db_path
self._session: aiohttp.ClientSession | None = None
self._db: sqlite3.Connection | None = None
async def startup(self) -> None:
self._session = aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=60),
)
self._db = sqlite3.connect(self._cursor_db_path)
self._db.execute("""
CREATE TABLE IF NOT EXISTS published_ids (
adapter TEXT NOT NULL,
event_id TEXT NOT NULL,
first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (adapter, event_id)
)
""")
self._db.execute("""
CREATE INDEX IF NOT EXISTS published_ids_last_seen
ON published_ids (last_seen)
""")
self._db.commit()
logger.info("SWPC kindex adapter started")
async def shutdown(self) -> None:
if self._session:
await self._session.close()
self._session = None
if self._db:
self._db.close()
self._db = None
logger.info("SWPC kindex adapter shut down")
async def apply_config(self, new_config: AdapterConfig) -> None:
logger.info("SWPC kindex config updated")
def is_published(self, event_id: str) -> bool:
if not self._db:
return False
cur = self._db.execute(
"SELECT 1 FROM published_ids WHERE adapter = ? AND event_id = ?",
(self.name, event_id),
)
return cur.fetchone() is not None
def mark_published(self, event_id: str) -> None:
if not self._db:
return
self._db.execute(
"""
INSERT INTO published_ids (adapter, event_id, first_seen, last_seen)
VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (adapter, event_id) DO UPDATE SET
last_seen = CURRENT_TIMESTAMP
""",
(self.name, event_id),
)
self._db.commit()
def sweep_old_ids(self) -> int:
if not self._db:
return 0
cur = self._db.execute(
"DELETE FROM published_ids WHERE adapter = ? AND last_seen < datetime('now', '-14 days')",
(self.name,),
)
self._db.commit()
count = cur.rowcount
if count > 0:
logger.info("SWPC kindex swept old dedup entries", extra={"count": count})
return count
def subject_for(self, event: Event) -> str:
return "central.space.kindex"
@retry(
stop=stop_after_attempt(3),
wait=wait_exponential_jitter(initial=1, max=30),
retry=retry_if_exception_type((aiohttp.ClientError, TimeoutError)),
)
async def _fetch(self) -> list[dict[str, Any]]:
if not self._session:
raise RuntimeError("Session not initialized")
async with self._session.get(
SWPC_KINDEX_URL, headers={"User-Agent": "Central/0.4"}
) as resp:
resp.raise_for_status()
data = await resp.json()
logger.info("SWPC kindex fetch completed", extra={"item_count": len(data)})
return data
async def poll(self) -> AsyncIterator[Event]:
if not self._db:
raise RuntimeError("Database not initialized")
try:
items = await self._fetch()
except Exception as e:
logger.error("SWPC kindex fetch failed", extra={"error": str(e)})
raise
events_yielded = 0
for item in items:
time_tag = item.get("time_tag")
kp = item.get("Kp")
if not time_tag or kp is None:
continue
event_id = time_tag
if self.is_published(event_id):
continue
event_time = parse_swpc_timestamp(time_tag, "kindex") or datetime.now(timezone.utc)
event = Event(
id=event_id,
adapter=self.name,
category="space.kindex",
time=event_time,
severity=severity_from_kp(kp),
geo=Geo(),
data={
"time_tag": time_tag,
"Kp": kp,
"a_running": item.get("a_running"),
"station_count": item.get("station_count"),
},
)
yield event
self.mark_published(event_id)
events_yielded += 1
self.sweep_old_ids()
logger.info("SWPC kindex poll completed", extra={"events_yielded": events_yielded})

View file

@ -0,0 +1,185 @@
"""NOAA SWPC GOES integral proton flux adapter."""
import logging
import sqlite3
from collections.abc import AsyncIterator
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import aiohttp
from tenacity import (
retry,
retry_if_exception_type,
stop_after_attempt,
wait_exponential_jitter,
)
from central.adapter import SourceAdapter
from central.adapters.swpc_common import (
SWPC_PROTONS_URL,
SWPCSettings,
parse_swpc_timestamp,
)
from central.config_models import AdapterConfig
from central.config_store import ConfigStore
from central.models import Event, Geo
logger = logging.getLogger(__name__)
class SWPCProtonsAdapter(SourceAdapter):
"""NOAA SWPC GOES integral proton flux adapter."""
name = "swpc_protons"
display_name = "NOAA SWPC — GOES Proton Flux"
description = "GOES primary satellite integral proton flux measurements (1-day window) from NOAA SWPC."
settings_schema = SWPCSettings
requires_api_key = None
api_key_field = None
wizard_order = None
default_cadence_s = 600
def __init__(
self,
config: AdapterConfig,
config_store: ConfigStore,
cursor_db_path: Path,
) -> None:
self._config_store = config_store
self._cursor_db_path = cursor_db_path
self._session: aiohttp.ClientSession | None = None
self._db: sqlite3.Connection | None = None
async def startup(self) -> None:
self._session = aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=60),
)
self._db = sqlite3.connect(self._cursor_db_path)
self._db.execute("""
CREATE TABLE IF NOT EXISTS published_ids (
adapter TEXT NOT NULL,
event_id TEXT NOT NULL,
first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (adapter, event_id)
)
""")
self._db.execute("""
CREATE INDEX IF NOT EXISTS published_ids_last_seen
ON published_ids (last_seen)
""")
self._db.commit()
logger.info("SWPC protons adapter started")
async def shutdown(self) -> None:
if self._session:
await self._session.close()
self._session = None
if self._db:
self._db.close()
self._db = None
logger.info("SWPC protons adapter shut down")
async def apply_config(self, new_config: AdapterConfig) -> None:
logger.info("SWPC protons config updated")
def is_published(self, event_id: str) -> bool:
if not self._db:
return False
cur = self._db.execute(
"SELECT 1 FROM published_ids WHERE adapter = ? AND event_id = ?",
(self.name, event_id),
)
return cur.fetchone() is not None
def mark_published(self, event_id: str) -> None:
if not self._db:
return
self._db.execute(
"""
INSERT INTO published_ids (adapter, event_id, first_seen, last_seen)
VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (adapter, event_id) DO UPDATE SET
last_seen = CURRENT_TIMESTAMP
""",
(self.name, event_id),
)
self._db.commit()
def sweep_old_ids(self) -> int:
if not self._db:
return 0
cur = self._db.execute(
"DELETE FROM published_ids WHERE adapter = ? AND last_seen < datetime('now', '-14 days')",
(self.name,),
)
self._db.commit()
count = cur.rowcount
if count > 0:
logger.info("SWPC protons swept old dedup entries", extra={"count": count})
return count
def subject_for(self, event: Event) -> str:
return "central.space.proton_flux"
@retry(
stop=stop_after_attempt(3),
wait=wait_exponential_jitter(initial=1, max=30),
retry=retry_if_exception_type((aiohttp.ClientError, TimeoutError)),
)
async def _fetch(self) -> list[dict[str, Any]]:
if not self._session:
raise RuntimeError("Session not initialized")
async with self._session.get(
SWPC_PROTONS_URL, headers={"User-Agent": "Central/0.4"}
) as resp:
resp.raise_for_status()
data = await resp.json()
logger.info("SWPC protons fetch completed", extra={"item_count": len(data)})
return data
async def poll(self) -> AsyncIterator[Event]:
if not self._db:
raise RuntimeError("Database not initialized")
try:
items = await self._fetch()
except Exception as e:
logger.error("SWPC protons fetch failed", extra={"error": str(e)})
raise
events_yielded = 0
for item in items:
time_tag = item.get("time_tag")
energy = item.get("energy")
if not time_tag or not energy:
continue
event_id = f"{time_tag}|{energy}"
if self.is_published(event_id):
continue
event_time = parse_swpc_timestamp(time_tag, "protons") or datetime.now(timezone.utc)
event = Event(
id=event_id,
adapter=self.name,
category="space.proton_flux",
time=event_time,
severity=0,
geo=Geo(),
data={
"time_tag": time_tag,
"satellite": item.get("satellite"),
"flux": item.get("flux"),
"energy": energy,
},
)
yield event
self.mark_published(event_id)
events_yielded += 1
self.sweep_old_ids()
logger.info("SWPC protons poll completed", extra={"events_yielded": events_yielded})

View file

@ -25,6 +25,7 @@ STREAMS = [
("CENTRAL_WX", "central.wx.>"),
("CENTRAL_FIRE", "central.fire.>"),
("CENTRAL_QUAKE", "central.quake.>"),
("CENTRAL_SPACE", "central.space.>"),
]
BATCH_SIZE = 100

View file

@ -64,7 +64,7 @@ def _adapter_classes() -> dict:
router = APIRouter()
# Streams to display on dashboard
DASHBOARD_STREAMS = ["CENTRAL_WX", "CENTRAL_FIRE", "CENTRAL_QUAKE", "CENTRAL_META"]
DASHBOARD_STREAMS = ["CENTRAL_WX", "CENTRAL_FIRE", "CENTRAL_QUAKE", "CENTRAL_SPACE", "CENTRAL_META"]
# Email validation regex (simple but effective)
ALIAS_REGEX = re.compile(r"^[a-zA-Z0-9_]+$")

View file

@ -29,6 +29,7 @@ STREAM_SUBJECTS = {
"CENTRAL_META": ["central.meta.>"],
"CENTRAL_FIRE": ["central.fire.>"],
"CENTRAL_QUAKE": ["central.quake.>"],
"CENTRAL_SPACE": ["central.space.>"],
}
# Recompute interval for stream max_bytes (1 hour)