mirror of
https://github.com/zvx-echo6/central.git
synced 2026-06-10 20:04:43 +02:00
feat(2-D): add NOAA SWPC space weather adapters (alerts, kindex, protons)
Three independent adapters sharing src/central/adapters/swpc_common.py, mirroring the WFIGS two-adapter pattern. Each adapter has its own row in config.adapters (ships disabled), its own cadence, and its own dedup state, so operators can independently enable/disable and so a broken upstream endpoint does not silently mask a healthy one. Subjects: swpc_alerts -> central.space.alert.<product_id_lower> swpc_kindex -> central.space.kindex swpc_protons -> central.space.proton_flux Dedup keys: alerts: product_id + issue_datetime kindex: time_tag protons: time_tag + energy Severity: G-scale on product_id for K0[5-9][AW] alerts (G1-G5 -> 1-4), G-scale on Kp for kindex, 0 for protons (raw flux carried in event.data). No geo on any SWPC events (centroid=None, regions=[], primary_region=None). No fall-off detection for alerts -- a single 115-row sample cannot confirm whether alerts disappear from the upstream JSON when expired; deferred to a later pass after 24h of observation. CENTRAL_SPACE stream seeded with 7-day retention / 1 GiB max_bytes, mirroring CENTRAL_FIRE / CENTRAL_QUAKE. STREAM_SUBJECTS, archive STREAMS, and DASHBOARD_STREAMS each pick up the new stream. Tests: 16 new cases in tests/test_swpc.py using real-shape frozen JSON fixtures (alerts product_ids EF3A/K05A/K07A; kindex Kp boundaries; protons composite dedup). Two existing tests updated for the new stream count (test_archive_multi_stream.test_streams_list_has_three_entries renamed to _has_four_entries; test_dashboard expects 5 streams not 4); added a test_streams_contains_central_space companion. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
0675a4214f
commit
72ec498365
12 changed files with 1007 additions and 5 deletions
186
src/central/adapters/swpc_alerts.py
Normal file
186
src/central/adapters/swpc_alerts.py
Normal file
|
|
@ -0,0 +1,186 @@
|
|||
"""NOAA SWPC space weather alerts adapter."""
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
from tenacity import (
|
||||
retry,
|
||||
retry_if_exception_type,
|
||||
stop_after_attempt,
|
||||
wait_exponential_jitter,
|
||||
)
|
||||
|
||||
from central.adapter import SourceAdapter
|
||||
from central.adapters.swpc_common import (
|
||||
SWPC_ALERTS_URL,
|
||||
SWPCSettings,
|
||||
parse_swpc_timestamp,
|
||||
severity_from_alert_product_id,
|
||||
)
|
||||
from central.config_models import AdapterConfig
|
||||
from central.config_store import ConfigStore
|
||||
from central.models import Event, Geo
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SWPCAlertsAdapter(SourceAdapter):
|
||||
"""NOAA SWPC space weather alerts adapter."""
|
||||
|
||||
name = "swpc_alerts"
|
||||
display_name = "NOAA SWPC — Space Weather Alerts"
|
||||
description = "Active NOAA SWPC space weather alerts, watches, warnings, and summaries."
|
||||
settings_schema = SWPCSettings
|
||||
requires_api_key = None
|
||||
api_key_field = None
|
||||
wizard_order = None
|
||||
default_cadence_s = 300
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: AdapterConfig,
|
||||
config_store: ConfigStore,
|
||||
cursor_db_path: Path,
|
||||
) -> None:
|
||||
self._config_store = config_store
|
||||
self._cursor_db_path = cursor_db_path
|
||||
self._session: aiohttp.ClientSession | None = None
|
||||
self._db: sqlite3.Connection | None = None
|
||||
|
||||
async def startup(self) -> None:
|
||||
self._session = aiohttp.ClientSession(
|
||||
timeout=aiohttp.ClientTimeout(total=60),
|
||||
)
|
||||
self._db = sqlite3.connect(self._cursor_db_path)
|
||||
self._db.execute("""
|
||||
CREATE TABLE IF NOT EXISTS published_ids (
|
||||
adapter TEXT NOT NULL,
|
||||
event_id TEXT NOT NULL,
|
||||
first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (adapter, event_id)
|
||||
)
|
||||
""")
|
||||
self._db.execute("""
|
||||
CREATE INDEX IF NOT EXISTS published_ids_last_seen
|
||||
ON published_ids (last_seen)
|
||||
""")
|
||||
self._db.commit()
|
||||
logger.info("SWPC alerts adapter started")
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
if self._session:
|
||||
await self._session.close()
|
||||
self._session = None
|
||||
if self._db:
|
||||
self._db.close()
|
||||
self._db = None
|
||||
logger.info("SWPC alerts adapter shut down")
|
||||
|
||||
async def apply_config(self, new_config: AdapterConfig) -> None:
|
||||
logger.info("SWPC alerts config updated")
|
||||
|
||||
def is_published(self, event_id: str) -> bool:
|
||||
if not self._db:
|
||||
return False
|
||||
cur = self._db.execute(
|
||||
"SELECT 1 FROM published_ids WHERE adapter = ? AND event_id = ?",
|
||||
(self.name, event_id),
|
||||
)
|
||||
return cur.fetchone() is not None
|
||||
|
||||
def mark_published(self, event_id: str) -> None:
|
||||
if not self._db:
|
||||
return
|
||||
self._db.execute(
|
||||
"""
|
||||
INSERT INTO published_ids (adapter, event_id, first_seen, last_seen)
|
||||
VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (adapter, event_id) DO UPDATE SET
|
||||
last_seen = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(self.name, event_id),
|
||||
)
|
||||
self._db.commit()
|
||||
|
||||
def sweep_old_ids(self) -> int:
|
||||
if not self._db:
|
||||
return 0
|
||||
cur = self._db.execute(
|
||||
"DELETE FROM published_ids WHERE adapter = ? AND last_seen < datetime('now', '-14 days')",
|
||||
(self.name,),
|
||||
)
|
||||
self._db.commit()
|
||||
count = cur.rowcount
|
||||
if count > 0:
|
||||
logger.info("SWPC alerts swept old dedup entries", extra={"count": count})
|
||||
return count
|
||||
|
||||
def subject_for(self, event: Event) -> str:
|
||||
product_id = event.data.get("product_id") or "unknown"
|
||||
return f"central.space.alert.{product_id.lower()}"
|
||||
|
||||
@retry(
|
||||
stop=stop_after_attempt(3),
|
||||
wait=wait_exponential_jitter(initial=1, max=30),
|
||||
retry=retry_if_exception_type((aiohttp.ClientError, TimeoutError)),
|
||||
)
|
||||
async def _fetch(self) -> list[dict[str, Any]]:
|
||||
if not self._session:
|
||||
raise RuntimeError("Session not initialized")
|
||||
async with self._session.get(
|
||||
SWPC_ALERTS_URL, headers={"User-Agent": "Central/0.4"}
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
data = await resp.json()
|
||||
logger.info("SWPC alerts fetch completed", extra={"item_count": len(data)})
|
||||
return data
|
||||
|
||||
async def poll(self) -> AsyncIterator[Event]:
|
||||
if not self._db:
|
||||
raise RuntimeError("Database not initialized")
|
||||
|
||||
try:
|
||||
items = await self._fetch()
|
||||
except Exception as e:
|
||||
logger.error("SWPC alerts fetch failed", extra={"error": str(e)})
|
||||
raise
|
||||
|
||||
events_yielded = 0
|
||||
for item in items:
|
||||
product_id = item.get("product_id")
|
||||
issue_dt_raw = item.get("issue_datetime")
|
||||
if not product_id or not issue_dt_raw:
|
||||
continue
|
||||
|
||||
event_id = f"{product_id}|{issue_dt_raw}"
|
||||
if self.is_published(event_id):
|
||||
continue
|
||||
|
||||
issue_dt = parse_swpc_timestamp(issue_dt_raw, "alerts") or datetime.now(timezone.utc)
|
||||
|
||||
event = Event(
|
||||
id=event_id,
|
||||
adapter=self.name,
|
||||
category="space.alert",
|
||||
time=issue_dt,
|
||||
severity=severity_from_alert_product_id(product_id),
|
||||
geo=Geo(),
|
||||
data={
|
||||
"product_id": product_id,
|
||||
"issue_datetime": issue_dt_raw,
|
||||
"message": item.get("message", ""),
|
||||
},
|
||||
)
|
||||
|
||||
yield event
|
||||
self.mark_published(event_id)
|
||||
events_yielded += 1
|
||||
|
||||
self.sweep_old_ids()
|
||||
logger.info("SWPC alerts poll completed", extra={"events_yielded": events_yielded})
|
||||
81
src/central/adapters/swpc_common.py
Normal file
81
src/central/adapters/swpc_common.py
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
"""Shared utilities for NOAA SWPC space weather adapters."""
|
||||
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
SWPC_ALERTS_URL = "https://services.swpc.noaa.gov/products/alerts.json"
|
||||
SWPC_KINDEX_URL = "https://services.swpc.noaa.gov/products/noaa-planetary-k-index.json"
|
||||
SWPC_PROTONS_URL = "https://services.swpc.noaa.gov/json/goes/primary/integral-protons-1-day.json"
|
||||
|
||||
|
||||
class SWPCSettings(BaseModel):
|
||||
"""Settings schema for SWPC adapters. No operator-tunable knobs today."""
|
||||
|
||||
|
||||
def parse_swpc_timestamp(raw: str | None, endpoint_kind: str) -> datetime | None:
|
||||
"""Normalize SWPC timestamp strings to UTC datetime.
|
||||
|
||||
endpoint_kind shapes:
|
||||
alerts -> "2026-05-19 05:14:59.780" (space-separated, no TZ; UTC per message body)
|
||||
kindex -> "2026-05-12T00:00:00" (ISO without TZ; UTC by convention)
|
||||
protons -> "2026-05-18T05:35:00Z" (ISO with Z)
|
||||
"""
|
||||
if not raw:
|
||||
return None
|
||||
if endpoint_kind == "alerts":
|
||||
try:
|
||||
dt = datetime.strptime(raw, "%Y-%m-%d %H:%M:%S.%f")
|
||||
except ValueError:
|
||||
dt = datetime.strptime(raw, "%Y-%m-%d %H:%M:%S")
|
||||
return dt.replace(tzinfo=timezone.utc)
|
||||
if endpoint_kind == "kindex":
|
||||
dt = datetime.fromisoformat(raw)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone(timezone.utc)
|
||||
if endpoint_kind == "protons":
|
||||
raw_norm = raw[:-1] + "+00:00" if raw.endswith("Z") else raw
|
||||
dt = datetime.fromisoformat(raw_norm)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone(timezone.utc)
|
||||
raise ValueError(f"unknown endpoint_kind: {endpoint_kind!r}")
|
||||
|
||||
|
||||
def severity_from_kp(kp: float | int | None) -> int:
|
||||
"""Map planetary K-index value (0-9) to severity 0-4 via the G-scale.
|
||||
|
||||
Kp 5 = G1 = severity 1, Kp 6 = G2 = severity 2, Kp 7 = G3 = severity 3,
|
||||
Kp 8 = G4 = severity 4, Kp 9 = G5 = severity 4 (capped).
|
||||
"""
|
||||
if kp is None:
|
||||
return 0
|
||||
if kp < 5:
|
||||
return 0
|
||||
if kp < 6:
|
||||
return 1
|
||||
if kp < 7:
|
||||
return 2
|
||||
if kp < 8:
|
||||
return 3
|
||||
return 4
|
||||
|
||||
|
||||
_ALERT_KP_PATTERN = re.compile(r"^K0([5-9])[AW]$")
|
||||
|
||||
|
||||
def severity_from_alert_product_id(product_id: str | None) -> int:
|
||||
"""Best-effort severity for an alert from its product_id G-scale.
|
||||
|
||||
Product IDs of form K0[5-9][AW] identify Kp-based geomagnetic storm
|
||||
alerts and warnings (K05A=G1, K06A=G2, K07A=G3, K08A=G4, K09A=G5).
|
||||
All other product IDs return 0.
|
||||
"""
|
||||
if not product_id:
|
||||
return 0
|
||||
m = _ALERT_KP_PATTERN.match(product_id.upper())
|
||||
if not m:
|
||||
return 0
|
||||
return severity_from_kp(int(m.group(1)))
|
||||
186
src/central/adapters/swpc_kindex.py
Normal file
186
src/central/adapters/swpc_kindex.py
Normal file
|
|
@ -0,0 +1,186 @@
|
|||
"""NOAA SWPC Planetary K-Index adapter."""
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
from tenacity import (
|
||||
retry,
|
||||
retry_if_exception_type,
|
||||
stop_after_attempt,
|
||||
wait_exponential_jitter,
|
||||
)
|
||||
|
||||
from central.adapter import SourceAdapter
|
||||
from central.adapters.swpc_common import (
|
||||
SWPC_KINDEX_URL,
|
||||
SWPCSettings,
|
||||
parse_swpc_timestamp,
|
||||
severity_from_kp,
|
||||
)
|
||||
from central.config_models import AdapterConfig
|
||||
from central.config_store import ConfigStore
|
||||
from central.models import Event, Geo
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SWPCKindexAdapter(SourceAdapter):
|
||||
"""NOAA SWPC planetary K-index adapter."""
|
||||
|
||||
name = "swpc_kindex"
|
||||
display_name = "NOAA SWPC — Planetary K-Index"
|
||||
description = "Planetary K-index measurements at 3-hour cadence from NOAA SWPC."
|
||||
settings_schema = SWPCSettings
|
||||
requires_api_key = None
|
||||
api_key_field = None
|
||||
wizard_order = None
|
||||
default_cadence_s = 600
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: AdapterConfig,
|
||||
config_store: ConfigStore,
|
||||
cursor_db_path: Path,
|
||||
) -> None:
|
||||
self._config_store = config_store
|
||||
self._cursor_db_path = cursor_db_path
|
||||
self._session: aiohttp.ClientSession | None = None
|
||||
self._db: sqlite3.Connection | None = None
|
||||
|
||||
async def startup(self) -> None:
|
||||
self._session = aiohttp.ClientSession(
|
||||
timeout=aiohttp.ClientTimeout(total=60),
|
||||
)
|
||||
self._db = sqlite3.connect(self._cursor_db_path)
|
||||
self._db.execute("""
|
||||
CREATE TABLE IF NOT EXISTS published_ids (
|
||||
adapter TEXT NOT NULL,
|
||||
event_id TEXT NOT NULL,
|
||||
first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (adapter, event_id)
|
||||
)
|
||||
""")
|
||||
self._db.execute("""
|
||||
CREATE INDEX IF NOT EXISTS published_ids_last_seen
|
||||
ON published_ids (last_seen)
|
||||
""")
|
||||
self._db.commit()
|
||||
logger.info("SWPC kindex adapter started")
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
if self._session:
|
||||
await self._session.close()
|
||||
self._session = None
|
||||
if self._db:
|
||||
self._db.close()
|
||||
self._db = None
|
||||
logger.info("SWPC kindex adapter shut down")
|
||||
|
||||
async def apply_config(self, new_config: AdapterConfig) -> None:
|
||||
logger.info("SWPC kindex config updated")
|
||||
|
||||
def is_published(self, event_id: str) -> bool:
|
||||
if not self._db:
|
||||
return False
|
||||
cur = self._db.execute(
|
||||
"SELECT 1 FROM published_ids WHERE adapter = ? AND event_id = ?",
|
||||
(self.name, event_id),
|
||||
)
|
||||
return cur.fetchone() is not None
|
||||
|
||||
def mark_published(self, event_id: str) -> None:
|
||||
if not self._db:
|
||||
return
|
||||
self._db.execute(
|
||||
"""
|
||||
INSERT INTO published_ids (adapter, event_id, first_seen, last_seen)
|
||||
VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (adapter, event_id) DO UPDATE SET
|
||||
last_seen = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(self.name, event_id),
|
||||
)
|
||||
self._db.commit()
|
||||
|
||||
def sweep_old_ids(self) -> int:
|
||||
if not self._db:
|
||||
return 0
|
||||
cur = self._db.execute(
|
||||
"DELETE FROM published_ids WHERE adapter = ? AND last_seen < datetime('now', '-14 days')",
|
||||
(self.name,),
|
||||
)
|
||||
self._db.commit()
|
||||
count = cur.rowcount
|
||||
if count > 0:
|
||||
logger.info("SWPC kindex swept old dedup entries", extra={"count": count})
|
||||
return count
|
||||
|
||||
def subject_for(self, event: Event) -> str:
|
||||
return "central.space.kindex"
|
||||
|
||||
@retry(
|
||||
stop=stop_after_attempt(3),
|
||||
wait=wait_exponential_jitter(initial=1, max=30),
|
||||
retry=retry_if_exception_type((aiohttp.ClientError, TimeoutError)),
|
||||
)
|
||||
async def _fetch(self) -> list[dict[str, Any]]:
|
||||
if not self._session:
|
||||
raise RuntimeError("Session not initialized")
|
||||
async with self._session.get(
|
||||
SWPC_KINDEX_URL, headers={"User-Agent": "Central/0.4"}
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
data = await resp.json()
|
||||
logger.info("SWPC kindex fetch completed", extra={"item_count": len(data)})
|
||||
return data
|
||||
|
||||
async def poll(self) -> AsyncIterator[Event]:
|
||||
if not self._db:
|
||||
raise RuntimeError("Database not initialized")
|
||||
|
||||
try:
|
||||
items = await self._fetch()
|
||||
except Exception as e:
|
||||
logger.error("SWPC kindex fetch failed", extra={"error": str(e)})
|
||||
raise
|
||||
|
||||
events_yielded = 0
|
||||
for item in items:
|
||||
time_tag = item.get("time_tag")
|
||||
kp = item.get("Kp")
|
||||
if not time_tag or kp is None:
|
||||
continue
|
||||
|
||||
event_id = time_tag
|
||||
if self.is_published(event_id):
|
||||
continue
|
||||
|
||||
event_time = parse_swpc_timestamp(time_tag, "kindex") or datetime.now(timezone.utc)
|
||||
|
||||
event = Event(
|
||||
id=event_id,
|
||||
adapter=self.name,
|
||||
category="space.kindex",
|
||||
time=event_time,
|
||||
severity=severity_from_kp(kp),
|
||||
geo=Geo(),
|
||||
data={
|
||||
"time_tag": time_tag,
|
||||
"Kp": kp,
|
||||
"a_running": item.get("a_running"),
|
||||
"station_count": item.get("station_count"),
|
||||
},
|
||||
)
|
||||
|
||||
yield event
|
||||
self.mark_published(event_id)
|
||||
events_yielded += 1
|
||||
|
||||
self.sweep_old_ids()
|
||||
logger.info("SWPC kindex poll completed", extra={"events_yielded": events_yielded})
|
||||
185
src/central/adapters/swpc_protons.py
Normal file
185
src/central/adapters/swpc_protons.py
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
"""NOAA SWPC GOES integral proton flux adapter."""
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
from tenacity import (
|
||||
retry,
|
||||
retry_if_exception_type,
|
||||
stop_after_attempt,
|
||||
wait_exponential_jitter,
|
||||
)
|
||||
|
||||
from central.adapter import SourceAdapter
|
||||
from central.adapters.swpc_common import (
|
||||
SWPC_PROTONS_URL,
|
||||
SWPCSettings,
|
||||
parse_swpc_timestamp,
|
||||
)
|
||||
from central.config_models import AdapterConfig
|
||||
from central.config_store import ConfigStore
|
||||
from central.models import Event, Geo
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SWPCProtonsAdapter(SourceAdapter):
|
||||
"""NOAA SWPC GOES integral proton flux adapter."""
|
||||
|
||||
name = "swpc_protons"
|
||||
display_name = "NOAA SWPC — GOES Proton Flux"
|
||||
description = "GOES primary satellite integral proton flux measurements (1-day window) from NOAA SWPC."
|
||||
settings_schema = SWPCSettings
|
||||
requires_api_key = None
|
||||
api_key_field = None
|
||||
wizard_order = None
|
||||
default_cadence_s = 600
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: AdapterConfig,
|
||||
config_store: ConfigStore,
|
||||
cursor_db_path: Path,
|
||||
) -> None:
|
||||
self._config_store = config_store
|
||||
self._cursor_db_path = cursor_db_path
|
||||
self._session: aiohttp.ClientSession | None = None
|
||||
self._db: sqlite3.Connection | None = None
|
||||
|
||||
async def startup(self) -> None:
|
||||
self._session = aiohttp.ClientSession(
|
||||
timeout=aiohttp.ClientTimeout(total=60),
|
||||
)
|
||||
self._db = sqlite3.connect(self._cursor_db_path)
|
||||
self._db.execute("""
|
||||
CREATE TABLE IF NOT EXISTS published_ids (
|
||||
adapter TEXT NOT NULL,
|
||||
event_id TEXT NOT NULL,
|
||||
first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (adapter, event_id)
|
||||
)
|
||||
""")
|
||||
self._db.execute("""
|
||||
CREATE INDEX IF NOT EXISTS published_ids_last_seen
|
||||
ON published_ids (last_seen)
|
||||
""")
|
||||
self._db.commit()
|
||||
logger.info("SWPC protons adapter started")
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
if self._session:
|
||||
await self._session.close()
|
||||
self._session = None
|
||||
if self._db:
|
||||
self._db.close()
|
||||
self._db = None
|
||||
logger.info("SWPC protons adapter shut down")
|
||||
|
||||
async def apply_config(self, new_config: AdapterConfig) -> None:
|
||||
logger.info("SWPC protons config updated")
|
||||
|
||||
def is_published(self, event_id: str) -> bool:
|
||||
if not self._db:
|
||||
return False
|
||||
cur = self._db.execute(
|
||||
"SELECT 1 FROM published_ids WHERE adapter = ? AND event_id = ?",
|
||||
(self.name, event_id),
|
||||
)
|
||||
return cur.fetchone() is not None
|
||||
|
||||
def mark_published(self, event_id: str) -> None:
|
||||
if not self._db:
|
||||
return
|
||||
self._db.execute(
|
||||
"""
|
||||
INSERT INTO published_ids (adapter, event_id, first_seen, last_seen)
|
||||
VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (adapter, event_id) DO UPDATE SET
|
||||
last_seen = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(self.name, event_id),
|
||||
)
|
||||
self._db.commit()
|
||||
|
||||
def sweep_old_ids(self) -> int:
|
||||
if not self._db:
|
||||
return 0
|
||||
cur = self._db.execute(
|
||||
"DELETE FROM published_ids WHERE adapter = ? AND last_seen < datetime('now', '-14 days')",
|
||||
(self.name,),
|
||||
)
|
||||
self._db.commit()
|
||||
count = cur.rowcount
|
||||
if count > 0:
|
||||
logger.info("SWPC protons swept old dedup entries", extra={"count": count})
|
||||
return count
|
||||
|
||||
def subject_for(self, event: Event) -> str:
|
||||
return "central.space.proton_flux"
|
||||
|
||||
@retry(
|
||||
stop=stop_after_attempt(3),
|
||||
wait=wait_exponential_jitter(initial=1, max=30),
|
||||
retry=retry_if_exception_type((aiohttp.ClientError, TimeoutError)),
|
||||
)
|
||||
async def _fetch(self) -> list[dict[str, Any]]:
|
||||
if not self._session:
|
||||
raise RuntimeError("Session not initialized")
|
||||
async with self._session.get(
|
||||
SWPC_PROTONS_URL, headers={"User-Agent": "Central/0.4"}
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
data = await resp.json()
|
||||
logger.info("SWPC protons fetch completed", extra={"item_count": len(data)})
|
||||
return data
|
||||
|
||||
async def poll(self) -> AsyncIterator[Event]:
|
||||
if not self._db:
|
||||
raise RuntimeError("Database not initialized")
|
||||
|
||||
try:
|
||||
items = await self._fetch()
|
||||
except Exception as e:
|
||||
logger.error("SWPC protons fetch failed", extra={"error": str(e)})
|
||||
raise
|
||||
|
||||
events_yielded = 0
|
||||
for item in items:
|
||||
time_tag = item.get("time_tag")
|
||||
energy = item.get("energy")
|
||||
if not time_tag or not energy:
|
||||
continue
|
||||
|
||||
event_id = f"{time_tag}|{energy}"
|
||||
if self.is_published(event_id):
|
||||
continue
|
||||
|
||||
event_time = parse_swpc_timestamp(time_tag, "protons") or datetime.now(timezone.utc)
|
||||
|
||||
event = Event(
|
||||
id=event_id,
|
||||
adapter=self.name,
|
||||
category="space.proton_flux",
|
||||
time=event_time,
|
||||
severity=0,
|
||||
geo=Geo(),
|
||||
data={
|
||||
"time_tag": time_tag,
|
||||
"satellite": item.get("satellite"),
|
||||
"flux": item.get("flux"),
|
||||
"energy": energy,
|
||||
},
|
||||
)
|
||||
|
||||
yield event
|
||||
self.mark_published(event_id)
|
||||
events_yielded += 1
|
||||
|
||||
self.sweep_old_ids()
|
||||
logger.info("SWPC protons poll completed", extra={"events_yielded": events_yielded})
|
||||
|
|
@ -25,6 +25,7 @@ STREAMS = [
|
|||
("CENTRAL_WX", "central.wx.>"),
|
||||
("CENTRAL_FIRE", "central.fire.>"),
|
||||
("CENTRAL_QUAKE", "central.quake.>"),
|
||||
("CENTRAL_SPACE", "central.space.>"),
|
||||
]
|
||||
|
||||
BATCH_SIZE = 100
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ def _adapter_classes() -> dict:
|
|||
router = APIRouter()
|
||||
|
||||
# Streams to display on dashboard
|
||||
DASHBOARD_STREAMS = ["CENTRAL_WX", "CENTRAL_FIRE", "CENTRAL_QUAKE", "CENTRAL_META"]
|
||||
DASHBOARD_STREAMS = ["CENTRAL_WX", "CENTRAL_FIRE", "CENTRAL_QUAKE", "CENTRAL_SPACE", "CENTRAL_META"]
|
||||
|
||||
# Email validation regex (simple but effective)
|
||||
ALIAS_REGEX = re.compile(r"^[a-zA-Z0-9_]+$")
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ STREAM_SUBJECTS = {
|
|||
"CENTRAL_META": ["central.meta.>"],
|
||||
"CENTRAL_FIRE": ["central.fire.>"],
|
||||
"CENTRAL_QUAKE": ["central.quake.>"],
|
||||
"CENTRAL_SPACE": ["central.space.>"],
|
||||
}
|
||||
|
||||
# Recompute interval for stream max_bytes (1 hour)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue