mirror of
https://github.com/zvx-echo6/central.git
synced 2026-05-21 18:14:44 +02:00
Compare commits
20 commits
87f46e8b35
...
37a778468d
| Author | SHA1 | Date | |
|---|---|---|---|
|
37a778468d |
|||
|
|
72ec498365 | ||
|
0675a4214f |
|||
|
|
4c1fdb8649 | ||
|
|
1ef19508a1 | ||
|
|
8751264f8c | ||
|
|
dfad7ef45d | ||
|
|
e0ffe686ec | ||
|
|
51be59ee02 | ||
|
|
4a209d3a03 | ||
|
|
045b8614e8 | ||
|
|
43bf973caf | ||
|
|
e8019a32b7 | ||
|
|
d0eeaa9d1a | ||
|
|
08eb729979 | ||
|
|
ce9f843ae0 | ||
|
|
d42b540e16 | ||
|
|
91f1d67abd | ||
|
|
bff6ccffff | ||
|
|
966661305f |
42 changed files with 5759 additions and 574 deletions
6
sql/migrations/015_add_adapters_last_error.sql
Normal file
6
sql/migrations/015_add_adapters_last_error.sql
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
-- Migration: 015_add_adapters_last_error
|
||||
-- Adds last_error column for adapter-side error reporting.
|
||||
-- Populated by supervisor when an adapter fails to start or apply config.
|
||||
|
||||
ALTER TABLE config.adapters
|
||||
ADD COLUMN IF NOT EXISTS last_error TEXT;
|
||||
37
sql/migrations/016_add_wfigs_adapters.sql
Normal file
37
sql/migrations/016_add_wfigs_adapters.sql
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
-- Migration: 016_add_wfigs_adapters
|
||||
-- Add WFIGS incident and perimeter adapters to config.adapters
|
||||
-- Idempotent: uses ON CONFLICT DO NOTHING
|
||||
|
||||
-- WFIGS Incidents adapter
|
||||
INSERT INTO config.adapters (name, enabled, cadence_s, settings)
|
||||
VALUES (
|
||||
'wfigs_incidents',
|
||||
false, -- Ships disabled; operator enables via GUI
|
||||
300,
|
||||
jsonb_build_object(
|
||||
'region', jsonb_build_object(
|
||||
'north', 49.0,
|
||||
'south', 31.0,
|
||||
'east', -102.0,
|
||||
'west', -124.0
|
||||
)
|
||||
)
|
||||
)
|
||||
ON CONFLICT (name) DO NOTHING;
|
||||
|
||||
-- WFIGS Perimeters adapter
|
||||
INSERT INTO config.adapters (name, enabled, cadence_s, settings)
|
||||
VALUES (
|
||||
'wfigs_perimeters',
|
||||
false, -- Ships disabled; operator enables via GUI
|
||||
300,
|
||||
jsonb_build_object(
|
||||
'region', jsonb_build_object(
|
||||
'north', 49.0,
|
||||
'south', 31.0,
|
||||
'east', -102.0,
|
||||
'west', -124.0
|
||||
)
|
||||
)
|
||||
)
|
||||
ON CONFLICT (name) DO NOTHING;
|
||||
19
sql/migrations/017_add_inciweb_adapter.sql
Normal file
19
sql/migrations/017_add_inciweb_adapter.sql
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
-- Migration: 017_add_inciweb_adapter
|
||||
-- Add InciWeb adapter to config.adapters
|
||||
-- Idempotent: uses ON CONFLICT DO NOTHING
|
||||
|
||||
INSERT INTO config.adapters (name, enabled, cadence_s, settings)
|
||||
VALUES (
|
||||
'inciweb',
|
||||
false, -- Ships disabled; operator enables via GUI
|
||||
600,
|
||||
jsonb_build_object(
|
||||
'region', jsonb_build_object(
|
||||
'north', 49.0,
|
||||
'south', 31.0,
|
||||
'east', -102.0,
|
||||
'west', -124.0
|
||||
)
|
||||
)
|
||||
)
|
||||
ON CONFLICT (name) DO NOTHING;
|
||||
11
sql/migrations/018_add_swpc_adapters.sql
Normal file
11
sql/migrations/018_add_swpc_adapters.sql
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
-- Migration: 018_add_swpc_adapters
|
||||
-- Add NOAA SWPC space weather adapters to config.adapters.
|
||||
-- All three ship disabled; operator enables individually via GUI.
|
||||
-- Idempotent: uses ON CONFLICT DO NOTHING.
|
||||
|
||||
INSERT INTO config.adapters (name, enabled, cadence_s, settings)
|
||||
VALUES
|
||||
('swpc_alerts', false, 300, '{}'::jsonb),
|
||||
('swpc_kindex', false, 600, '{}'::jsonb),
|
||||
('swpc_protons', false, 600, '{}'::jsonb)
|
||||
ON CONFLICT (name) DO NOTHING;
|
||||
8
sql/migrations/019_add_central_space_stream.sql
Normal file
8
sql/migrations/019_add_central_space_stream.sql
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
-- Migration: 019_add_central_space_stream
|
||||
-- Seeds the CENTRAL_SPACE JetStream stream row for central.space.> subjects.
|
||||
-- 7-day retention, 1 GiB max_bytes (clamped by supervisor recompute) -- mirrors CENTRAL_FIRE / CENTRAL_QUAKE.
|
||||
-- Idempotent: uses ON CONFLICT DO NOTHING.
|
||||
|
||||
INSERT INTO config.streams (name, max_age_s, max_bytes)
|
||||
VALUES ('CENTRAL_SPACE', 604800, 1073741824)
|
||||
ON CONFLICT (name) DO NOTHING;
|
||||
|
|
@ -34,6 +34,10 @@ class SourceAdapter(ABC):
|
|||
description: str
|
||||
settings_schema: type[BaseModel]
|
||||
requires_api_key: str | None = None
|
||||
api_key_field: str | None = None
|
||||
"""Names the settings_schema field that holds an api_key alias reference, if any.
|
||||
The GUI renders this field as a select populated from config.api_keys;
|
||||
the wizard validates it against staged api_keys state."""
|
||||
wizard_order: int | None = None
|
||||
default_cadence_s: int
|
||||
|
||||
|
|
|
|||
34
src/central/adapter_discovery.py
Normal file
34
src/central/adapter_discovery.py
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
"""Adapter discovery utilities."""
|
||||
|
||||
import importlib
|
||||
import logging
|
||||
import pkgutil
|
||||
|
||||
import central.adapters
|
||||
from central.adapter import SourceAdapter
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def discover_adapters() -> dict[str, type[SourceAdapter]]:
|
||||
"""Auto-discover adapter classes from central.adapters package."""
|
||||
registry: dict[str, type[SourceAdapter]] = {}
|
||||
for module_info in pkgutil.iter_modules(central.adapters.__path__):
|
||||
try:
|
||||
module = importlib.import_module(f"central.adapters.{module_info.name}")
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Failed to import adapter module",
|
||||
extra={"module": module_info.name, "error": str(e)},
|
||||
)
|
||||
continue
|
||||
for attr_name in dir(module):
|
||||
attr = getattr(module, attr_name)
|
||||
if (
|
||||
isinstance(attr, type)
|
||||
and issubclass(attr, SourceAdapter)
|
||||
and attr is not SourceAdapter
|
||||
and hasattr(attr, "name")
|
||||
):
|
||||
registry[attr.name] = attr
|
||||
return registry
|
||||
|
|
@ -7,7 +7,7 @@ from collections.abc import AsyncIterator
|
|||
from datetime import datetime, timezone
|
||||
from io import StringIO
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Any, Literal
|
||||
|
||||
import aiohttp
|
||||
from tenacity import (
|
||||
|
|
@ -54,7 +54,7 @@ SEVERITY_MAP = {
|
|||
class FIRMSSettings(BaseModel):
|
||||
"""Settings schema for FIRMS adapter."""
|
||||
api_key_alias: str = "firms"
|
||||
satellites: list[str] = ["VIIRS_SNPP_NRT", "VIIRS_NOAA20_NRT"]
|
||||
satellites: list[Literal["VIIRS_SNPP_NRT", "VIIRS_NOAA20_NRT", "VIIRS_NOAA21_NRT"]] = ["VIIRS_SNPP_NRT", "VIIRS_NOAA20_NRT"]
|
||||
region: RegionConfig | None = None
|
||||
|
||||
|
||||
|
|
@ -66,6 +66,7 @@ class FIRMSAdapter(SourceAdapter):
|
|||
description = "Near-real-time satellite-detected fire hotspots from NASA FIRMS."
|
||||
settings_schema = FIRMSSettings
|
||||
requires_api_key = "firms"
|
||||
api_key_field = "api_key_alias"
|
||||
wizard_order = 2
|
||||
default_cadence_s = 300
|
||||
|
||||
|
|
|
|||
477
src/central/adapters/inciweb.py
Normal file
477
src/central/adapters/inciweb.py
Normal file
|
|
@ -0,0 +1,477 @@
|
|||
"""InciWeb adapter for wildfire narrative updates."""
|
||||
|
||||
import html
|
||||
import logging
|
||||
import re
|
||||
import sqlite3
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import datetime, timezone
|
||||
from email.utils import parsedate_to_datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from xml.etree import ElementTree as ET
|
||||
|
||||
import aiohttp
|
||||
from pydantic import BaseModel
|
||||
from tenacity import (
|
||||
retry,
|
||||
retry_if_exception_type,
|
||||
stop_after_attempt,
|
||||
wait_exponential_jitter,
|
||||
)
|
||||
|
||||
from central.adapter import SourceAdapter
|
||||
from central.config_models import AdapterConfig, RegionConfig
|
||||
from central.config_store import ConfigStore
|
||||
from central.models import Event, Geo
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# InciWeb RSS feed URL
|
||||
INCIWEB_RSS_URL = "https://inciweb.wildfire.gov/incidents/rss.xml"
|
||||
|
||||
# State name to 2-letter code mapping
|
||||
STATE_NAME_TO_CODE = {
|
||||
"alabama": "AL", "alaska": "AK", "arizona": "AZ", "arkansas": "AR",
|
||||
"california": "CA", "colorado": "CO", "connecticut": "CT", "delaware": "DE",
|
||||
"florida": "FL", "georgia": "GA", "hawaii": "HI", "idaho": "ID",
|
||||
"illinois": "IL", "indiana": "IN", "iowa": "IA", "kansas": "KS",
|
||||
"kentucky": "KY", "louisiana": "LA", "maine": "ME", "maryland": "MD",
|
||||
"massachusetts": "MA", "michigan": "MI", "minnesota": "MN", "mississippi": "MS",
|
||||
"missouri": "MO", "montana": "MT", "nebraska": "NE", "nevada": "NV",
|
||||
"new hampshire": "NH", "new jersey": "NJ", "new mexico": "NM", "new york": "NY",
|
||||
"north carolina": "NC", "north dakota": "ND", "ohio": "OH", "oklahoma": "OK",
|
||||
"oregon": "OR", "pennsylvania": "PA", "rhode island": "RI", "south carolina": "SC",
|
||||
"south dakota": "SD", "tennessee": "TN", "texas": "TX", "utah": "UT",
|
||||
"vermont": "VT", "virginia": "VA", "washington": "WA", "west virginia": "WV",
|
||||
"wisconsin": "WI", "wyoming": "WY", "district of columbia": "DC",
|
||||
"puerto rico": "PR", "guam": "GU", "virgin islands": "VI",
|
||||
"american samoa": "AS", "northern mariana islands": "MP",
|
||||
}
|
||||
|
||||
|
||||
def parse_coordinates_from_description(description: str) -> tuple[float, float] | None:
|
||||
"""
|
||||
Parse latitude/longitude from InciWeb description text.
|
||||
|
||||
Format: "Latitude: 47° 3 17 Longitude: 91° 38 6"
|
||||
InciWeb uses unsigned values for US coordinates (west longitude implied).
|
||||
Returns (lon, lat) tuple or None if not found.
|
||||
"""
|
||||
# Pattern for degree/minute/second format
|
||||
lat_pattern = r"Latitude:\s*(-?\d+)°\s*(\d+)\s*(\d+(?:\.\d+)?)"
|
||||
lon_pattern = r"Longitude:\s*(-?\d+)°\s*(\d+)\s*(\d+(?:\.\d+)?)"
|
||||
|
||||
lat_match = re.search(lat_pattern, description)
|
||||
lon_match = re.search(lon_pattern, description)
|
||||
|
||||
if not lat_match or not lon_match:
|
||||
return None
|
||||
|
||||
try:
|
||||
lat_deg = int(lat_match.group(1))
|
||||
lat_min = int(lat_match.group(2))
|
||||
lat_sec = float(lat_match.group(3))
|
||||
|
||||
lon_deg = int(lon_match.group(1))
|
||||
lon_min = int(lon_match.group(2))
|
||||
lon_sec = float(lon_match.group(3))
|
||||
|
||||
# Convert to decimal degrees
|
||||
# Latitude: positive in northern hemisphere
|
||||
if lat_deg >= 0:
|
||||
lat = lat_deg + lat_min / 60 + lat_sec / 3600
|
||||
else:
|
||||
lat = lat_deg - lat_min / 60 - lat_sec / 3600
|
||||
|
||||
# Longitude: InciWeb gives unsigned values for US west longitudes
|
||||
# Make negative for western hemisphere (US coordinates)
|
||||
lon = lon_deg + lon_min / 60 + lon_sec / 3600
|
||||
if lon > 0:
|
||||
lon = -lon # US longitudes are west (negative)
|
||||
|
||||
return (lon, lat)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def parse_state_from_description(description: str) -> str | None:
|
||||
"""
|
||||
Parse state name from InciWeb description text.
|
||||
|
||||
Format: "State: Minnesota" or "State: New Mexico"
|
||||
Returns 2-letter state code or None if not found.
|
||||
|
||||
Design note: State is parsed from the description rather than the title
|
||||
because InciWeb titles use unit code prefixes (e.g., "MNMNS Stewart Trail",
|
||||
"CACNP Santa Rosa Island Fire") which are not reliable state indicators.
|
||||
The description has a structured "State: <name>" field that reliably
|
||||
identifies the state for all incidents.
|
||||
"""
|
||||
pattern = r"State:\s*([A-Za-z\s]+?)(?:\n|---|$)"
|
||||
match = re.search(pattern, description)
|
||||
|
||||
if not match:
|
||||
return None
|
||||
|
||||
state_name = match.group(1).strip().lower()
|
||||
return STATE_NAME_TO_CODE.get(state_name)
|
||||
|
||||
|
||||
def strip_html(html_text: str) -> str:
|
||||
"""
|
||||
Strip HTML tags and decode entities to plain text.
|
||||
"""
|
||||
# Decode HTML entities (handles & < > etc.)
|
||||
text = html.unescape(html_text)
|
||||
|
||||
# Handle specifically (not a standard Python html entity)
|
||||
text = text.replace(" ", " ")
|
||||
text = text.replace("\xa0", " ") # Non-breaking space character
|
||||
|
||||
# Remove HTML tags
|
||||
text = re.sub(r"<[^>]+>", "", text)
|
||||
|
||||
# Normalize whitespace
|
||||
text = re.sub(r"\s+", " ", text)
|
||||
|
||||
return text.strip()
|
||||
|
||||
|
||||
def point_in_bbox(
|
||||
lon: float,
|
||||
lat: float,
|
||||
west: float,
|
||||
south: float,
|
||||
east: float,
|
||||
north: float,
|
||||
) -> bool:
|
||||
"""Check if a point is within a bounding box."""
|
||||
return west <= lon <= east and south <= lat <= north
|
||||
|
||||
|
||||
class InciWebSettings(BaseModel):
|
||||
"""Settings schema for InciWeb adapter."""
|
||||
|
||||
region: RegionConfig | None = None
|
||||
|
||||
|
||||
class InciWebAdapter(SourceAdapter):
|
||||
"""NIFC InciWeb wildfire narrative adapter."""
|
||||
|
||||
name = "inciweb"
|
||||
display_name = "NIFC InciWeb — Wildfire Narrative"
|
||||
description = (
|
||||
"Narrative wildfire updates from InciWeb. Editorial; lower precision "
|
||||
"than WFIGS. Use as supplementary context."
|
||||
)
|
||||
settings_schema = InciWebSettings
|
||||
requires_api_key = None
|
||||
api_key_field = None
|
||||
wizard_order = None # Ships disabled
|
||||
default_cadence_s = 600
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: AdapterConfig,
|
||||
config_store: ConfigStore,
|
||||
cursor_db_path: Path,
|
||||
) -> None:
|
||||
self._config_store = config_store
|
||||
self._cursor_db_path = cursor_db_path
|
||||
self._session: aiohttp.ClientSession | None = None
|
||||
self._db: sqlite3.Connection | None = None
|
||||
|
||||
# Conditional fetch state
|
||||
self._last_modified: str | None = None
|
||||
self._etag: str | None = None
|
||||
|
||||
# Parse region from settings
|
||||
region_dict = config.settings.get("region")
|
||||
if region_dict:
|
||||
self.region: RegionConfig | None = RegionConfig(**region_dict)
|
||||
else:
|
||||
self.region = None
|
||||
|
||||
async def startup(self) -> None:
|
||||
"""Initialize HTTP session and SQLite connection."""
|
||||
self._session = aiohttp.ClientSession(
|
||||
timeout=aiohttp.ClientTimeout(total=60),
|
||||
)
|
||||
self._db = sqlite3.connect(self._cursor_db_path)
|
||||
|
||||
# Create table for dedup tracking
|
||||
self._db.execute("""
|
||||
CREATE TABLE IF NOT EXISTS published_ids (
|
||||
adapter TEXT NOT NULL,
|
||||
event_id TEXT NOT NULL,
|
||||
first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (adapter, event_id)
|
||||
)
|
||||
""")
|
||||
self._db.execute("""
|
||||
CREATE INDEX IF NOT EXISTS published_ids_last_seen
|
||||
ON published_ids (last_seen)
|
||||
""")
|
||||
self._db.commit()
|
||||
|
||||
logger.info(
|
||||
"InciWeb adapter started",
|
||||
extra={"region": self.region.model_dump() if self.region else None},
|
||||
)
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
"""Close HTTP session and SQLite connection."""
|
||||
if self._session:
|
||||
await self._session.close()
|
||||
self._session = None
|
||||
if self._db:
|
||||
self._db.close()
|
||||
self._db = None
|
||||
logger.info("InciWeb adapter shut down")
|
||||
|
||||
async def apply_config(self, new_config: AdapterConfig) -> None:
|
||||
"""Apply new configuration from hot-reload."""
|
||||
region_dict = new_config.settings.get("region")
|
||||
if region_dict:
|
||||
self.region = RegionConfig(**region_dict)
|
||||
else:
|
||||
self.region = None
|
||||
logger.info(
|
||||
"InciWeb config updated",
|
||||
extra={"region": self.region.model_dump() if self.region else None},
|
||||
)
|
||||
|
||||
def is_published(self, event_id: str) -> bool:
|
||||
"""Check if an event has already been published."""
|
||||
if not self._db:
|
||||
return False
|
||||
cur = self._db.execute(
|
||||
"SELECT 1 FROM published_ids WHERE adapter = ? AND event_id = ?",
|
||||
(self.name, event_id),
|
||||
)
|
||||
return cur.fetchone() is not None
|
||||
|
||||
def mark_published(self, event_id: str) -> None:
|
||||
"""Mark an event as published."""
|
||||
if not self._db:
|
||||
return
|
||||
self._db.execute(
|
||||
"""
|
||||
INSERT INTO published_ids (adapter, event_id, first_seen, last_seen)
|
||||
VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (adapter, event_id) DO UPDATE SET
|
||||
last_seen = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(self.name, event_id),
|
||||
)
|
||||
self._db.commit()
|
||||
|
||||
def bump_last_seen(self, event_id: str) -> None:
|
||||
"""Bump the last_seen timestamp for an event."""
|
||||
if not self._db:
|
||||
return
|
||||
self._db.execute(
|
||||
"UPDATE published_ids SET last_seen = CURRENT_TIMESTAMP WHERE adapter = ? AND event_id = ?",
|
||||
(self.name, event_id),
|
||||
)
|
||||
self._db.commit()
|
||||
|
||||
def sweep_old_ids(self) -> int:
|
||||
"""Remove published_ids older than 14 days. Returns count deleted."""
|
||||
if not self._db:
|
||||
return 0
|
||||
cur = self._db.execute(
|
||||
"DELETE FROM published_ids WHERE adapter = ? AND last_seen < datetime('now', '-14 days')",
|
||||
(self.name,),
|
||||
)
|
||||
self._db.commit()
|
||||
count = cur.rowcount
|
||||
if count > 0:
|
||||
logger.info("InciWeb swept old dedup entries", extra={"count": count})
|
||||
return count
|
||||
|
||||
def subject_for(self, event: Event) -> str:
|
||||
"""Compute NATS subject for an event."""
|
||||
state = event.geo.primary_region
|
||||
if state and state.startswith("US-") and len(state) == 5:
|
||||
state_code = state[3:].lower()
|
||||
return f"central.fire.narrative.inciweb.{state_code}"
|
||||
return "central.fire.narrative.inciweb.unknown"
|
||||
|
||||
@retry(
|
||||
stop=stop_after_attempt(3),
|
||||
wait=wait_exponential_jitter(initial=1, max=30),
|
||||
retry=retry_if_exception_type((aiohttp.ClientError, TimeoutError)),
|
||||
)
|
||||
async def _fetch_rss(self) -> list[dict[str, Any]]:
|
||||
"""Fetch and parse RSS feed from InciWeb."""
|
||||
if not self._session:
|
||||
raise RuntimeError("Session not initialized")
|
||||
|
||||
# Build request headers with conditional fetch support
|
||||
headers = {"User-Agent": "Central/0.4"}
|
||||
if self._last_modified:
|
||||
headers["If-Modified-Since"] = self._last_modified
|
||||
if self._etag:
|
||||
headers["If-None-Match"] = self._etag
|
||||
|
||||
async with self._session.get(INCIWEB_RSS_URL, headers=headers) as resp:
|
||||
# Handle 304 Not Modified
|
||||
if resp.status == 304:
|
||||
logger.info("InciWeb not modified")
|
||||
return []
|
||||
|
||||
resp.raise_for_status()
|
||||
|
||||
# Capture conditional fetch headers for next request
|
||||
self._last_modified = resp.headers.get("Last-Modified")
|
||||
self._etag = resp.headers.get("ETag")
|
||||
|
||||
content = await resp.text()
|
||||
|
||||
# Parse RSS XML
|
||||
items = []
|
||||
try:
|
||||
root = ET.fromstring(content)
|
||||
channel = root.find("channel")
|
||||
if channel is None:
|
||||
return []
|
||||
|
||||
for item_elem in channel.findall("item"):
|
||||
item: dict[str, Any] = {}
|
||||
|
||||
title = item_elem.find("title")
|
||||
item["title"] = title.text if title is not None and title.text else ""
|
||||
|
||||
link = item_elem.find("link")
|
||||
item["link"] = link.text if link is not None and link.text else ""
|
||||
|
||||
description = item_elem.find("description")
|
||||
item["description"] = description.text if description is not None and description.text else ""
|
||||
|
||||
pub_date = item_elem.find("pubDate")
|
||||
item["pubDate"] = pub_date.text if pub_date is not None and pub_date.text else ""
|
||||
|
||||
guid = item_elem.find("guid")
|
||||
item["guid"] = guid.text if guid is not None and guid.text else ""
|
||||
|
||||
# Check for dc:creator
|
||||
creator = item_elem.find("{http://purl.org/dc/elements/1.1/}creator")
|
||||
item["creator"] = creator.text if creator is not None and creator.text else ""
|
||||
|
||||
items.append(item)
|
||||
|
||||
except ET.ParseError as e:
|
||||
logger.error("InciWeb RSS parse error", extra={"error": str(e)})
|
||||
raise
|
||||
|
||||
logger.info(
|
||||
"InciWeb fetch completed",
|
||||
extra={"item_count": len(items)},
|
||||
)
|
||||
return items
|
||||
|
||||
async def poll(self) -> AsyncIterator[Event]:
|
||||
"""Poll InciWeb for narrative updates."""
|
||||
if not self._db:
|
||||
raise RuntimeError("Database not initialized")
|
||||
|
||||
# Fetch RSS feed
|
||||
try:
|
||||
items = await self._fetch_rss()
|
||||
except Exception as e:
|
||||
logger.error("InciWeb fetch failed", extra={"error": str(e)})
|
||||
raise
|
||||
|
||||
events_yielded = 0
|
||||
|
||||
for item in items:
|
||||
guid = item.get("guid", "")
|
||||
if not guid:
|
||||
continue
|
||||
|
||||
# Dedup: skip if already published
|
||||
if self.is_published(guid):
|
||||
self.bump_last_seen(guid)
|
||||
continue
|
||||
|
||||
description_html = item.get("description", "")
|
||||
|
||||
# Parse coordinates from description
|
||||
centroid = parse_coordinates_from_description(description_html)
|
||||
|
||||
# Post-filter: skip if point outside region bbox
|
||||
if self.region and centroid:
|
||||
lon, lat = centroid
|
||||
if not point_in_bbox(
|
||||
lon, lat,
|
||||
self.region.west, self.region.south,
|
||||
self.region.east, self.region.north,
|
||||
):
|
||||
continue
|
||||
|
||||
# Parse state from description
|
||||
state_code = parse_state_from_description(description_html)
|
||||
|
||||
# Build regions
|
||||
if state_code:
|
||||
regions = [f"US-{state_code}"]
|
||||
primary_region = f"US-{state_code}"
|
||||
else:
|
||||
regions = []
|
||||
primary_region = None
|
||||
|
||||
# Parse pubDate (RFC 822 format)
|
||||
pub_date_str = item.get("pubDate", "")
|
||||
try:
|
||||
event_time = parsedate_to_datetime(pub_date_str)
|
||||
# Ensure UTC
|
||||
if event_time.tzinfo is None:
|
||||
event_time = event_time.replace(tzinfo=timezone.utc)
|
||||
else:
|
||||
event_time = event_time.astimezone(timezone.utc)
|
||||
except (ValueError, TypeError):
|
||||
event_time = datetime.now(timezone.utc)
|
||||
|
||||
# Build geo
|
||||
geo = Geo(
|
||||
centroid=centroid,
|
||||
bbox=(centroid[0], centroid[1], centroid[0], centroid[1]) if centroid else None,
|
||||
regions=regions,
|
||||
primary_region=primary_region,
|
||||
)
|
||||
|
||||
# Strip HTML from description
|
||||
description_plain = strip_html(description_html)
|
||||
|
||||
# Build event
|
||||
event = Event(
|
||||
id=guid,
|
||||
adapter=self.name,
|
||||
category="fire.narrative.inciweb",
|
||||
time=event_time,
|
||||
severity=0, # Narrative; not authoritative
|
||||
geo=geo,
|
||||
data={
|
||||
"title": item.get("title", ""),
|
||||
"description": description_plain,
|
||||
"description_html": description_html,
|
||||
"url": item.get("link", ""),
|
||||
"guid": guid,
|
||||
"raw": item,
|
||||
},
|
||||
)
|
||||
|
||||
yield event
|
||||
self.mark_published(guid)
|
||||
events_yielded += 1
|
||||
|
||||
# Periodic cleanup of old entries
|
||||
self.sweep_old_ids()
|
||||
|
||||
logger.info(
|
||||
"InciWeb poll completed",
|
||||
extra={"events_yielded": events_yielded},
|
||||
)
|
||||
|
|
@ -19,7 +19,7 @@ from tenacity import (
|
|||
|
||||
from central import __version__
|
||||
from central.adapter import SourceAdapter
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from central.config_models import AdapterConfig, RegionConfig
|
||||
from central.config_store import ConfigStore
|
||||
|
|
@ -193,7 +193,11 @@ def _build_regions(same_codes: list[str], ugc_codes: list[str]) -> list[str]:
|
|||
|
||||
class NWSSettings(BaseModel):
|
||||
"""Settings schema for NWS adapter."""
|
||||
contact_email: str = ""
|
||||
contact_email: str = Field(
|
||||
default="",
|
||||
pattern=r"^$|^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$",
|
||||
description="Contact email for NWS API User-Agent header",
|
||||
)
|
||||
region: RegionConfig | None = None
|
||||
|
||||
|
||||
|
|
|
|||
186
src/central/adapters/swpc_alerts.py
Normal file
186
src/central/adapters/swpc_alerts.py
Normal file
|
|
@ -0,0 +1,186 @@
|
|||
"""NOAA SWPC space weather alerts adapter."""
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
from tenacity import (
|
||||
retry,
|
||||
retry_if_exception_type,
|
||||
stop_after_attempt,
|
||||
wait_exponential_jitter,
|
||||
)
|
||||
|
||||
from central.adapter import SourceAdapter
|
||||
from central.adapters.swpc_common import (
|
||||
SWPC_ALERTS_URL,
|
||||
SWPCSettings,
|
||||
parse_swpc_timestamp,
|
||||
severity_from_alert_product_id,
|
||||
)
|
||||
from central.config_models import AdapterConfig
|
||||
from central.config_store import ConfigStore
|
||||
from central.models import Event, Geo
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SWPCAlertsAdapter(SourceAdapter):
|
||||
"""NOAA SWPC space weather alerts adapter."""
|
||||
|
||||
name = "swpc_alerts"
|
||||
display_name = "NOAA SWPC — Space Weather Alerts"
|
||||
description = "Active NOAA SWPC space weather alerts, watches, warnings, and summaries."
|
||||
settings_schema = SWPCSettings
|
||||
requires_api_key = None
|
||||
api_key_field = None
|
||||
wizard_order = None
|
||||
default_cadence_s = 300
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: AdapterConfig,
|
||||
config_store: ConfigStore,
|
||||
cursor_db_path: Path,
|
||||
) -> None:
|
||||
self._config_store = config_store
|
||||
self._cursor_db_path = cursor_db_path
|
||||
self._session: aiohttp.ClientSession | None = None
|
||||
self._db: sqlite3.Connection | None = None
|
||||
|
||||
async def startup(self) -> None:
|
||||
self._session = aiohttp.ClientSession(
|
||||
timeout=aiohttp.ClientTimeout(total=60),
|
||||
)
|
||||
self._db = sqlite3.connect(self._cursor_db_path)
|
||||
self._db.execute("""
|
||||
CREATE TABLE IF NOT EXISTS published_ids (
|
||||
adapter TEXT NOT NULL,
|
||||
event_id TEXT NOT NULL,
|
||||
first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (adapter, event_id)
|
||||
)
|
||||
""")
|
||||
self._db.execute("""
|
||||
CREATE INDEX IF NOT EXISTS published_ids_last_seen
|
||||
ON published_ids (last_seen)
|
||||
""")
|
||||
self._db.commit()
|
||||
logger.info("SWPC alerts adapter started")
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
if self._session:
|
||||
await self._session.close()
|
||||
self._session = None
|
||||
if self._db:
|
||||
self._db.close()
|
||||
self._db = None
|
||||
logger.info("SWPC alerts adapter shut down")
|
||||
|
||||
async def apply_config(self, new_config: AdapterConfig) -> None:
|
||||
logger.info("SWPC alerts config updated")
|
||||
|
||||
def is_published(self, event_id: str) -> bool:
|
||||
if not self._db:
|
||||
return False
|
||||
cur = self._db.execute(
|
||||
"SELECT 1 FROM published_ids WHERE adapter = ? AND event_id = ?",
|
||||
(self.name, event_id),
|
||||
)
|
||||
return cur.fetchone() is not None
|
||||
|
||||
def mark_published(self, event_id: str) -> None:
|
||||
if not self._db:
|
||||
return
|
||||
self._db.execute(
|
||||
"""
|
||||
INSERT INTO published_ids (adapter, event_id, first_seen, last_seen)
|
||||
VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (adapter, event_id) DO UPDATE SET
|
||||
last_seen = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(self.name, event_id),
|
||||
)
|
||||
self._db.commit()
|
||||
|
||||
def sweep_old_ids(self) -> int:
|
||||
if not self._db:
|
||||
return 0
|
||||
cur = self._db.execute(
|
||||
"DELETE FROM published_ids WHERE adapter = ? AND last_seen < datetime('now', '-14 days')",
|
||||
(self.name,),
|
||||
)
|
||||
self._db.commit()
|
||||
count = cur.rowcount
|
||||
if count > 0:
|
||||
logger.info("SWPC alerts swept old dedup entries", extra={"count": count})
|
||||
return count
|
||||
|
||||
def subject_for(self, event: Event) -> str:
|
||||
product_id = event.data.get("product_id") or "unknown"
|
||||
return f"central.space.alert.{product_id.lower()}"
|
||||
|
||||
@retry(
|
||||
stop=stop_after_attempt(3),
|
||||
wait=wait_exponential_jitter(initial=1, max=30),
|
||||
retry=retry_if_exception_type((aiohttp.ClientError, TimeoutError)),
|
||||
)
|
||||
async def _fetch(self) -> list[dict[str, Any]]:
|
||||
if not self._session:
|
||||
raise RuntimeError("Session not initialized")
|
||||
async with self._session.get(
|
||||
SWPC_ALERTS_URL, headers={"User-Agent": "Central/0.4"}
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
data = await resp.json()
|
||||
logger.info("SWPC alerts fetch completed", extra={"item_count": len(data)})
|
||||
return data
|
||||
|
||||
async def poll(self) -> AsyncIterator[Event]:
|
||||
if not self._db:
|
||||
raise RuntimeError("Database not initialized")
|
||||
|
||||
try:
|
||||
items = await self._fetch()
|
||||
except Exception as e:
|
||||
logger.error("SWPC alerts fetch failed", extra={"error": str(e)})
|
||||
raise
|
||||
|
||||
events_yielded = 0
|
||||
for item in items:
|
||||
product_id = item.get("product_id")
|
||||
issue_dt_raw = item.get("issue_datetime")
|
||||
if not product_id or not issue_dt_raw:
|
||||
continue
|
||||
|
||||
event_id = f"{product_id}|{issue_dt_raw}"
|
||||
if self.is_published(event_id):
|
||||
continue
|
||||
|
||||
issue_dt = parse_swpc_timestamp(issue_dt_raw, "alerts") or datetime.now(timezone.utc)
|
||||
|
||||
event = Event(
|
||||
id=event_id,
|
||||
adapter=self.name,
|
||||
category="space.alert",
|
||||
time=issue_dt,
|
||||
severity=severity_from_alert_product_id(product_id),
|
||||
geo=Geo(),
|
||||
data={
|
||||
"product_id": product_id,
|
||||
"issue_datetime": issue_dt_raw,
|
||||
"message": item.get("message", ""),
|
||||
},
|
||||
)
|
||||
|
||||
yield event
|
||||
self.mark_published(event_id)
|
||||
events_yielded += 1
|
||||
|
||||
self.sweep_old_ids()
|
||||
logger.info("SWPC alerts poll completed", extra={"events_yielded": events_yielded})
|
||||
81
src/central/adapters/swpc_common.py
Normal file
81
src/central/adapters/swpc_common.py
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
"""Shared utilities for NOAA SWPC space weather adapters."""
|
||||
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
SWPC_ALERTS_URL = "https://services.swpc.noaa.gov/products/alerts.json"
|
||||
SWPC_KINDEX_URL = "https://services.swpc.noaa.gov/products/noaa-planetary-k-index.json"
|
||||
SWPC_PROTONS_URL = "https://services.swpc.noaa.gov/json/goes/primary/integral-protons-1-day.json"
|
||||
|
||||
|
||||
class SWPCSettings(BaseModel):
|
||||
"""Settings schema for SWPC adapters. No operator-tunable knobs today."""
|
||||
|
||||
|
||||
def parse_swpc_timestamp(raw: str | None, endpoint_kind: str) -> datetime | None:
|
||||
"""Normalize SWPC timestamp strings to UTC datetime.
|
||||
|
||||
endpoint_kind shapes:
|
||||
alerts -> "2026-05-19 05:14:59.780" (space-separated, no TZ; UTC per message body)
|
||||
kindex -> "2026-05-12T00:00:00" (ISO without TZ; UTC by convention)
|
||||
protons -> "2026-05-18T05:35:00Z" (ISO with Z)
|
||||
"""
|
||||
if not raw:
|
||||
return None
|
||||
if endpoint_kind == "alerts":
|
||||
try:
|
||||
dt = datetime.strptime(raw, "%Y-%m-%d %H:%M:%S.%f")
|
||||
except ValueError:
|
||||
dt = datetime.strptime(raw, "%Y-%m-%d %H:%M:%S")
|
||||
return dt.replace(tzinfo=timezone.utc)
|
||||
if endpoint_kind == "kindex":
|
||||
dt = datetime.fromisoformat(raw)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone(timezone.utc)
|
||||
if endpoint_kind == "protons":
|
||||
raw_norm = raw[:-1] + "+00:00" if raw.endswith("Z") else raw
|
||||
dt = datetime.fromisoformat(raw_norm)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone(timezone.utc)
|
||||
raise ValueError(f"unknown endpoint_kind: {endpoint_kind!r}")
|
||||
|
||||
|
||||
def severity_from_kp(kp: float | int | None) -> int:
|
||||
"""Map planetary K-index value (0-9) to severity 0-4 via the G-scale.
|
||||
|
||||
Kp 5 = G1 = severity 1, Kp 6 = G2 = severity 2, Kp 7 = G3 = severity 3,
|
||||
Kp 8 = G4 = severity 4, Kp 9 = G5 = severity 4 (capped).
|
||||
"""
|
||||
if kp is None:
|
||||
return 0
|
||||
if kp < 5:
|
||||
return 0
|
||||
if kp < 6:
|
||||
return 1
|
||||
if kp < 7:
|
||||
return 2
|
||||
if kp < 8:
|
||||
return 3
|
||||
return 4
|
||||
|
||||
|
||||
_ALERT_KP_PATTERN = re.compile(r"^K0([5-9])[AW]$")
|
||||
|
||||
|
||||
def severity_from_alert_product_id(product_id: str | None) -> int:
|
||||
"""Best-effort severity for an alert from its product_id G-scale.
|
||||
|
||||
Product IDs of form K0[5-9][AW] identify Kp-based geomagnetic storm
|
||||
alerts and warnings (K05A=G1, K06A=G2, K07A=G3, K08A=G4, K09A=G5).
|
||||
All other product IDs return 0.
|
||||
"""
|
||||
if not product_id:
|
||||
return 0
|
||||
m = _ALERT_KP_PATTERN.match(product_id.upper())
|
||||
if not m:
|
||||
return 0
|
||||
return severity_from_kp(int(m.group(1)))
|
||||
186
src/central/adapters/swpc_kindex.py
Normal file
186
src/central/adapters/swpc_kindex.py
Normal file
|
|
@ -0,0 +1,186 @@
|
|||
"""NOAA SWPC Planetary K-Index adapter."""
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
from tenacity import (
|
||||
retry,
|
||||
retry_if_exception_type,
|
||||
stop_after_attempt,
|
||||
wait_exponential_jitter,
|
||||
)
|
||||
|
||||
from central.adapter import SourceAdapter
|
||||
from central.adapters.swpc_common import (
|
||||
SWPC_KINDEX_URL,
|
||||
SWPCSettings,
|
||||
parse_swpc_timestamp,
|
||||
severity_from_kp,
|
||||
)
|
||||
from central.config_models import AdapterConfig
|
||||
from central.config_store import ConfigStore
|
||||
from central.models import Event, Geo
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SWPCKindexAdapter(SourceAdapter):
|
||||
"""NOAA SWPC planetary K-index adapter."""
|
||||
|
||||
name = "swpc_kindex"
|
||||
display_name = "NOAA SWPC — Planetary K-Index"
|
||||
description = "Planetary K-index measurements at 3-hour cadence from NOAA SWPC."
|
||||
settings_schema = SWPCSettings
|
||||
requires_api_key = None
|
||||
api_key_field = None
|
||||
wizard_order = None
|
||||
default_cadence_s = 600
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: AdapterConfig,
|
||||
config_store: ConfigStore,
|
||||
cursor_db_path: Path,
|
||||
) -> None:
|
||||
self._config_store = config_store
|
||||
self._cursor_db_path = cursor_db_path
|
||||
self._session: aiohttp.ClientSession | None = None
|
||||
self._db: sqlite3.Connection | None = None
|
||||
|
||||
async def startup(self) -> None:
|
||||
self._session = aiohttp.ClientSession(
|
||||
timeout=aiohttp.ClientTimeout(total=60),
|
||||
)
|
||||
self._db = sqlite3.connect(self._cursor_db_path)
|
||||
self._db.execute("""
|
||||
CREATE TABLE IF NOT EXISTS published_ids (
|
||||
adapter TEXT NOT NULL,
|
||||
event_id TEXT NOT NULL,
|
||||
first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (adapter, event_id)
|
||||
)
|
||||
""")
|
||||
self._db.execute("""
|
||||
CREATE INDEX IF NOT EXISTS published_ids_last_seen
|
||||
ON published_ids (last_seen)
|
||||
""")
|
||||
self._db.commit()
|
||||
logger.info("SWPC kindex adapter started")
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
if self._session:
|
||||
await self._session.close()
|
||||
self._session = None
|
||||
if self._db:
|
||||
self._db.close()
|
||||
self._db = None
|
||||
logger.info("SWPC kindex adapter shut down")
|
||||
|
||||
async def apply_config(self, new_config: AdapterConfig) -> None:
|
||||
logger.info("SWPC kindex config updated")
|
||||
|
||||
def is_published(self, event_id: str) -> bool:
|
||||
if not self._db:
|
||||
return False
|
||||
cur = self._db.execute(
|
||||
"SELECT 1 FROM published_ids WHERE adapter = ? AND event_id = ?",
|
||||
(self.name, event_id),
|
||||
)
|
||||
return cur.fetchone() is not None
|
||||
|
||||
def mark_published(self, event_id: str) -> None:
|
||||
if not self._db:
|
||||
return
|
||||
self._db.execute(
|
||||
"""
|
||||
INSERT INTO published_ids (adapter, event_id, first_seen, last_seen)
|
||||
VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (adapter, event_id) DO UPDATE SET
|
||||
last_seen = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(self.name, event_id),
|
||||
)
|
||||
self._db.commit()
|
||||
|
||||
def sweep_old_ids(self) -> int:
|
||||
if not self._db:
|
||||
return 0
|
||||
cur = self._db.execute(
|
||||
"DELETE FROM published_ids WHERE adapter = ? AND last_seen < datetime('now', '-14 days')",
|
||||
(self.name,),
|
||||
)
|
||||
self._db.commit()
|
||||
count = cur.rowcount
|
||||
if count > 0:
|
||||
logger.info("SWPC kindex swept old dedup entries", extra={"count": count})
|
||||
return count
|
||||
|
||||
def subject_for(self, event: Event) -> str:
|
||||
return "central.space.kindex"
|
||||
|
||||
@retry(
|
||||
stop=stop_after_attempt(3),
|
||||
wait=wait_exponential_jitter(initial=1, max=30),
|
||||
retry=retry_if_exception_type((aiohttp.ClientError, TimeoutError)),
|
||||
)
|
||||
async def _fetch(self) -> list[dict[str, Any]]:
|
||||
if not self._session:
|
||||
raise RuntimeError("Session not initialized")
|
||||
async with self._session.get(
|
||||
SWPC_KINDEX_URL, headers={"User-Agent": "Central/0.4"}
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
data = await resp.json()
|
||||
logger.info("SWPC kindex fetch completed", extra={"item_count": len(data)})
|
||||
return data
|
||||
|
||||
async def poll(self) -> AsyncIterator[Event]:
|
||||
if not self._db:
|
||||
raise RuntimeError("Database not initialized")
|
||||
|
||||
try:
|
||||
items = await self._fetch()
|
||||
except Exception as e:
|
||||
logger.error("SWPC kindex fetch failed", extra={"error": str(e)})
|
||||
raise
|
||||
|
||||
events_yielded = 0
|
||||
for item in items:
|
||||
time_tag = item.get("time_tag")
|
||||
kp = item.get("Kp")
|
||||
if not time_tag or kp is None:
|
||||
continue
|
||||
|
||||
event_id = time_tag
|
||||
if self.is_published(event_id):
|
||||
continue
|
||||
|
||||
event_time = parse_swpc_timestamp(time_tag, "kindex") or datetime.now(timezone.utc)
|
||||
|
||||
event = Event(
|
||||
id=event_id,
|
||||
adapter=self.name,
|
||||
category="space.kindex",
|
||||
time=event_time,
|
||||
severity=severity_from_kp(kp),
|
||||
geo=Geo(),
|
||||
data={
|
||||
"time_tag": time_tag,
|
||||
"Kp": kp,
|
||||
"a_running": item.get("a_running"),
|
||||
"station_count": item.get("station_count"),
|
||||
},
|
||||
)
|
||||
|
||||
yield event
|
||||
self.mark_published(event_id)
|
||||
events_yielded += 1
|
||||
|
||||
self.sweep_old_ids()
|
||||
logger.info("SWPC kindex poll completed", extra={"events_yielded": events_yielded})
|
||||
185
src/central/adapters/swpc_protons.py
Normal file
185
src/central/adapters/swpc_protons.py
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
"""NOAA SWPC GOES integral proton flux adapter."""
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
from tenacity import (
|
||||
retry,
|
||||
retry_if_exception_type,
|
||||
stop_after_attempt,
|
||||
wait_exponential_jitter,
|
||||
)
|
||||
|
||||
from central.adapter import SourceAdapter
|
||||
from central.adapters.swpc_common import (
|
||||
SWPC_PROTONS_URL,
|
||||
SWPCSettings,
|
||||
parse_swpc_timestamp,
|
||||
)
|
||||
from central.config_models import AdapterConfig
|
||||
from central.config_store import ConfigStore
|
||||
from central.models import Event, Geo
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SWPCProtonsAdapter(SourceAdapter):
|
||||
"""NOAA SWPC GOES integral proton flux adapter."""
|
||||
|
||||
name = "swpc_protons"
|
||||
display_name = "NOAA SWPC — GOES Proton Flux"
|
||||
description = "GOES primary satellite integral proton flux measurements (1-day window) from NOAA SWPC."
|
||||
settings_schema = SWPCSettings
|
||||
requires_api_key = None
|
||||
api_key_field = None
|
||||
wizard_order = None
|
||||
default_cadence_s = 600
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: AdapterConfig,
|
||||
config_store: ConfigStore,
|
||||
cursor_db_path: Path,
|
||||
) -> None:
|
||||
self._config_store = config_store
|
||||
self._cursor_db_path = cursor_db_path
|
||||
self._session: aiohttp.ClientSession | None = None
|
||||
self._db: sqlite3.Connection | None = None
|
||||
|
||||
async def startup(self) -> None:
|
||||
self._session = aiohttp.ClientSession(
|
||||
timeout=aiohttp.ClientTimeout(total=60),
|
||||
)
|
||||
self._db = sqlite3.connect(self._cursor_db_path)
|
||||
self._db.execute("""
|
||||
CREATE TABLE IF NOT EXISTS published_ids (
|
||||
adapter TEXT NOT NULL,
|
||||
event_id TEXT NOT NULL,
|
||||
first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (adapter, event_id)
|
||||
)
|
||||
""")
|
||||
self._db.execute("""
|
||||
CREATE INDEX IF NOT EXISTS published_ids_last_seen
|
||||
ON published_ids (last_seen)
|
||||
""")
|
||||
self._db.commit()
|
||||
logger.info("SWPC protons adapter started")
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
if self._session:
|
||||
await self._session.close()
|
||||
self._session = None
|
||||
if self._db:
|
||||
self._db.close()
|
||||
self._db = None
|
||||
logger.info("SWPC protons adapter shut down")
|
||||
|
||||
async def apply_config(self, new_config: AdapterConfig) -> None:
|
||||
logger.info("SWPC protons config updated")
|
||||
|
||||
def is_published(self, event_id: str) -> bool:
|
||||
if not self._db:
|
||||
return False
|
||||
cur = self._db.execute(
|
||||
"SELECT 1 FROM published_ids WHERE adapter = ? AND event_id = ?",
|
||||
(self.name, event_id),
|
||||
)
|
||||
return cur.fetchone() is not None
|
||||
|
||||
def mark_published(self, event_id: str) -> None:
|
||||
if not self._db:
|
||||
return
|
||||
self._db.execute(
|
||||
"""
|
||||
INSERT INTO published_ids (adapter, event_id, first_seen, last_seen)
|
||||
VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (adapter, event_id) DO UPDATE SET
|
||||
last_seen = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(self.name, event_id),
|
||||
)
|
||||
self._db.commit()
|
||||
|
||||
def sweep_old_ids(self) -> int:
|
||||
if not self._db:
|
||||
return 0
|
||||
cur = self._db.execute(
|
||||
"DELETE FROM published_ids WHERE adapter = ? AND last_seen < datetime('now', '-14 days')",
|
||||
(self.name,),
|
||||
)
|
||||
self._db.commit()
|
||||
count = cur.rowcount
|
||||
if count > 0:
|
||||
logger.info("SWPC protons swept old dedup entries", extra={"count": count})
|
||||
return count
|
||||
|
||||
def subject_for(self, event: Event) -> str:
|
||||
return "central.space.proton_flux"
|
||||
|
||||
@retry(
|
||||
stop=stop_after_attempt(3),
|
||||
wait=wait_exponential_jitter(initial=1, max=30),
|
||||
retry=retry_if_exception_type((aiohttp.ClientError, TimeoutError)),
|
||||
)
|
||||
async def _fetch(self) -> list[dict[str, Any]]:
|
||||
if not self._session:
|
||||
raise RuntimeError("Session not initialized")
|
||||
async with self._session.get(
|
||||
SWPC_PROTONS_URL, headers={"User-Agent": "Central/0.4"}
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
data = await resp.json()
|
||||
logger.info("SWPC protons fetch completed", extra={"item_count": len(data)})
|
||||
return data
|
||||
|
||||
async def poll(self) -> AsyncIterator[Event]:
|
||||
if not self._db:
|
||||
raise RuntimeError("Database not initialized")
|
||||
|
||||
try:
|
||||
items = await self._fetch()
|
||||
except Exception as e:
|
||||
logger.error("SWPC protons fetch failed", extra={"error": str(e)})
|
||||
raise
|
||||
|
||||
events_yielded = 0
|
||||
for item in items:
|
||||
time_tag = item.get("time_tag")
|
||||
energy = item.get("energy")
|
||||
if not time_tag or not energy:
|
||||
continue
|
||||
|
||||
event_id = f"{time_tag}|{energy}"
|
||||
if self.is_published(event_id):
|
||||
continue
|
||||
|
||||
event_time = parse_swpc_timestamp(time_tag, "protons") or datetime.now(timezone.utc)
|
||||
|
||||
event = Event(
|
||||
id=event_id,
|
||||
adapter=self.name,
|
||||
category="space.proton_flux",
|
||||
time=event_time,
|
||||
severity=0,
|
||||
geo=Geo(),
|
||||
data={
|
||||
"time_tag": time_tag,
|
||||
"satellite": item.get("satellite"),
|
||||
"flux": item.get("flux"),
|
||||
"energy": energy,
|
||||
},
|
||||
)
|
||||
|
||||
yield event
|
||||
self.mark_published(event_id)
|
||||
events_yielded += 1
|
||||
|
||||
self.sweep_old_ids()
|
||||
logger.info("SWPC protons poll completed", extra={"events_yielded": events_yielded})
|
||||
|
|
@ -5,7 +5,7 @@ import sqlite3
|
|||
from collections.abc import AsyncIterator
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Any, Literal
|
||||
|
||||
import aiohttp
|
||||
from shapely.geometry import Point, box as shapely_box
|
||||
|
|
@ -64,7 +64,7 @@ def magnitude_to_severity(mag: float) -> int:
|
|||
|
||||
class USGSQuakeSettings(BaseModel):
|
||||
"""Settings schema for USGS quake adapter."""
|
||||
feed: str = "all_hour"
|
||||
feed: Literal["all_hour", "all_day", "all_week", "all_month"] = "all_hour"
|
||||
region: RegionConfig | None = None
|
||||
|
||||
|
||||
|
|
|
|||
242
src/central/adapters/wfigs_common.py
Normal file
242
src/central/adapters/wfigs_common.py
Normal file
|
|
@ -0,0 +1,242 @@
|
|||
"""Shared utilities for WFIGS (Wildland Fire Interagency Geospatial Services) adapters."""
|
||||
|
||||
import sqlite3
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
# WFIGS FeatureServer endpoints
|
||||
WFIGS_INCIDENTS_URL = (
|
||||
"https://services3.arcgis.com/T4QMspbfLg3qTGWY/ArcGIS/rest/services/"
|
||||
"WFIGS_Incident_Locations_Current/FeatureServer/0/query"
|
||||
)
|
||||
WFIGS_PERIMETERS_URL = (
|
||||
"https://services3.arcgis.com/T4QMspbfLg3qTGWY/ArcGIS/rest/services/"
|
||||
"WFIGS_Interagency_Perimeters_Current/FeatureServer/0/query"
|
||||
)
|
||||
|
||||
# Fall-off sweep window: 14 days (matches WFIGS's longest fall-off: large fires)
|
||||
FALLOFF_WINDOW_DAYS = 14
|
||||
|
||||
# Incident type code mappings (WFIGS uses 2-letter codes)
|
||||
INCIDENT_TYPE_MAP = {
|
||||
"WF": "wildfire",
|
||||
"RX": "prescribed_fire",
|
||||
"CX": "complex",
|
||||
"FA": "false_alarm",
|
||||
}
|
||||
|
||||
|
||||
def normalize_state(state: str | None) -> str | None:
|
||||
"""Strip 'US-' prefix from POOState (ISO 3166-2 -> 2-letter)."""
|
||||
if not state:
|
||||
return None
|
||||
if state.startswith("US-") and len(state) == 5:
|
||||
return state[3:]
|
||||
if len(state) == 2:
|
||||
return state
|
||||
return state # unknown shape, pass through
|
||||
|
||||
|
||||
def normalize_incident_type(code: str | None) -> str:
|
||||
"""Map IncidentTypeCategory code to a readable name."""
|
||||
if not code:
|
||||
return "unknown"
|
||||
upper = code.upper()
|
||||
if upper in INCIDENT_TYPE_MAP:
|
||||
return INCIDENT_TYPE_MAP[upper]
|
||||
return code.lower()
|
||||
|
||||
|
||||
def severity_from_acres(acres: float | None) -> int:
|
||||
"""Map DailyAcres to severity level 0-4."""
|
||||
if acres is None or acres == 0:
|
||||
return 0
|
||||
if acres < 10:
|
||||
return 1
|
||||
if acres < 100:
|
||||
return 2
|
||||
if acres < 1000:
|
||||
return 3
|
||||
return 4
|
||||
|
||||
|
||||
def parse_wfigs_timestamp(epoch_ms: int | None) -> datetime | None:
|
||||
"""Parse WFIGS epoch milliseconds to UTC datetime."""
|
||||
if epoch_ms is None:
|
||||
return None
|
||||
return datetime.fromtimestamp(epoch_ms / 1000, tz=timezone.utc)
|
||||
|
||||
|
||||
def build_regions(state: str | None, county: str | None) -> tuple[list[str], str | None]:
|
||||
"""
|
||||
Build geo.regions list and primary_region from POOState and POOCounty.
|
||||
|
||||
Expects normalized 2-letter state codes (e.g., "MT" not "US-MT").
|
||||
Returns (regions, primary_region).
|
||||
"""
|
||||
if not state:
|
||||
return [], None
|
||||
|
||||
state_upper = state.upper()
|
||||
if county:
|
||||
# Normalize county: remove spaces, uppercase
|
||||
county_normalized = county.replace(" ", "_").upper()
|
||||
region = f"US-{state_upper}-{county_normalized}"
|
||||
return [region], region
|
||||
else:
|
||||
region = f"US-{state_upper}"
|
||||
return [region], region
|
||||
|
||||
|
||||
def subject_suffix(state: str | None, county: str | None) -> str:
|
||||
"""
|
||||
Build subject suffix from state and county.
|
||||
|
||||
Expects normalized 2-letter state codes.
|
||||
Returns lowercase state.county (county with spaces→underscores).
|
||||
Falls back to "unknown" if state is not available.
|
||||
"""
|
||||
if not state:
|
||||
return "unknown"
|
||||
|
||||
state_lower = state.lower()
|
||||
if county:
|
||||
county_lower = county.lower().replace(" ", "_")
|
||||
return f"{state_lower}.{county_lower}"
|
||||
return state_lower
|
||||
|
||||
|
||||
def init_observed_table(db: sqlite3.Connection) -> None:
|
||||
"""Create the wfigs_observed table if it doesn't exist."""
|
||||
db.execute("""
|
||||
CREATE TABLE IF NOT EXISTS wfigs_observed (
|
||||
layer TEXT NOT NULL,
|
||||
irwin_id TEXT NOT NULL,
|
||||
last_observed_at TEXT NOT NULL,
|
||||
state TEXT,
|
||||
county TEXT,
|
||||
PRIMARY KEY (layer, irwin_id)
|
||||
)
|
||||
""")
|
||||
db.commit()
|
||||
|
||||
|
||||
def get_observed_guids(db: sqlite3.Connection, layer: str) -> dict[str, tuple[str, str | None, str | None]]:
|
||||
"""
|
||||
Get all observed IRWIN GUIDs for a layer.
|
||||
|
||||
Returns dict mapping irwin_id -> (last_observed_at, state, county).
|
||||
"""
|
||||
cursor = db.execute(
|
||||
"SELECT irwin_id, last_observed_at, state, county FROM wfigs_observed WHERE layer = ?",
|
||||
(layer,),
|
||||
)
|
||||
return {row[0]: (row[1], row[2], row[3]) for row in cursor.fetchall()}
|
||||
|
||||
|
||||
def update_observed(
|
||||
db: sqlite3.Connection,
|
||||
layer: str,
|
||||
current_guids: dict[str, tuple[str | None, str | None]],
|
||||
) -> None:
|
||||
"""
|
||||
Update the observed table with current poll's GUIDs.
|
||||
|
||||
current_guids: dict mapping irwin_id -> (state, county)
|
||||
"""
|
||||
now_iso = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
# Use INSERT OR REPLACE to upsert
|
||||
for irwin_id, (state, county) in current_guids.items():
|
||||
db.execute(
|
||||
"""
|
||||
INSERT OR REPLACE INTO wfigs_observed (layer, irwin_id, last_observed_at, state, county)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
""",
|
||||
(layer, irwin_id, now_iso, state, county),
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
def delete_observed(db: sqlite3.Connection, layer: str, irwin_ids: set[str]) -> None:
|
||||
"""Delete fallen-off GUIDs from the observed table."""
|
||||
for irwin_id in irwin_ids:
|
||||
db.execute(
|
||||
"DELETE FROM wfigs_observed WHERE layer = ? AND irwin_id = ?",
|
||||
(layer, irwin_id),
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
def cleanup_old_observed(db: sqlite3.Connection, layer: str, days: int = FALLOFF_WINDOW_DAYS) -> None:
|
||||
"""Remove observed entries older than the sweep window."""
|
||||
cutoff = datetime.now(timezone.utc).isoformat()
|
||||
db.execute(
|
||||
f"""
|
||||
DELETE FROM wfigs_observed
|
||||
WHERE layer = ?
|
||||
AND datetime(last_observed_at) < datetime(?, '-{days} days')
|
||||
""",
|
||||
(layer, cutoff),
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
def point_in_bbox(
|
||||
lon: float,
|
||||
lat: float,
|
||||
west: float,
|
||||
south: float,
|
||||
east: float,
|
||||
north: float,
|
||||
) -> bool:
|
||||
"""Check if a point is within a bounding box."""
|
||||
return west <= lon <= east and south <= lat <= north
|
||||
|
||||
|
||||
def polygon_intersects_bbox(
|
||||
geometry: dict[str, Any],
|
||||
west: float,
|
||||
south: float,
|
||||
east: float,
|
||||
north: float,
|
||||
) -> bool:
|
||||
"""
|
||||
Check if a GeoJSON geometry intersects a bounding box.
|
||||
|
||||
Uses shapely for accurate polygon intersection.
|
||||
"""
|
||||
try:
|
||||
from shapely.geometry import box, shape
|
||||
|
||||
bbox_polygon = box(west, south, east, north)
|
||||
geom = shape(geometry)
|
||||
return bbox_polygon.intersects(geom)
|
||||
except Exception:
|
||||
# If shapely fails, fall back to centroid check
|
||||
if geometry.get("type") == "Point":
|
||||
coords = geometry.get("coordinates", [])
|
||||
if len(coords) >= 2:
|
||||
return point_in_bbox(coords[0], coords[1], west, south, east, north)
|
||||
return True # Include if we can't determine
|
||||
|
||||
|
||||
def extract_centroid(geometry: dict[str, Any]) -> tuple[float, float] | None:
|
||||
"""Extract centroid from GeoJSON geometry."""
|
||||
if not geometry:
|
||||
return None
|
||||
|
||||
geom_type = geometry.get("type")
|
||||
coords = geometry.get("coordinates")
|
||||
|
||||
if geom_type == "Point" and coords and len(coords) >= 2:
|
||||
return (coords[0], coords[1])
|
||||
|
||||
# For polygons, use shapely to compute centroid
|
||||
try:
|
||||
from shapely.geometry import shape
|
||||
geom = shape(geometry)
|
||||
centroid = geom.centroid
|
||||
return (centroid.x, centroid.y)
|
||||
except Exception:
|
||||
return None
|
||||
383
src/central/adapters/wfigs_incidents.py
Normal file
383
src/central/adapters/wfigs_incidents.py
Normal file
|
|
@ -0,0 +1,383 @@
|
|||
"""WFIGS Incidents adapter for wildfire incident locations."""
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
from pydantic import BaseModel
|
||||
from tenacity import (
|
||||
retry,
|
||||
retry_if_exception_type,
|
||||
stop_after_attempt,
|
||||
wait_exponential_jitter,
|
||||
)
|
||||
|
||||
from central.adapter import SourceAdapter
|
||||
from central.adapters.wfigs_common import (
|
||||
WFIGS_INCIDENTS_URL,
|
||||
build_regions,
|
||||
cleanup_old_observed,
|
||||
delete_observed,
|
||||
extract_centroid,
|
||||
get_observed_guids,
|
||||
init_observed_table,
|
||||
normalize_incident_type,
|
||||
normalize_state,
|
||||
parse_wfigs_timestamp,
|
||||
point_in_bbox,
|
||||
severity_from_acres,
|
||||
subject_suffix,
|
||||
update_observed,
|
||||
)
|
||||
from central.config_models import AdapterConfig, RegionConfig
|
||||
from central.config_store import ConfigStore
|
||||
from central.models import Event, Geo
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
LAYER_NAME = "incidents"
|
||||
|
||||
|
||||
class WFIGSIncidentsSettings(BaseModel):
|
||||
"""Settings schema for WFIGS Incidents adapter."""
|
||||
|
||||
region: RegionConfig | None = None
|
||||
|
||||
|
||||
class WFIGSIncidentsAdapter(SourceAdapter):
|
||||
"""NIFC WFIGS wildfire incidents adapter."""
|
||||
|
||||
name = "wfigs_incidents"
|
||||
display_name = "NIFC WFIGS — Wildfire Incidents"
|
||||
description = "Active wildfire incident locations from NIFC WFIGS."
|
||||
settings_schema = WFIGSIncidentsSettings
|
||||
requires_api_key = None
|
||||
api_key_field = None
|
||||
wizard_order = None # Not in setup wizard
|
||||
default_cadence_s = 300
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: AdapterConfig,
|
||||
config_store: ConfigStore,
|
||||
cursor_db_path: Path,
|
||||
) -> None:
|
||||
self._config_store = config_store
|
||||
self._cursor_db_path = cursor_db_path
|
||||
self._session: aiohttp.ClientSession | None = None
|
||||
self._db: sqlite3.Connection | None = None
|
||||
self._last_poll_time: datetime | None = None
|
||||
|
||||
# Parse region from settings
|
||||
region_dict = config.settings.get("region")
|
||||
if region_dict:
|
||||
self.region: RegionConfig | None = RegionConfig(**region_dict)
|
||||
else:
|
||||
self.region = None
|
||||
|
||||
async def startup(self) -> None:
|
||||
"""Initialize HTTP session and SQLite connection."""
|
||||
self._session = aiohttp.ClientSession(
|
||||
timeout=aiohttp.ClientTimeout(total=60),
|
||||
)
|
||||
self._db = sqlite3.connect(self._cursor_db_path)
|
||||
|
||||
# Create tables for dedup and fall-off tracking
|
||||
self._db.execute("""
|
||||
CREATE TABLE IF NOT EXISTS published_ids (
|
||||
adapter TEXT NOT NULL,
|
||||
event_id TEXT NOT NULL,
|
||||
first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (adapter, event_id)
|
||||
)
|
||||
""")
|
||||
self._db.execute("""
|
||||
CREATE INDEX IF NOT EXISTS published_ids_last_seen
|
||||
ON published_ids (last_seen)
|
||||
""")
|
||||
init_observed_table(self._db)
|
||||
self._db.commit()
|
||||
|
||||
logger.info(
|
||||
"WFIGS incidents adapter started",
|
||||
extra={"region": self.region.model_dump() if self.region else None},
|
||||
)
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
"""Close HTTP session and SQLite connection."""
|
||||
if self._session:
|
||||
await self._session.close()
|
||||
self._session = None
|
||||
if self._db:
|
||||
self._db.close()
|
||||
self._db = None
|
||||
logger.info("WFIGS incidents adapter shut down")
|
||||
|
||||
async def apply_config(self, new_config: AdapterConfig) -> None:
|
||||
"""Apply new configuration from hot-reload."""
|
||||
region_dict = new_config.settings.get("region")
|
||||
if region_dict:
|
||||
self.region = RegionConfig(**region_dict)
|
||||
else:
|
||||
self.region = None
|
||||
logger.info(
|
||||
"WFIGS incidents config updated",
|
||||
extra={"region": self.region.model_dump() if self.region else None},
|
||||
)
|
||||
|
||||
def is_published(self, event_id: str) -> bool:
|
||||
"""Check if an event has already been published."""
|
||||
if not self._db:
|
||||
return False
|
||||
cur = self._db.execute(
|
||||
"SELECT 1 FROM published_ids WHERE adapter = ? AND event_id = ?",
|
||||
(self.name, event_id),
|
||||
)
|
||||
return cur.fetchone() is not None
|
||||
|
||||
def mark_published(self, event_id: str) -> None:
|
||||
"""Mark an event as published."""
|
||||
if not self._db:
|
||||
return
|
||||
self._db.execute(
|
||||
"""
|
||||
INSERT INTO published_ids (adapter, event_id, first_seen, last_seen)
|
||||
VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (adapter, event_id) DO UPDATE SET
|
||||
last_seen = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(self.name, event_id),
|
||||
)
|
||||
self._db.commit()
|
||||
|
||||
def bump_last_seen(self, event_id: str) -> None:
|
||||
"""Bump the last_seen timestamp for an event."""
|
||||
if not self._db:
|
||||
return
|
||||
self._db.execute(
|
||||
"UPDATE published_ids SET last_seen = CURRENT_TIMESTAMP WHERE adapter = ? AND event_id = ?",
|
||||
(self.name, event_id),
|
||||
)
|
||||
self._db.commit()
|
||||
|
||||
def sweep_old_ids(self) -> int:
|
||||
"""Remove published_ids older than 14 days. Returns count deleted."""
|
||||
if not self._db:
|
||||
return 0
|
||||
cur = self._db.execute(
|
||||
"DELETE FROM published_ids WHERE adapter = ? AND last_seen < datetime('now', '-14 days')",
|
||||
(self.name,),
|
||||
)
|
||||
self._db.commit()
|
||||
count = cur.rowcount
|
||||
if count > 0:
|
||||
logger.info("WFIGS incidents swept old dedup entries", extra={"count": count})
|
||||
return count
|
||||
|
||||
def subject_for(self, event: Event) -> str:
|
||||
"""Compute NATS subject for an event."""
|
||||
# Removal events have a different subject pattern
|
||||
if event.category.startswith("fire.incident.removed"):
|
||||
state = event.data.get("state", "").lower() or "unknown"
|
||||
return f"central.fire.incident.removed.{state}"
|
||||
|
||||
# Regular incidents: central.fire.incident.<state>.<county>
|
||||
# POOState is already normalized (2-letter code)
|
||||
state = event.data.get("POOState")
|
||||
county = event.data.get("POOCounty")
|
||||
suffix = subject_suffix(state, county)
|
||||
return f"central.fire.incident.{suffix}"
|
||||
|
||||
@retry(
|
||||
stop=stop_after_attempt(3),
|
||||
wait=wait_exponential_jitter(initial=1, max=30),
|
||||
retry=retry_if_exception_type((aiohttp.ClientError, TimeoutError)),
|
||||
)
|
||||
async def _fetch_features(self) -> list[dict[str, Any]]:
|
||||
"""Fetch features from WFIGS FeatureServer."""
|
||||
if not self._session:
|
||||
raise RuntimeError("Session not initialized")
|
||||
|
||||
# Build query params
|
||||
params: dict[str, str] = {
|
||||
"outFields": "*",
|
||||
"returnGeometry": "true",
|
||||
"f": "geojson",
|
||||
}
|
||||
|
||||
# Time filter: only fetch modified since last poll
|
||||
if self._last_poll_time:
|
||||
iso_time = self._last_poll_time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
params["where"] = f"ModifiedOnDateTime > timestamp '{iso_time}'"
|
||||
else:
|
||||
params["where"] = "1=1"
|
||||
|
||||
# Bbox filter if region configured
|
||||
if self.region:
|
||||
bbox = f"{self.region.west},{self.region.south},{self.region.east},{self.region.north}"
|
||||
params["geometry"] = bbox
|
||||
params["geometryType"] = "esriGeometryEnvelope"
|
||||
params["spatialRel"] = "esriSpatialRelIntersects"
|
||||
params["inSR"] = "4326"
|
||||
|
||||
async with self._session.get(WFIGS_INCIDENTS_URL, params=params) as resp:
|
||||
resp.raise_for_status()
|
||||
data = await resp.json()
|
||||
|
||||
features = data.get("features", [])
|
||||
logger.info(
|
||||
"WFIGS incidents fetch completed",
|
||||
extra={"feature_count": len(features)},
|
||||
)
|
||||
return features
|
||||
|
||||
async def poll(self) -> AsyncIterator[Event]:
|
||||
"""Poll WFIGS for incident updates."""
|
||||
if not self._db:
|
||||
raise RuntimeError("Database not initialized")
|
||||
|
||||
# Fetch features from upstream
|
||||
try:
|
||||
features = await self._fetch_features()
|
||||
except Exception as e:
|
||||
logger.error("WFIGS incidents fetch failed", extra={"error": str(e)})
|
||||
raise
|
||||
|
||||
# Get previous poll's observed GUIDs for fall-off detection
|
||||
observed_before = get_observed_guids(self._db, LAYER_NAME)
|
||||
|
||||
# Process features and track current GUIDs
|
||||
current_guids: dict[str, tuple[str | None, str | None]] = {}
|
||||
events_yielded = 0
|
||||
|
||||
for feature in features:
|
||||
props = feature.get("properties", {})
|
||||
geometry = feature.get("geometry")
|
||||
|
||||
irwin_id = props.get("IrwinID")
|
||||
if not irwin_id:
|
||||
continue
|
||||
|
||||
# Extract location
|
||||
centroid = extract_centroid(geometry)
|
||||
|
||||
# Post-filter: skip if outside region bbox
|
||||
if self.region and centroid:
|
||||
lon, lat = centroid
|
||||
if not point_in_bbox(
|
||||
lon, lat,
|
||||
self.region.west, self.region.south,
|
||||
self.region.east, self.region.north,
|
||||
):
|
||||
continue
|
||||
|
||||
# Normalize at parse boundary
|
||||
state_raw = props.get("POOState")
|
||||
state = normalize_state(state_raw)
|
||||
county = props.get("POOCounty")
|
||||
incident_type_raw = props.get("IncidentTypeCategory")
|
||||
incident_type = normalize_incident_type(incident_type_raw)
|
||||
|
||||
# Track this GUID as observed (for fall-off detection)
|
||||
# Store normalized state for consistency
|
||||
current_guids[irwin_id] = (state, county)
|
||||
|
||||
# Parse fields
|
||||
discovery_time = parse_wfigs_timestamp(props.get("FireDiscoveryDateTime"))
|
||||
daily_acres = props.get("DailyAcres")
|
||||
|
||||
# Build regions (expects normalized 2-letter state code)
|
||||
regions, primary_region = build_regions(state, county)
|
||||
|
||||
# Build geo
|
||||
if centroid:
|
||||
geo = Geo(
|
||||
centroid=centroid,
|
||||
bbox=(centroid[0], centroid[1], centroid[0], centroid[1]),
|
||||
regions=regions,
|
||||
primary_region=primary_region,
|
||||
)
|
||||
else:
|
||||
geo = Geo(regions=regions, primary_region=primary_region)
|
||||
|
||||
# Build event with normalized values in data
|
||||
event = Event(
|
||||
id=irwin_id,
|
||||
adapter=self.name,
|
||||
category=f"fire.incident.{incident_type}",
|
||||
time=discovery_time or datetime.now(timezone.utc),
|
||||
severity=severity_from_acres(daily_acres),
|
||||
geo=geo,
|
||||
data={
|
||||
"IrwinID": irwin_id,
|
||||
"IncidentName": props.get("IncidentName"),
|
||||
"IncidentTypeCategory": incident_type,
|
||||
"IncidentTypeCategory_raw": incident_type_raw,
|
||||
"DailyAcres": daily_acres,
|
||||
"PercentContained": props.get("PercentContained"),
|
||||
"FireDiscoveryDateTime": props.get("FireDiscoveryDateTime"),
|
||||
"ModifiedOnDateTime": props.get("ModifiedOnDateTime"),
|
||||
"POOState": state,
|
||||
"POOState_raw": state_raw,
|
||||
"POOCounty": county,
|
||||
"raw": props,
|
||||
},
|
||||
)
|
||||
|
||||
yield event
|
||||
events_yielded += 1
|
||||
|
||||
# Detect fall-offs: GUIDs in previous but not current
|
||||
fallen_off = set(observed_before.keys()) - set(current_guids.keys())
|
||||
|
||||
for irwin_id in fallen_off:
|
||||
last_observed, state, county = observed_before[irwin_id]
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
removal_event = Event(
|
||||
id=f"{irwin_id}:removed:{now.isoformat()}",
|
||||
adapter=self.name,
|
||||
category="fire.incident.removed",
|
||||
time=now,
|
||||
severity=0,
|
||||
geo=Geo(),
|
||||
data={
|
||||
"irwin_id": irwin_id,
|
||||
"last_observed_at": last_observed,
|
||||
"state": state,
|
||||
"county": county,
|
||||
"reason": "fallen_off_current_service",
|
||||
},
|
||||
)
|
||||
|
||||
yield removal_event
|
||||
events_yielded += 1
|
||||
logger.info(
|
||||
"WFIGS incident fall-off detected",
|
||||
extra={"irwin_id": irwin_id, "state": state},
|
||||
)
|
||||
|
||||
# Update observed table
|
||||
update_observed(self._db, LAYER_NAME, current_guids)
|
||||
delete_observed(self._db, LAYER_NAME, fallen_off)
|
||||
|
||||
# Periodic cleanup of old entries
|
||||
cleanup_old_observed(self._db, LAYER_NAME)
|
||||
self.sweep_old_ids()
|
||||
|
||||
# Update last poll time
|
||||
self._last_poll_time = datetime.now(timezone.utc)
|
||||
|
||||
logger.info(
|
||||
"WFIGS incidents poll completed",
|
||||
extra={
|
||||
"events_yielded": events_yielded,
|
||||
"current_observed": len(current_guids),
|
||||
"fallen_off": len(fallen_off),
|
||||
},
|
||||
)
|
||||
397
src/central/adapters/wfigs_perimeters.py
Normal file
397
src/central/adapters/wfigs_perimeters.py
Normal file
|
|
@ -0,0 +1,397 @@
|
|||
"""WFIGS Perimeters adapter for wildfire perimeter polygons."""
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
from pydantic import BaseModel
|
||||
from tenacity import (
|
||||
retry,
|
||||
retry_if_exception_type,
|
||||
stop_after_attempt,
|
||||
wait_exponential_jitter,
|
||||
)
|
||||
|
||||
from central.adapter import SourceAdapter
|
||||
from central.adapters.wfigs_common import (
|
||||
WFIGS_PERIMETERS_URL,
|
||||
build_regions,
|
||||
cleanup_old_observed,
|
||||
delete_observed,
|
||||
extract_centroid,
|
||||
get_observed_guids,
|
||||
init_observed_table,
|
||||
normalize_incident_type,
|
||||
normalize_state,
|
||||
parse_wfigs_timestamp,
|
||||
polygon_intersects_bbox,
|
||||
severity_from_acres,
|
||||
subject_suffix,
|
||||
update_observed,
|
||||
)
|
||||
from central.config_models import AdapterConfig, RegionConfig
|
||||
from central.config_store import ConfigStore
|
||||
from central.models import Event, Geo
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
LAYER_NAME = "perimeters"
|
||||
|
||||
|
||||
class WFIGSPerimetersSettings(BaseModel):
|
||||
"""Settings schema for WFIGS Perimeters adapter."""
|
||||
|
||||
region: RegionConfig | None = None
|
||||
|
||||
|
||||
class WFIGSPerimetersAdapter(SourceAdapter):
|
||||
"""NIFC WFIGS wildfire perimeters adapter."""
|
||||
|
||||
name = "wfigs_perimeters"
|
||||
display_name = "NIFC WFIGS — Wildfire Perimeters"
|
||||
description = "Active wildfire perimeter polygons from NIFC WFIGS."
|
||||
settings_schema = WFIGSPerimetersSettings
|
||||
requires_api_key = None
|
||||
api_key_field = None
|
||||
wizard_order = None # Not in setup wizard
|
||||
default_cadence_s = 300
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: AdapterConfig,
|
||||
config_store: ConfigStore,
|
||||
cursor_db_path: Path,
|
||||
) -> None:
|
||||
self._config_store = config_store
|
||||
self._cursor_db_path = cursor_db_path
|
||||
self._session: aiohttp.ClientSession | None = None
|
||||
self._db: sqlite3.Connection | None = None
|
||||
self._last_poll_time: datetime | None = None
|
||||
|
||||
# Parse region from settings
|
||||
region_dict = config.settings.get("region")
|
||||
if region_dict:
|
||||
self.region: RegionConfig | None = RegionConfig(**region_dict)
|
||||
else:
|
||||
self.region = None
|
||||
|
||||
async def startup(self) -> None:
|
||||
"""Initialize HTTP session and SQLite connection."""
|
||||
self._session = aiohttp.ClientSession(
|
||||
timeout=aiohttp.ClientTimeout(total=120), # Longer timeout for large polygons
|
||||
)
|
||||
self._db = sqlite3.connect(self._cursor_db_path)
|
||||
|
||||
# Create tables for dedup and fall-off tracking
|
||||
self._db.execute("""
|
||||
CREATE TABLE IF NOT EXISTS published_ids (
|
||||
adapter TEXT NOT NULL,
|
||||
event_id TEXT NOT NULL,
|
||||
first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (adapter, event_id)
|
||||
)
|
||||
""")
|
||||
self._db.execute("""
|
||||
CREATE INDEX IF NOT EXISTS published_ids_last_seen
|
||||
ON published_ids (last_seen)
|
||||
""")
|
||||
init_observed_table(self._db)
|
||||
self._db.commit()
|
||||
|
||||
logger.info(
|
||||
"WFIGS perimeters adapter started",
|
||||
extra={"region": self.region.model_dump() if self.region else None},
|
||||
)
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
"""Close HTTP session and SQLite connection."""
|
||||
if self._session:
|
||||
await self._session.close()
|
||||
self._session = None
|
||||
if self._db:
|
||||
self._db.close()
|
||||
self._db = None
|
||||
logger.info("WFIGS perimeters adapter shut down")
|
||||
|
||||
async def apply_config(self, new_config: AdapterConfig) -> None:
|
||||
"""Apply new configuration from hot-reload."""
|
||||
region_dict = new_config.settings.get("region")
|
||||
if region_dict:
|
||||
self.region = RegionConfig(**region_dict)
|
||||
else:
|
||||
self.region = None
|
||||
logger.info(
|
||||
"WFIGS perimeters config updated",
|
||||
extra={"region": self.region.model_dump() if self.region else None},
|
||||
)
|
||||
|
||||
def is_published(self, event_id: str) -> bool:
|
||||
"""Check if an event has already been published."""
|
||||
if not self._db:
|
||||
return False
|
||||
cur = self._db.execute(
|
||||
"SELECT 1 FROM published_ids WHERE adapter = ? AND event_id = ?",
|
||||
(self.name, event_id),
|
||||
)
|
||||
return cur.fetchone() is not None
|
||||
|
||||
def mark_published(self, event_id: str) -> None:
|
||||
"""Mark an event as published."""
|
||||
if not self._db:
|
||||
return
|
||||
self._db.execute(
|
||||
"""
|
||||
INSERT INTO published_ids (adapter, event_id, first_seen, last_seen)
|
||||
VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (adapter, event_id) DO UPDATE SET
|
||||
last_seen = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(self.name, event_id),
|
||||
)
|
||||
self._db.commit()
|
||||
|
||||
def bump_last_seen(self, event_id: str) -> None:
|
||||
"""Bump the last_seen timestamp for an event."""
|
||||
if not self._db:
|
||||
return
|
||||
self._db.execute(
|
||||
"UPDATE published_ids SET last_seen = CURRENT_TIMESTAMP WHERE adapter = ? AND event_id = ?",
|
||||
(self.name, event_id),
|
||||
)
|
||||
self._db.commit()
|
||||
|
||||
def sweep_old_ids(self) -> int:
|
||||
"""Remove published_ids older than 14 days. Returns count deleted."""
|
||||
if not self._db:
|
||||
return 0
|
||||
cur = self._db.execute(
|
||||
"DELETE FROM published_ids WHERE adapter = ? AND last_seen < datetime('now', '-14 days')",
|
||||
(self.name,),
|
||||
)
|
||||
self._db.commit()
|
||||
count = cur.rowcount
|
||||
if count > 0:
|
||||
logger.info("WFIGS perimeters swept old dedup entries", extra={"count": count})
|
||||
return count
|
||||
|
||||
def subject_for(self, event: Event) -> str:
|
||||
"""Compute NATS subject for an event."""
|
||||
# Removal events have a different subject pattern
|
||||
if event.category.startswith("fire.perimeter.removed"):
|
||||
state = event.data.get("state", "").lower() or "unknown"
|
||||
return f"central.fire.perimeter.removed.{state}"
|
||||
|
||||
# Regular perimeters: central.fire.perimeter.<state>.<county>
|
||||
# POOState is already normalized (2-letter code)
|
||||
state = event.data.get("POOState")
|
||||
county = event.data.get("POOCounty")
|
||||
suffix = subject_suffix(state, county)
|
||||
return f"central.fire.perimeter.{suffix}"
|
||||
|
||||
@retry(
|
||||
stop=stop_after_attempt(3),
|
||||
wait=wait_exponential_jitter(initial=1, max=30),
|
||||
retry=retry_if_exception_type((aiohttp.ClientError, TimeoutError)),
|
||||
)
|
||||
async def _fetch_features(self) -> list[dict[str, Any]]:
|
||||
"""Fetch features from WFIGS FeatureServer."""
|
||||
if not self._session:
|
||||
raise RuntimeError("Session not initialized")
|
||||
|
||||
# Build query params
|
||||
params: dict[str, str] = {
|
||||
"outFields": "*",
|
||||
"returnGeometry": "true",
|
||||
"f": "geojson",
|
||||
}
|
||||
|
||||
# Time filter: only fetch modified since last poll
|
||||
# Note: perimeters use attr_ModifiedOnDateTime_dt field
|
||||
if self._last_poll_time:
|
||||
iso_time = self._last_poll_time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
params["where"] = f"attr_ModifiedOnDateTime_dt > timestamp '{iso_time}'"
|
||||
else:
|
||||
params["where"] = "1=1"
|
||||
|
||||
# Bbox filter if region configured
|
||||
if self.region:
|
||||
bbox = f"{self.region.west},{self.region.south},{self.region.east},{self.region.north}"
|
||||
params["geometry"] = bbox
|
||||
params["geometryType"] = "esriGeometryEnvelope"
|
||||
params["spatialRel"] = "esriSpatialRelIntersects"
|
||||
params["inSR"] = "4326"
|
||||
|
||||
async with self._session.get(WFIGS_PERIMETERS_URL, params=params) as resp:
|
||||
resp.raise_for_status()
|
||||
data = await resp.json()
|
||||
|
||||
features = data.get("features", [])
|
||||
logger.info(
|
||||
"WFIGS perimeters fetch completed",
|
||||
extra={"feature_count": len(features)},
|
||||
)
|
||||
return features
|
||||
|
||||
async def poll(self) -> AsyncIterator[Event]:
|
||||
"""Poll WFIGS for perimeter updates."""
|
||||
if not self._db:
|
||||
raise RuntimeError("Database not initialized")
|
||||
|
||||
# Fetch features from upstream
|
||||
try:
|
||||
features = await self._fetch_features()
|
||||
except Exception as e:
|
||||
logger.error("WFIGS perimeters fetch failed", extra={"error": str(e)})
|
||||
raise
|
||||
|
||||
# Get previous poll's observed GUIDs for fall-off detection
|
||||
observed_before = get_observed_guids(self._db, LAYER_NAME)
|
||||
|
||||
# Process features and track current GUIDs
|
||||
current_guids: dict[str, tuple[str | None, str | None]] = {}
|
||||
events_yielded = 0
|
||||
|
||||
for feature in features:
|
||||
props = feature.get("properties", {})
|
||||
geometry = feature.get("geometry")
|
||||
|
||||
# WFIGS Perimeters use prefixed field names (attr_*, poly_*)
|
||||
irwin_id = props.get("attr_IrwinID") or props.get("poly_IRWINID")
|
||||
if not irwin_id:
|
||||
continue
|
||||
|
||||
# Post-filter: skip if geometry doesn't intersect region bbox
|
||||
if self.region and geometry:
|
||||
if not polygon_intersects_bbox(
|
||||
geometry,
|
||||
self.region.west, self.region.south,
|
||||
self.region.east, self.region.north,
|
||||
):
|
||||
continue
|
||||
|
||||
# Normalize at parse boundary
|
||||
state_raw = props.get("attr_POOState")
|
||||
state = normalize_state(state_raw)
|
||||
county = props.get("attr_POOCounty")
|
||||
incident_type_raw = props.get("attr_IncidentTypeCategory")
|
||||
incident_type = normalize_incident_type(incident_type_raw)
|
||||
|
||||
# Track this GUID as observed (for fall-off detection)
|
||||
# Store normalized state for consistency
|
||||
current_guids[irwin_id] = (state, county)
|
||||
|
||||
# Parse fields using prefixed names
|
||||
discovery_time = parse_wfigs_timestamp(props.get("attr_FireDiscoveryDateTime"))
|
||||
# Use poly_GISAcres or attr_IncidentSize for acreage
|
||||
daily_acres = props.get("attr_IncidentSize") or props.get("poly_GISAcres")
|
||||
|
||||
# Build regions (expects normalized 2-letter state code)
|
||||
regions, primary_region = build_regions(state, county)
|
||||
|
||||
# Extract centroid for geo
|
||||
centroid = extract_centroid(geometry)
|
||||
|
||||
# Build bbox from geometry if available
|
||||
bbox = None
|
||||
if geometry:
|
||||
try:
|
||||
from shapely.geometry import shape
|
||||
geom = shape(geometry)
|
||||
bounds = geom.bounds # (minx, miny, maxx, maxy)
|
||||
bbox = (bounds[0], bounds[1], bounds[2], bounds[3])
|
||||
except Exception:
|
||||
if centroid:
|
||||
bbox = (centroid[0], centroid[1], centroid[0], centroid[1])
|
||||
|
||||
# Build geo
|
||||
geo = Geo(
|
||||
centroid=centroid,
|
||||
bbox=bbox,
|
||||
regions=regions,
|
||||
primary_region=primary_region,
|
||||
)
|
||||
|
||||
# Build event with geometry in data
|
||||
# Use normalized field names in event data for consistency
|
||||
event = Event(
|
||||
id=irwin_id,
|
||||
adapter=self.name,
|
||||
category=f"fire.perimeter.{incident_type}",
|
||||
time=discovery_time or datetime.now(timezone.utc),
|
||||
severity=severity_from_acres(daily_acres),
|
||||
geo=geo,
|
||||
data={
|
||||
"IrwinID": irwin_id,
|
||||
"IncidentName": props.get("attr_IncidentName") or props.get("poly_IncidentName"),
|
||||
"IncidentTypeCategory": incident_type,
|
||||
"IncidentTypeCategory_raw": incident_type_raw,
|
||||
"DailyAcres": props.get("attr_IncidentSize"),
|
||||
"GISAcres": props.get("poly_GISAcres"),
|
||||
"PercentContained": props.get("attr_PercentContained"),
|
||||
"FireDiscoveryDateTime": props.get("attr_FireDiscoveryDateTime"),
|
||||
"ModifiedOnDateTime": props.get("attr_ModifiedOnDateTime_dt"),
|
||||
"POOState": state,
|
||||
"POOState_raw": state_raw,
|
||||
"POOCounty": county,
|
||||
"geometry": geometry, # Full GeoJSON polygon
|
||||
"raw": props,
|
||||
},
|
||||
)
|
||||
|
||||
yield event
|
||||
events_yielded += 1
|
||||
|
||||
# Detect fall-offs: GUIDs in previous but not current
|
||||
fallen_off = set(observed_before.keys()) - set(current_guids.keys())
|
||||
|
||||
for irwin_id in fallen_off:
|
||||
last_observed, state, county = observed_before[irwin_id]
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
removal_event = Event(
|
||||
id=f"{irwin_id}:removed:{now.isoformat()}",
|
||||
adapter=self.name,
|
||||
category="fire.perimeter.removed",
|
||||
time=now,
|
||||
severity=0,
|
||||
geo=Geo(),
|
||||
data={
|
||||
"irwin_id": irwin_id,
|
||||
"last_observed_at": last_observed,
|
||||
"state": state,
|
||||
"county": county,
|
||||
"reason": "fallen_off_current_service",
|
||||
},
|
||||
)
|
||||
|
||||
yield removal_event
|
||||
events_yielded += 1
|
||||
logger.info(
|
||||
"WFIGS perimeter fall-off detected",
|
||||
extra={"irwin_id": irwin_id, "state": state},
|
||||
)
|
||||
|
||||
# Update observed table
|
||||
update_observed(self._db, LAYER_NAME, current_guids)
|
||||
delete_observed(self._db, LAYER_NAME, fallen_off)
|
||||
|
||||
# Periodic cleanup of old entries
|
||||
cleanup_old_observed(self._db, LAYER_NAME)
|
||||
self.sweep_old_ids()
|
||||
|
||||
# Update last poll time
|
||||
self._last_poll_time = datetime.now(timezone.utc)
|
||||
|
||||
logger.info(
|
||||
"WFIGS perimeters poll completed",
|
||||
extra={
|
||||
"events_yielded": events_yielded,
|
||||
"current_observed": len(current_guids),
|
||||
"fallen_off": len(fallen_off),
|
||||
},
|
||||
)
|
||||
|
|
@ -25,6 +25,7 @@ STREAMS = [
|
|||
("CENTRAL_WX", "central.wx.>"),
|
||||
("CENTRAL_FIRE", "central.fire.>"),
|
||||
("CENTRAL_QUAKE", "central.quake.>"),
|
||||
("CENTRAL_SPACE", "central.space.>"),
|
||||
]
|
||||
|
||||
BATCH_SIZE = 100
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ class AdapterConfig(BaseModel):
|
|||
|
||||
name: str = Field(description="Unique adapter identifier")
|
||||
enabled: bool = Field(default=True, description="Whether adapter is active")
|
||||
cadence_s: int = Field(description="Poll interval in seconds")
|
||||
cadence_s: int = Field(ge=10, description="Poll interval in seconds")
|
||||
settings: dict[str, Any] = Field(
|
||||
default_factory=dict, description="Adapter-specific settings"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -241,6 +241,14 @@ class ConfigStore:
|
|||
)
|
||||
return result == "DELETE 1"
|
||||
|
||||
async def set_adapter_last_error(self, name: str, error: str | None) -> None:
|
||||
"""Set or clear the last_error field on an adapter row."""
|
||||
async with self._pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"UPDATE config.adapters SET last_error = $1 WHERE name = $2",
|
||||
error, name,
|
||||
)
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Change notifications
|
||||
# -------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -247,18 +247,37 @@ def _create_app() -> FastAPI:
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
# Import helper functions for valid values
|
||||
from central.gui.routes import _get_valid_satellites, _get_valid_feeds
|
||||
# Add field descriptors to adapters
|
||||
from central.gui.routes import _adapter_classes
|
||||
from central.gui.form_descriptors import describe_fields
|
||||
adapter_classes = _adapter_classes()
|
||||
wizard_adapters = sorted(
|
||||
[(name, cls) for name, cls in adapter_classes.items() if cls.wizard_order is not None],
|
||||
key=lambda nc: nc[1].wizard_order
|
||||
)
|
||||
# Rebuild adapters with fields
|
||||
enriched_adapters = []
|
||||
for name, cls in wizard_adapters:
|
||||
adapter_data = next((a for a in adapters if a["name"] == name), None)
|
||||
if adapter_data:
|
||||
settings_dict = adapter_data.get("settings", {})
|
||||
fields = describe_fields(cls.settings_schema, settings_dict)
|
||||
enriched_adapters.append({
|
||||
"name": name,
|
||||
"display_name": cls.display_name,
|
||||
"enabled": adapter_data.get("enabled", False),
|
||||
"cadence_s": adapter_data.get("cadence_s", 300),
|
||||
"settings": settings_dict,
|
||||
"fields": fields,
|
||||
})
|
||||
|
||||
response = templates.TemplateResponse(
|
||||
request=request,
|
||||
name="setup_adapters.html",
|
||||
context={
|
||||
"csrf_token": csrf_token,
|
||||
"adapters": adapters,
|
||||
"adapters": enriched_adapters,
|
||||
"api_keys": api_keys,
|
||||
"valid_satellites": _get_valid_satellites(),
|
||||
"valid_feeds": sorted(_get_valid_feeds()),
|
||||
"tile_url": tile_url,
|
||||
"tile_attribution": tile_attribution,
|
||||
"error": error_msg,
|
||||
|
|
|
|||
163
src/central/gui/form_descriptors.py
Normal file
163
src/central/gui/form_descriptors.py
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
"""Form field descriptors for adapter settings.
|
||||
|
||||
If a second nested settings type beyond RegionConfig appears,
|
||||
refactor this helper to recurse over nested models.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Literal, Union, get_args, get_origin
|
||||
|
||||
from pydantic import BaseModel
|
||||
from pydantic.fields import FieldInfo
|
||||
from pydantic_core import PydanticUndefined
|
||||
|
||||
from central.config_models import RegionConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class FieldDescriptor:
|
||||
"""Describes a form field for rendering."""
|
||||
name: str
|
||||
label: str
|
||||
widget: str # "text", "number", "checkbox", "csv", "select", "checkboxes", "region"
|
||||
current_value: Any
|
||||
default: Any
|
||||
description: str
|
||||
required: bool
|
||||
options: list[str] | None = None # For select/checkboxes widgets
|
||||
|
||||
|
||||
def _is_literal(tp: type) -> bool:
|
||||
"""Check if a type is a Literal type."""
|
||||
return get_origin(tp) is Literal
|
||||
|
||||
|
||||
def _get_literal_values(tp: type) -> list[str]:
|
||||
"""Extract the literal values from a Literal type."""
|
||||
return list(get_args(tp))
|
||||
|
||||
|
||||
def _type_to_widget_and_options(field_name: str, field_type: type) -> tuple[str, list[str] | None]:
|
||||
"""Map a Python type to a widget type and optional options list.
|
||||
|
||||
Returns:
|
||||
Tuple of (widget_type, options_list_or_none)
|
||||
"""
|
||||
# Handle Optional/Union types
|
||||
origin = get_origin(field_type)
|
||||
args = get_args(field_type)
|
||||
|
||||
# Check for Optional[X] (Union[X, None])
|
||||
if origin is Union or (origin is not None and type(None) in args):
|
||||
# Get the non-None type
|
||||
non_none_args = [a for a in args if a is not type(None)]
|
||||
if non_none_args:
|
||||
inner_type = non_none_args[0]
|
||||
# Recursively determine widget for the inner type
|
||||
return _type_to_widget_and_options(field_name, inner_type)
|
||||
|
||||
# Check for Literal type (single select)
|
||||
if _is_literal(field_type):
|
||||
options = _get_literal_values(field_type)
|
||||
return "select", [str(o) for o in options]
|
||||
|
||||
# Direct type checks
|
||||
if field_type is str:
|
||||
return "text", None
|
||||
if field_type is int:
|
||||
return "number", None
|
||||
if field_type is bool:
|
||||
return "checkbox", None
|
||||
if field_type is RegionConfig:
|
||||
return "region", None
|
||||
|
||||
# Check for list types
|
||||
if origin is list:
|
||||
inner_type = args[0] if args else None
|
||||
|
||||
# list[Literal[...]] -> checkboxes
|
||||
if inner_type is not None and _is_literal(inner_type):
|
||||
options = _get_literal_values(inner_type)
|
||||
return "checkboxes", [str(o) for o in options]
|
||||
|
||||
# list[str] -> csv
|
||||
if inner_type is str:
|
||||
return "csv", None
|
||||
|
||||
raise NotImplementedError(
|
||||
f"Field '{field_name}' has unsupported list type: list[{inner_type.__name__ if inner_type else '?'}]"
|
||||
)
|
||||
|
||||
# Check if it's a BaseModel subclass (nested model other than RegionConfig)
|
||||
if isinstance(field_type, type) and issubclass(field_type, BaseModel):
|
||||
raise NotImplementedError(
|
||||
f"Field '{field_name}' has unsupported nested type: {field_type.__name__}. "
|
||||
f"If a second nested type beyond RegionConfig is needed, "
|
||||
f"refactor describe_fields to recurse over nested models."
|
||||
)
|
||||
|
||||
raise NotImplementedError(
|
||||
f"Field '{field_name}' has unsupported type: {field_type}"
|
||||
)
|
||||
|
||||
|
||||
def _name_to_label(name: str) -> str:
|
||||
"""Convert field name to human-readable label."""
|
||||
return name.replace("_", " ").title()
|
||||
|
||||
|
||||
def _is_undefined(value: Any) -> bool:
|
||||
"""Check if a value is Pydantic's undefined sentinel."""
|
||||
return value is PydanticUndefined
|
||||
|
||||
|
||||
def describe_fields(model_cls: type[BaseModel], current: dict) -> list[FieldDescriptor]:
|
||||
"""Generate field descriptors for a Pydantic model.
|
||||
|
||||
Args:
|
||||
model_cls: The Pydantic model class (e.g., NWSSettings)
|
||||
current: Current settings values from the database
|
||||
|
||||
Returns:
|
||||
List of FieldDescriptor objects for rendering the form
|
||||
"""
|
||||
descriptors = []
|
||||
|
||||
for field_name, field_info in model_cls.model_fields.items():
|
||||
# Get the field type
|
||||
field_type = field_info.annotation
|
||||
|
||||
# Determine widget and options
|
||||
widget, options = _type_to_widget_and_options(field_name, field_type)
|
||||
|
||||
# Get current value, falling back to default
|
||||
if field_name in current:
|
||||
current_value = current[field_name]
|
||||
elif not _is_undefined(field_info.default):
|
||||
current_value = field_info.default
|
||||
else:
|
||||
current_value = None
|
||||
|
||||
# Get default
|
||||
default = field_info.default if not _is_undefined(field_info.default) else None
|
||||
|
||||
# Get description
|
||||
description = ""
|
||||
if field_info.description:
|
||||
description = field_info.description
|
||||
|
||||
# Determine if required (no default and not Optional)
|
||||
required = _is_undefined(field_info.default) and field_info.is_required()
|
||||
|
||||
descriptors.append(FieldDescriptor(
|
||||
name=field_name,
|
||||
label=_name_to_label(field_name),
|
||||
widget=widget,
|
||||
current_value=current_value,
|
||||
default=default,
|
||||
description=description,
|
||||
required=required,
|
||||
options=options,
|
||||
))
|
||||
|
||||
return descriptors
|
||||
|
|
@ -9,6 +9,7 @@ from typing import Any
|
|||
|
||||
logger = logging.getLogger("central.gui.routes")
|
||||
|
||||
|
||||
from fastapi import APIRouter, Depends, Form, Request
|
||||
from fastapi.responses import HTMLResponse, RedirectResponse, Response
|
||||
from central.bootstrap_config import get_settings
|
||||
|
|
@ -43,12 +44,27 @@ from central.gui.audit import (
|
|||
SYSTEM_UPDATE,
|
||||
write_audit,
|
||||
)
|
||||
from functools import cache
|
||||
|
||||
from central.gui.db import get_pool
|
||||
from central.gui.form_descriptors import describe_fields, FieldDescriptor
|
||||
from central.adapter_discovery import discover_adapters
|
||||
from pydantic import ValidationError
|
||||
|
||||
@cache
|
||||
def _adapter_classes() -> dict:
|
||||
"""Cached adapter class discovery.
|
||||
|
||||
GUI is a separate process from supervisor; walks pkgutil itself.
|
||||
Python's import cache makes subsequent calls free.
|
||||
"""
|
||||
return discover_adapters()
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# Streams to display on dashboard
|
||||
DASHBOARD_STREAMS = ["CENTRAL_WX", "CENTRAL_FIRE", "CENTRAL_QUAKE", "CENTRAL_META"]
|
||||
DASHBOARD_STREAMS = ["CENTRAL_WX", "CENTRAL_FIRE", "CENTRAL_QUAKE", "CENTRAL_SPACE", "CENTRAL_META"]
|
||||
|
||||
# Email validation regex (simple but effective)
|
||||
ALIAS_REGEX = re.compile(r"^[a-zA-Z0-9_]+$")
|
||||
|
|
@ -57,18 +73,6 @@ ALIAS_REGEX = re.compile(r"^[a-zA-Z0-9_]+$")
|
|||
EMAIL_REGEX = re.compile(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$")
|
||||
|
||||
|
||||
def _get_valid_satellites() -> list[str]:
|
||||
"""Get valid satellite identifiers from firms adapter."""
|
||||
from central.adapters.firms import SATELLITE_SHORT
|
||||
return list(SATELLITE_SHORT.keys())
|
||||
|
||||
|
||||
def _get_valid_feeds() -> set[str]:
|
||||
"""Get valid feed values from usgs_quake adapter."""
|
||||
from central.adapters.usgs_quake import VALID_FEEDS
|
||||
return VALID_FEEDS
|
||||
|
||||
|
||||
def _get_templates():
|
||||
"""Get templates instance (deferred import to avoid circular)."""
|
||||
from central.gui import templates
|
||||
|
|
@ -631,18 +635,36 @@ async def setup_adapters_form(request: Request) -> HTMLResponse:
|
|||
templates = _get_templates()
|
||||
pool = get_pool()
|
||||
|
||||
# Get wizard adapters (filtered by wizard_order)
|
||||
adapter_classes = _adapter_classes()
|
||||
wizard_adapters = sorted(
|
||||
[(name, cls) for name, cls in adapter_classes.items() if cls.wizard_order is not None],
|
||||
key=lambda nc: nc[1].wizard_order
|
||||
)
|
||||
|
||||
# Pre-fill from cookie state or DB defaults
|
||||
if state.adapters:
|
||||
adapters = []
|
||||
for name in ["firms", "nws", "usgs_quake"]:
|
||||
for name, cls in wizard_adapters:
|
||||
if name in state.adapters:
|
||||
a = state.adapters[name]
|
||||
adapters.append({
|
||||
"name": name,
|
||||
"enabled": a["enabled"],
|
||||
"cadence_s": a["cadence_s"],
|
||||
"settings": a["settings"],
|
||||
})
|
||||
settings_dict = a["settings"]
|
||||
else:
|
||||
settings_dict = {}
|
||||
fields = describe_fields(cls.settings_schema, settings_dict)
|
||||
# Swap widget for api_key_field to api_key_select
|
||||
if cls.api_key_field is not None:
|
||||
for f in fields:
|
||||
if f.name == cls.api_key_field:
|
||||
f.widget = "api_key_select"
|
||||
adapters.append({
|
||||
"name": name,
|
||||
"display_name": cls.display_name,
|
||||
"enabled": a["enabled"] if name in state.adapters else False,
|
||||
"cadence_s": a["cadence_s"] if name in state.adapters else 300,
|
||||
"settings": settings_dict,
|
||||
"fields": fields,
|
||||
})
|
||||
else:
|
||||
async with pool.acquire() as conn:
|
||||
rows = await conn.fetch(
|
||||
|
|
@ -652,15 +674,33 @@ async def setup_adapters_form(request: Request) -> HTMLResponse:
|
|||
ORDER BY name
|
||||
"""
|
||||
)
|
||||
adapters = []
|
||||
for row in rows:
|
||||
settings_data = row["settings"] or {}
|
||||
adapters.append({
|
||||
"name": row["name"],
|
||||
"enabled": row["enabled"],
|
||||
"cadence_s": row["cadence_s"],
|
||||
"settings": settings_data,
|
||||
})
|
||||
db_adapters = {row["name"]: row for row in rows}
|
||||
|
||||
adapters = []
|
||||
for name, cls in wizard_adapters:
|
||||
if name in db_adapters:
|
||||
row = db_adapters[name]
|
||||
settings_dict = row["settings"] or {}
|
||||
enabled = row["enabled"]
|
||||
cadence_s = row["cadence_s"]
|
||||
else:
|
||||
settings_dict = {}
|
||||
enabled = False
|
||||
cadence_s = 300
|
||||
fields = describe_fields(cls.settings_schema, settings_dict)
|
||||
# Swap widget for api_key_field to api_key_select
|
||||
if cls.api_key_field is not None:
|
||||
for f in fields:
|
||||
if f.name == cls.api_key_field:
|
||||
f.widget = "api_key_select"
|
||||
adapters.append({
|
||||
"name": name,
|
||||
"display_name": cls.display_name,
|
||||
"enabled": enabled,
|
||||
"cadence_s": cadence_s,
|
||||
"settings": settings_dict,
|
||||
"fields": fields,
|
||||
})
|
||||
|
||||
# Get API keys from wizard state (not DB)
|
||||
api_keys = [{"alias": k["alias"]} for k in state.api_keys]
|
||||
|
|
@ -685,8 +725,6 @@ async def setup_adapters_form(request: Request) -> HTMLResponse:
|
|||
"csrf_token": csrf_token,
|
||||
"adapters": adapters,
|
||||
"api_keys": api_keys,
|
||||
"valid_satellites": _get_valid_satellites(),
|
||||
"valid_feeds": sorted(_get_valid_feeds()),
|
||||
"tile_url": tile_url,
|
||||
"tile_attribution": tile_attribution,
|
||||
"error": None,
|
||||
|
|
@ -739,7 +777,14 @@ async def setup_adapters_submit(request: Request) -> Response:
|
|||
"settings": row["settings"] or {},
|
||||
}
|
||||
|
||||
for adapter_name in ["firms", "nws", "usgs_quake"]:
|
||||
# Get wizard adapters (filtered by wizard_order)
|
||||
adapter_classes = _adapter_classes()
|
||||
wizard_adapters = sorted(
|
||||
[(name, cls) for name, cls in adapter_classes.items() if cls.wizard_order is not None],
|
||||
key=lambda nc: nc[1].wizard_order
|
||||
)
|
||||
|
||||
for adapter_name, adapter_cls in wizard_adapters:
|
||||
current = current_adapters.get(adapter_name, {"enabled": False, "cadence_s": 300, "settings": {}})
|
||||
current_settings = current.get("settings", {})
|
||||
new_settings = dict(current_settings)
|
||||
|
|
@ -747,83 +792,108 @@ async def setup_adapters_submit(request: Request) -> Response:
|
|||
# Parse enabled
|
||||
enabled = f"{adapter_name}_enabled" in form
|
||||
|
||||
# Parse cadence
|
||||
# Parse cadence using AdapterConfig field constraint
|
||||
cadence_str = form.get(f"{adapter_name}_cadence_s", "")
|
||||
try:
|
||||
cadence_s = int(cadence_str)
|
||||
if cadence_s < 60 or cadence_s > 3600:
|
||||
errors[f"{adapter_name}_cadence_s"] = "Cadence must be between 60 and 3600 seconds"
|
||||
from central.config_models import AdapterConfig
|
||||
min_cadence = AdapterConfig.model_fields["cadence_s"].metadata[0].ge
|
||||
if cadence_s < min_cadence:
|
||||
errors[f"{adapter_name}_cadence_s"] = (
|
||||
f"Input should be greater than or equal to {min_cadence}"
|
||||
)
|
||||
except ValueError:
|
||||
errors[f"{adapter_name}_cadence_s"] = "Cadence must be a valid integer"
|
||||
cadence_s = current.get("cadence_s", 300)
|
||||
|
||||
# Adapter-specific validation
|
||||
if adapter_name == "nws":
|
||||
contact_email = form.get(f"{adapter_name}_contact_email", "").strip()
|
||||
if enabled:
|
||||
if not contact_email:
|
||||
errors[f"{adapter_name}_contact_email"] = "Contact email is required when enabled"
|
||||
elif not EMAIL_REGEX.match(contact_email):
|
||||
errors[f"{adapter_name}_contact_email"] = "Invalid email format"
|
||||
# Generic field parsing using describe_fields
|
||||
fields = describe_fields(adapter_cls.settings_schema, current_settings)
|
||||
for field in fields:
|
||||
form_key = f"{adapter_name}_{field.name}"
|
||||
|
||||
if field.widget == "text":
|
||||
value = form.get(form_key, "").strip()
|
||||
new_settings[field.name] = value if value else current_settings.get(field.name)
|
||||
|
||||
elif field.widget == "api_key_select":
|
||||
# API key alias field - stored as text, validated post-loop
|
||||
value = form.get(form_key, "").strip()
|
||||
new_settings[field.name] = value if value else None
|
||||
|
||||
elif field.widget == "number":
|
||||
value_str = form.get(form_key, "").strip()
|
||||
if value_str:
|
||||
try:
|
||||
new_settings[field.name] = int(value_str)
|
||||
except ValueError:
|
||||
errors[form_key] = f"{field.label} must be a valid number"
|
||||
else:
|
||||
new_settings["contact_email"] = contact_email
|
||||
else:
|
||||
new_settings["contact_email"] = contact_email if contact_email else current_settings.get("contact_email")
|
||||
new_settings[field.name] = current_settings.get(field.name)
|
||||
|
||||
elif adapter_name == "firms":
|
||||
api_key_alias = form.get(f"{adapter_name}_api_key_alias", "").strip()
|
||||
satellites = form.getlist(f"{adapter_name}_satellites")
|
||||
elif field.widget == "checkbox":
|
||||
new_settings[field.name] = form_key in form
|
||||
|
||||
if api_key_alias:
|
||||
# Validate against wizard state keys
|
||||
if not any(k["alias"] == api_key_alias for k in state.api_keys):
|
||||
errors[f"{adapter_name}_api_key_alias"] = f"API key alias does not exist"
|
||||
elif field.widget == "csv":
|
||||
value = form.get(form_key, "").strip()
|
||||
if value:
|
||||
new_settings[field.name] = [v.strip() for v in value.split(",") if v.strip()]
|
||||
else:
|
||||
new_settings["api_key_alias"] = api_key_alias
|
||||
else:
|
||||
new_settings["api_key_alias"] = None
|
||||
new_settings[field.name] = []
|
||||
|
||||
# Validate satellites
|
||||
valid_sats = set(_get_valid_satellites())
|
||||
invalid_sats = [s for s in satellites if s not in valid_sats]
|
||||
if invalid_sats:
|
||||
errors[f"{adapter_name}_satellites"] = f"Invalid satellites: " + ", ".join(invalid_sats)
|
||||
else:
|
||||
new_settings["satellites"] = satellites
|
||||
elif field.widget == "select":
|
||||
value = form.get(form_key, "").strip()
|
||||
if value and field.options and value not in field.options:
|
||||
errors[form_key] = f"Invalid {field.label.lower()}"
|
||||
else:
|
||||
new_settings[field.name] = value
|
||||
|
||||
elif adapter_name == "usgs_quake":
|
||||
feed = form.get(f"{adapter_name}_feed", "").strip()
|
||||
valid_feeds = _get_valid_feeds()
|
||||
if feed not in valid_feeds:
|
||||
errors[f"{adapter_name}_feed"] = "Invalid feed"
|
||||
else:
|
||||
new_settings["feed"] = feed
|
||||
elif field.widget == "checkboxes":
|
||||
# Use getlist for checkbox groups - absence means empty list
|
||||
values = form.getlist(form_key)
|
||||
if field.options:
|
||||
invalid = [v for v in values if v not in field.options]
|
||||
if invalid:
|
||||
errors[form_key] = f"Invalid values: {', '.join(invalid)}"
|
||||
else:
|
||||
new_settings[field.name] = values
|
||||
else:
|
||||
new_settings[field.name] = values
|
||||
|
||||
# Region validation (all adapters)
|
||||
region_north_str = form.get(f"{adapter_name}_region_north", "").strip()
|
||||
region_south_str = form.get(f"{adapter_name}_region_south", "").strip()
|
||||
region_east_str = form.get(f"{adapter_name}_region_east", "").strip()
|
||||
region_west_str = form.get(f"{adapter_name}_region_west", "").strip()
|
||||
elif field.widget == "region":
|
||||
# Region validation via RegionConfig model
|
||||
from central.config_models import RegionConfig
|
||||
region_north_str = form.get(f"{adapter_name}_{field.name}_north", "").strip()
|
||||
region_south_str = form.get(f"{adapter_name}_{field.name}_south", "").strip()
|
||||
region_east_str = form.get(f"{adapter_name}_{field.name}_east", "").strip()
|
||||
region_west_str = form.get(f"{adapter_name}_{field.name}_west", "").strip()
|
||||
|
||||
try:
|
||||
region_model = RegionConfig(
|
||||
north=float(region_north_str),
|
||||
south=float(region_south_str),
|
||||
east=float(region_east_str),
|
||||
west=float(region_west_str),
|
||||
)
|
||||
new_settings[field.name] = region_model.model_dump()
|
||||
except (ValueError, ValidationError) as e:
|
||||
errors[f"{adapter_name}_{field.name}"] = str(e)
|
||||
|
||||
# Run Pydantic validation on assembled settings to catch Literal violations etc.
|
||||
try:
|
||||
region_north = float(region_north_str)
|
||||
region_south = float(region_south_str)
|
||||
region_east = float(region_east_str)
|
||||
region_west = float(region_west_str)
|
||||
adapter_cls.settings_schema(**new_settings)
|
||||
except ValidationError as e:
|
||||
for err in e.errors():
|
||||
loc = err["loc"][0] if err["loc"] else "unknown"
|
||||
errors[f"{adapter_name}_{loc}"] = err["msg"]
|
||||
|
||||
if not (-90 <= region_south < region_north <= 90):
|
||||
errors[f"{adapter_name}_region"] = "Invalid latitude: south < north, both -90 to 90"
|
||||
elif not (-180 <= region_west < region_east <= 180):
|
||||
errors[f"{adapter_name}_region"] = "Invalid longitude: west < east, both -180 to 180"
|
||||
else:
|
||||
new_settings["region"] = {
|
||||
"north": region_north,
|
||||
"south": region_south,
|
||||
"east": region_east,
|
||||
"west": region_west,
|
||||
}
|
||||
except ValueError:
|
||||
errors[f"{adapter_name}_region"] = "Region coordinates must be valid numbers"
|
||||
# Generic api_key_field validation against wizard state
|
||||
if adapter_cls.api_key_field is not None:
|
||||
field_value = new_settings.get(adapter_cls.api_key_field)
|
||||
if field_value:
|
||||
if not any(k["alias"] == field_value for k in state.api_keys):
|
||||
errors[f"{adapter_name}_{adapter_cls.api_key_field}"] = (
|
||||
"API key alias does not exist"
|
||||
)
|
||||
|
||||
new_adapters[adapter_name] = {
|
||||
"enabled": enabled,
|
||||
|
|
@ -833,12 +903,23 @@ async def setup_adapters_submit(request: Request) -> Response:
|
|||
|
||||
# If errors, re-render
|
||||
if errors:
|
||||
adapters = [
|
||||
{"name": name, "enabled": new_adapters[name]["enabled"],
|
||||
"cadence_s": new_adapters[name]["cadence_s"],
|
||||
"settings": new_adapters[name]["settings"]}
|
||||
for name in ["firms", "nws", "usgs_quake"]
|
||||
]
|
||||
adapters = []
|
||||
for name, cls in wizard_adapters:
|
||||
settings_dict = new_adapters[name]["settings"]
|
||||
fields = describe_fields(cls.settings_schema, settings_dict)
|
||||
# Swap widget for api_key_field to api_key_select
|
||||
if cls.api_key_field is not None:
|
||||
for f in fields:
|
||||
if f.name == cls.api_key_field:
|
||||
f.widget = "api_key_select"
|
||||
adapters.append({
|
||||
"name": name,
|
||||
"display_name": cls.display_name,
|
||||
"enabled": new_adapters[name]["enabled"],
|
||||
"cadence_s": new_adapters[name]["cadence_s"],
|
||||
"settings": settings_dict,
|
||||
"fields": fields,
|
||||
})
|
||||
api_keys = [{"alias": k["alias"]} for k in state.api_keys]
|
||||
|
||||
if state.system:
|
||||
|
|
@ -856,8 +937,6 @@ async def setup_adapters_submit(request: Request) -> Response:
|
|||
"csrf_token": csrf_token,
|
||||
"adapters": adapters,
|
||||
"api_keys": api_keys,
|
||||
"valid_satellites": _get_valid_satellites(),
|
||||
"valid_feeds": sorted(_get_valid_feeds()),
|
||||
"tile_url": tile_url,
|
||||
"tile_attribution": tile_attribution,
|
||||
"error": "Please fix the errors below.",
|
||||
|
|
@ -898,10 +977,20 @@ async def setup_finish_form(request: Request) -> HTMLResponse:
|
|||
|
||||
adapters = []
|
||||
if state.adapters:
|
||||
for name in ["firms", "nws", "usgs_quake"]:
|
||||
adapter_classes = _adapter_classes()
|
||||
wizard_adapters = sorted(
|
||||
[(name, cls) for name, cls in adapter_classes.items() if cls.wizard_order is not None],
|
||||
key=lambda nc: nc[1].wizard_order
|
||||
)
|
||||
for name, cls in wizard_adapters:
|
||||
if name in state.adapters:
|
||||
a = state.adapters[name]
|
||||
adapters.append({"name": name, "enabled": a["enabled"], "cadence_s": a["cadence_s"]})
|
||||
adapters.append({
|
||||
"name": name,
|
||||
"display_name": cls.display_name,
|
||||
"enabled": a["enabled"],
|
||||
"cadence_s": a["cadence_s"],
|
||||
})
|
||||
|
||||
csrf_token, signed_token = reuse_or_generate_pre_auth_csrf(request, settings.csrf_secret)
|
||||
response = templates.TemplateResponse(
|
||||
|
|
@ -1229,27 +1318,45 @@ async def adapters_list(
|
|||
templates = _get_templates()
|
||||
pool = get_pool()
|
||||
operator = request.state.operator
|
||||
adapter_classes = _adapter_classes()
|
||||
|
||||
async with pool.acquire() as conn:
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
SELECT name, enabled, cadence_s, settings, paused_at, updated_at
|
||||
SELECT name, enabled, cadence_s, settings, paused_at, updated_at, last_error
|
||||
FROM config.adapters
|
||||
ORDER BY name
|
||||
"""
|
||||
)
|
||||
|
||||
adapters = []
|
||||
for row in rows:
|
||||
settings = row["settings"] or {}
|
||||
adapters.append({
|
||||
"name": row["name"],
|
||||
"enabled": row["enabled"],
|
||||
"cadence_s": row["cadence_s"],
|
||||
"settings": settings,
|
||||
"paused_at": row["paused_at"],
|
||||
"updated_at": row["updated_at"],
|
||||
})
|
||||
adapters = []
|
||||
for row in rows:
|
||||
settings = row["settings"] or {}
|
||||
adapter_cls = adapter_classes.get(row["name"])
|
||||
|
||||
# Check if required API key is missing
|
||||
api_key_missing = False
|
||||
requires_api_key_alias = None
|
||||
if adapter_cls and adapter_cls.requires_api_key is not None:
|
||||
requires_api_key_alias = adapter_cls.requires_api_key
|
||||
has_key = await conn.fetchval(
|
||||
"SELECT 1 FROM config.api_keys WHERE alias = $1",
|
||||
requires_api_key_alias,
|
||||
)
|
||||
api_key_missing = not has_key
|
||||
|
||||
adapters.append({
|
||||
"name": row["name"],
|
||||
"display_name": getattr(adapter_cls, "display_name", row["name"]) if adapter_cls else row["name"],
|
||||
"enabled": row["enabled"],
|
||||
"cadence_s": row["cadence_s"],
|
||||
"settings": settings,
|
||||
"paused_at": row["paused_at"],
|
||||
"updated_at": row["updated_at"],
|
||||
"last_error": row["last_error"],
|
||||
"api_key_missing": api_key_missing,
|
||||
"requires_api_key_alias": requires_api_key_alias,
|
||||
})
|
||||
|
||||
csrf_token = request.state.csrf_token
|
||||
response = templates.TemplateResponse(
|
||||
|
|
@ -1275,10 +1382,14 @@ async def adapters_edit_form(
|
|||
pool = get_pool()
|
||||
operator = request.state.operator
|
||||
|
||||
# Look up the adapter class
|
||||
adapter_classes = _adapter_classes()
|
||||
adapter_cls = adapter_classes.get(name)
|
||||
|
||||
async with pool.acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
SELECT name, enabled, cadence_s, settings, paused_at, updated_at
|
||||
SELECT name, enabled, cadence_s, settings, paused_at, updated_at, last_error
|
||||
FROM config.adapters
|
||||
WHERE name = $1
|
||||
""",
|
||||
|
|
@ -1288,11 +1399,6 @@ async def adapters_edit_form(
|
|||
if row is None:
|
||||
return Response(status_code=404, content="Adapter not found")
|
||||
|
||||
# Get API keys for firms dropdown
|
||||
api_keys = await conn.fetch(
|
||||
"SELECT alias FROM config.api_keys ORDER BY alias"
|
||||
)
|
||||
|
||||
# Get map tile settings from config.system
|
||||
sys_row = await conn.fetchrow(
|
||||
"SELECT map_tile_url, map_attribution FROM config.system WHERE id = true"
|
||||
|
|
@ -1301,15 +1407,48 @@ async def adapters_edit_form(
|
|||
tile_attribution = sys_row["map_attribution"] if sys_row else "© OpenStreetMap contributors"
|
||||
|
||||
settings = row["settings"] or {}
|
||||
|
||||
# Build adapter dict with class metadata
|
||||
adapter = {
|
||||
"name": row["name"],
|
||||
"display_name": getattr(adapter_cls, "display_name", row["name"]) if adapter_cls else row["name"],
|
||||
"description": getattr(adapter_cls, "description", "") if adapter_cls else "",
|
||||
"enabled": row["enabled"],
|
||||
"cadence_s": row["cadence_s"],
|
||||
"settings": settings,
|
||||
"paused_at": row["paused_at"],
|
||||
"updated_at": row["updated_at"],
|
||||
"last_error": row["last_error"],
|
||||
}
|
||||
|
||||
# Generate field descriptors if we have the adapter class
|
||||
fields = []
|
||||
if adapter_cls and hasattr(adapter_cls, "settings_schema"):
|
||||
fields = describe_fields(adapter_cls.settings_schema, settings)
|
||||
# Swap widget for api_key_field to api_key_select
|
||||
if adapter_cls.api_key_field is not None:
|
||||
for f in fields:
|
||||
if f.name == adapter_cls.api_key_field:
|
||||
f.widget = "api_key_select"
|
||||
|
||||
# Fetch API keys for api_key_select widget
|
||||
api_keys = []
|
||||
async with pool.acquire() as conn:
|
||||
api_key_rows = await conn.fetch("SELECT alias FROM config.api_keys ORDER BY alias")
|
||||
api_keys = [{"alias": r["alias"]} for r in api_key_rows]
|
||||
|
||||
# Check if required API key is missing
|
||||
api_key_missing = False
|
||||
requires_api_key_alias = None
|
||||
if adapter_cls and adapter_cls.requires_api_key is not None:
|
||||
requires_api_key_alias = adapter_cls.requires_api_key
|
||||
async with pool.acquire() as conn:
|
||||
has_key = await conn.fetchval(
|
||||
"SELECT 1 FROM config.api_keys WHERE alias = $1",
|
||||
requires_api_key_alias,
|
||||
)
|
||||
api_key_missing = not has_key
|
||||
|
||||
csrf_token = request.state.csrf_token
|
||||
response = templates.TemplateResponse(
|
||||
request=request,
|
||||
|
|
@ -1318,13 +1457,14 @@ async def adapters_edit_form(
|
|||
"operator": operator,
|
||||
"csrf_token": csrf_token,
|
||||
"adapter": adapter,
|
||||
"fields": fields,
|
||||
"api_keys": api_keys,
|
||||
"errors": None,
|
||||
"form_data": None,
|
||||
"api_keys": [{"alias": k["alias"]} for k in api_keys],
|
||||
"valid_satellites": _get_valid_satellites(),
|
||||
"valid_feeds": sorted(_get_valid_feeds()),
|
||||
"tile_url": tile_url,
|
||||
"tile_attribution": tile_attribution,
|
||||
"api_key_missing": api_key_missing,
|
||||
"requires_api_key_alias": requires_api_key_alias,
|
||||
},
|
||||
)
|
||||
return response
|
||||
|
|
@ -1347,24 +1487,27 @@ async def adapters_edit_submit(
|
|||
if not form_csrf or form_csrf != request.state.csrf_token:
|
||||
raise CsrfValidationError("Invalid CSRF token")
|
||||
|
||||
# Parse form data
|
||||
form = await request.form()
|
||||
# Look up the adapter class
|
||||
adapter_classes = _adapter_classes()
|
||||
adapter_cls = adapter_classes.get(name)
|
||||
|
||||
# Parse common form fields
|
||||
enabled = "enabled" in form
|
||||
cadence_s_str = form.get("cadence_s", "")
|
||||
|
||||
# Build form_data for re-render on error
|
||||
errors: dict[str, str] = {}
|
||||
form_data: dict[str, Any] = {
|
||||
"enabled": enabled,
|
||||
"cadence_s": cadence_s_str,
|
||||
}
|
||||
|
||||
errors: dict[str, str] = {}
|
||||
|
||||
# Validate cadence_s
|
||||
# Validate cadence_s using AdapterConfig field constraint (ge=10)
|
||||
try:
|
||||
cadence_s = int(cadence_s_str)
|
||||
if cadence_s < 60 or cadence_s > 3600:
|
||||
errors["cadence_s"] = "Cadence must be between 60 and 3600 seconds"
|
||||
from central.config_models import AdapterConfig
|
||||
min_cadence = AdapterConfig.model_fields["cadence_s"].metadata[0].ge
|
||||
if cadence_s < min_cadence:
|
||||
errors["cadence_s"] = f"Input should be greater than or equal to {min_cadence}"
|
||||
except ValueError:
|
||||
errors["cadence_s"] = "Cadence must be a valid integer"
|
||||
cadence_s = 0
|
||||
|
|
@ -1373,7 +1516,7 @@ async def adapters_edit_submit(
|
|||
# Get current adapter state
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
SELECT name, enabled, cadence_s, settings, paused_at, updated_at
|
||||
SELECT name, enabled, cadence_s, settings, paused_at, updated_at, last_error
|
||||
FROM config.adapters
|
||||
WHERE name = $1
|
||||
""",
|
||||
|
|
@ -1384,103 +1527,113 @@ async def adapters_edit_submit(
|
|||
return Response(status_code=404, content="Adapter not found")
|
||||
|
||||
current_settings = row["settings"] or {}
|
||||
new_settings = dict(current_settings)
|
||||
|
||||
# Adapter-specific validation and settings update
|
||||
if name == "nws":
|
||||
contact_email = form.get("contact_email", "").strip()
|
||||
form_data["contact_email"] = contact_email
|
||||
if not contact_email:
|
||||
errors["contact_email"] = "Contact email is required"
|
||||
elif not EMAIL_REGEX.match(contact_email):
|
||||
errors["contact_email"] = "Invalid email format"
|
||||
# Parse and validate settings via Pydantic if we have the adapter class
|
||||
new_settings = {}
|
||||
if adapter_cls and hasattr(adapter_cls, "settings_schema"):
|
||||
schema = adapter_cls.settings_schema
|
||||
fields = describe_fields(schema, current_settings)
|
||||
|
||||
# Parse form values based on widget type
|
||||
parsed_values = {}
|
||||
for field in fields:
|
||||
raw = form.get(field.name, "")
|
||||
form_data[field.name] = raw
|
||||
|
||||
if field.widget == "text":
|
||||
parsed_values[field.name] = raw.strip() if raw else None
|
||||
elif field.widget == "number":
|
||||
try:
|
||||
parsed_values[field.name] = int(raw) if raw else None
|
||||
except ValueError:
|
||||
errors[field.name] = f"{field.label} must be a number"
|
||||
elif field.widget == "checkbox":
|
||||
parsed_values[field.name] = field.name in form
|
||||
elif field.widget == "csv":
|
||||
if raw.strip():
|
||||
parsed_values[field.name] = [v.strip() for v in raw.split(",") if v.strip()]
|
||||
else:
|
||||
parsed_values[field.name] = []
|
||||
elif field.widget == "select":
|
||||
value = raw.strip() if raw else None
|
||||
if value and field.options and value not in field.options:
|
||||
errors[field.name] = f"Invalid {field.label.lower()}"
|
||||
else:
|
||||
parsed_values[field.name] = value
|
||||
elif field.widget == "checkboxes":
|
||||
# Use getlist for checkbox groups
|
||||
values = form.getlist(field.name)
|
||||
form_data[field.name] = values # Override raw value
|
||||
if field.options:
|
||||
invalid = [v for v in values if v not in field.options]
|
||||
if invalid:
|
||||
errors[field.name] = f"Invalid values: {', '.join(invalid)}"
|
||||
else:
|
||||
parsed_values[field.name] = values
|
||||
else:
|
||||
parsed_values[field.name] = values
|
||||
elif field.widget == "api_key_select":
|
||||
# API key select - validate against existing keys
|
||||
value = raw.strip() if raw else None
|
||||
parsed_values[field.name] = value
|
||||
elif field.widget == "region":
|
||||
# Region handled separately below
|
||||
pass
|
||||
|
||||
# Handle region fields (common pattern)
|
||||
region_north_str = form.get("region_north", "").strip()
|
||||
region_south_str = form.get("region_south", "").strip()
|
||||
region_east_str = form.get("region_east", "").strip()
|
||||
region_west_str = form.get("region_west", "").strip()
|
||||
|
||||
form_data["region_north"] = region_north_str
|
||||
form_data["region_south"] = region_south_str
|
||||
form_data["region_east"] = region_east_str
|
||||
form_data["region_west"] = region_west_str
|
||||
|
||||
# Check if any region field has a value
|
||||
has_region = any([region_north_str, region_south_str, region_east_str, region_west_str])
|
||||
|
||||
if has_region:
|
||||
try:
|
||||
region_north = float(region_north_str)
|
||||
region_south = float(region_south_str)
|
||||
region_east = float(region_east_str)
|
||||
region_west = float(region_west_str)
|
||||
|
||||
if not (-90 <= region_south < region_north <= 90):
|
||||
errors["region"] = "Invalid latitude: south must be less than north, both between -90 and 90"
|
||||
elif not (-180 <= region_west < region_east <= 180):
|
||||
errors["region"] = "Invalid longitude: west must be less than east, both between -180 and 180"
|
||||
else:
|
||||
parsed_values["region"] = {
|
||||
"north": region_north,
|
||||
"south": region_south,
|
||||
"east": region_east,
|
||||
"west": region_west,
|
||||
}
|
||||
except ValueError:
|
||||
errors["region"] = "Region coordinates must be valid numbers"
|
||||
else:
|
||||
new_settings["contact_email"] = contact_email
|
||||
parsed_values["region"] = None
|
||||
|
||||
elif name == "firms":
|
||||
api_key_alias = form.get("api_key_alias", "").strip()
|
||||
satellites = form.getlist("satellites")
|
||||
form_data["api_key_alias"] = api_key_alias
|
||||
form_data["satellites"] = satellites
|
||||
|
||||
# Validate api_key_alias if set
|
||||
if api_key_alias:
|
||||
key_exists = await conn.fetchrow(
|
||||
"SELECT 1 FROM config.api_keys WHERE alias = $1",
|
||||
api_key_alias,
|
||||
)
|
||||
if not key_exists:
|
||||
errors["api_key_alias"] = f"API key alias '{api_key_alias}' does not exist"
|
||||
else:
|
||||
new_settings["api_key_alias"] = api_key_alias
|
||||
else:
|
||||
new_settings["api_key_alias"] = None
|
||||
|
||||
# Validate satellites
|
||||
valid_sats = set(_get_valid_satellites())
|
||||
invalid_sats = [s for s in satellites if s not in valid_sats]
|
||||
if invalid_sats:
|
||||
errors["satellites"] = f"Invalid satellites: {', '.join(invalid_sats)}"
|
||||
else:
|
||||
new_settings["satellites"] = satellites
|
||||
|
||||
elif name == "usgs_quake":
|
||||
feed = form.get("feed", "").strip()
|
||||
form_data["feed"] = feed
|
||||
valid_feeds = _get_valid_feeds()
|
||||
if feed not in valid_feeds:
|
||||
errors["feed"] = f"Invalid feed. Must be one of: {', '.join(sorted(valid_feeds))}"
|
||||
else:
|
||||
new_settings["feed"] = feed
|
||||
|
||||
# Region validation (applies to all adapters)
|
||||
region_north_str = form.get("region_north", "").strip()
|
||||
region_south_str = form.get("region_south", "").strip()
|
||||
region_east_str = form.get("region_east", "").strip()
|
||||
region_west_str = form.get("region_west", "").strip()
|
||||
|
||||
form_data["region_north"] = region_north_str
|
||||
form_data["region_south"] = region_south_str
|
||||
form_data["region_east"] = region_east_str
|
||||
form_data["region_west"] = region_west_str
|
||||
|
||||
try:
|
||||
region_north = float(region_north_str)
|
||||
region_south = float(region_south_str)
|
||||
region_east = float(region_east_str)
|
||||
region_west = float(region_west_str)
|
||||
|
||||
# Validate latitude bounds
|
||||
if not (-90 <= region_south < region_north <= 90):
|
||||
errors["region"] = "Invalid latitude: south must be less than north, both between -90 and 90"
|
||||
# Validate longitude bounds
|
||||
elif not (-180 <= region_west < region_east <= 180):
|
||||
errors["region"] = "Invalid longitude: west must be less than east, both between -180 and 180"
|
||||
else:
|
||||
new_settings["region"] = {
|
||||
"north": region_north,
|
||||
"south": region_south,
|
||||
"east": region_east,
|
||||
"west": region_west,
|
||||
}
|
||||
except ValueError:
|
||||
errors["region"] = "Region coordinates must be valid numbers"
|
||||
# Only validate with Pydantic if no parse errors
|
||||
if not errors:
|
||||
try:
|
||||
# Filter out None values for optional fields without defaults
|
||||
validated_data = {k: v for k, v in parsed_values.items() if v is not None}
|
||||
validated = schema(**validated_data)
|
||||
new_settings = validated.model_dump(mode="json")
|
||||
except ValidationError as e:
|
||||
for err in e.errors():
|
||||
field_name = err["loc"][0] if err["loc"] else "unknown"
|
||||
errors[str(field_name)] = err["msg"]
|
||||
else:
|
||||
# No schema - just preserve existing settings
|
||||
new_settings = dict(current_settings)
|
||||
|
||||
# If there are errors, re-render the form
|
||||
if errors:
|
||||
adapter = {
|
||||
"name": row["name"],
|
||||
"enabled": row["enabled"],
|
||||
"cadence_s": row["cadence_s"],
|
||||
"settings": current_settings,
|
||||
"paused_at": row["paused_at"],
|
||||
"updated_at": row["updated_at"],
|
||||
}
|
||||
|
||||
api_keys = await conn.fetch(
|
||||
"SELECT alias FROM config.api_keys ORDER BY alias"
|
||||
)
|
||||
|
||||
# Get map tile settings for re-render
|
||||
sys_row = await conn.fetchrow(
|
||||
"SELECT map_tile_url, map_attribution FROM config.system WHERE id = true"
|
||||
|
|
@ -1488,6 +1641,42 @@ async def adapters_edit_submit(
|
|||
tile_url = sys_row["map_tile_url"] if sys_row else "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png"
|
||||
tile_attribution = sys_row["map_attribution"] if sys_row else "© OpenStreetMap contributors"
|
||||
|
||||
adapter = {
|
||||
"name": row["name"],
|
||||
"display_name": getattr(adapter_cls, "display_name", row["name"]) if adapter_cls else row["name"],
|
||||
"description": getattr(adapter_cls, "description", "") if adapter_cls else "",
|
||||
"enabled": row["enabled"],
|
||||
"cadence_s": row["cadence_s"],
|
||||
"settings": current_settings,
|
||||
"paused_at": row["paused_at"],
|
||||
"updated_at": row["updated_at"],
|
||||
"last_error": row["last_error"],
|
||||
}
|
||||
|
||||
fields = []
|
||||
if adapter_cls and hasattr(adapter_cls, "settings_schema"):
|
||||
fields = describe_fields(adapter_cls.settings_schema, current_settings)
|
||||
# Swap widget for api_key_field to api_key_select
|
||||
if adapter_cls.api_key_field is not None:
|
||||
for f in fields:
|
||||
if f.name == adapter_cls.api_key_field:
|
||||
f.widget = "api_key_select"
|
||||
|
||||
# Fetch API keys for api_key_select widget
|
||||
api_key_rows = await conn.fetch("SELECT alias FROM config.api_keys ORDER BY alias")
|
||||
api_keys = [{"alias": r["alias"]} for r in api_key_rows]
|
||||
|
||||
# Check if required API key is missing
|
||||
api_key_missing = False
|
||||
requires_api_key_alias = None
|
||||
if adapter_cls and adapter_cls.requires_api_key is not None:
|
||||
requires_api_key_alias = adapter_cls.requires_api_key
|
||||
has_key = await conn.fetchval(
|
||||
"SELECT 1 FROM config.api_keys WHERE alias = $1",
|
||||
requires_api_key_alias,
|
||||
)
|
||||
api_key_missing = not has_key
|
||||
|
||||
csrf_token = request.state.csrf_token
|
||||
response = templates.TemplateResponse(
|
||||
request=request,
|
||||
|
|
@ -1496,13 +1685,14 @@ async def adapters_edit_submit(
|
|||
"operator": operator,
|
||||
"csrf_token": csrf_token,
|
||||
"adapter": adapter,
|
||||
"fields": fields,
|
||||
"api_keys": api_keys,
|
||||
"errors": errors,
|
||||
"form_data": form_data,
|
||||
"api_keys": [{"alias": k["alias"]} for k in api_keys],
|
||||
"valid_satellites": _get_valid_satellites(),
|
||||
"valid_feeds": sorted(_get_valid_feeds()),
|
||||
"tile_url": tile_url,
|
||||
"tile_attribution": tile_attribution,
|
||||
"api_key_missing": api_key_missing,
|
||||
"requires_api_key_alias": requires_api_key_alias,
|
||||
},
|
||||
status_code=200,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{% extends "base.html" %}
|
||||
|
||||
{% block title %}Central — Edit {{ adapter.name }}{% endblock %}
|
||||
{% block title %}Central — Edit {{ adapter.display_name }}{% endblock %}
|
||||
|
||||
{% block head %}
|
||||
<link rel="stylesheet" href="https://unpkg.com/leaflet@1.9.4/dist/leaflet.css" integrity="sha256-p4NxAoJBhIIN+hmNHrzRCf9tD/miZyoHS5obTRR9BMY=" crossorigin="">
|
||||
|
|
@ -10,35 +10,173 @@
|
|||
{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<h1>Edit Adapter: {{ adapter.name }}</h1>
|
||||
<h1>{{ adapter.display_name }}</h1>
|
||||
<p class="secondary">{{ adapter.description }}</p>
|
||||
|
||||
{% if adapter.paused_at %}
|
||||
<article aria-label="Adapter Paused" style="background-color: var(--pico-mark-background-color); margin-bottom: 1rem;">
|
||||
<strong>⏸️ Paused</strong> since {{ adapter.paused_at }}
|
||||
</article>
|
||||
{% endif %}
|
||||
|
||||
{% if adapter.last_error %}
|
||||
<article aria-label="Last Error" style="background-color: var(--pico-del-color); margin-bottom: 1rem;">
|
||||
<strong>Last Error:</strong> {{ adapter.last_error }}
|
||||
</article>
|
||||
{% endif %}
|
||||
|
||||
{% if api_key_missing %}
|
||||
<article aria-label="API Key Required" style="background-color: var(--pico-mark-background-color); margin-bottom: 1rem;">
|
||||
<strong>⚠️ API Key Required:</strong> This adapter requires the <code>{{ requires_api_key_alias }}</code> API key to be configured before it can be enabled.
|
||||
<a href="/api-keys">Configure API Keys</a>
|
||||
</article>
|
||||
{% endif %}
|
||||
|
||||
<form method="post" action="/adapters/{{ adapter.name }}">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
|
||||
|
||||
<fieldset>
|
||||
<legend>Universal Settings</legend>
|
||||
<legend>Core Settings</legend>
|
||||
|
||||
<label>
|
||||
<input type="checkbox" name="enabled" {% if adapter.enabled %}checked{% endif %}>
|
||||
Enabled
|
||||
<input type="checkbox" name="enabled" {% if form_data %}{% if form_data.enabled %}checked{% endif %}{% elif adapter.enabled %}checked{% endif %}{% if api_key_missing %} disabled{% endif %}>
|
||||
Enabled{% if api_key_missing %} <small>(requires API key)</small>{% endif %}
|
||||
</label>
|
||||
|
||||
<label for="cadence_s">Cadence (seconds)</label>
|
||||
<input type="number" id="cadence_s" name="cadence_s" value="{{ form_data.cadence_s if form_data else adapter.cadence_s }}" min="60" max="3600" required>
|
||||
<input type="number" id="cadence_s" name="cadence_s"
|
||||
value="{{ form_data.cadence_s if form_data else adapter.cadence_s }}"
|
||||
required>
|
||||
{% if errors and errors.cadence_s %}
|
||||
<small style="color: var(--pico-color-red-500);">{{ errors.cadence_s }}</small>
|
||||
{% endif %}
|
||||
</fieldset>
|
||||
|
||||
{% if fields %}
|
||||
<fieldset>
|
||||
<legend>Adapter-Specific Settings</legend>
|
||||
{% include "adapters_edit_" + adapter.name + ".html" %}
|
||||
</fieldset>
|
||||
<legend>Adapter Settings</legend>
|
||||
|
||||
{% for field in fields %}
|
||||
{% if field.widget == "region" %}
|
||||
{# Region is rendered in a separate fieldset below #}
|
||||
{% elif field.widget == "text" %}
|
||||
<label for="{{ field.name }}">{{ field.label }}</label>
|
||||
<input type="text" id="{{ field.name }}" name="{{ field.name }}"
|
||||
value="{{ form_data[field.name] if form_data and field.name in form_data else field.current_value or '' }}"
|
||||
{% if field.required %}required{% endif %}>
|
||||
{% if field.description %}
|
||||
<small>{{ field.description }}</small>
|
||||
{% endif %}
|
||||
{% if errors and errors[field.name] %}
|
||||
<small style="color: var(--pico-color-red-500);">{{ errors[field.name] }}</small>
|
||||
{% endif %}
|
||||
|
||||
{% elif field.widget == "number" %}
|
||||
<label for="{{ field.name }}">{{ field.label }}</label>
|
||||
<input type="number" id="{{ field.name }}" name="{{ field.name }}"
|
||||
value="{{ form_data[field.name] if form_data and field.name in form_data else field.current_value or '' }}"
|
||||
{% if field.required %}required{% endif %}>
|
||||
{% if field.description %}
|
||||
<small>{{ field.description }}</small>
|
||||
{% endif %}
|
||||
{% if errors and errors[field.name] %}
|
||||
<small style="color: var(--pico-color-red-500);">{{ errors[field.name] }}</small>
|
||||
{% endif %}
|
||||
|
||||
{% elif field.widget == "checkbox" %}
|
||||
<label>
|
||||
<input type="checkbox" name="{{ field.name }}"
|
||||
{% if form_data and field.name in form_data %}
|
||||
{% if form_data[field.name] %}checked{% endif %}
|
||||
{% elif field.current_value %}checked{% endif %}>
|
||||
{{ field.label }}
|
||||
</label>
|
||||
{% if field.description %}
|
||||
<small>{{ field.description }}</small>
|
||||
{% endif %}
|
||||
{% if errors and errors[field.name] %}
|
||||
<small style="color: var(--pico-color-red-500);">{{ errors[field.name] }}</small>
|
||||
{% endif %}
|
||||
|
||||
{% elif field.widget == "csv" %}
|
||||
<label for="{{ field.name }}">{{ field.label }}</label>
|
||||
<input type="text" id="{{ field.name }}" name="{{ field.name }}"
|
||||
value="{{ form_data[field.name] if form_data and field.name in form_data else (field.current_value | join(',') if field.current_value else '') }}"
|
||||
{% if field.required %}required{% endif %}>
|
||||
<small>Comma-separated values{% if field.description %} — {{ field.description }}{% endif %}</small>
|
||||
{% if errors and errors[field.name] %}
|
||||
<small style="color: var(--pico-color-red-500);">{{ errors[field.name] }}</small>
|
||||
{% endif %}
|
||||
|
||||
{% elif field.widget == "select" %}
|
||||
<label for="{{ field.name }}">{{ field.label }}</label>
|
||||
<select id="{{ field.name }}" name="{{ field.name }}">
|
||||
{% for opt in field.options %}
|
||||
<option value="{{ opt }}"
|
||||
{% if (form_data[field.name] if form_data and field.name in form_data else field.current_value) == opt %}selected{% endif %}>
|
||||
{{ opt }}
|
||||
</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
{% if field.description %}
|
||||
<small>{{ field.description }}</small>
|
||||
{% endif %}
|
||||
{% if errors and errors[field.name] %}
|
||||
<small style="color: var(--pico-color-red-500);">{{ errors[field.name] }}</small>
|
||||
{% endif %}
|
||||
|
||||
{% elif field.widget == "checkboxes" %}
|
||||
<label>{{ field.label }}</label>
|
||||
{% set current_values = form_data.getlist(field.name) if form_data and form_data.getlist else (field.current_value or []) %}
|
||||
{% for opt in field.options %}
|
||||
<label style="display: inline-block; margin-right: 1rem;">
|
||||
<input type="checkbox" name="{{ field.name }}" value="{{ opt }}"
|
||||
{% if opt in current_values %}checked{% endif %}>
|
||||
{{ opt }}
|
||||
</label>
|
||||
{% endfor %}
|
||||
{% if field.description %}
|
||||
<small style="display: block;">{{ field.description }}</small>
|
||||
{% endif %}
|
||||
{% if errors and errors[field.name] %}
|
||||
<small style="color: var(--pico-color-red-500); display: block;">{{ errors[field.name] }}</small>
|
||||
{% endif %}
|
||||
|
||||
{% elif field.widget == "api_key_select" %}
|
||||
<label for="{{ field.name }}">{{ field.label }}</label>
|
||||
<select id="{{ field.name }}" name="{{ field.name }}">
|
||||
<option value="">(none)</option>
|
||||
{% for key in api_keys %}
|
||||
<option value="{{ key.alias }}"
|
||||
{% if (form_data[field.name] if form_data and field.name in form_data else field.current_value) == key.alias %}selected{% endif %}>
|
||||
{{ key.alias }}
|
||||
</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
{% if field.description %}
|
||||
<small>{{ field.description }}</small>
|
||||
{% endif %}
|
||||
{% if errors and errors[field.name] %}
|
||||
<small style="color: var(--pico-color-red-500);">{{ errors[field.name] }}</small>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</fieldset>
|
||||
{% endif %}
|
||||
|
||||
{% set has_region = namespace(value=false) %}
|
||||
{% for field in fields %}
|
||||
{% if field.widget == "region" %}
|
||||
{% set has_region.value = true %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
{% if has_region.value %}
|
||||
<fieldset>
|
||||
<legend>Region</legend>
|
||||
{% include "_region_picker.html" %}
|
||||
</fieldset>
|
||||
{% endif %}
|
||||
|
||||
<button type="submit">Save Changes</button>
|
||||
<a href="/adapters" role="button" class="outline">Cancel</a>
|
||||
|
|
|
|||
|
|
@ -1,21 +0,0 @@
|
|||
<label for="api_key_alias">API Key Alias</label>
|
||||
<select id="api_key_alias" name="api_key_alias">
|
||||
<option value="" {% if not (form_data.api_key_alias if form_data else adapter.settings.api_key_alias) %}selected{% endif %}>(none)</option>
|
||||
{% for key in api_keys %}
|
||||
<option value="{{ key.alias }}" {% if (form_data.api_key_alias if form_data else adapter.settings.api_key_alias) == key.alias %}selected{% endif %}>{{ key.alias }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
{% if errors and errors.api_key_alias %}
|
||||
<small style="color: var(--pico-color-red-500);">{{ errors.api_key_alias }}</small>
|
||||
{% endif %}
|
||||
|
||||
<label>Satellites</label>
|
||||
{% for sat in valid_satellites %}
|
||||
<label>
|
||||
<input type="checkbox" name="satellites" value="{{ sat }}" {% if sat in (form_data.satellites if form_data else adapter.settings.satellites or []) %}checked{% endif %}>
|
||||
{{ sat }}
|
||||
</label>
|
||||
{% endfor %}
|
||||
{% if errors and errors.satellites %}
|
||||
<small style="color: var(--pico-color-red-500);">{{ errors.satellites }}</small>
|
||||
{% endif %}
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
<label for="contact_email">Contact Email</label>
|
||||
<input type="email" id="contact_email" name="contact_email" value="{{ form_data.contact_email if form_data else adapter.settings.contact_email }}" required>
|
||||
{% if errors and errors.contact_email %}
|
||||
<small style="color: var(--pico-color-red-500);">{{ errors.contact_email }}</small>
|
||||
{% endif %}
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
<label for="feed">Feed</label>
|
||||
<select id="feed" name="feed" required>
|
||||
{% for f in valid_feeds %}
|
||||
<option value="{{ f }}" {% if (form_data.feed if form_data else adapter.settings.feed) == f %}selected{% endif %}>{{ f }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
{% if errors and errors.feed %}
|
||||
<small style="color: var(--pico-color-red-500);">{{ errors.feed }}</small>
|
||||
{% endif %}
|
||||
|
|
@ -17,7 +17,12 @@
|
|||
<tbody>
|
||||
{% for adapter in adapters %}
|
||||
<tr>
|
||||
<td>{{ adapter.name }}</td>
|
||||
<td>
|
||||
{{ adapter.display_name or adapter.name }}
|
||||
{% if adapter.api_key_missing %}
|
||||
<span style="color: var(--pico-color-orange-500); margin-left: 0.5rem;" title="Missing API key: {{ adapter.requires_api_key_alias }}">⚠️ API Key Missing</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>{% if adapter.enabled %}Yes{% else %}No{% endif %}</td>
|
||||
<td>{{ adapter.cadence_s }}s</td>
|
||||
<td>{{ adapter.updated_at.strftime('%Y-%m-%d %H:%M') if adapter.updated_at else '—' }}</td>
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@
|
|||
|
||||
{% for adapter in adapters %}
|
||||
<details open style="margin-bottom: 2rem;">
|
||||
<summary><strong>{{ adapter.name }}</strong></summary>
|
||||
<summary><strong>{{ adapter.display_name or adapter.name }}</strong></summary>
|
||||
|
||||
<div style="padding: 1rem; border-left: 3px solid var(--pico-primary);">
|
||||
<label>
|
||||
|
|
@ -44,100 +44,158 @@
|
|||
|
||||
<label for="{{ adapter.name }}_cadence_s">Cadence (seconds)</label>
|
||||
<input type="number" id="{{ adapter.name }}_cadence_s" name="{{ adapter.name }}_cadence_s"
|
||||
value="{{ form_data.get(adapter.name + '_cadence_s') if form_data else adapter.cadence_s }}"
|
||||
min="60" max="3600">
|
||||
value="{{ form_data.get(adapter.name + '_cadence_s') if form_data else adapter.cadence_s }}">
|
||||
{% if errors and errors.get(adapter.name + '_cadence_s') %}
|
||||
<small style="color: var(--pico-color-red-500);">{{ errors[adapter.name + '_cadence_s'] }}</small>
|
||||
{% endif %}
|
||||
|
||||
{% if adapter.name == 'nws' %}
|
||||
<label for="{{ adapter.name }}_contact_email">Contact Email</label>
|
||||
<input type="email" id="{{ adapter.name }}_contact_email" name="{{ adapter.name }}_contact_email"
|
||||
value="{{ form_data.get(adapter.name + '_contact_email') if form_data else adapter.settings.contact_email }}">
|
||||
{% if errors and errors.get(adapter.name + '_contact_email') %}
|
||||
<small style="color: var(--pico-color-red-500);">{{ errors[adapter.name + '_contact_email'] }}</small>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% for field in adapter.fields %}
|
||||
{% set form_key = adapter.name + '_' + field.name %}
|
||||
|
||||
{% if adapter.name == 'firms' %}
|
||||
<label for="{{ adapter.name }}_api_key_alias">API Key Alias</label>
|
||||
<select id="{{ adapter.name }}_api_key_alias" name="{{ adapter.name }}_api_key_alias">
|
||||
<option value="">(none)</option>
|
||||
{% for key in api_keys %}
|
||||
<option value="{{ key.alias }}"
|
||||
{% if (form_data.get(adapter.name + '_api_key_alias') if form_data else adapter.settings.api_key_alias) == key.alias %}selected{% endif %}>
|
||||
{{ key.alias }}
|
||||
</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
{% if errors and errors.get(adapter.name + '_api_key_alias') %}
|
||||
<small style="color: var(--pico-color-red-500);">{{ errors[adapter.name + '_api_key_alias'] }}</small>
|
||||
{% endif %}
|
||||
{% if field.widget == "text" %}
|
||||
<label for="{{ form_key }}">{{ field.label }}</label>
|
||||
<input type="text" id="{{ form_key }}" name="{{ form_key }}"
|
||||
value="{{ form_data.get(form_key) if form_data else field.current_value or '' }}"
|
||||
{% if field.required %}required{% endif %}>
|
||||
{% if field.description %}
|
||||
<small>{{ field.description }}</small>
|
||||
{% endif %}
|
||||
{% if errors and errors.get(form_key) %}
|
||||
<small style="color: var(--pico-color-red-500);">{{ errors[form_key] }}</small>
|
||||
{% endif %}
|
||||
|
||||
<label>Satellites</label>
|
||||
{% for sat in valid_satellites %}
|
||||
<label style="display: inline-block; margin-right: 1rem;">
|
||||
<input type="checkbox" name="{{ adapter.name }}_satellites" value="{{ sat }}"
|
||||
{% if sat in (form_data.getlist(adapter.name + '_satellites') if form_data else adapter.settings.satellites or []) %}checked{% endif %}>
|
||||
{{ sat }}
|
||||
</label>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% elif field.widget == "api_key_select" %}
|
||||
<label for="{{ form_key }}">{{ field.label }}</label>
|
||||
<select id="{{ form_key }}" name="{{ form_key }}">
|
||||
<option value="">(none)</option>
|
||||
{% for key in api_keys %}
|
||||
<option value="{{ key.alias }}"
|
||||
{% if (form_data.get(form_key) if form_data else field.current_value) == key.alias %}selected{% endif %}>
|
||||
{{ key.alias }}
|
||||
</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
{% if field.description %}
|
||||
<small>{{ field.description }}</small>
|
||||
{% endif %}
|
||||
{% if errors and errors.get(form_key) %}
|
||||
<small style="color: var(--pico-color-red-500);">{{ errors[form_key] }}</small>
|
||||
{% endif %}
|
||||
|
||||
{% if adapter.name == 'usgs_quake' %}
|
||||
<label for="{{ adapter.name }}_feed">Feed</label>
|
||||
<select id="{{ adapter.name }}_feed" name="{{ adapter.name }}_feed">
|
||||
{% for f in valid_feeds %}
|
||||
<option value="{{ f }}"
|
||||
{% if (form_data.get(adapter.name + '_feed') if form_data else adapter.settings.feed) == f %}selected{% endif %}>
|
||||
{{ f }}
|
||||
</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
{% if errors and errors.get(adapter.name + '_feed') %}
|
||||
<small style="color: var(--pico-color-red-500);">{{ errors[adapter.name + '_feed'] }}</small>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% elif field.widget == "number" %}
|
||||
<label for="{{ form_key }}">{{ field.label }}</label>
|
||||
<input type="number" id="{{ form_key }}" name="{{ form_key }}"
|
||||
value="{{ form_data.get(form_key) if form_data else field.current_value or '' }}"
|
||||
{% if field.required %}required{% endif %}>
|
||||
{% if field.description %}
|
||||
<small>{{ field.description }}</small>
|
||||
{% endif %}
|
||||
{% if errors and errors.get(form_key) %}
|
||||
<small style="color: var(--pico-color-red-500);">{{ errors[form_key] }}</small>
|
||||
{% endif %}
|
||||
|
||||
<h4>Region</h4>
|
||||
{% set region = form_data if form_data else adapter.settings.region %}
|
||||
<div id="region-picker-{{ adapter.name }}"
|
||||
data-adapter="{{ adapter.name }}"
|
||||
data-north="{{ form_data.get(adapter.name + '_region_north') if form_data else (adapter.settings.region.north if adapter.settings.region else 49.5) }}"
|
||||
data-south="{{ form_data.get(adapter.name + '_region_south') if form_data else (adapter.settings.region.south if adapter.settings.region else 31.0) }}"
|
||||
data-east="{{ form_data.get(adapter.name + '_region_east') if form_data else (adapter.settings.region.east if adapter.settings.region else -102.0) }}"
|
||||
data-west="{{ form_data.get(adapter.name + '_region_west') if form_data else (adapter.settings.region.west if adapter.settings.region else -124.5) }}"
|
||||
data-tile-url="{{ tile_url }}"
|
||||
data-tile-attr="{{ tile_attribution }}">
|
||||
{% elif field.widget == "checkbox" %}
|
||||
<label>
|
||||
<input type="checkbox" name="{{ form_key }}"
|
||||
{% if form_data and form_data.get(form_key) %}checked
|
||||
{% elif not form_data and field.current_value %}checked{% endif %}>
|
||||
{{ field.label }}
|
||||
</label>
|
||||
{% if field.description %}
|
||||
<small>{{ field.description }}</small>
|
||||
{% endif %}
|
||||
{% if errors and errors.get(form_key) %}
|
||||
<small style="color: var(--pico-color-red-500);">{{ errors[form_key] }}</small>
|
||||
{% endif %}
|
||||
|
||||
<div id="region-map-{{ adapter.name }}" style="height: 300px; margin-bottom: 1rem;"></div>
|
||||
{% elif field.widget == "csv" %}
|
||||
<label for="{{ form_key }}">{{ field.label }}</label>
|
||||
<input type="text" id="{{ form_key }}" name="{{ form_key }}"
|
||||
value="{{ form_data.get(form_key) if form_data else (field.current_value | join(',') if field.current_value else '') }}"
|
||||
{% if field.required %}required{% endif %}>
|
||||
<small>Comma-separated values{% if field.description %} — {{ field.description }}{% endif %}</small>
|
||||
{% if errors and errors.get(form_key) %}
|
||||
<small style="color: var(--pico-color-red-500);">{{ errors[form_key] }}</small>
|
||||
{% endif %}
|
||||
|
||||
<div class="grid">
|
||||
<div>
|
||||
<label>North</label>
|
||||
<input type="number" name="{{ adapter.name }}_region_north" step="0.0001" min="-90" max="90" readonly
|
||||
value="{{ form_data.get(adapter.name + '_region_north') if form_data else (adapter.settings.region.north if adapter.settings.region else 49.5) }}">
|
||||
{% elif field.widget == "select" %}
|
||||
<label for="{{ form_key }}">{{ field.label }}</label>
|
||||
<select id="{{ form_key }}" name="{{ form_key }}">
|
||||
{% for opt in field.options %}
|
||||
<option value="{{ opt }}"
|
||||
{% if (form_data.get(form_key) if form_data else field.current_value) == opt %}selected{% endif %}>
|
||||
{{ opt }}
|
||||
</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
{% if field.description %}
|
||||
<small>{{ field.description }}</small>
|
||||
{% endif %}
|
||||
{% if errors and errors.get(form_key) %}
|
||||
<small style="color: var(--pico-color-red-500);">{{ errors[form_key] }}</small>
|
||||
{% endif %}
|
||||
|
||||
{% elif field.widget == "checkboxes" %}
|
||||
<label>{{ field.label }}</label>
|
||||
{% set current_values = form_data.getlist(form_key) if form_data else (field.current_value or []) %}
|
||||
{% for opt in field.options %}
|
||||
<label style="display: inline-block; margin-right: 1rem;">
|
||||
<input type="checkbox" name="{{ form_key }}" value="{{ opt }}"
|
||||
{% if opt in current_values %}checked{% endif %}>
|
||||
{{ opt }}
|
||||
</label>
|
||||
{% endfor %}
|
||||
{% if field.description %}
|
||||
<small style="display: block;">{{ field.description }}</small>
|
||||
{% endif %}
|
||||
{% if errors and errors.get(form_key) %}
|
||||
<small style="color: var(--pico-color-red-500); display: block;">{{ errors[form_key] }}</small>
|
||||
{% endif %}
|
||||
|
||||
{% elif field.widget == "region" %}
|
||||
<h4>Region</h4>
|
||||
{% set region_key = adapter.name + '_' + field.name %}
|
||||
{% set region = field.current_value or {} %}
|
||||
<div id="region-picker-{{ adapter.name }}"
|
||||
data-adapter="{{ adapter.name }}"
|
||||
data-field="{{ field.name }}"
|
||||
data-north="{{ form_data.get(region_key + '_north') if form_data else (region.north if region else 49.5) }}"
|
||||
data-south="{{ form_data.get(region_key + '_south') if form_data else (region.south if region else 31.0) }}"
|
||||
data-east="{{ form_data.get(region_key + '_east') if form_data else (region.east if region else -102.0) }}"
|
||||
data-west="{{ form_data.get(region_key + '_west') if form_data else (region.west if region else -124.5) }}"
|
||||
data-tile-url="{{ tile_url }}"
|
||||
data-tile-attr="{{ tile_attribution }}">
|
||||
|
||||
<div id="region-map-{{ adapter.name }}" style="height: 300px; margin-bottom: 1rem;"></div>
|
||||
|
||||
<div class="grid">
|
||||
<div>
|
||||
<label>North</label>
|
||||
<input type="number" name="{{ region_key }}_north" step="0.0001" min="-90" max="90" readonly
|
||||
value="{{ form_data.get(region_key + '_north') if form_data else (region.north if region else 49.5) }}">
|
||||
</div>
|
||||
<div>
|
||||
<label>South</label>
|
||||
<input type="number" name="{{ region_key }}_south" step="0.0001" min="-90" max="90" readonly
|
||||
value="{{ form_data.get(region_key + '_south') if form_data else (region.south if region else 31.0) }}">
|
||||
</div>
|
||||
<div>
|
||||
<label>East</label>
|
||||
<input type="number" name="{{ region_key }}_east" step="0.0001" min="-180" max="180" readonly
|
||||
value="{{ form_data.get(region_key + '_east') if form_data else (region.east if region else -102.0) }}">
|
||||
</div>
|
||||
<div>
|
||||
<label>West</label>
|
||||
<input type="number" name="{{ region_key }}_west" step="0.0001" min="-180" max="180" readonly
|
||||
value="{{ form_data.get(region_key + '_west') if form_data else (region.west if region else -124.5) }}">
|
||||
</div>
|
||||
</div>
|
||||
{% if errors and errors.get(region_key) %}
|
||||
<small style="color: var(--pico-color-red-500);">{{ errors[region_key] }}</small>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div>
|
||||
<label>South</label>
|
||||
<input type="number" name="{{ adapter.name }}_region_south" step="0.0001" min="-90" max="90" readonly
|
||||
value="{{ form_data.get(adapter.name + '_region_south') if form_data else (adapter.settings.region.south if adapter.settings.region else 31.0) }}">
|
||||
</div>
|
||||
<div>
|
||||
<label>East</label>
|
||||
<input type="number" name="{{ adapter.name }}_region_east" step="0.0001" min="-180" max="180" readonly
|
||||
value="{{ form_data.get(adapter.name + '_region_east') if form_data else (adapter.settings.region.east if adapter.settings.region else -102.0) }}">
|
||||
</div>
|
||||
<div>
|
||||
<label>West</label>
|
||||
<input type="number" name="{{ adapter.name }}_region_west" step="0.0001" min="-180" max="180" readonly
|
||||
value="{{ form_data.get(adapter.name + '_region_west') if form_data else (adapter.settings.region.west if adapter.settings.region else -124.5) }}">
|
||||
</div>
|
||||
</div>
|
||||
{% if errors and errors.get(adapter.name + '_region') %}
|
||||
<small style="color: var(--pico-color-red-500);">{{ errors[adapter.name + '_region'] }}</small>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</details>
|
||||
{% endfor %}
|
||||
|
|
@ -151,11 +209,12 @@
|
|||
|
||||
<script>
|
||||
document.addEventListener('DOMContentLoaded', function() {
|
||||
const adapters = ['nws', 'firms', 'usgs_quake'];
|
||||
// Find all region pickers dynamically
|
||||
const regionPickers = document.querySelectorAll('[id^="region-picker-"]');
|
||||
|
||||
adapters.forEach(function(adapterName) {
|
||||
const container = document.getElementById('region-picker-' + adapterName);
|
||||
if (!container) return;
|
||||
regionPickers.forEach(function(container) {
|
||||
const adapterName = container.dataset.adapter;
|
||||
const fieldName = container.dataset.field || 'region';
|
||||
|
||||
const savedNorth = parseFloat(container.dataset.north);
|
||||
const savedSouth = parseFloat(container.dataset.south);
|
||||
|
|
@ -215,10 +274,11 @@ document.addEventListener('DOMContentLoaded', function() {
|
|||
|
||||
rectangle.editing.enable();
|
||||
|
||||
const northInput = container.querySelector('input[name="' + adapterName + '_region_north"]');
|
||||
const southInput = container.querySelector('input[name="' + adapterName + '_region_south"]');
|
||||
const eastInput = container.querySelector('input[name="' + adapterName + '_region_east"]');
|
||||
const westInput = container.querySelector('input[name="' + adapterName + '_region_west"]');
|
||||
const inputPrefix = adapterName + '_' + fieldName;
|
||||
const northInput = container.querySelector('input[name="' + inputPrefix + '_north"]');
|
||||
const southInput = container.querySelector('input[name="' + inputPrefix + '_south"]');
|
||||
const eastInput = container.querySelector('input[name="' + inputPrefix + '_east"]');
|
||||
const westInput = container.querySelector('input[name="' + inputPrefix + '_west"]');
|
||||
|
||||
function updateInputs() {
|
||||
const b = rectangle.getBounds();
|
||||
|
|
|
|||
|
|
@ -13,41 +13,14 @@ from typing import Any
|
|||
import nats
|
||||
from nats.js import JetStreamContext
|
||||
|
||||
import importlib
|
||||
import pkgutil
|
||||
|
||||
from central.adapter import SourceAdapter
|
||||
from central.adapter_discovery import discover_adapters
|
||||
from central.cloudevents_wire import wrap_event
|
||||
from central.config_models import AdapterConfig
|
||||
from central.config_source import ConfigSource, DbConfigSource
|
||||
from central.config_store import ConfigStore
|
||||
from central.bootstrap_config import get_settings
|
||||
from central.stream_manager import StreamManager
|
||||
import central.adapters
|
||||
|
||||
def discover_adapters() -> dict[str, type[SourceAdapter]]:
|
||||
"""Auto-discover adapter classes from central.adapters package."""
|
||||
registry: dict[str, type[SourceAdapter]] = {}
|
||||
for module_info in pkgutil.iter_modules(central.adapters.__path__):
|
||||
try:
|
||||
module = importlib.import_module(f"central.adapters.{module_info.name}")
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Failed to import adapter module",
|
||||
extra={"module": module_info.name, "error": str(e)},
|
||||
)
|
||||
continue
|
||||
for attr_name in dir(module):
|
||||
attr = getattr(module, attr_name)
|
||||
if (
|
||||
isinstance(attr, type)
|
||||
and issubclass(attr, SourceAdapter)
|
||||
and attr is not SourceAdapter
|
||||
and hasattr(attr, "name")
|
||||
):
|
||||
registry[attr.name] = attr
|
||||
return registry
|
||||
|
||||
CURSOR_DB_PATH = Path("/var/lib/central/cursors.db")
|
||||
|
||||
# Stream subject mappings
|
||||
|
|
@ -56,6 +29,7 @@ STREAM_SUBJECTS = {
|
|||
"CENTRAL_META": ["central.meta.>"],
|
||||
"CENTRAL_FIRE": ["central.fire.>"],
|
||||
"CENTRAL_QUAKE": ["central.quake.>"],
|
||||
"CENTRAL_SPACE": ["central.space.>"],
|
||||
}
|
||||
|
||||
# Recompute interval for stream max_bytes (1 hour)
|
||||
|
|
@ -293,6 +267,23 @@ class Supervisor:
|
|||
If the adapter was previously stopped (state exists but task is not running),
|
||||
reuses the existing state to preserve last_completed_poll for rate limiting.
|
||||
"""
|
||||
# API key precondition
|
||||
adapter_cls = self._adapters.get(config.name)
|
||||
if adapter_cls is not None and adapter_cls.requires_api_key is not None:
|
||||
alias = adapter_cls.requires_api_key
|
||||
key_value = await self._config_store.get_api_key(alias)
|
||||
if not key_value:
|
||||
error_msg = f"missing api key: {alias}"
|
||||
logger.warning(
|
||||
"Adapter cannot start - api key missing",
|
||||
extra={"adapter": config.name, "alias": alias},
|
||||
)
|
||||
await self._config_store.set_adapter_last_error(config.name, error_msg)
|
||||
return
|
||||
|
||||
# Clear any stale last_error before proceeding
|
||||
await self._config_store.set_adapter_last_error(config.name, None)
|
||||
|
||||
existing_state = self._adapter_states.get(config.name)
|
||||
|
||||
if existing_state is not None:
|
||||
|
|
|
|||
|
|
@ -42,9 +42,9 @@ class TestAdaptersListAuthenticated:
|
|||
|
||||
mock_conn = AsyncMock()
|
||||
mock_conn.fetch.return_value = [
|
||||
{"name": "firms", "enabled": True, "cadence_s": 300, "settings": {"api_key_alias": "firms"}, "paused_at": None, "updated_at": None},
|
||||
{"name": "nws", "enabled": True, "cadence_s": 60, "settings": {"contact_email": "test@test.com"}, "paused_at": None, "updated_at": None},
|
||||
{"name": "usgs_quake", "enabled": True, "cadence_s": 120, "settings": {"feed": "all_hour"}, "paused_at": None, "updated_at": None},
|
||||
{"name": "firms", "enabled": True, "cadence_s": 300, "settings": {"api_key_alias": "firms"}, "paused_at": None, "updated_at": None, "last_error": None},
|
||||
{"name": "nws", "enabled": True, "cadence_s": 60, "settings": {"contact_email": "test@test.com"}, "paused_at": None, "updated_at": None, "last_error": None},
|
||||
{"name": "usgs_quake", "enabled": True, "cadence_s": 120, "settings": {"feed": "all_hour"}, "paused_at": None, "updated_at": None, "last_error": None},
|
||||
]
|
||||
|
||||
mock_pool = MagicMock()
|
||||
|
|
@ -55,9 +55,22 @@ class TestAdaptersListAuthenticated:
|
|||
mock_response = MagicMock()
|
||||
mock_templates.TemplateResponse.return_value = mock_response
|
||||
|
||||
# Mock adapter classes
|
||||
mock_firms_cls = MagicMock()
|
||||
mock_firms_cls.requires_api_key = "firms"
|
||||
mock_firms_cls.display_name = "FIRMS"
|
||||
mock_nws_cls = MagicMock()
|
||||
mock_nws_cls.requires_api_key = None
|
||||
mock_nws_cls.display_name = "NWS"
|
||||
mock_usgs_cls = MagicMock()
|
||||
mock_usgs_cls.requires_api_key = None
|
||||
mock_usgs_cls.display_name = "USGS Quake"
|
||||
mock_adapter_classes = {"firms": mock_firms_cls, "nws": mock_nws_cls, "usgs_quake": mock_usgs_cls}
|
||||
|
||||
with patch("central.gui.routes._get_templates", return_value=mock_templates):
|
||||
with patch("central.gui.routes.get_pool", return_value=mock_pool):
|
||||
result = await adapters_list(mock_request)
|
||||
with patch("central.gui.routes._adapter_classes", return_value=mock_adapter_classes):
|
||||
result = await adapters_list(mock_request)
|
||||
|
||||
# Verify template was called with adapters
|
||||
call_args = mock_templates.TemplateResponse.call_args
|
||||
|
|
@ -78,6 +91,7 @@ class TestAdaptersEditForm:
|
|||
|
||||
mock_request = MagicMock()
|
||||
mock_request.state.operator = MagicMock(id=1, username="testop")
|
||||
mock_request.state.csrf_token = "test_csrf"
|
||||
|
||||
mock_conn = AsyncMock()
|
||||
mock_conn.fetchrow.side_effect = [
|
||||
|
|
@ -88,10 +102,10 @@ class TestAdaptersEditForm:
|
|||
"settings": {"contact_email": "test@example.com", "region": {"north": 49, "south": 24, "east": -66, "west": -125}},
|
||||
"paused_at": None,
|
||||
"updated_at": None,
|
||||
"last_error": None,
|
||||
},
|
||||
{"map_tile_url": "https://tile.example.com/{z}/{x}/{y}.png", "map_attribution": "Test"},
|
||||
]
|
||||
mock_conn.fetch.return_value = [] # No API keys
|
||||
|
||||
mock_pool = MagicMock()
|
||||
mock_pool.acquire.return_value.__aenter__ = AsyncMock(return_value=mock_conn)
|
||||
|
|
@ -109,6 +123,8 @@ class TestAdaptersEditForm:
|
|||
context = call_args.kwargs.get("context", call_args[1].get("context"))
|
||||
assert context["adapter"]["name"] == "nws"
|
||||
assert context["adapter"]["settings"]["contact_email"] == "test@example.com"
|
||||
# Verify fields are generated
|
||||
assert "fields" in context
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_adapters_edit_nonexistent_returns_404(self):
|
||||
|
|
@ -167,6 +183,7 @@ class TestAdaptersEditSubmit:
|
|||
"settings": {"contact_email": "old@example.com", "region": {"north": 49, "south": 24, "east": -66, "west": -125}},
|
||||
"paused_at": None,
|
||||
"updated_at": None,
|
||||
"last_error": None,
|
||||
}
|
||||
mock_conn.execute = AsyncMock()
|
||||
|
||||
|
|
@ -185,17 +202,17 @@ class TestAdaptersEditSubmit:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_adapters_edit_invalid_cadence_shows_error(self):
|
||||
"""POST /adapters/nws with cadence_s=30 shows error, no DB update."""
|
||||
"""POST /adapters/nws with cadence_s=5 shows error, no DB update."""
|
||||
from central.gui.routes import adapters_edit_submit
|
||||
|
||||
mock_request = MagicMock()
|
||||
mock_request.state.operator = MagicMock(id=1, username="testop")
|
||||
mock_request.state.csrf_token = "test_csrf_token"
|
||||
|
||||
mock_form = MagicMock()
|
||||
mock_request.state.csrf_token = "test_csrf_token"
|
||||
mock_form.get.side_effect = lambda k, d="": {
|
||||
"csrf_token": "test_csrf_token",
|
||||
"cadence_s": "30",
|
||||
"cadence_s": "5",
|
||||
"contact_email": "test@example.com",
|
||||
"region_north": "49.0",
|
||||
"region_south": "24.0",
|
||||
|
|
@ -215,10 +232,10 @@ class TestAdaptersEditSubmit:
|
|||
"settings": {"contact_email": "test@example.com", "region": {"north": 49, "south": 24, "east": -66, "west": -125}},
|
||||
"paused_at": None,
|
||||
"updated_at": None,
|
||||
"last_error": None,
|
||||
},
|
||||
{"map_tile_url": None, "map_attribution": None}, # system settings for re-render
|
||||
]
|
||||
mock_conn.fetch.return_value = []
|
||||
|
||||
mock_pool = MagicMock()
|
||||
mock_pool.acquire.return_value.__aenter__ = AsyncMock(return_value=mock_conn)
|
||||
|
|
@ -237,117 +254,7 @@ class TestAdaptersEditSubmit:
|
|||
call_args = mock_templates.TemplateResponse.call_args
|
||||
context = call_args.kwargs.get("context", call_args[1].get("context"))
|
||||
assert "cadence_s" in context["errors"]
|
||||
assert "60" in context["errors"]["cadence_s"] or "3600" in context["errors"]["cadence_s"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_adapters_edit_firms_unknown_api_key_shows_error(self):
|
||||
"""POST /adapters/firms with unknown api_key_alias shows error."""
|
||||
from central.gui.routes import adapters_edit_submit
|
||||
|
||||
mock_request = MagicMock()
|
||||
mock_request.state.operator = MagicMock(id=1, username="testop")
|
||||
|
||||
mock_form = MagicMock()
|
||||
mock_request.state.csrf_token = "test_csrf_token"
|
||||
mock_form.get.side_effect = lambda k, d="": {
|
||||
"csrf_token": "test_csrf_token",
|
||||
"cadence_s": "300",
|
||||
"api_key_alias": "nonexistent_key",
|
||||
"region_north": "49.5",
|
||||
"region_south": "31.0",
|
||||
"region_east": "-102.0",
|
||||
"region_west": "-124.5",
|
||||
}.get(k, d)
|
||||
mock_form.getlist.return_value = ["VIIRS_SNPP_NRT"]
|
||||
mock_form.__contains__ = lambda self, k: k == "enabled"
|
||||
mock_request.form = AsyncMock(return_value=mock_form)
|
||||
|
||||
mock_conn = AsyncMock()
|
||||
mock_conn.fetchrow.side_effect = [
|
||||
{ # First call: get adapter
|
||||
"name": "firms",
|
||||
"enabled": True,
|
||||
"cadence_s": 300,
|
||||
"settings": {"api_key_alias": "firms", "satellites": ["VIIRS_SNPP_NRT"], "region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}},
|
||||
"paused_at": None,
|
||||
"updated_at": None,
|
||||
},
|
||||
None, # Second call: check api_key exists - returns None
|
||||
{"map_tile_url": None, "map_attribution": None}, # system settings for re-render
|
||||
]
|
||||
mock_conn.fetch.return_value = []
|
||||
|
||||
mock_pool = MagicMock()
|
||||
mock_pool.acquire.return_value.__aenter__ = AsyncMock(return_value=mock_conn)
|
||||
mock_pool.acquire.return_value.__aexit__ = AsyncMock(return_value=None)
|
||||
|
||||
mock_templates = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_templates.TemplateResponse.return_value = mock_response
|
||||
|
||||
with patch("central.gui.routes._get_templates", return_value=mock_templates):
|
||||
with patch("central.gui.routes.get_pool", return_value=mock_pool):
|
||||
result = await adapters_edit_submit(mock_request, "firms")
|
||||
|
||||
call_args = mock_templates.TemplateResponse.call_args
|
||||
context = call_args.kwargs.get("context", call_args[1].get("context"))
|
||||
assert "api_key_alias" in context["errors"]
|
||||
assert "nonexistent_key" in context["errors"]["api_key_alias"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_adapters_edit_usgs_unknown_feed_shows_error(self):
|
||||
"""POST /adapters/usgs_quake with unknown feed shows error."""
|
||||
from central.gui.routes import adapters_edit_submit
|
||||
|
||||
mock_request = MagicMock()
|
||||
mock_request.state.operator = MagicMock(id=1, username="testop")
|
||||
|
||||
mock_form = MagicMock()
|
||||
mock_request.state.csrf_token = "test_csrf_token"
|
||||
mock_form.get.side_effect = lambda k, d="": {
|
||||
"csrf_token": "test_csrf_token",
|
||||
"cadence_s": "120",
|
||||
"feed": "invalid_feed",
|
||||
"region_north": "49.0",
|
||||
"region_south": "24.0",
|
||||
"region_east": "-66.0",
|
||||
"region_west": "-125.0",
|
||||
}.get(k, d)
|
||||
mock_form.getlist.return_value = []
|
||||
mock_form.__contains__ = lambda self, k: k == "enabled"
|
||||
mock_request.form = AsyncMock(return_value=mock_form)
|
||||
|
||||
mock_conn = AsyncMock()
|
||||
mock_conn.fetchrow.side_effect = [
|
||||
{
|
||||
"name": "usgs_quake",
|
||||
"enabled": True,
|
||||
"cadence_s": 120,
|
||||
"settings": {"feed": "all_hour", "region": {"north": 49, "south": 24, "east": -66, "west": -125}},
|
||||
"paused_at": None,
|
||||
"updated_at": None,
|
||||
},
|
||||
{"map_tile_url": None, "map_attribution": None}, # system settings for re-render
|
||||
]
|
||||
mock_conn.fetch.return_value = []
|
||||
|
||||
mock_pool = MagicMock()
|
||||
mock_pool.acquire.return_value.__aenter__ = AsyncMock(return_value=mock_conn)
|
||||
mock_pool.acquire.return_value.__aexit__ = AsyncMock(return_value=None)
|
||||
|
||||
mock_templates = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_templates.TemplateResponse.return_value = mock_response
|
||||
|
||||
with patch("central.gui.routes._get_templates", return_value=mock_templates):
|
||||
with patch("central.gui.routes.get_pool", return_value=mock_pool):
|
||||
result = await adapters_edit_submit(mock_request, "usgs_quake")
|
||||
|
||||
call_args = mock_templates.TemplateResponse.call_args
|
||||
context = call_args.kwargs.get("context", call_args[1].get("context"))
|
||||
assert "feed" in context["errors"]
|
||||
assert "10" in context["errors"]["cadence_s"]
|
||||
|
||||
|
||||
class TestAdaptersAudit:
|
||||
|
|
@ -384,6 +291,7 @@ class TestAdaptersAudit:
|
|||
"settings": {"contact_email": "old@example.com", "region": {"north": 49, "south": 24, "east": -66, "west": -125}},
|
||||
"paused_at": None,
|
||||
"updated_at": None,
|
||||
"last_error": None,
|
||||
}
|
||||
mock_conn.execute = AsyncMock()
|
||||
|
||||
|
|
@ -407,8 +315,6 @@ class TestAdaptersAudit:
|
|||
assert captured_audit["target"] == "nws"
|
||||
assert captured_audit["before"]["cadence_s"] == 60
|
||||
assert captured_audit["after"]["cadence_s"] == 120
|
||||
assert captured_audit["before"]["settings"]["contact_email"] == "old@example.com"
|
||||
assert captured_audit["after"]["settings"]["contact_email"] == "new@example.com"
|
||||
|
||||
|
||||
class TestAdaptersJsonbRegression:
|
||||
|
|
@ -449,6 +355,7 @@ class TestAdaptersJsonbRegression:
|
|||
"settings": {"contact_email": "old@example.com", "region": {"north": 49, "south": 24, "east": -66, "west": -125}}, # dict, as asyncpg returns
|
||||
"paused_at": None,
|
||||
"updated_at": None,
|
||||
"last_error": None,
|
||||
}
|
||||
mock_conn.execute = AsyncMock()
|
||||
|
||||
|
|
@ -468,7 +375,6 @@ class TestAdaptersJsonbRegression:
|
|||
# CRITICAL: settings must be a dict, NOT a string
|
||||
# If json.dumps() was called, this would be a str like {contact_email: ...}
|
||||
assert isinstance(settings_arg, dict), f"settings should be dict, got {type(settings_arg)}: {settings_arg}"
|
||||
assert settings_arg["contact_email"] == "test@example.com"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_audit_before_after_passed_as_dict(self):
|
||||
|
|
@ -501,6 +407,7 @@ class TestAdaptersJsonbRegression:
|
|||
"settings": {"contact_email": "old@example.com", "region": {"north": 49, "south": 24, "east": -66, "west": -125}}, # dict
|
||||
"paused_at": None,
|
||||
"updated_at": None,
|
||||
"last_error": None,
|
||||
}
|
||||
mock_conn.execute = AsyncMock()
|
||||
|
||||
|
|
@ -523,3 +430,47 @@ class TestAdaptersJsonbRegression:
|
|||
assert isinstance(captured_audit["after"], dict), f"after should be dict, got {type(captured_audit['after'])}"
|
||||
assert isinstance(captured_audit["before"]["settings"], dict), "before.settings should be dict"
|
||||
assert isinstance(captured_audit["after"]["settings"], dict), "after.settings should be dict"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_adapters_edit_fetches_api_keys_into_context(self):
|
||||
"""GET /adapters/firms includes api_keys from database in context."""
|
||||
from central.gui.routes import adapters_edit_form
|
||||
|
||||
mock_request = MagicMock()
|
||||
mock_request.state.operator = MagicMock(id=1, username="testop")
|
||||
mock_request.state.csrf_token = "test_csrf_token"
|
||||
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.fetchrow = AsyncMock(side_effect=[
|
||||
# Adapter row
|
||||
{"name": "firms", "enabled": True, "cadence_s": 300, "settings": {},
|
||||
"paused_at": None, "updated_at": None, "last_error": None},
|
||||
# System row
|
||||
{"map_tile_url": "https://tile.example.com", "map_attribution": "Test"},
|
||||
])
|
||||
mock_conn.fetch = AsyncMock(return_value=[
|
||||
{"alias": "firms_key"},
|
||||
{"alias": "other_key"},
|
||||
])
|
||||
mock_conn.__aenter__ = AsyncMock(return_value=mock_conn)
|
||||
mock_conn.__aexit__ = AsyncMock()
|
||||
|
||||
mock_pool = MagicMock()
|
||||
mock_pool.acquire = MagicMock(return_value=mock_conn)
|
||||
|
||||
mock_templates = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_templates.TemplateResponse.return_value = mock_response
|
||||
|
||||
with patch("central.gui.routes._get_templates", return_value=mock_templates):
|
||||
with patch("central.gui.routes.get_pool", return_value=mock_pool):
|
||||
result = await adapters_edit_form(mock_request, "firms")
|
||||
|
||||
call_args = mock_templates.TemplateResponse.call_args
|
||||
context = call_args.kwargs.get("context", call_args[1].get("context"))
|
||||
|
||||
assert "api_keys" in context
|
||||
assert len(context["api_keys"]) == 2
|
||||
assert context["api_keys"][0]["alias"] == "firms_key"
|
||||
assert context["api_keys"][1]["alias"] == "other_key"
|
||||
|
||||
|
|
|
|||
|
|
@ -29,9 +29,9 @@ class TestConsumerNaming:
|
|||
class TestStreamsConfiguration:
|
||||
"""Test streams configuration."""
|
||||
|
||||
def test_streams_list_has_three_entries(self):
|
||||
"""STREAMS list has three event-bearing streams."""
|
||||
assert len(STREAMS) == 3
|
||||
def test_streams_list_has_four_entries(self):
|
||||
"""STREAMS list has four event-bearing streams."""
|
||||
assert len(STREAMS) == 4
|
||||
|
||||
def test_streams_contains_central_wx(self):
|
||||
"""STREAMS contains CENTRAL_WX with correct filter."""
|
||||
|
|
@ -45,6 +45,10 @@ class TestStreamsConfiguration:
|
|||
"""STREAMS contains CENTRAL_QUAKE with correct filter."""
|
||||
assert ("CENTRAL_QUAKE", "central.quake.>") in STREAMS
|
||||
|
||||
def test_streams_contains_central_space(self):
|
||||
"""STREAMS contains CENTRAL_SPACE with correct filter."""
|
||||
assert ("CENTRAL_SPACE", "central.space.>") in STREAMS
|
||||
|
||||
def test_streams_excludes_central_meta(self):
|
||||
"""STREAMS does not contain CENTRAL_META (status messages only)."""
|
||||
stream_names = [s[0] for s in STREAMS]
|
||||
|
|
|
|||
|
|
@ -205,7 +205,7 @@ class TestDashboardStreamsIsolation:
|
|||
call_args = mock_templates.TemplateResponse.call_args
|
||||
context = call_args.kwargs.get("context", call_args[1].get("context"))
|
||||
streams = context["streams"]
|
||||
assert len(streams) == 4
|
||||
assert len(streams) == 5
|
||||
fire_stream = next(s for s in streams if s["name"] == "CENTRAL_FIRE")
|
||||
assert fire_stream.get("error") == "unavailable"
|
||||
wx_stream = next(s for s in streams if s["name"] == "CENTRAL_WX")
|
||||
|
|
|
|||
238
tests/test_form_descriptors.py
Normal file
238
tests/test_form_descriptors.py
Normal file
|
|
@ -0,0 +1,238 @@
|
|||
"""Tests for form_descriptors module."""
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
|
||||
from central.gui.form_descriptors import describe_fields, FieldDescriptor, _type_to_widget_and_options
|
||||
from central.config_models import RegionConfig
|
||||
|
||||
|
||||
class SimpleSettings(BaseModel):
|
||||
"""Simple settings model for testing."""
|
||||
name: str
|
||||
count: int
|
||||
enabled: bool
|
||||
|
||||
|
||||
class SettingsWithOptional(BaseModel):
|
||||
"""Settings with optional fields."""
|
||||
required_field: str
|
||||
optional_field: Optional[str] = None
|
||||
with_default: str = "default_value"
|
||||
|
||||
|
||||
class SettingsWithList(BaseModel):
|
||||
"""Settings with list field."""
|
||||
tags: list[str]
|
||||
|
||||
|
||||
class SettingsWithRegion(BaseModel):
|
||||
"""Settings with region config."""
|
||||
region: Optional[RegionConfig] = None
|
||||
|
||||
|
||||
class TestTypeToWidget:
|
||||
"""Tests for _type_to_widget_and_options function."""
|
||||
|
||||
def test_str_maps_to_text(self):
|
||||
assert _type_to_widget_and_options("field", str) == ("text", None)
|
||||
|
||||
def test_int_maps_to_number(self):
|
||||
assert _type_to_widget_and_options("field", int) == ("number", None)
|
||||
|
||||
def test_bool_maps_to_checkbox(self):
|
||||
assert _type_to_widget_and_options("field", bool) == ("checkbox", None)
|
||||
|
||||
def test_list_str_maps_to_csv(self):
|
||||
assert _type_to_widget_and_options("field", list[str]) == ("csv", None)
|
||||
|
||||
def test_region_config_maps_to_region(self):
|
||||
assert _type_to_widget_and_options("field", RegionConfig) == ("region", None)
|
||||
|
||||
def test_optional_region_maps_to_region(self):
|
||||
assert _type_to_widget_and_options("field", Optional[RegionConfig]) == ("region", None)
|
||||
|
||||
def test_optional_str_maps_to_text(self):
|
||||
"""Optional[str] should map to text widget."""
|
||||
assert _type_to_widget_and_options("field", Optional[str]) == ("text", None)
|
||||
|
||||
def test_optional_int_maps_to_number(self):
|
||||
"""Optional[int] should map to number widget."""
|
||||
assert _type_to_widget_and_options("field", Optional[int]) == ("number", None)
|
||||
|
||||
def test_unsupported_type_raises(self):
|
||||
class CustomType:
|
||||
pass
|
||||
with pytest.raises(NotImplementedError):
|
||||
_type_to_widget_and_options("field", CustomType)
|
||||
|
||||
|
||||
class TestDescribeFields:
|
||||
"""Tests for describe_fields function."""
|
||||
|
||||
def test_simple_model_fields(self):
|
||||
"""describe_fields returns correct descriptors for simple model."""
|
||||
fields = describe_fields(SimpleSettings, {"name": "test", "count": 5, "enabled": True})
|
||||
|
||||
assert len(fields) == 3
|
||||
|
||||
name_field = next(f for f in fields if f.name == "name")
|
||||
assert name_field.label == "Name"
|
||||
assert name_field.widget == "text"
|
||||
assert name_field.current_value == "test"
|
||||
|
||||
count_field = next(f for f in fields if f.name == "count")
|
||||
assert count_field.label == "Count"
|
||||
assert count_field.widget == "number"
|
||||
assert count_field.current_value == 5
|
||||
|
||||
enabled_field = next(f for f in fields if f.name == "enabled")
|
||||
assert enabled_field.label == "Enabled"
|
||||
assert enabled_field.widget == "checkbox"
|
||||
assert enabled_field.current_value is True
|
||||
|
||||
def test_uses_current_values(self):
|
||||
"""Current values from dict are used."""
|
||||
fields = describe_fields(SimpleSettings, {"name": "current_name", "count": 42, "enabled": False})
|
||||
|
||||
name_field = next(f for f in fields if f.name == "name")
|
||||
assert name_field.current_value == "current_name"
|
||||
|
||||
count_field = next(f for f in fields if f.name == "count")
|
||||
assert count_field.current_value == 42
|
||||
|
||||
def test_missing_values_use_defaults(self):
|
||||
"""Missing values fall back to model defaults."""
|
||||
fields = describe_fields(SettingsWithOptional, {"required_field": "value"})
|
||||
|
||||
optional_field = next(f for f in fields if f.name == "optional_field")
|
||||
assert optional_field.current_value is None
|
||||
assert optional_field.widget == "text" # Optional[str] -> text
|
||||
|
||||
default_field = next(f for f in fields if f.name == "with_default")
|
||||
assert default_field.current_value == "default_value"
|
||||
|
||||
def test_list_field_returns_csv_widget(self):
|
||||
"""List[str] fields get csv widget."""
|
||||
fields = describe_fields(SettingsWithList, {"tags": ["a", "b", "c"]})
|
||||
|
||||
tags_field = next(f for f in fields if f.name == "tags")
|
||||
assert tags_field.widget == "csv"
|
||||
assert tags_field.current_value == ["a", "b", "c"]
|
||||
|
||||
def test_region_field_returns_region_widget(self):
|
||||
"""RegionConfig fields get region widget."""
|
||||
fields = describe_fields(SettingsWithRegion, {
|
||||
"region": {"north": 50.0, "south": 40.0, "east": -100.0, "west": -120.0}
|
||||
})
|
||||
|
||||
region_field = next(f for f in fields if f.name == "region")
|
||||
assert region_field.widget == "region"
|
||||
|
||||
def test_empty_current_dict(self):
|
||||
"""Works with empty current values dict."""
|
||||
fields = describe_fields(SettingsWithOptional, {})
|
||||
|
||||
required_field = next(f for f in fields if f.name == "required_field")
|
||||
assert required_field.current_value is None
|
||||
assert required_field.widget == "text"
|
||||
|
||||
def test_field_descriptor_attributes(self):
|
||||
"""FieldDescriptor has all expected attributes."""
|
||||
fields = describe_fields(SimpleSettings, {"name": "test", "count": 1, "enabled": True})
|
||||
field = fields[0]
|
||||
|
||||
assert hasattr(field, "name")
|
||||
assert hasattr(field, "label")
|
||||
assert hasattr(field, "widget")
|
||||
assert hasattr(field, "current_value")
|
||||
assert hasattr(field, "default")
|
||||
assert hasattr(field, "description")
|
||||
assert hasattr(field, "required")
|
||||
|
||||
|
||||
class TestRealAdapterSchemas:
|
||||
"""Test with actual adapter settings schemas."""
|
||||
|
||||
def test_nws_settings(self):
|
||||
"""NWSSettings generates correct field descriptors."""
|
||||
from central.adapters.nws import NWSSettings
|
||||
|
||||
fields = describe_fields(NWSSettings, {"contact_email": "test@example.com"})
|
||||
|
||||
assert len(fields) >= 1
|
||||
email_field = next(f for f in fields if f.name == "contact_email")
|
||||
assert email_field.widget == "text"
|
||||
assert email_field.current_value == "test@example.com"
|
||||
|
||||
def test_firms_settings(self):
|
||||
"""FIRMSSettings generates correct field descriptors."""
|
||||
from central.adapters.firms import FIRMSSettings
|
||||
|
||||
fields = describe_fields(FIRMSSettings, {
|
||||
"api_key_alias": "firms_key",
|
||||
"satellites": ["VIIRS_SNPP_NRT"]
|
||||
})
|
||||
|
||||
key_field = next(f for f in fields if f.name == "api_key_alias")
|
||||
assert key_field.widget == "text"
|
||||
|
||||
sat_field = next(f for f in fields if f.name == "satellites")
|
||||
assert sat_field.widget == "checkboxes"
|
||||
assert sat_field.current_value == ["VIIRS_SNPP_NRT"]
|
||||
assert sat_field.options is not None
|
||||
assert "VIIRS_SNPP_NRT" in sat_field.options
|
||||
|
||||
def test_usgs_quake_settings(self):
|
||||
"""USGSQuakeSettings generates correct field descriptors."""
|
||||
from central.adapters.usgs_quake import USGSQuakeSettings
|
||||
|
||||
fields = describe_fields(USGSQuakeSettings, {"feed": "all_hour"})
|
||||
|
||||
feed_field = next(f for f in fields if f.name == "feed")
|
||||
assert feed_field.widget == "select"
|
||||
assert feed_field.current_value == "all_hour"
|
||||
assert feed_field.options is not None
|
||||
assert "all_hour" in feed_field.options
|
||||
assert "all_day" in feed_field.options
|
||||
|
||||
def test_all_adapters_have_region_field(self):
|
||||
"""All adapter settings schemas include region field."""
|
||||
from central.adapters.nws import NWSSettings
|
||||
from central.adapters.firms import FIRMSSettings
|
||||
from central.adapters.usgs_quake import USGSQuakeSettings
|
||||
|
||||
for schema in [NWSSettings, FIRMSSettings, USGSQuakeSettings]:
|
||||
fields = describe_fields(schema, {})
|
||||
region_field = next((f for f in fields if f.name == "region"), None)
|
||||
assert region_field is not None, f"{schema.__name__} should have region field"
|
||||
assert region_field.widget == "region"
|
||||
|
||||
class TestLiteralTypes:
|
||||
"""Tests for Literal type support."""
|
||||
|
||||
def test_literal_maps_to_select(self):
|
||||
"""Literal type maps to select widget with options."""
|
||||
from typing import Literal
|
||||
|
||||
widget, options = _type_to_widget_and_options("field", Literal["a", "b", "c"])
|
||||
assert widget == "select"
|
||||
assert options == ["a", "b", "c"]
|
||||
|
||||
def test_list_literal_maps_to_checkboxes(self):
|
||||
"""list[Literal] maps to checkboxes widget with options."""
|
||||
from typing import Literal
|
||||
|
||||
widget, options = _type_to_widget_and_options("field", list[Literal["x", "y", "z"]])
|
||||
assert widget == "checkboxes"
|
||||
assert options == ["x", "y", "z"]
|
||||
|
||||
def test_optional_literal_maps_to_select(self):
|
||||
"""Optional[Literal] maps to select widget."""
|
||||
from typing import Literal, Optional
|
||||
|
||||
widget, options = _type_to_widget_and_options("field", Optional[Literal["one", "two"]])
|
||||
assert widget == "select"
|
||||
assert options == ["one", "two"]
|
||||
|
||||
599
tests/test_inciweb.py
Normal file
599
tests/test_inciweb.py
Normal file
|
|
@ -0,0 +1,599 @@
|
|||
"""Tests for InciWeb adapter."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from central.config_models import AdapterConfig
|
||||
from central.models import Event, Geo
|
||||
|
||||
|
||||
# Real RSS snippet from InciWeb (frozen fixture)
|
||||
SAMPLE_RSS_CONTENT = """<?xml version="1.0" encoding="utf-8"?>
|
||||
<rss xmlns:dc="http://purl.org/dc/elements/1.1/" version="2.0" xml:base="http://inciweb.wildfire.gov/">
|
||||
<channel>
|
||||
<title>InciWeb</title>
|
||||
<link>http://inciweb.wildfire.gov/</link>
|
||||
<description>Inciweb Fire Incidents</description>
|
||||
<language>en</language>
|
||||
<item>
|
||||
<title>MNMNS Stewart Trail</title>
|
||||
<link>http://inciweb.wildfire.gov/incident-information/mnmns-stewart-trail</link>
|
||||
<description>Last updated: 2026-05-18
|
||||
|
||||
---
|
||||
|
||||
The type of incident is Wildfire and involves the following unit(s) Minnesota Department of Natural Resources.
|
||||
|
||||
---
|
||||
|
||||
State: Minnesota
|
||||
|
||||
---
|
||||
|
||||
Coordinates:
|
||||
|
||||
Latitude: 47° 3 17 Longitude: 91° 38 6
|
||||
|
||||
---
|
||||
|
||||
NOTE: All fire perimeters and points are approximations.
|
||||
|
||||
---
|
||||
|
||||
Incident Overview: The Stewart Trail Fire was detected during the afternoon hours on Friday, May 15, 2026.&nbsp;A temporary flight restriction (TFR) is in place.</description>
|
||||
<pubDate>Fri, 15 May 2026 08:48:11 EDT</pubDate>
|
||||
<dc:creator>llangeberg</dc:creator>
|
||||
<guid isPermaLink="false">327828</guid>
|
||||
</item>
|
||||
<item>
|
||||
<title>CACNP Santa Rosa Island Fire</title>
|
||||
<link>http://inciweb.wildfire.gov/incident-information/cacnp-santa-rosa-island-fire</link>
|
||||
<description>Last updated: 2026-05-18
|
||||
|
||||
---
|
||||
|
||||
The type of incident is Wildfire and involves the following unit(s) Channel Islands National Park.
|
||||
|
||||
---
|
||||
|
||||
State: California
|
||||
|
||||
---
|
||||
|
||||
Coordinates:
|
||||
|
||||
Latitude: 33° 55 2 Longitude: 120° 5 10
|
||||
|
||||
---
|
||||
|
||||
NOTE: All fire perimeters and points are approximations.
|
||||
|
||||
---
|
||||
|
||||
Incident Overview: On Friday, May 15, 2026, an aircraft flying over Santa Rosa Island in Channel Islands National Park reported a wildfire.<br><p>This is a <strong>full-suppression</strong> human-caused wildfire and is under investigation.</p>&nbsp;</description>
|
||||
<pubDate>Sat, 16 May 2026 12:09:07 EDT</pubDate>
|
||||
<dc:creator>mtheune</dc:creator>
|
||||
<guid isPermaLink="false">327838</guid>
|
||||
</item>
|
||||
<item>
|
||||
<title>Some Fire Without Coordinates</title>
|
||||
<link>http://inciweb.wildfire.gov/incident-information/no-coords-fire</link>
|
||||
<description>Last updated: 2026-05-18
|
||||
|
||||
---
|
||||
|
||||
The type of incident is Wildfire.
|
||||
|
||||
---
|
||||
|
||||
State: Unknown State
|
||||
|
||||
---
|
||||
|
||||
Incident Overview: This is a test incident without coordinates.</description>
|
||||
<pubDate>Mon, 18 May 2026 09:00:00 EDT</pubDate>
|
||||
<dc:creator>test</dc:creator>
|
||||
<guid isPermaLink="false">999999</guid>
|
||||
</item>
|
||||
<item>
|
||||
<title>Florida Fire Outside Bbox</title>
|
||||
<link>http://inciweb.wildfire.gov/incident-information/florida-fire</link>
|
||||
<description>Last updated: 2026-05-18
|
||||
|
||||
---
|
||||
|
||||
State: Florida
|
||||
|
||||
---
|
||||
|
||||
Coordinates:
|
||||
|
||||
Latitude: 26° 0 0 Longitude: 80° 0 0
|
||||
|
||||
---
|
||||
|
||||
Incident Overview: This fire is in Florida, outside the CONUS west bbox.</description>
|
||||
<pubDate>Mon, 18 May 2026 10:00:00 EDT</pubDate>
|
||||
<dc:creator>test</dc:creator>
|
||||
<guid isPermaLink="false">888888</guid>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>"""
|
||||
|
||||
|
||||
class TestInciWebHelpers:
|
||||
"""Tests for InciWeb helper functions."""
|
||||
|
||||
def test_parse_coordinates_from_description(self):
|
||||
"""Parse coordinates from description text."""
|
||||
from central.adapters.inciweb import parse_coordinates_from_description
|
||||
|
||||
description = """Coordinates:
|
||||
|
||||
Latitude: 47° 3 17 Longitude: 91° 38 6"""
|
||||
|
||||
result = parse_coordinates_from_description(description)
|
||||
assert result is not None
|
||||
lon, lat = result
|
||||
# 47° 3' 17" = 47.054722...
|
||||
assert 47.0 < lat < 47.1
|
||||
# 91° 38' 6" = -91.635 (west longitude)
|
||||
assert -92.0 < lon < -91.0
|
||||
|
||||
def test_parse_coordinates_no_match(self):
|
||||
"""No coordinates in description returns None."""
|
||||
from central.adapters.inciweb import parse_coordinates_from_description
|
||||
|
||||
result = parse_coordinates_from_description("No coordinates here")
|
||||
assert result is None
|
||||
|
||||
def test_parse_state_from_description(self):
|
||||
"""Parse state name and return 2-letter code."""
|
||||
from central.adapters.inciweb import parse_state_from_description
|
||||
|
||||
description = """---
|
||||
|
||||
State: Minnesota
|
||||
|
||||
---"""
|
||||
assert parse_state_from_description(description) == "MN"
|
||||
|
||||
def test_parse_state_from_description_new_mexico(self):
|
||||
"""Parse multi-word state name."""
|
||||
from central.adapters.inciweb import parse_state_from_description
|
||||
|
||||
description = """State: New Mexico
|
||||
|
||||
---"""
|
||||
assert parse_state_from_description(description) == "NM"
|
||||
|
||||
def test_parse_state_from_description_no_match(self):
|
||||
"""Unknown state name returns None."""
|
||||
from central.adapters.inciweb import parse_state_from_description
|
||||
|
||||
description = """State: Unknown State
|
||||
|
||||
---"""
|
||||
assert parse_state_from_description(description) is None
|
||||
|
||||
def test_strip_html(self):
|
||||
"""HTML tags are stripped, entities decoded."""
|
||||
from central.adapters.inciweb import strip_html
|
||||
|
||||
html = "This is &nbsp;a <strong>test</strong> with <br>line breaks."
|
||||
result = strip_html(html)
|
||||
assert "<" not in result
|
||||
assert ">" not in result
|
||||
assert " " not in result
|
||||
assert "&" not in result
|
||||
assert "test" in result
|
||||
|
||||
|
||||
class TestInciWebAdapter:
|
||||
"""Tests for InciWeb adapter."""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_config(self) -> AdapterConfig:
|
||||
return AdapterConfig(
|
||||
name="inciweb",
|
||||
enabled=True,
|
||||
cadence_s=600,
|
||||
settings={
|
||||
"region": {"north": 49.0, "south": 31.0, "east": -102.0, "west": -124.0}
|
||||
},
|
||||
updated_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
@pytest.fixture
|
||||
def mock_config_no_region(self) -> AdapterConfig:
|
||||
return AdapterConfig(
|
||||
name="inciweb",
|
||||
enabled=True,
|
||||
cadence_s=600,
|
||||
settings={},
|
||||
updated_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
@pytest.fixture
|
||||
def mock_config_store(self) -> MagicMock:
|
||||
return MagicMock()
|
||||
|
||||
@pytest.fixture
|
||||
def cursor_db_path(self, tmp_path: Path) -> Path:
|
||||
return tmp_path / "cursors.db"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_normalization_with_georss_point(
|
||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
||||
):
|
||||
"""Items with coordinates are correctly normalized."""
|
||||
from central.adapters.inciweb import InciWebAdapter
|
||||
|
||||
adapter = InciWebAdapter(mock_config, mock_config_store, cursor_db_path)
|
||||
await adapter.startup()
|
||||
|
||||
mock_response = AsyncMock()
|
||||
mock_response.raise_for_status = MagicMock()
|
||||
mock_response.text = AsyncMock(return_value=SAMPLE_RSS_CONTENT)
|
||||
|
||||
with patch.object(adapter._session, "get", return_value=AsyncMock(__aenter__=AsyncMock(return_value=mock_response), __aexit__=AsyncMock())):
|
||||
events = [e async for e in adapter.poll()]
|
||||
|
||||
await adapter.shutdown()
|
||||
|
||||
# Bbox is west=-124, east=-102 (CONUS west)
|
||||
# Minnesota at -91 longitude is OUTSIDE bbox (east of -102)
|
||||
# California at -120 longitude is INSIDE bbox
|
||||
# Florida at -80 longitude is OUTSIDE bbox
|
||||
# Unknown state without coords passes through
|
||||
assert len(events) == 2
|
||||
|
||||
# Check California event
|
||||
ca_event = next(e for e in events if e.data["guid"] == "327838")
|
||||
assert ca_event.id == "327838"
|
||||
assert ca_event.adapter == "inciweb"
|
||||
assert ca_event.category == "fire.narrative.inciweb"
|
||||
assert ca_event.severity == 0
|
||||
assert ca_event.geo.primary_region == "US-CA"
|
||||
assert ca_event.geo.centroid is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_normalization_without_georss_point(
|
||||
self, mock_config_no_region: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
||||
):
|
||||
"""Items without coordinates have centroid=None."""
|
||||
from central.adapters.inciweb import InciWebAdapter
|
||||
|
||||
adapter = InciWebAdapter(mock_config_no_region, mock_config_store, cursor_db_path)
|
||||
await adapter.startup()
|
||||
|
||||
mock_response = AsyncMock()
|
||||
mock_response.raise_for_status = MagicMock()
|
||||
mock_response.text = AsyncMock(return_value=SAMPLE_RSS_CONTENT)
|
||||
|
||||
with patch.object(adapter._session, "get", return_value=AsyncMock(__aenter__=AsyncMock(return_value=mock_response), __aexit__=AsyncMock())):
|
||||
events = [e async for e in adapter.poll()]
|
||||
|
||||
await adapter.shutdown()
|
||||
|
||||
# All 4 items pass (no region filter)
|
||||
assert len(events) == 4
|
||||
|
||||
# Check item without coords
|
||||
no_coords_event = next(e for e in events if e.data["guid"] == "999999")
|
||||
assert no_coords_event.geo.centroid is None
|
||||
assert no_coords_event.geo.regions == []
|
||||
assert no_coords_event.geo.primary_region is None
|
||||
|
||||
def test_state_parse_from_title(
|
||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
||||
):
|
||||
"""State parsing from description produces correct region."""
|
||||
from central.adapters.inciweb import parse_state_from_description
|
||||
|
||||
# Test California
|
||||
assert parse_state_from_description("State: California\n") == "CA"
|
||||
# Test Minnesota
|
||||
assert parse_state_from_description("State: Minnesota\n---") == "MN"
|
||||
# Test multi-word
|
||||
assert parse_state_from_description("State: New York\n") == "NY"
|
||||
# Test unknown
|
||||
assert parse_state_from_description("State: Narnia\n") is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_html_stripping(
|
||||
self, mock_config_no_region: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
||||
):
|
||||
"""HTML is stripped from description, raw preserved in description_html."""
|
||||
from central.adapters.inciweb import InciWebAdapter
|
||||
|
||||
adapter = InciWebAdapter(mock_config_no_region, mock_config_store, cursor_db_path)
|
||||
await adapter.startup()
|
||||
|
||||
mock_response = AsyncMock()
|
||||
mock_response.raise_for_status = MagicMock()
|
||||
mock_response.text = AsyncMock(return_value=SAMPLE_RSS_CONTENT)
|
||||
|
||||
with patch.object(adapter._session, "get", return_value=AsyncMock(__aenter__=AsyncMock(return_value=mock_response), __aexit__=AsyncMock())):
|
||||
events = [e async for e in adapter.poll()]
|
||||
|
||||
await adapter.shutdown()
|
||||
|
||||
# California item has HTML tags in description
|
||||
ca_event = next(e for e in events if e.data["guid"] == "327838")
|
||||
|
||||
# Plain text should not have HTML tags
|
||||
assert "<br>" not in ca_event.data["description"]
|
||||
assert "<p>" not in ca_event.data["description"]
|
||||
assert "<strong>" not in ca_event.data["description"]
|
||||
assert " " not in ca_event.data["description"]
|
||||
|
||||
# Raw HTML should be preserved
|
||||
assert "<br>" in ca_event.data["description_html"] or "<br>" in ca_event.data["description_html"]
|
||||
|
||||
def test_subject_for_with_state(
|
||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
||||
):
|
||||
"""subject_for returns correct subject with state."""
|
||||
from central.adapters.inciweb import InciWebAdapter
|
||||
|
||||
adapter = InciWebAdapter(mock_config, mock_config_store, cursor_db_path)
|
||||
|
||||
event = Event(
|
||||
id="test-id",
|
||||
adapter="inciweb",
|
||||
category="fire.narrative.inciweb",
|
||||
time=datetime.now(timezone.utc),
|
||||
severity=0,
|
||||
geo=Geo(primary_region="US-CA"),
|
||||
data={"title": "Test Fire", "description": "Test"},
|
||||
)
|
||||
|
||||
subject = adapter.subject_for(event)
|
||||
assert subject == "central.fire.narrative.inciweb.ca"
|
||||
|
||||
def test_subject_for_without_state(
|
||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
||||
):
|
||||
"""subject_for returns unknown when no state."""
|
||||
from central.adapters.inciweb import InciWebAdapter
|
||||
|
||||
adapter = InciWebAdapter(mock_config, mock_config_store, cursor_db_path)
|
||||
|
||||
event = Event(
|
||||
id="test-id",
|
||||
adapter="inciweb",
|
||||
category="fire.narrative.inciweb",
|
||||
time=datetime.now(timezone.utc),
|
||||
severity=0,
|
||||
geo=Geo(),
|
||||
data={"title": "Test Fire", "description": "Test"},
|
||||
)
|
||||
|
||||
subject = adapter.subject_for(event)
|
||||
assert subject == "central.fire.narrative.inciweb.unknown"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dedup_same_guid(
|
||||
self, mock_config_no_region: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
||||
):
|
||||
"""is_published/mark_published provides dedup functionality."""
|
||||
from central.adapters.inciweb import InciWebAdapter
|
||||
|
||||
adapter = InciWebAdapter(mock_config_no_region, mock_config_store, cursor_db_path)
|
||||
await adapter.startup()
|
||||
|
||||
# Initially not published
|
||||
assert adapter.is_published("327828") is False
|
||||
|
||||
# Mark as published
|
||||
adapter.mark_published("327828")
|
||||
|
||||
# Now it should be published
|
||||
assert adapter.is_published("327828") is True
|
||||
|
||||
await adapter.shutdown()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bbox_filters_point_outside(
|
||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
||||
):
|
||||
"""Items with coords outside bbox are filtered; items without coords pass."""
|
||||
from central.adapters.inciweb import InciWebAdapter
|
||||
|
||||
adapter = InciWebAdapter(mock_config, mock_config_store, cursor_db_path)
|
||||
await adapter.startup()
|
||||
|
||||
mock_response = AsyncMock()
|
||||
mock_response.raise_for_status = MagicMock()
|
||||
mock_response.text = AsyncMock(return_value=SAMPLE_RSS_CONTENT)
|
||||
|
||||
with patch.object(adapter._session, "get", return_value=AsyncMock(__aenter__=AsyncMock(return_value=mock_response), __aexit__=AsyncMock())):
|
||||
events = [e async for e in adapter.poll()]
|
||||
|
||||
await adapter.shutdown()
|
||||
|
||||
# Florida (-80 longitude) should be filtered out
|
||||
guids = {e.data["guid"] for e in events}
|
||||
assert "888888" not in guids # Florida, outside bbox
|
||||
|
||||
# Item without coords should pass through
|
||||
assert "999999" in guids
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_config_region_change(
|
||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
||||
):
|
||||
"""apply_config updates region."""
|
||||
from central.adapters.inciweb import InciWebAdapter
|
||||
|
||||
adapter = InciWebAdapter(mock_config, mock_config_store, cursor_db_path)
|
||||
|
||||
assert adapter.region is not None
|
||||
assert adapter.region.north == 49.0
|
||||
|
||||
new_config = AdapterConfig(
|
||||
name="inciweb",
|
||||
enabled=True,
|
||||
cadence_s=600,
|
||||
settings={
|
||||
"region": {"north": 50.0, "south": 35.0, "east": -100.0, "west": -120.0}
|
||||
},
|
||||
updated_at=datetime.now(timezone.utc),
|
||||
)
|
||||
await adapter.apply_config(new_config)
|
||||
|
||||
assert adapter.region.north == 50.0
|
||||
assert adapter.region.south == 35.0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dedup_in_poll_loop(
|
||||
self, mock_config_no_region: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
||||
):
|
||||
"""Dedup integration: second poll with same items yields zero events."""
|
||||
from central.adapters.inciweb import InciWebAdapter
|
||||
|
||||
adapter = InciWebAdapter(mock_config_no_region, mock_config_store, cursor_db_path)
|
||||
await adapter.startup()
|
||||
|
||||
# Single-item RSS for clarity
|
||||
single_item_rss = """<?xml version="1.0" encoding="utf-8"?>
|
||||
<rss xmlns:dc="http://purl.org/dc/elements/1.1/" version="2.0">
|
||||
<channel>
|
||||
<title>InciWeb</title>
|
||||
<item>
|
||||
<title>Test Fire</title>
|
||||
<link>http://inciweb.wildfire.gov/test</link>
|
||||
<description>State: California</description>
|
||||
<pubDate>Mon, 18 May 2026 09:00:00 EDT</pubDate>
|
||||
<guid isPermaLink="false">DEDUP-TEST-001</guid>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>"""
|
||||
|
||||
def make_mock_response():
|
||||
mock_response = AsyncMock()
|
||||
mock_response.status = 200
|
||||
mock_response.raise_for_status = MagicMock()
|
||||
mock_response.text = AsyncMock(return_value=single_item_rss)
|
||||
mock_response.headers = {"Last-Modified": None, "ETag": None}
|
||||
return mock_response
|
||||
|
||||
# First poll: should yield 1 event
|
||||
with patch.object(
|
||||
adapter._session, "get",
|
||||
return_value=AsyncMock(
|
||||
__aenter__=AsyncMock(return_value=make_mock_response()),
|
||||
__aexit__=AsyncMock()
|
||||
)
|
||||
):
|
||||
events_first = [e async for e in adapter.poll()]
|
||||
|
||||
assert len(events_first) == 1
|
||||
assert events_first[0].data["guid"] == "DEDUP-TEST-001"
|
||||
|
||||
# Verify mark_published was called
|
||||
assert adapter.is_published("DEDUP-TEST-001") is True
|
||||
|
||||
# Second poll: same item should be skipped (dedup)
|
||||
with patch.object(
|
||||
adapter._session, "get",
|
||||
return_value=AsyncMock(
|
||||
__aenter__=AsyncMock(return_value=make_mock_response()),
|
||||
__aexit__=AsyncMock()
|
||||
)
|
||||
):
|
||||
events_second = [e async for e in adapter.poll()]
|
||||
|
||||
assert len(events_second) == 0 # Dedup prevents re-yield
|
||||
|
||||
await adapter.shutdown()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_conditional_304_yields_zero(
|
||||
self, mock_config_no_region: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
||||
):
|
||||
"""HTTP 304 Not Modified returns empty list and yields zero events."""
|
||||
from central.adapters.inciweb import InciWebAdapter
|
||||
|
||||
adapter = InciWebAdapter(mock_config_no_region, mock_config_store, cursor_db_path)
|
||||
await adapter.startup()
|
||||
|
||||
# Mock 304 response
|
||||
mock_response = AsyncMock()
|
||||
mock_response.status = 304
|
||||
mock_response.raise_for_status = MagicMock()
|
||||
|
||||
with patch.object(
|
||||
adapter._session, "get",
|
||||
return_value=AsyncMock(
|
||||
__aenter__=AsyncMock(return_value=mock_response),
|
||||
__aexit__=AsyncMock()
|
||||
)
|
||||
):
|
||||
events = [e async for e in adapter.poll()]
|
||||
|
||||
assert len(events) == 0
|
||||
|
||||
await adapter.shutdown()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_conditional_headers_sent_after_first_poll(
|
||||
self, mock_config_no_region: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
||||
):
|
||||
"""Conditional fetch headers sent on second poll after first captures them."""
|
||||
from central.adapters.inciweb import InciWebAdapter
|
||||
|
||||
adapter = InciWebAdapter(mock_config_no_region, mock_config_store, cursor_db_path)
|
||||
await adapter.startup()
|
||||
|
||||
# First response with Last-Modified and ETag
|
||||
first_response = AsyncMock()
|
||||
first_response.status = 200
|
||||
first_response.raise_for_status = MagicMock()
|
||||
first_response.text = AsyncMock(return_value="""<?xml version="1.0"?>
|
||||
<rss version="2.0"><channel><title>Test</title></channel></rss>""")
|
||||
first_response.headers = {
|
||||
"Last-Modified": "Tue, 19 May 2026 03:00:00 GMT",
|
||||
"ETag": "\"abc123\"",
|
||||
}
|
||||
|
||||
# Track headers sent on second request
|
||||
captured_headers = {}
|
||||
|
||||
def capture_get(*args, **kwargs):
|
||||
captured_headers.update(kwargs.get("headers", {}))
|
||||
second_response = AsyncMock()
|
||||
second_response.status = 304
|
||||
second_response.raise_for_status = MagicMock()
|
||||
return AsyncMock(
|
||||
__aenter__=AsyncMock(return_value=second_response),
|
||||
__aexit__=AsyncMock()
|
||||
)
|
||||
|
||||
# First poll
|
||||
with patch.object(
|
||||
adapter._session, "get",
|
||||
return_value=AsyncMock(
|
||||
__aenter__=AsyncMock(return_value=first_response),
|
||||
__aexit__=AsyncMock()
|
||||
)
|
||||
):
|
||||
[e async for e in adapter.poll()]
|
||||
|
||||
# Verify adapter captured the headers
|
||||
assert adapter._last_modified == "Tue, 19 May 2026 03:00:00 GMT"
|
||||
assert adapter._etag == "\"abc123\""
|
||||
|
||||
# Second poll with header capture
|
||||
with patch.object(adapter._session, "get", side_effect=capture_get):
|
||||
[e async for e in adapter.poll()]
|
||||
|
||||
# Verify conditional headers were sent
|
||||
assert captured_headers.get("If-Modified-Since") == "Tue, 19 May 2026 03:00:00 GMT"
|
||||
assert captured_headers.get("If-None-Match") == "\"abc123\""
|
||||
|
||||
await adapter.shutdown()
|
||||
|
|
@ -21,6 +21,7 @@ class TestRegionPickerInTemplate:
|
|||
|
||||
mock_request = MagicMock()
|
||||
mock_request.state.operator = MagicMock(id=1, username="testop")
|
||||
mock_request.state.csrf_token = "test_csrf"
|
||||
|
||||
mock_conn = AsyncMock()
|
||||
mock_conn.fetchrow.side_effect = [
|
||||
|
|
@ -35,13 +36,13 @@ class TestRegionPickerInTemplate:
|
|||
},
|
||||
"paused_at": None,
|
||||
"updated_at": None,
|
||||
"last_error": None,
|
||||
},
|
||||
{ # System settings row
|
||||
"map_tile_url": "https://tile.example.com/{z}/{x}/{y}.png",
|
||||
"map_attribution": "Test Attribution",
|
||||
},
|
||||
]
|
||||
mock_conn.fetch.return_value = [{"alias": "firms"}]
|
||||
|
||||
mock_pool = MagicMock()
|
||||
mock_pool.acquire.return_value.__aenter__ = AsyncMock(return_value=mock_conn)
|
||||
|
|
@ -80,27 +81,26 @@ class TestRegionValidation:
|
|||
"csrf_token": "test_csrf_token",
|
||||
"cadence_s": "300",
|
||||
"api_key_alias": "firms",
|
||||
"satellites": "VIIRS_SNPP_NRT",
|
||||
"region_north": "45.0",
|
||||
"region_south": "35.0",
|
||||
"region_east": "-100.0",
|
||||
"region_west": "-120.0",
|
||||
}.get(k, d)
|
||||
mock_form.getlist.return_value = ["VIIRS_SNPP_NRT"]
|
||||
mock_form.getlist.return_value = []
|
||||
mock_form.__contains__ = lambda self, k: k == "enabled"
|
||||
mock_request.form = AsyncMock(return_value=mock_form)
|
||||
|
||||
mock_conn = AsyncMock()
|
||||
mock_conn.fetchrow.side_effect = [
|
||||
{ # Adapter row
|
||||
"name": "firms",
|
||||
"enabled": True,
|
||||
"cadence_s": 300,
|
||||
"settings": {"api_key_alias": "firms", "region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}},
|
||||
"paused_at": None,
|
||||
"updated_at": None,
|
||||
},
|
||||
{"id": 1}, # api_key exists check
|
||||
]
|
||||
mock_conn.fetchrow.return_value = {
|
||||
"name": "firms",
|
||||
"enabled": True,
|
||||
"cadence_s": 300,
|
||||
"settings": {"api_key_alias": "firms", "satellites": ["VIIRS_SNPP_NRT"], "region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}},
|
||||
"paused_at": None,
|
||||
"updated_at": None,
|
||||
"last_error": None,
|
||||
}
|
||||
mock_conn.execute = AsyncMock()
|
||||
|
||||
mock_pool = MagicMock()
|
||||
|
|
@ -139,12 +139,13 @@ class TestRegionValidation:
|
|||
"csrf_token": "test_csrf_token",
|
||||
"cadence_s": "300",
|
||||
"api_key_alias": "firms",
|
||||
"satellites": "VIIRS_SNPP_NRT",
|
||||
"region_north": "30.0", # Less than south!
|
||||
"region_south": "35.0",
|
||||
"region_east": "-100.0",
|
||||
"region_west": "-120.0",
|
||||
}.get(k, d)
|
||||
mock_form.getlist.return_value = ["VIIRS_SNPP_NRT"]
|
||||
mock_form.getlist.return_value = []
|
||||
mock_form.__contains__ = lambda self, k: k == "enabled"
|
||||
mock_request.form = AsyncMock(return_value=mock_form)
|
||||
|
||||
|
|
@ -154,14 +155,13 @@ class TestRegionValidation:
|
|||
"name": "firms",
|
||||
"enabled": True,
|
||||
"cadence_s": 300,
|
||||
"settings": {"api_key_alias": "firms", "region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}},
|
||||
"settings": {"api_key_alias": "firms", "satellites": ["VIIRS_SNPP_NRT"], "region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}},
|
||||
"paused_at": None,
|
||||
"updated_at": None,
|
||||
"last_error": None,
|
||||
},
|
||||
{"id": 1}, # api_key exists
|
||||
{"map_tile_url": None, "map_attribution": None}, # system settings for re-render
|
||||
]
|
||||
mock_conn.fetch.return_value = [{"alias": "firms"}]
|
||||
|
||||
mock_pool = MagicMock()
|
||||
mock_pool.acquire.return_value.__aenter__ = AsyncMock(return_value=mock_conn)
|
||||
|
|
@ -195,12 +195,13 @@ class TestRegionValidation:
|
|||
"csrf_token": "test_csrf_token",
|
||||
"cadence_s": "300",
|
||||
"api_key_alias": "firms",
|
||||
"satellites": "VIIRS_SNPP_NRT",
|
||||
"region_north": "45.0",
|
||||
"region_south": "35.0",
|
||||
"region_east": "-130.0", # Less than west!
|
||||
"region_west": "-120.0",
|
||||
}.get(k, d)
|
||||
mock_form.getlist.return_value = ["VIIRS_SNPP_NRT"]
|
||||
mock_form.getlist.return_value = []
|
||||
mock_form.__contains__ = lambda self, k: k == "enabled"
|
||||
mock_request.form = AsyncMock(return_value=mock_form)
|
||||
|
||||
|
|
@ -210,14 +211,13 @@ class TestRegionValidation:
|
|||
"name": "firms",
|
||||
"enabled": True,
|
||||
"cadence_s": 300,
|
||||
"settings": {"api_key_alias": "firms", "region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}},
|
||||
"settings": {"api_key_alias": "firms", "satellites": ["VIIRS_SNPP_NRT"], "region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}},
|
||||
"paused_at": None,
|
||||
"updated_at": None,
|
||||
"last_error": None,
|
||||
},
|
||||
{"id": 1},
|
||||
{"map_tile_url": None, "map_attribution": None},
|
||||
]
|
||||
mock_conn.fetch.return_value = [{"alias": "firms"}]
|
||||
|
||||
mock_pool = MagicMock()
|
||||
mock_pool.acquire.return_value.__aenter__ = AsyncMock(return_value=mock_conn)
|
||||
|
|
@ -251,12 +251,13 @@ class TestRegionValidation:
|
|||
"csrf_token": "test_csrf_token",
|
||||
"cadence_s": "300",
|
||||
"api_key_alias": "firms",
|
||||
"satellites": "VIIRS_SNPP_NRT",
|
||||
"region_north": "95.0", # > 90!
|
||||
"region_south": "35.0",
|
||||
"region_east": "-100.0",
|
||||
"region_west": "-120.0",
|
||||
}.get(k, d)
|
||||
mock_form.getlist.return_value = ["VIIRS_SNPP_NRT"]
|
||||
mock_form.getlist.return_value = []
|
||||
mock_form.__contains__ = lambda self, k: k == "enabled"
|
||||
mock_request.form = AsyncMock(return_value=mock_form)
|
||||
|
||||
|
|
@ -266,14 +267,13 @@ class TestRegionValidation:
|
|||
"name": "firms",
|
||||
"enabled": True,
|
||||
"cadence_s": 300,
|
||||
"settings": {"api_key_alias": "firms", "region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}},
|
||||
"settings": {"api_key_alias": "firms", "satellites": ["VIIRS_SNPP_NRT"], "region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}},
|
||||
"paused_at": None,
|
||||
"updated_at": None,
|
||||
"last_error": None,
|
||||
},
|
||||
{"id": 1},
|
||||
{"map_tile_url": None, "map_attribution": None},
|
||||
]
|
||||
mock_conn.fetch.return_value = [{"alias": "firms"}]
|
||||
|
||||
mock_pool = MagicMock()
|
||||
mock_pool.acquire.return_value.__aenter__ = AsyncMock(return_value=mock_conn)
|
||||
|
|
@ -310,30 +310,30 @@ class TestRegionAuditLog:
|
|||
"csrf_token": "test_csrf_token",
|
||||
"cadence_s": "300",
|
||||
"api_key_alias": "firms",
|
||||
"satellites": "VIIRS_SNPP_NRT",
|
||||
"region_north": "45.0",
|
||||
"region_south": "35.0",
|
||||
"region_east": "-100.0",
|
||||
"region_west": "-120.0",
|
||||
}.get(k, d)
|
||||
mock_form.getlist.return_value = ["VIIRS_SNPP_NRT"]
|
||||
mock_form.getlist.return_value = []
|
||||
mock_form.__contains__ = lambda self, k: k == "enabled"
|
||||
mock_request.form = AsyncMock(return_value=mock_form)
|
||||
|
||||
mock_conn = AsyncMock()
|
||||
mock_conn.fetchrow.side_effect = [
|
||||
{
|
||||
"name": "firms",
|
||||
"enabled": True,
|
||||
"cadence_s": 300,
|
||||
"settings": {
|
||||
"api_key_alias": "firms",
|
||||
"region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}
|
||||
},
|
||||
"paused_at": None,
|
||||
"updated_at": None,
|
||||
mock_conn.fetchrow.return_value = {
|
||||
"name": "firms",
|
||||
"enabled": True,
|
||||
"cadence_s": 300,
|
||||
"settings": {
|
||||
"api_key_alias": "firms",
|
||||
"satellites": ["VIIRS_SNPP_NRT"],
|
||||
"region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}
|
||||
},
|
||||
{"id": 1},
|
||||
]
|
||||
"paused_at": None,
|
||||
"updated_at": None,
|
||||
"last_error": None,
|
||||
}
|
||||
mock_conn.execute = AsyncMock()
|
||||
|
||||
mock_pool = MagicMock()
|
||||
|
|
|
|||
361
tests/test_requires_api_key.py
Normal file
361
tests/test_requires_api_key.py
Normal file
|
|
@ -0,0 +1,361 @@
|
|||
"""Tests for requires_api_key enforcement."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from central.config_models import AdapterConfig
|
||||
|
||||
|
||||
class TestConfigStoreSetAdapterLastError:
|
||||
"""Tests for ConfigStore.set_adapter_last_error method."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_set_adapter_last_error_updates_row(self):
|
||||
"""set_adapter_last_error should update the last_error column."""
|
||||
from central.config_store import ConfigStore
|
||||
|
||||
mock_pool = MagicMock()
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.execute = AsyncMock()
|
||||
mock_conn.__aenter__ = AsyncMock(return_value=mock_conn)
|
||||
mock_conn.__aexit__ = AsyncMock()
|
||||
mock_pool.acquire = MagicMock(return_value=mock_conn)
|
||||
|
||||
config_store = ConfigStore.__new__(ConfigStore)
|
||||
config_store._pool = mock_pool
|
||||
|
||||
await config_store.set_adapter_last_error("firms", "missing api key: firms")
|
||||
|
||||
mock_conn.execute.assert_called_once()
|
||||
call_args = mock_conn.execute.call_args[0]
|
||||
assert "UPDATE config.adapters SET last_error" in call_args[0]
|
||||
assert call_args[1] == "missing api key: firms"
|
||||
assert call_args[2] == "firms"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clear_adapter_last_error(self):
|
||||
"""set_adapter_last_error with None should clear the error."""
|
||||
from central.config_store import ConfigStore
|
||||
|
||||
mock_pool = MagicMock()
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.execute = AsyncMock()
|
||||
mock_conn.__aenter__ = AsyncMock(return_value=mock_conn)
|
||||
mock_conn.__aexit__ = AsyncMock()
|
||||
mock_pool.acquire = MagicMock(return_value=mock_conn)
|
||||
|
||||
config_store = ConfigStore.__new__(ConfigStore)
|
||||
config_store._pool = mock_pool
|
||||
|
||||
await config_store.set_adapter_last_error("firms", None)
|
||||
|
||||
mock_conn.execute.assert_called_once()
|
||||
call_args = mock_conn.execute.call_args[0]
|
||||
assert call_args[1] is None
|
||||
assert call_args[2] == "firms"
|
||||
|
||||
|
||||
class TestRoutesApiKeyMissing:
|
||||
"""Tests for routes api_key_missing computation."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_adapters_list_includes_api_key_missing_flag(self):
|
||||
"""adapters_list should compute api_key_missing for each adapter."""
|
||||
from central.gui.routes import adapters_list
|
||||
|
||||
mock_request = MagicMock()
|
||||
mock_request.state = MagicMock()
|
||||
mock_request.state.operator = {"username": "test"}
|
||||
mock_request.state.csrf_token = "test_token"
|
||||
|
||||
mock_pool = MagicMock()
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.fetch = AsyncMock(return_value=[
|
||||
{"name": "firms", "enabled": False, "cadence_s": 300, "settings": {}, "paused_at": None, "updated_at": None, "last_error": None},
|
||||
])
|
||||
mock_conn.fetchval = AsyncMock(return_value=None) # No API key exists
|
||||
mock_conn.__aenter__ = AsyncMock(return_value=mock_conn)
|
||||
mock_conn.__aexit__ = AsyncMock()
|
||||
mock_pool.acquire = MagicMock(return_value=mock_conn)
|
||||
|
||||
# Mock adapter class with requires_api_key
|
||||
mock_firms_cls = MagicMock()
|
||||
mock_firms_cls.requires_api_key = "firms"
|
||||
mock_firms_cls.display_name = "FIRMS"
|
||||
|
||||
with patch("central.gui.routes._get_templates") as mock_templates:
|
||||
with patch("central.gui.routes.get_pool", return_value=mock_pool):
|
||||
with patch("central.gui.routes._adapter_classes", return_value={"firms": mock_firms_cls}):
|
||||
mock_template_response = MagicMock()
|
||||
mock_templates.return_value.TemplateResponse = MagicMock(return_value=mock_template_response)
|
||||
|
||||
await adapters_list(mock_request)
|
||||
|
||||
# Check the context passed to template
|
||||
call_kwargs = mock_templates.return_value.TemplateResponse.call_args[1]
|
||||
adapters = call_kwargs["context"]["adapters"]
|
||||
|
||||
assert len(adapters) == 1
|
||||
assert adapters[0]["api_key_missing"] is True
|
||||
assert adapters[0]["requires_api_key_alias"] == "firms"
|
||||
|
||||
|
||||
class TestAdapterClassRequiresApiKey:
|
||||
"""Tests for adapter class requires_api_key attribute."""
|
||||
|
||||
def test_firms_adapter_requires_api_key(self):
|
||||
"""FIRMS adapter should declare requires_api_key."""
|
||||
from central.adapters.firms import FIRMSAdapter
|
||||
assert FIRMSAdapter.requires_api_key == "firms"
|
||||
|
||||
def test_nws_adapter_no_requires_api_key(self):
|
||||
"""NWS adapter should not require an API key."""
|
||||
from central.adapters.nws import NWSAdapter
|
||||
assert NWSAdapter.requires_api_key is None
|
||||
|
||||
def test_usgs_quake_adapter_no_requires_api_key(self):
|
||||
"""USGS Quake adapter should not require an API key."""
|
||||
from central.adapters.usgs_quake import USGSQuakeAdapter
|
||||
assert USGSQuakeAdapter.requires_api_key is None
|
||||
|
||||
|
||||
class TestSupervisorApiKeyPrecondition:
|
||||
"""Tests for supervisor API key precondition check in _start_adapter."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_adapter_refuses_when_required_key_missing(self, tmp_path: Path):
|
||||
"""Adapter with requires_api_key but missing key should not start."""
|
||||
from central.supervisor import Supervisor
|
||||
from central.adapters.firms import FIRMSAdapter
|
||||
|
||||
# Create mock config store
|
||||
mock_config_store = MagicMock()
|
||||
mock_config_store.get_api_key = AsyncMock(return_value=None) # Key missing
|
||||
mock_config_store.set_adapter_last_error = AsyncMock()
|
||||
|
||||
# Create mock NATS
|
||||
mock_nats = MagicMock()
|
||||
mock_nats.publish = AsyncMock()
|
||||
|
||||
# Build supervisor with FIRMS adapter
|
||||
supervisor = Supervisor.__new__(Supervisor)
|
||||
supervisor._config_store = mock_config_store
|
||||
supervisor._adapters = {"firms": FIRMSAdapter}
|
||||
supervisor._adapter_states = {}
|
||||
supervisor._nats = mock_nats
|
||||
supervisor._cursor_db_path = tmp_path / "cursors.db"
|
||||
supervisor._log = MagicMock()
|
||||
|
||||
config = AdapterConfig(
|
||||
name="firms",
|
||||
enabled=True,
|
||||
cadence_s=300,
|
||||
settings={"api_key_alias": "firms", "satellites": ["VIIRS_SNPP_NRT"]},
|
||||
updated_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
await supervisor._start_adapter(config)
|
||||
|
||||
# Should have checked for key
|
||||
mock_config_store.get_api_key.assert_called_once_with("firms")
|
||||
|
||||
# Should have set error
|
||||
mock_config_store.set_adapter_last_error.assert_called_once()
|
||||
args = mock_config_store.set_adapter_last_error.call_args[0]
|
||||
assert args[0] == "firms"
|
||||
assert "missing api key" in args[1].lower()
|
||||
|
||||
# Should NOT have created adapter state (adapter did not start)
|
||||
assert "firms" not in supervisor._adapter_states
|
||||
|
||||
# Should NOT have published to NATS
|
||||
mock_nats.publish.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_adapter_succeeds_after_key_added_and_clears_last_error(self, tmp_path: Path):
|
||||
"""Adapter with requires_api_key and key present should start and clear last_error."""
|
||||
from central.supervisor import Supervisor
|
||||
from central.adapters.firms import FIRMSAdapter
|
||||
|
||||
# Create mock config store with key present
|
||||
mock_config_store = MagicMock()
|
||||
mock_config_store.get_api_key = AsyncMock(return_value="encrypted-firms-key")
|
||||
mock_config_store.set_adapter_last_error = AsyncMock()
|
||||
|
||||
# Create mock NATS
|
||||
mock_nats = MagicMock()
|
||||
mock_nats.publish = AsyncMock()
|
||||
|
||||
# Build supervisor with FIRMS adapter
|
||||
supervisor = Supervisor.__new__(Supervisor)
|
||||
supervisor._config_store = mock_config_store
|
||||
supervisor._adapters = {"firms": FIRMSAdapter}
|
||||
supervisor._adapter_states = {}
|
||||
supervisor._nats = mock_nats
|
||||
supervisor._cursor_db_path = tmp_path / "cursors.db"
|
||||
supervisor._log = MagicMock()
|
||||
|
||||
config = AdapterConfig(
|
||||
name="firms",
|
||||
enabled=True,
|
||||
cadence_s=300,
|
||||
settings={"api_key_alias": "firms", "satellites": ["VIIRS_SNPP_NRT"]},
|
||||
updated_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
# Mock the adapter instantiation to avoid actual HTTP calls
|
||||
with patch.object(FIRMSAdapter, "__init__", return_value=None):
|
||||
with patch.object(FIRMSAdapter, "startup", new_callable=AsyncMock):
|
||||
await supervisor._start_adapter(config)
|
||||
|
||||
# Should have checked for key
|
||||
mock_config_store.get_api_key.assert_called_once_with("firms")
|
||||
|
||||
# Should have cleared any stale error (called with None)
|
||||
mock_config_store.set_adapter_last_error.assert_called_once_with("firms", None)
|
||||
|
||||
# Should have created adapter state
|
||||
assert "firms" in supervisor._adapter_states
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_adapter_does_not_check_when_no_requires_api_key(self, tmp_path: Path):
|
||||
"""Adapter without requires_api_key should skip the API key check."""
|
||||
from central.supervisor import Supervisor
|
||||
from central.adapters.nws import NWSAdapter
|
||||
|
||||
# Create mock config store
|
||||
mock_config_store = MagicMock()
|
||||
mock_config_store.get_api_key = AsyncMock()
|
||||
mock_config_store.set_adapter_last_error = AsyncMock()
|
||||
|
||||
# Create mock NATS
|
||||
mock_nats = MagicMock()
|
||||
mock_nats.publish = AsyncMock()
|
||||
|
||||
# Build supervisor with NWS adapter (no requires_api_key)
|
||||
supervisor = Supervisor.__new__(Supervisor)
|
||||
supervisor._config_store = mock_config_store
|
||||
supervisor._adapters = {"nws": NWSAdapter}
|
||||
supervisor._adapter_states = {}
|
||||
supervisor._nats = mock_nats
|
||||
supervisor._cursor_db_path = tmp_path / "cursors.db"
|
||||
supervisor._log = MagicMock()
|
||||
|
||||
config = AdapterConfig(
|
||||
name="nws",
|
||||
enabled=True,
|
||||
cadence_s=300,
|
||||
settings={"contact_email": "test@example.com"},
|
||||
updated_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
# Mock the adapter instantiation to avoid actual HTTP calls
|
||||
with patch.object(NWSAdapter, "__init__", return_value=None):
|
||||
with patch.object(NWSAdapter, "startup", new_callable=AsyncMock):
|
||||
await supervisor._start_adapter(config)
|
||||
|
||||
# Should NOT have called get_api_key (no requires_api_key)
|
||||
mock_config_store.get_api_key.assert_not_called()
|
||||
|
||||
# Should have cleared stale error (routine clear)
|
||||
mock_config_store.set_adapter_last_error.assert_called_once_with("nws", None)
|
||||
|
||||
# Should have created adapter state
|
||||
assert "nws" in supervisor._adapter_states
|
||||
|
||||
|
||||
class TestAdaptersEditSubmitErrorRerender:
|
||||
"""Tests for adapters_edit_submit error re-render including api_key_missing."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_adapters_edit_submit_error_rerender_includes_api_key_missing(self):
|
||||
"""Error re-render on /adapters/firms should include api_key_missing in context."""
|
||||
from central.gui.routes import adapters_edit_submit
|
||||
from pydantic import BaseModel
|
||||
from typing import Literal
|
||||
|
||||
mock_request = MagicMock()
|
||||
mock_request.state = MagicMock()
|
||||
mock_request.state.operator = {"username": "test"}
|
||||
mock_request.state.csrf_token = "test_token"
|
||||
|
||||
# Mock form with invalid cadence (below minimum of 10)
|
||||
mock_form = MagicMock()
|
||||
def form_get(k, d=""):
|
||||
values = {
|
||||
"csrf_token": "test_token",
|
||||
"cadence_s": "5", # Invalid - below minimum
|
||||
"api_key_alias": "firms",
|
||||
"satellites": "",
|
||||
"region_north": "",
|
||||
"region_south": "",
|
||||
"region_east": "",
|
||||
"region_west": "",
|
||||
}
|
||||
return values.get(k, d)
|
||||
mock_form.get = MagicMock(side_effect=form_get)
|
||||
mock_form.getlist = MagicMock(return_value=["VIIRS_SNPP_NRT"])
|
||||
mock_form.__contains__ = lambda self, k: k == "enabled"
|
||||
mock_request.form = AsyncMock(return_value=mock_form)
|
||||
|
||||
mock_pool = MagicMock()
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.fetchrow = AsyncMock(side_effect=[
|
||||
# First call: adapter row
|
||||
{
|
||||
"name": "firms",
|
||||
"enabled": False,
|
||||
"cadence_s": 300,
|
||||
"settings": {"api_key_alias": "firms", "satellites": ["VIIRS_SNPP_NRT"]},
|
||||
"paused_at": None,
|
||||
"updated_at": datetime.now(timezone.utc),
|
||||
"last_error": None,
|
||||
},
|
||||
# Second call: system row for map tiles
|
||||
{"map_tile_url": "https://tile.example.com/{z}/{x}/{y}.png", "map_attribution": "Test"},
|
||||
])
|
||||
mock_conn.fetchval = AsyncMock(return_value=None) # No API key exists
|
||||
mock_conn.fetch = AsyncMock(return_value=[]) # No API keys
|
||||
mock_conn.__aenter__ = AsyncMock(return_value=mock_conn)
|
||||
mock_conn.__aexit__ = AsyncMock()
|
||||
mock_pool.acquire = MagicMock(return_value=mock_conn)
|
||||
|
||||
# Mock FIRMS adapter class
|
||||
class MockFIRMSSettings(BaseModel):
|
||||
api_key_alias: str = ""
|
||||
satellites: list[Literal["VIIRS_SNPP_NRT", "VIIRS_NOAA20_NRT"]] = []
|
||||
|
||||
mock_firms_cls = MagicMock()
|
||||
mock_firms_cls.requires_api_key = "firms"
|
||||
mock_firms_cls.api_key_field = "api_key_alias"
|
||||
mock_firms_cls.display_name = "FIRMS"
|
||||
mock_firms_cls.description = "Fire detection"
|
||||
mock_firms_cls.settings_schema = MockFIRMSSettings
|
||||
|
||||
with patch("central.gui.routes._get_templates") as mock_templates:
|
||||
with patch("central.gui.routes.get_pool", return_value=mock_pool):
|
||||
with patch("central.gui.routes._adapter_classes", return_value={"firms": mock_firms_cls}):
|
||||
with patch("central.gui.routes.describe_fields", return_value=[]):
|
||||
mock_template_response = MagicMock()
|
||||
mock_template_response.status_code = 200
|
||||
mock_templates.return_value.TemplateResponse = MagicMock(return_value=mock_template_response)
|
||||
|
||||
result = await adapters_edit_submit(mock_request, "firms")
|
||||
|
||||
# Verify TemplateResponse was called (error re-render)
|
||||
assert mock_templates.return_value.TemplateResponse.called
|
||||
|
||||
# Check the context passed to template
|
||||
call_kwargs = mock_templates.return_value.TemplateResponse.call_args[1]
|
||||
context = call_kwargs["context"]
|
||||
|
||||
# Should have errors (invalid cadence)
|
||||
assert context.get("errors") is not None
|
||||
assert "cadence_s" in context["errors"]
|
||||
|
||||
# Should include api_key_missing
|
||||
assert context["api_key_missing"] is True
|
||||
assert context["requires_api_key_alias"] == "firms"
|
||||
|
|
@ -94,6 +94,8 @@ class MockConfigSource:
|
|||
class MockNWSAdapter:
|
||||
"""Mock NWSAdapter that tracks poll calls and allows control."""
|
||||
|
||||
requires_api_key = None # Mock adapters don't require API keys
|
||||
|
||||
def __init__(self, config, config_store, cursor_db_path) -> None:
|
||||
self.config = config
|
||||
self._config_store = config_store
|
||||
|
|
@ -152,6 +154,8 @@ def mock_config_store():
|
|||
store = MagicMock()
|
||||
store.list_streams = AsyncMock(return_value=[])
|
||||
store.get_stream = AsyncMock(return_value=None)
|
||||
store.set_adapter_last_error = AsyncMock()
|
||||
store.get_api_key = AsyncMock(return_value=None)
|
||||
return store
|
||||
|
||||
|
||||
|
|
|
|||
339
tests/test_swpc.py
Normal file
339
tests/test_swpc.py
Normal file
|
|
@ -0,0 +1,339 @@
|
|||
"""Tests for NOAA SWPC space weather adapters."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from central.config_models import AdapterConfig
|
||||
from central.models import Event
|
||||
|
||||
|
||||
# Frozen fixtures captured from upstream feeds; real shapes.
|
||||
SAMPLE_ALERTS = [
|
||||
{
|
||||
"product_id": "EF3A",
|
||||
"issue_datetime": "2026-05-19 05:14:59.780",
|
||||
"message": (
|
||||
"Space Weather Message Code: ALTEF3\r\nSerial Number: 3689\r\n"
|
||||
"Issue Time: 2026 May 19 0514 UTC\r\n\r\n"
|
||||
"ALERT: Electron 2MeV Integral Flux exceeded 1000pfu \n"
|
||||
"Threshold Reached: 2026 May 16 1740 UTC\n"
|
||||
"Station: GOES-19\n"
|
||||
),
|
||||
},
|
||||
{
|
||||
"product_id": "K05A",
|
||||
"issue_datetime": "2026-05-15 14:30:00.000",
|
||||
"message": (
|
||||
"Space Weather Message Code: ALTK05\r\nSerial Number: 100\r\n"
|
||||
"Issue Time: 2026 May 15 1430 UTC\r\n\r\n"
|
||||
"ALERT: Geomagnetic K-index of 5\n"
|
||||
),
|
||||
},
|
||||
{
|
||||
"product_id": "K07A",
|
||||
"issue_datetime": "2026-05-15 18:00:00.000",
|
||||
"message": "Space Weather Message Code: ALTK07\r\nSerial Number: 101\r\n",
|
||||
},
|
||||
]
|
||||
|
||||
SAMPLE_KINDEX = [
|
||||
{"time_tag": "2026-05-12T00:00:00", "Kp": 0.67, "a_running": 3, "station_count": 8},
|
||||
{"time_tag": "2026-05-12T03:00:00", "Kp": 5.33, "a_running": 30, "station_count": 8},
|
||||
{"time_tag": "2026-05-12T06:00:00", "Kp": 8.0, "a_running": 100, "station_count": 8},
|
||||
]
|
||||
|
||||
SAMPLE_PROTONS = [
|
||||
{"time_tag": "2026-05-18T05:35:00Z", "satellite": 19, "flux": 7.09, "energy": ">=1 MeV"},
|
||||
{"time_tag": "2026-05-18T05:35:00Z", "satellite": 19, "flux": 0.21, "energy": ">=10 MeV"},
|
||||
{"time_tag": "2026-05-18T05:40:00Z", "satellite": 19, "flux": 7.10, "energy": ">=1 MeV"},
|
||||
]
|
||||
|
||||
|
||||
def _config(name: str, cadence: int) -> AdapterConfig:
|
||||
return AdapterConfig(
|
||||
name=name,
|
||||
enabled=True,
|
||||
cadence_s=cadence,
|
||||
settings={},
|
||||
updated_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
|
||||
class TestSWPCCommon:
|
||||
"""Tests for swpc_common helpers."""
|
||||
|
||||
def test_parse_swpc_timestamp_alerts(self):
|
||||
from central.adapters.swpc_common import parse_swpc_timestamp
|
||||
|
||||
dt = parse_swpc_timestamp("2026-05-19 05:14:59.780", "alerts")
|
||||
assert dt == datetime(2026, 5, 19, 5, 14, 59, 780000, tzinfo=timezone.utc)
|
||||
|
||||
def test_parse_swpc_timestamp_alerts_no_fraction(self):
|
||||
from central.adapters.swpc_common import parse_swpc_timestamp
|
||||
|
||||
dt = parse_swpc_timestamp("2026-05-19 05:14:59", "alerts")
|
||||
assert dt == datetime(2026, 5, 19, 5, 14, 59, tzinfo=timezone.utc)
|
||||
|
||||
def test_parse_swpc_timestamp_kindex(self):
|
||||
from central.adapters.swpc_common import parse_swpc_timestamp
|
||||
|
||||
dt = parse_swpc_timestamp("2026-05-12T03:00:00", "kindex")
|
||||
assert dt == datetime(2026, 5, 12, 3, 0, 0, tzinfo=timezone.utc)
|
||||
|
||||
def test_parse_swpc_timestamp_protons(self):
|
||||
from central.adapters.swpc_common import parse_swpc_timestamp
|
||||
|
||||
dt = parse_swpc_timestamp("2026-05-18T05:35:00Z", "protons")
|
||||
assert dt == datetime(2026, 5, 18, 5, 35, 0, tzinfo=timezone.utc)
|
||||
|
||||
def test_parse_swpc_timestamp_empty(self):
|
||||
from central.adapters.swpc_common import parse_swpc_timestamp
|
||||
|
||||
assert parse_swpc_timestamp("", "alerts") is None
|
||||
assert parse_swpc_timestamp(None, "alerts") is None
|
||||
|
||||
def test_severity_from_kp_boundaries(self):
|
||||
from central.adapters.swpc_common import severity_from_kp
|
||||
|
||||
assert severity_from_kp(None) == 0
|
||||
assert severity_from_kp(0) == 0
|
||||
assert severity_from_kp(4.5) == 0
|
||||
assert severity_from_kp(4.9) == 0
|
||||
assert severity_from_kp(5.0) == 1
|
||||
assert severity_from_kp(5.99) == 1
|
||||
assert severity_from_kp(6.0) == 2
|
||||
assert severity_from_kp(6.99) == 2
|
||||
assert severity_from_kp(7.0) == 3
|
||||
assert severity_from_kp(7.99) == 3
|
||||
assert severity_from_kp(8.0) == 4
|
||||
assert severity_from_kp(9.0) == 4
|
||||
|
||||
def test_severity_from_alert_product_id(self):
|
||||
from central.adapters.swpc_common import severity_from_alert_product_id
|
||||
|
||||
assert severity_from_alert_product_id(None) == 0
|
||||
assert severity_from_alert_product_id("") == 0
|
||||
assert severity_from_alert_product_id("EF3A") == 0
|
||||
assert severity_from_alert_product_id("BHIS") == 0
|
||||
assert severity_from_alert_product_id("K04A") == 0
|
||||
assert severity_from_alert_product_id("K05A") == 1
|
||||
assert severity_from_alert_product_id("K05W") == 1
|
||||
assert severity_from_alert_product_id("K06A") == 2
|
||||
assert severity_from_alert_product_id("K07A") == 3
|
||||
assert severity_from_alert_product_id("K08A") == 4
|
||||
assert severity_from_alert_product_id("K09A") == 4
|
||||
|
||||
|
||||
class TestSWPCAlertsAdapter:
|
||||
"""Tests for SWPCAlertsAdapter."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_alerts_normalization(self, tmp_path: Path):
|
||||
from central.adapters.swpc_alerts import SWPCAlertsAdapter
|
||||
|
||||
adapter = SWPCAlertsAdapter(
|
||||
_config("swpc_alerts", 300), MagicMock(), tmp_path / "cursors.db"
|
||||
)
|
||||
adapter._fetch = AsyncMock(return_value=SAMPLE_ALERTS)
|
||||
|
||||
await adapter.startup()
|
||||
events: list[Event] = [e async for e in adapter.poll()]
|
||||
await adapter.shutdown()
|
||||
|
||||
assert len(events) == 3
|
||||
|
||||
ef3a = events[0]
|
||||
assert ef3a.adapter == "swpc_alerts"
|
||||
assert ef3a.category == "space.alert"
|
||||
assert ef3a.id == "EF3A|2026-05-19 05:14:59.780"
|
||||
assert ef3a.time == datetime(2026, 5, 19, 5, 14, 59, 780000, tzinfo=timezone.utc)
|
||||
assert ef3a.severity == 0
|
||||
assert ef3a.data["product_id"] == "EF3A"
|
||||
assert ef3a.geo.centroid is None
|
||||
assert ef3a.geo.regions == []
|
||||
assert ef3a.geo.primary_region is None
|
||||
|
||||
k05a = events[1]
|
||||
assert k05a.severity == 1
|
||||
k07a = events[2]
|
||||
assert k07a.severity == 3
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_alerts_dedup(self, tmp_path: Path):
|
||||
from central.adapters.swpc_alerts import SWPCAlertsAdapter
|
||||
|
||||
adapter = SWPCAlertsAdapter(
|
||||
_config("swpc_alerts", 300), MagicMock(), tmp_path / "cursors.db"
|
||||
)
|
||||
adapter._fetch = AsyncMock(return_value=SAMPLE_ALERTS)
|
||||
|
||||
await adapter.startup()
|
||||
first_pass = [e async for e in adapter.poll()]
|
||||
second_pass = [e async for e in adapter.poll()]
|
||||
await adapter.shutdown()
|
||||
|
||||
assert len(first_pass) == 3
|
||||
assert len(second_pass) == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_alerts_subject_for(self, tmp_path: Path):
|
||||
from central.adapters.swpc_alerts import SWPCAlertsAdapter
|
||||
from central.models import Geo
|
||||
|
||||
adapter = SWPCAlertsAdapter(
|
||||
_config("swpc_alerts", 300), MagicMock(), tmp_path / "cursors.db"
|
||||
)
|
||||
event = Event(
|
||||
id="EF3A|2026-05-19 05:14:59.780",
|
||||
adapter="swpc_alerts",
|
||||
category="space.alert",
|
||||
time=datetime(2026, 5, 19, 5, 14, 59, tzinfo=timezone.utc),
|
||||
severity=0,
|
||||
geo=Geo(),
|
||||
data={"product_id": "EF3A"},
|
||||
)
|
||||
assert adapter.subject_for(event) == "central.space.alert.ef3a"
|
||||
|
||||
event_k = Event(
|
||||
id="K05A|...",
|
||||
adapter="swpc_alerts",
|
||||
category="space.alert",
|
||||
time=datetime(2026, 5, 15, tzinfo=timezone.utc),
|
||||
severity=1,
|
||||
geo=Geo(),
|
||||
data={"product_id": "K05A"},
|
||||
)
|
||||
assert adapter.subject_for(event_k) == "central.space.alert.k05a"
|
||||
|
||||
|
||||
class TestSWPCKindexAdapter:
|
||||
"""Tests for SWPCKindexAdapter."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_kindex_normalization(self, tmp_path: Path):
|
||||
from central.adapters.swpc_kindex import SWPCKindexAdapter
|
||||
|
||||
adapter = SWPCKindexAdapter(
|
||||
_config("swpc_kindex", 600), MagicMock(), tmp_path / "cursors.db"
|
||||
)
|
||||
adapter._fetch = AsyncMock(return_value=SAMPLE_KINDEX)
|
||||
|
||||
await adapter.startup()
|
||||
events = [e async for e in adapter.poll()]
|
||||
await adapter.shutdown()
|
||||
|
||||
assert len(events) == 3
|
||||
quiet, g1, g4 = events
|
||||
assert quiet.category == "space.kindex"
|
||||
assert quiet.id == "2026-05-12T00:00:00"
|
||||
assert quiet.severity == 0
|
||||
assert quiet.data["Kp"] == 0.67
|
||||
assert g1.severity == 1
|
||||
assert g4.severity == 4
|
||||
assert g4.time == datetime(2026, 5, 12, 6, 0, 0, tzinfo=timezone.utc)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_kindex_dedup(self, tmp_path: Path):
|
||||
from central.adapters.swpc_kindex import SWPCKindexAdapter
|
||||
|
||||
adapter = SWPCKindexAdapter(
|
||||
_config("swpc_kindex", 600), MagicMock(), tmp_path / "cursors.db"
|
||||
)
|
||||
adapter._fetch = AsyncMock(return_value=SAMPLE_KINDEX)
|
||||
|
||||
await adapter.startup()
|
||||
first_pass = [e async for e in adapter.poll()]
|
||||
second_pass = [e async for e in adapter.poll()]
|
||||
await adapter.shutdown()
|
||||
|
||||
assert len(first_pass) == 3
|
||||
assert len(second_pass) == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_kindex_subject_for(self, tmp_path: Path):
|
||||
from central.adapters.swpc_kindex import SWPCKindexAdapter
|
||||
from central.models import Geo
|
||||
|
||||
adapter = SWPCKindexAdapter(
|
||||
_config("swpc_kindex", 600), MagicMock(), tmp_path / "cursors.db"
|
||||
)
|
||||
event = Event(
|
||||
id="2026-05-12T03:00:00",
|
||||
adapter="swpc_kindex",
|
||||
category="space.kindex",
|
||||
time=datetime(2026, 5, 12, 3, tzinfo=timezone.utc),
|
||||
severity=1,
|
||||
geo=Geo(),
|
||||
data={"Kp": 5.33},
|
||||
)
|
||||
assert adapter.subject_for(event) == "central.space.kindex"
|
||||
|
||||
|
||||
class TestSWPCProtonsAdapter:
|
||||
"""Tests for SWPCProtonsAdapter."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_protons_normalization(self, tmp_path: Path):
|
||||
from central.adapters.swpc_protons import SWPCProtonsAdapter
|
||||
|
||||
adapter = SWPCProtonsAdapter(
|
||||
_config("swpc_protons", 600), MagicMock(), tmp_path / "cursors.db"
|
||||
)
|
||||
adapter._fetch = AsyncMock(return_value=SAMPLE_PROTONS)
|
||||
|
||||
await adapter.startup()
|
||||
events = [e async for e in adapter.poll()]
|
||||
await adapter.shutdown()
|
||||
|
||||
assert len(events) == 3
|
||||
first = events[0]
|
||||
assert first.category == "space.proton_flux"
|
||||
assert first.id == "2026-05-18T05:35:00Z|>=1 MeV"
|
||||
assert first.severity == 0
|
||||
assert first.data["energy"] == ">=1 MeV"
|
||||
assert first.data["flux"] == 7.09
|
||||
assert first.time == datetime(2026, 5, 18, 5, 35, 0, tzinfo=timezone.utc)
|
||||
assert first.geo.centroid is None
|
||||
assert first.geo.regions == []
|
||||
|
||||
# Same time_tag, different energy -> distinct event_id
|
||||
assert events[1].id == "2026-05-18T05:35:00Z|>=10 MeV"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_protons_dedup(self, tmp_path: Path):
|
||||
from central.adapters.swpc_protons import SWPCProtonsAdapter
|
||||
|
||||
adapter = SWPCProtonsAdapter(
|
||||
_config("swpc_protons", 600), MagicMock(), tmp_path / "cursors.db"
|
||||
)
|
||||
adapter._fetch = AsyncMock(return_value=SAMPLE_PROTONS)
|
||||
|
||||
await adapter.startup()
|
||||
first_pass = [e async for e in adapter.poll()]
|
||||
second_pass = [e async for e in adapter.poll()]
|
||||
await adapter.shutdown()
|
||||
|
||||
assert len(first_pass) == 3
|
||||
assert len(second_pass) == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_protons_subject_for(self, tmp_path: Path):
|
||||
from central.adapters.swpc_protons import SWPCProtonsAdapter
|
||||
from central.models import Geo
|
||||
|
||||
adapter = SWPCProtonsAdapter(
|
||||
_config("swpc_protons", 600), MagicMock(), tmp_path / "cursors.db"
|
||||
)
|
||||
event = Event(
|
||||
id="2026-05-18T05:35:00Z|>=10 MeV",
|
||||
adapter="swpc_protons",
|
||||
category="space.proton_flux",
|
||||
time=datetime(2026, 5, 18, 5, 35, 0, tzinfo=timezone.utc),
|
||||
severity=0,
|
||||
geo=Geo(),
|
||||
data={"energy": ">=10 MeV", "flux": 0.21},
|
||||
)
|
||||
assert adapter.subject_for(event) == "central.space.proton_flux"
|
||||
534
tests/test_wfigs.py
Normal file
534
tests/test_wfigs.py
Normal file
|
|
@ -0,0 +1,534 @@
|
|||
"""Tests for WFIGS adapters."""
|
||||
|
||||
import sqlite3
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from central.config_models import AdapterConfig, RegionConfig
|
||||
from central.models import Event, Geo
|
||||
|
||||
|
||||
# Sample GeoJSON response with incidents using real WFIGS format
|
||||
# Note: POOState comes as ISO 3166-2 ("US-MT"), IncidentTypeCategory as codes ("WF")
|
||||
SAMPLE_INCIDENTS_RESPONSE = {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [-113.5, 48.5]},
|
||||
"properties": {
|
||||
"IrwinID": "GUID-001-GLACIER",
|
||||
"IncidentName": "Glacier Fire",
|
||||
"IncidentTypeCategory": "WF", # Real format: 2-letter code
|
||||
"DailyAcres": 150,
|
||||
"PercentContained": 25,
|
||||
"FireDiscoveryDateTime": 1716000000000,
|
||||
"ModifiedOnDateTime": 1716100000000,
|
||||
"POOState": "US-MT", # Real format: ISO 3166-2
|
||||
"POOCounty": "Glacier",
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [-116.5, 43.5]},
|
||||
"properties": {
|
||||
"IrwinID": "GUID-002-OWYHEE",
|
||||
"IncidentName": "Owyhee Rx",
|
||||
"IncidentTypeCategory": "RX", # Prescribed fire
|
||||
"DailyAcres": 5,
|
||||
"PercentContained": 100,
|
||||
"FireDiscoveryDateTime": 1716200000000,
|
||||
"ModifiedOnDateTime": 1716300000000,
|
||||
"POOState": "US-ID",
|
||||
"POOCounty": "Owyhee",
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [-80.0, 26.0]},
|
||||
"properties": {
|
||||
"IrwinID": "GUID-003-FLORIDA",
|
||||
"IncidentName": "Florida Fire",
|
||||
"IncidentTypeCategory": "WF",
|
||||
"DailyAcres": 50,
|
||||
"PercentContained": 0,
|
||||
"FireDiscoveryDateTime": 1716400000000,
|
||||
"ModifiedOnDateTime": 1716500000000,
|
||||
"POOState": "US-FL",
|
||||
"POOCounty": "Miami-Dade",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
# Perimeters API uses prefixed field names (attr_*, poly_*)
|
||||
SAMPLE_PERIMETERS_RESPONSE = {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [[
|
||||
[-113.6, 48.4],
|
||||
[-113.4, 48.4],
|
||||
[-113.4, 48.6],
|
||||
[-113.6, 48.6],
|
||||
[-113.6, 48.4],
|
||||
]],
|
||||
},
|
||||
"properties": {
|
||||
"attr_IrwinID": "GUID-001-GLACIER",
|
||||
"attr_IncidentName": "Glacier Fire",
|
||||
"attr_IncidentTypeCategory": "WF", # Real format: 2-letter code
|
||||
"attr_IncidentSize": 150,
|
||||
"poly_GISAcres": 148.5,
|
||||
"attr_PercentContained": 25,
|
||||
"attr_FireDiscoveryDateTime": 1716000000000,
|
||||
"attr_ModifiedOnDateTime_dt": 1716100000000,
|
||||
"attr_POOState": "US-MT", # Real format: ISO 3166-2
|
||||
"attr_POOCounty": "Glacier",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class TestWFIGSCommon:
|
||||
"""Tests for WFIGS common utilities."""
|
||||
|
||||
def test_severity_from_acres_none(self):
|
||||
from central.adapters.wfigs_common import severity_from_acres
|
||||
assert severity_from_acres(None) == 0
|
||||
assert severity_from_acres(0) == 0
|
||||
|
||||
def test_severity_from_acres_small(self):
|
||||
from central.adapters.wfigs_common import severity_from_acres
|
||||
assert severity_from_acres(5) == 1
|
||||
assert severity_from_acres(9.9) == 1
|
||||
|
||||
def test_severity_from_acres_medium(self):
|
||||
from central.adapters.wfigs_common import severity_from_acres
|
||||
assert severity_from_acres(10) == 2
|
||||
assert severity_from_acres(99) == 2
|
||||
|
||||
def test_severity_from_acres_large(self):
|
||||
from central.adapters.wfigs_common import severity_from_acres
|
||||
assert severity_from_acres(100) == 3
|
||||
assert severity_from_acres(999) == 3
|
||||
|
||||
def test_severity_from_acres_very_large(self):
|
||||
from central.adapters.wfigs_common import severity_from_acres
|
||||
assert severity_from_acres(1000) == 4
|
||||
assert severity_from_acres(100000) == 4
|
||||
|
||||
def test_parse_wfigs_timestamp(self):
|
||||
from central.adapters.wfigs_common import parse_wfigs_timestamp
|
||||
ts = parse_wfigs_timestamp(1716000000000)
|
||||
assert ts is not None
|
||||
assert ts.tzinfo == timezone.utc
|
||||
assert ts.year == 2024
|
||||
|
||||
def test_parse_wfigs_timestamp_none(self):
|
||||
from central.adapters.wfigs_common import parse_wfigs_timestamp
|
||||
assert parse_wfigs_timestamp(None) is None
|
||||
|
||||
def test_build_regions_full(self):
|
||||
from central.adapters.wfigs_common import build_regions
|
||||
# Expects normalized 2-letter state code
|
||||
regions, primary = build_regions("MT", "Glacier")
|
||||
assert regions == ["US-MT-GLACIER"]
|
||||
assert primary == "US-MT-GLACIER"
|
||||
|
||||
def test_build_regions_state_only(self):
|
||||
from central.adapters.wfigs_common import build_regions
|
||||
regions, primary = build_regions("MT", None)
|
||||
assert regions == ["US-MT"]
|
||||
assert primary == "US-MT"
|
||||
|
||||
def test_build_regions_none(self):
|
||||
from central.adapters.wfigs_common import build_regions
|
||||
regions, primary = build_regions(None, None)
|
||||
assert regions == []
|
||||
assert primary is None
|
||||
|
||||
def test_subject_suffix(self):
|
||||
from central.adapters.wfigs_common import subject_suffix
|
||||
# Expects normalized 2-letter state code
|
||||
assert subject_suffix("MT", "Glacier") == "mt.glacier"
|
||||
assert subject_suffix("ID", "Ada County") == "id.ada_county"
|
||||
assert subject_suffix("ID", None) == "id"
|
||||
assert subject_suffix(None, None) == "unknown"
|
||||
|
||||
def test_point_in_bbox(self):
|
||||
from central.adapters.wfigs_common import point_in_bbox
|
||||
assert point_in_bbox(-116.5, 43.5, -124, 31, -102, 49) is True
|
||||
assert point_in_bbox(-80.0, 26.0, -124, 31, -102, 49) is False
|
||||
|
||||
# Normalization tests
|
||||
def test_normalize_state_iso_3166(self):
|
||||
"""normalize_state strips US- prefix from ISO 3166-2 codes."""
|
||||
from central.adapters.wfigs_common import normalize_state
|
||||
assert normalize_state("US-MT") == "MT"
|
||||
assert normalize_state("US-ID") == "ID"
|
||||
assert normalize_state("US-CA") == "CA"
|
||||
|
||||
def test_normalize_state_already_2letter(self):
|
||||
"""normalize_state passes through 2-letter codes."""
|
||||
from central.adapters.wfigs_common import normalize_state
|
||||
assert normalize_state("MT") == "MT"
|
||||
assert normalize_state("ID") == "ID"
|
||||
|
||||
def test_normalize_state_none_empty(self):
|
||||
"""normalize_state handles None and empty strings."""
|
||||
from central.adapters.wfigs_common import normalize_state
|
||||
assert normalize_state(None) is None
|
||||
assert normalize_state("") is None
|
||||
|
||||
def test_normalize_state_unknown_format(self):
|
||||
"""normalize_state passes through unknown formats."""
|
||||
from central.adapters.wfigs_common import normalize_state
|
||||
assert normalize_state("Montana") == "Montana"
|
||||
assert normalize_state("US-MONTANA") == "US-MONTANA"
|
||||
|
||||
def test_normalize_incident_type_wf(self):
|
||||
"""normalize_incident_type maps WF to wildfire."""
|
||||
from central.adapters.wfigs_common import normalize_incident_type
|
||||
assert normalize_incident_type("WF") == "wildfire"
|
||||
assert normalize_incident_type("wf") == "wildfire"
|
||||
|
||||
def test_normalize_incident_type_rx(self):
|
||||
"""normalize_incident_type maps RX to prescribed_fire."""
|
||||
from central.adapters.wfigs_common import normalize_incident_type
|
||||
assert normalize_incident_type("RX") == "prescribed_fire"
|
||||
assert normalize_incident_type("rx") == "prescribed_fire"
|
||||
|
||||
def test_normalize_incident_type_cx(self):
|
||||
"""normalize_incident_type maps CX to complex."""
|
||||
from central.adapters.wfigs_common import normalize_incident_type
|
||||
assert normalize_incident_type("CX") == "complex"
|
||||
|
||||
def test_normalize_incident_type_fa(self):
|
||||
"""normalize_incident_type maps FA to false_alarm."""
|
||||
from central.adapters.wfigs_common import normalize_incident_type
|
||||
assert normalize_incident_type("FA") == "false_alarm"
|
||||
|
||||
def test_normalize_incident_type_unknown_code(self):
|
||||
"""normalize_incident_type lowercases unknown codes."""
|
||||
from central.adapters.wfigs_common import normalize_incident_type
|
||||
assert normalize_incident_type("UNKNOWN_CODE") == "unknown_code"
|
||||
assert normalize_incident_type("Wildfire") == "wildfire"
|
||||
|
||||
def test_normalize_incident_type_none(self):
|
||||
"""normalize_incident_type returns unknown for None."""
|
||||
from central.adapters.wfigs_common import normalize_incident_type
|
||||
assert normalize_incident_type(None) == "unknown"
|
||||
assert normalize_incident_type("") == "unknown"
|
||||
|
||||
|
||||
class TestWFIGSIncidentsAdapter:
|
||||
"""Tests for WFIGS Incidents adapter."""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_config(self) -> AdapterConfig:
|
||||
return AdapterConfig(
|
||||
name="wfigs_incidents",
|
||||
enabled=True,
|
||||
cadence_s=300,
|
||||
settings={
|
||||
"region": {"north": 49.0, "south": 31.0, "east": -102.0, "west": -124.0}
|
||||
},
|
||||
updated_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
@pytest.fixture
|
||||
def mock_config_store(self) -> MagicMock:
|
||||
return MagicMock()
|
||||
|
||||
@pytest.fixture
|
||||
def cursor_db_path(self, tmp_path: Path) -> Path:
|
||||
return tmp_path / "cursors.db"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_normalization_incidents(
|
||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
||||
):
|
||||
"""Incidents are correctly normalized to Events."""
|
||||
from central.adapters.wfigs_incidents import WFIGSIncidentsAdapter
|
||||
|
||||
adapter = WFIGSIncidentsAdapter(mock_config, mock_config_store, cursor_db_path)
|
||||
await adapter.startup()
|
||||
|
||||
mock_response = AsyncMock()
|
||||
mock_response.raise_for_status = MagicMock()
|
||||
mock_response.json = AsyncMock(return_value=SAMPLE_INCIDENTS_RESPONSE)
|
||||
|
||||
with patch.object(adapter._session, "get", return_value=AsyncMock(__aenter__=AsyncMock(return_value=mock_response), __aexit__=AsyncMock())):
|
||||
events = [e async for e in adapter.poll()]
|
||||
|
||||
await adapter.shutdown()
|
||||
|
||||
# Should have 2 events (Florida filtered out by bbox)
|
||||
assert len(events) == 2
|
||||
|
||||
# First event: Glacier Fire
|
||||
event = events[0]
|
||||
assert event.id == "GUID-001-GLACIER"
|
||||
assert event.adapter == "wfigs_incidents"
|
||||
# Category uses normalized incident type
|
||||
assert event.category == "fire.incident.wildfire" # NOT fire.incident.wf
|
||||
assert event.severity == 3 # 150 acres = severity 3 (100-999 range)
|
||||
# Region uses normalized state (no double US-)
|
||||
assert event.geo.primary_region == "US-MT-GLACIER" # NOT US-US-MT-GLACIER
|
||||
# Data contains both normalized and raw values
|
||||
assert event.data["POOState"] == "MT" # normalized
|
||||
assert event.data["POOState_raw"] == "US-MT" # raw
|
||||
assert event.data["IncidentTypeCategory"] == "wildfire" # normalized
|
||||
assert event.data["IncidentTypeCategory_raw"] == "WF" # raw
|
||||
|
||||
# Second event: Owyhee Rx
|
||||
event2 = events[1]
|
||||
assert event2.category == "fire.incident.prescribed_fire" # NOT fire.incident.rx
|
||||
assert event2.data["POOState"] == "ID"
|
||||
assert event2.data["POOState_raw"] == "US-ID"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_is_published_dedup(
|
||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
||||
):
|
||||
"""is_published/mark_published provides dedup functionality."""
|
||||
from central.adapters.wfigs_incidents import WFIGSIncidentsAdapter
|
||||
|
||||
adapter = WFIGSIncidentsAdapter(mock_config, mock_config_store, cursor_db_path)
|
||||
await adapter.startup()
|
||||
|
||||
# Initially not published
|
||||
assert adapter.is_published("test-id") is False
|
||||
|
||||
# Mark as published
|
||||
adapter.mark_published("test-id")
|
||||
|
||||
# Now it should be published
|
||||
assert adapter.is_published("test-id") is True
|
||||
|
||||
await adapter.shutdown()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fall_off_emits_removal(
|
||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
||||
):
|
||||
"""Fall-off detection emits removal events."""
|
||||
from central.adapters.wfigs_incidents import WFIGSIncidentsAdapter
|
||||
|
||||
adapter = WFIGSIncidentsAdapter(mock_config, mock_config_store, cursor_db_path)
|
||||
await adapter.startup()
|
||||
|
||||
# First poll with 2 incidents
|
||||
mock_response1 = AsyncMock()
|
||||
mock_response1.raise_for_status = MagicMock()
|
||||
mock_response1.json = AsyncMock(return_value=SAMPLE_INCIDENTS_RESPONSE)
|
||||
|
||||
# Second poll with only 1 incident (GUID-002 fell off)
|
||||
reduced_response = {
|
||||
"type": "FeatureCollection",
|
||||
"features": [SAMPLE_INCIDENTS_RESPONSE["features"][0]],
|
||||
}
|
||||
mock_response2 = AsyncMock()
|
||||
mock_response2.raise_for_status = MagicMock()
|
||||
mock_response2.json = AsyncMock(return_value=reduced_response)
|
||||
|
||||
with patch.object(adapter._session, "get", return_value=AsyncMock(__aenter__=AsyncMock(return_value=mock_response1), __aexit__=AsyncMock())):
|
||||
events1 = [e async for e in adapter.poll()]
|
||||
|
||||
with patch.object(adapter._session, "get", return_value=AsyncMock(__aenter__=AsyncMock(return_value=mock_response2), __aexit__=AsyncMock())):
|
||||
events2 = [e async for e in adapter.poll()]
|
||||
|
||||
await adapter.shutdown()
|
||||
|
||||
# First poll: 2 incident events
|
||||
assert len(events1) == 2
|
||||
|
||||
# Second poll: 1 incident (seen again) + 1 removal for GUID-002
|
||||
# The incident event is yielded (supervisor does dedup via is_published)
|
||||
# The removal is yielded for GUID-002
|
||||
removal_events = [e for e in events2 if e.category == "fire.incident.removed"]
|
||||
assert len(removal_events) == 1
|
||||
assert removal_events[0].data["irwin_id"] == "GUID-002-OWYHEE"
|
||||
|
||||
def test_subject_for_incidents_normalized(
|
||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
||||
):
|
||||
"""subject_for uses normalized state codes."""
|
||||
from central.adapters.wfigs_incidents import WFIGSIncidentsAdapter
|
||||
|
||||
adapter = WFIGSIncidentsAdapter(mock_config, mock_config_store, cursor_db_path)
|
||||
|
||||
# Event data contains normalized state (MT not US-MT)
|
||||
event = Event(
|
||||
id="test-id",
|
||||
adapter="wfigs_incidents",
|
||||
category="fire.incident.wildfire",
|
||||
time=datetime.now(timezone.utc),
|
||||
severity=2,
|
||||
geo=Geo(primary_region="US-MT-GLACIER"),
|
||||
data={"POOState": "MT", "POOCounty": "Glacier"},
|
||||
)
|
||||
|
||||
subject = adapter.subject_for(event)
|
||||
# Subject uses normalized state: mt.glacier not us-mt.glacier
|
||||
assert subject == "central.fire.incident.mt.glacier"
|
||||
|
||||
def test_subject_for_removal(
|
||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
||||
):
|
||||
from central.adapters.wfigs_incidents import WFIGSIncidentsAdapter
|
||||
|
||||
adapter = WFIGSIncidentsAdapter(mock_config, mock_config_store, cursor_db_path)
|
||||
|
||||
event = Event(
|
||||
id="test-id:removed:2024-01-01",
|
||||
adapter="wfigs_incidents",
|
||||
category="fire.incident.removed",
|
||||
time=datetime.now(timezone.utc),
|
||||
severity=0,
|
||||
geo=Geo(),
|
||||
data={"irwin_id": "test-id", "state": "MT"},
|
||||
)
|
||||
|
||||
subject = adapter.subject_for(event)
|
||||
assert subject == "central.fire.incident.removed.mt"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bbox_post_filter(
|
||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
||||
):
|
||||
"""Features outside bbox are filtered out."""
|
||||
from central.adapters.wfigs_incidents import WFIGSIncidentsAdapter
|
||||
|
||||
adapter = WFIGSIncidentsAdapter(mock_config, mock_config_store, cursor_db_path)
|
||||
await adapter.startup()
|
||||
|
||||
mock_response = AsyncMock()
|
||||
mock_response.raise_for_status = MagicMock()
|
||||
mock_response.json = AsyncMock(return_value=SAMPLE_INCIDENTS_RESPONSE)
|
||||
|
||||
with patch.object(adapter._session, "get", return_value=AsyncMock(__aenter__=AsyncMock(return_value=mock_response), __aexit__=AsyncMock())):
|
||||
events = [e async for e in adapter.poll()]
|
||||
|
||||
await adapter.shutdown()
|
||||
|
||||
# Florida incident should be filtered out
|
||||
assert len(events) == 2
|
||||
irwin_ids = {e.id for e in events}
|
||||
assert "GUID-003-FLORIDA" not in irwin_ids
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_config_region_change(
|
||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
||||
):
|
||||
from central.adapters.wfigs_incidents import WFIGSIncidentsAdapter
|
||||
|
||||
adapter = WFIGSIncidentsAdapter(mock_config, mock_config_store, cursor_db_path)
|
||||
|
||||
assert adapter.region.north == 49.0
|
||||
|
||||
new_config = AdapterConfig(
|
||||
name="wfigs_incidents",
|
||||
enabled=True,
|
||||
cadence_s=300,
|
||||
settings={
|
||||
"region": {"north": 50.0, "south": 35.0, "east": -100.0, "west": -120.0}
|
||||
},
|
||||
updated_at=datetime.now(timezone.utc),
|
||||
)
|
||||
await adapter.apply_config(new_config)
|
||||
|
||||
assert adapter.region.north == 50.0
|
||||
assert adapter.region.south == 35.0
|
||||
|
||||
|
||||
class TestWFIGSPerimetersAdapter:
|
||||
"""Tests for WFIGS Perimeters adapter."""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_config(self) -> AdapterConfig:
|
||||
return AdapterConfig(
|
||||
name="wfigs_perimeters",
|
||||
enabled=True,
|
||||
cadence_s=300,
|
||||
settings={
|
||||
"region": {"north": 49.0, "south": 31.0, "east": -102.0, "west": -124.0}
|
||||
},
|
||||
updated_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
@pytest.fixture
|
||||
def mock_config_store(self) -> MagicMock:
|
||||
return MagicMock()
|
||||
|
||||
@pytest.fixture
|
||||
def cursor_db_path(self, tmp_path: Path) -> Path:
|
||||
return tmp_path / "cursors.db"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_normalization_perimeters(
|
||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
||||
):
|
||||
"""Perimeters are correctly normalized to Events with geometry."""
|
||||
from central.adapters.wfigs_perimeters import WFIGSPerimetersAdapter
|
||||
|
||||
adapter = WFIGSPerimetersAdapter(mock_config, mock_config_store, cursor_db_path)
|
||||
await adapter.startup()
|
||||
|
||||
mock_response = AsyncMock()
|
||||
mock_response.raise_for_status = MagicMock()
|
||||
mock_response.json = AsyncMock(return_value=SAMPLE_PERIMETERS_RESPONSE)
|
||||
|
||||
with patch.object(adapter._session, "get", return_value=AsyncMock(__aenter__=AsyncMock(return_value=mock_response), __aexit__=AsyncMock())):
|
||||
events = [e async for e in adapter.poll()]
|
||||
|
||||
await adapter.shutdown()
|
||||
|
||||
assert len(events) == 1
|
||||
|
||||
event = events[0]
|
||||
assert event.id == "GUID-001-GLACIER"
|
||||
assert event.adapter == "wfigs_perimeters"
|
||||
# Category uses normalized incident type
|
||||
assert event.category == "fire.perimeter.wildfire" # NOT fire.perimeter.wf
|
||||
# Region uses normalized state (no double US-)
|
||||
assert event.geo.primary_region == "US-MT-GLACIER" # NOT US-US-MT-GLACIER
|
||||
# Data contains both normalized and raw values
|
||||
assert event.data["POOState"] == "MT" # normalized
|
||||
assert event.data["POOState_raw"] == "US-MT" # raw
|
||||
assert event.data["IncidentTypeCategory"] == "wildfire" # normalized
|
||||
assert event.data["IncidentTypeCategory_raw"] == "WF" # raw
|
||||
# Geometry is included
|
||||
assert "geometry" in event.data
|
||||
assert event.data["geometry"]["type"] == "Polygon"
|
||||
|
||||
def test_subject_for_perimeters_normalized(
|
||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
||||
):
|
||||
"""subject_for uses normalized state codes."""
|
||||
from central.adapters.wfigs_perimeters import WFIGSPerimetersAdapter
|
||||
|
||||
adapter = WFIGSPerimetersAdapter(mock_config, mock_config_store, cursor_db_path)
|
||||
|
||||
# Event data contains normalized state (MT not US-MT)
|
||||
event = Event(
|
||||
id="test-id",
|
||||
adapter="wfigs_perimeters",
|
||||
category="fire.perimeter.wildfire",
|
||||
time=datetime.now(timezone.utc),
|
||||
severity=2,
|
||||
geo=Geo(primary_region="US-MT-GLACIER"),
|
||||
data={"POOState": "MT", "POOCounty": "Glacier", "geometry": {}},
|
||||
)
|
||||
|
||||
subject = adapter.subject_for(event)
|
||||
# Subject uses normalized state: mt.glacier not us-mt.glacier
|
||||
assert subject == "central.fire.perimeter.mt.glacier"
|
||||
|
|
@ -199,3 +199,357 @@ class TestSetupGateMiddlewareWizard:
|
|||
response = client.get("/setup/operator")
|
||||
assert response.status_code == 302
|
||||
assert response.headers["location"] == "/"
|
||||
|
||||
class TestSetupAdaptersErrorRerender:
|
||||
"""Test wizard adapters form error re-render path."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_cadence_rerenders_with_error(self):
|
||||
"""POST /setup/adapters with cadence_s=5 re-renders form with error, no DB write."""
|
||||
from central.gui.routes import setup_adapters_submit
|
||||
|
||||
mock_request = MagicMock()
|
||||
mock_request.cookies = {}
|
||||
mock_request.state = MagicMock()
|
||||
|
||||
# Mock form data with invalid cadence
|
||||
mock_form = MagicMock()
|
||||
mock_form.get.side_effect = lambda k, d="": {
|
||||
"csrf_token": "test_csrf_token",
|
||||
"nws_enabled": "on",
|
||||
"nws_cadence_s": "5", # Invalid: below ge=10
|
||||
"nws_contact_email": "test@example.com",
|
||||
"nws_region_north": "49.0",
|
||||
"nws_region_south": "31.0",
|
||||
"nws_region_east": "-102.0",
|
||||
"nws_region_west": "-124.0",
|
||||
"firms_cadence_s": "300",
|
||||
"firms_region_north": "49.0",
|
||||
"firms_region_south": "31.0",
|
||||
"firms_region_east": "-102.0",
|
||||
"firms_region_west": "-124.0",
|
||||
"usgs_quake_cadence_s": "300",
|
||||
"usgs_quake_feed": "all_hour",
|
||||
"usgs_quake_region_north": "49.0",
|
||||
"usgs_quake_region_south": "31.0",
|
||||
"usgs_quake_region_east": "-102.0",
|
||||
"usgs_quake_region_west": "-124.0",
|
||||
}.get(k, d)
|
||||
mock_form.getlist.side_effect = lambda k: {
|
||||
"firms_satellites": ["VIIRS_SNPP_NRT"],
|
||||
}.get(k, [])
|
||||
mock_form.__contains__ = lambda self, k: k in ["nws_enabled"]
|
||||
|
||||
mock_request.form = AsyncMock(return_value=mock_form)
|
||||
|
||||
# Mock wizard state
|
||||
mock_state = MagicMock()
|
||||
mock_state.operator = {"username": "test", "password_hash": "hash"}
|
||||
mock_state.api_keys = []
|
||||
mock_state.adapters = None
|
||||
mock_state.system = None
|
||||
|
||||
# Mock pool with no actual DB access (should not be called for writes)
|
||||
mock_pool = MagicMock()
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.fetch = AsyncMock(return_value=[
|
||||
{"name": "nws", "enabled": False, "cadence_s": 300, "settings": {}},
|
||||
{"name": "firms", "enabled": False, "cadence_s": 300, "settings": {}},
|
||||
{"name": "usgs_quake", "enabled": False, "cadence_s": 300, "settings": {}},
|
||||
])
|
||||
mock_conn.__aenter__ = AsyncMock(return_value=mock_conn)
|
||||
mock_conn.__aexit__ = AsyncMock()
|
||||
mock_pool.acquire = MagicMock(return_value=mock_conn)
|
||||
|
||||
mock_templates = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_templates.TemplateResponse.return_value = mock_response
|
||||
|
||||
with patch("central.gui.routes._get_templates", return_value=mock_templates):
|
||||
with patch("central.gui.routes.get_pool", return_value=mock_pool):
|
||||
with patch("central.gui.routes.get_settings") as mock_settings:
|
||||
mock_settings.return_value.csrf_secret = "testsecret12345678901234567890ab"
|
||||
with patch("central.gui.routes.validate_pre_auth_csrf", return_value=True):
|
||||
with patch("central.gui.wizard.get_wizard_state", return_value=mock_state):
|
||||
with patch("central.gui.routes.reuse_or_generate_pre_auth_csrf", return_value=("csrf", None)):
|
||||
result = await setup_adapters_submit(mock_request)
|
||||
|
||||
# Should return 200 (re-render), not 302 (redirect)
|
||||
assert result.status_code == 200
|
||||
|
||||
# Check that template was called with errors
|
||||
call_args = mock_templates.TemplateResponse.call_args
|
||||
context = call_args.kwargs.get("context", call_args[1].get("context"))
|
||||
|
||||
assert context["error"] == "Please fix the errors below."
|
||||
assert "errors" in context
|
||||
assert context["errors"] is not None
|
||||
assert "nws_cadence_s" in context["errors"]
|
||||
assert "10" in context["errors"]["nws_cadence_s"] # Should mention min value
|
||||
|
||||
# Verify adapters have correct shape (with fields)
|
||||
assert "adapters" in context
|
||||
for adapter in context["adapters"]:
|
||||
assert "name" in adapter
|
||||
assert "display_name" in adapter
|
||||
assert "enabled" in adapter
|
||||
assert "cadence_s" in adapter
|
||||
assert "settings" in adapter
|
||||
assert "fields" in adapter
|
||||
|
||||
# Verify no DB execute was called (no writes)
|
||||
mock_conn.execute.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_region_bounds_shows_pydantic_error(self):
|
||||
"""POST /setup/adapters with inverted region bounds shows RegionConfig error."""
|
||||
from central.gui.routes import setup_adapters_submit
|
||||
|
||||
mock_request = MagicMock()
|
||||
mock_request.cookies = {}
|
||||
mock_request.state = MagicMock()
|
||||
|
||||
# Mock form data with inverted region (south > north)
|
||||
mock_form = MagicMock()
|
||||
mock_form.get.side_effect = lambda k, d="": {
|
||||
"csrf_token": "test_csrf_token",
|
||||
"nws_cadence_s": "300",
|
||||
"nws_contact_email": "test@example.com",
|
||||
"nws_region_north": "10.0", # Invalid: north < south
|
||||
"nws_region_south": "20.0",
|
||||
"nws_region_east": "-102.0",
|
||||
"nws_region_west": "-124.0",
|
||||
"firms_cadence_s": "300",
|
||||
"firms_region_north": "49.0",
|
||||
"firms_region_south": "31.0",
|
||||
"firms_region_east": "-102.0",
|
||||
"firms_region_west": "-124.0",
|
||||
"usgs_quake_cadence_s": "300",
|
||||
"usgs_quake_feed": "all_hour",
|
||||
"usgs_quake_region_north": "49.0",
|
||||
"usgs_quake_region_south": "31.0",
|
||||
"usgs_quake_region_east": "-102.0",
|
||||
"usgs_quake_region_west": "-124.0",
|
||||
}.get(k, d)
|
||||
mock_form.getlist.side_effect = lambda k: {
|
||||
"firms_satellites": ["VIIRS_SNPP_NRT"],
|
||||
}.get(k, [])
|
||||
mock_form.__contains__ = lambda self, k: False
|
||||
|
||||
mock_request.form = AsyncMock(return_value=mock_form)
|
||||
|
||||
mock_state = MagicMock()
|
||||
mock_state.operator = {"username": "test", "password_hash": "hash"}
|
||||
mock_state.api_keys = []
|
||||
mock_state.adapters = None
|
||||
mock_state.system = None
|
||||
|
||||
mock_pool = MagicMock()
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.fetch = AsyncMock(return_value=[
|
||||
{"name": "nws", "enabled": False, "cadence_s": 300, "settings": {}},
|
||||
{"name": "firms", "enabled": False, "cadence_s": 300, "settings": {}},
|
||||
{"name": "usgs_quake", "enabled": False, "cadence_s": 300, "settings": {}},
|
||||
])
|
||||
mock_conn.__aenter__ = AsyncMock(return_value=mock_conn)
|
||||
mock_conn.__aexit__ = AsyncMock()
|
||||
mock_pool.acquire = MagicMock(return_value=mock_conn)
|
||||
|
||||
mock_templates = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_templates.TemplateResponse.return_value = mock_response
|
||||
|
||||
with patch("central.gui.routes._get_templates", return_value=mock_templates):
|
||||
with patch("central.gui.routes.get_pool", return_value=mock_pool):
|
||||
with patch("central.gui.routes.get_settings") as mock_settings:
|
||||
mock_settings.return_value.csrf_secret = "testsecret12345678901234567890ab"
|
||||
with patch("central.gui.routes.validate_pre_auth_csrf", return_value=True):
|
||||
with patch("central.gui.wizard.get_wizard_state", return_value=mock_state):
|
||||
with patch("central.gui.routes.reuse_or_generate_pre_auth_csrf", return_value=("csrf", None)):
|
||||
result = await setup_adapters_submit(mock_request)
|
||||
|
||||
assert result.status_code == 200
|
||||
|
||||
call_args = mock_templates.TemplateResponse.call_args
|
||||
context = call_args.kwargs.get("context", call_args[1].get("context"))
|
||||
|
||||
assert context["errors"] is not None
|
||||
assert "nws_region" in context["errors"]
|
||||
# Error should come from RegionConfig validator, mentioning bounds
|
||||
assert "north" in context["errors"]["nws_region"].lower() or "south" in context["errors"]["nws_region"].lower()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_contact_email_via_pydantic_pattern(self):
|
||||
"""POST /setup/adapters with NWS contact_email='not-an-email' shows Pydantic pattern error."""
|
||||
from central.gui.routes import setup_adapters_submit
|
||||
|
||||
mock_request = MagicMock()
|
||||
mock_request.cookies = {}
|
||||
mock_request.state = MagicMock()
|
||||
|
||||
mock_form = MagicMock()
|
||||
mock_form.get.side_effect = lambda k, d="": {
|
||||
"csrf_token": "test_csrf_token",
|
||||
"nws_enabled": "on",
|
||||
"nws_cadence_s": "300",
|
||||
"nws_contact_email": "not-an-email", # Invalid email format
|
||||
"nws_region_north": "49.0",
|
||||
"nws_region_south": "31.0",
|
||||
"nws_region_east": "-102.0",
|
||||
"nws_region_west": "-124.0",
|
||||
"firms_cadence_s": "300",
|
||||
"firms_region_north": "49.0",
|
||||
"firms_region_south": "31.0",
|
||||
"firms_region_east": "-102.0",
|
||||
"firms_region_west": "-124.0",
|
||||
"usgs_quake_cadence_s": "300",
|
||||
"usgs_quake_feed": "all_hour",
|
||||
"usgs_quake_region_north": "49.0",
|
||||
"usgs_quake_region_south": "31.0",
|
||||
"usgs_quake_region_east": "-102.0",
|
||||
"usgs_quake_region_west": "-124.0",
|
||||
}.get(k, d)
|
||||
mock_form.getlist.side_effect = lambda k: {
|
||||
"firms_satellites": ["VIIRS_SNPP_NRT"],
|
||||
}.get(k, [])
|
||||
mock_form.__contains__ = lambda self, k: k in ["nws_enabled"]
|
||||
|
||||
mock_request.form = AsyncMock(return_value=mock_form)
|
||||
|
||||
mock_state = MagicMock()
|
||||
mock_state.operator = {"username": "test", "password_hash": "hash"}
|
||||
mock_state.api_keys = []
|
||||
mock_state.adapters = None
|
||||
mock_state.system = None
|
||||
|
||||
mock_pool = MagicMock()
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.fetch = AsyncMock(return_value=[
|
||||
{"name": "nws", "enabled": False, "cadence_s": 300, "settings": {}},
|
||||
{"name": "firms", "enabled": False, "cadence_s": 300, "settings": {}},
|
||||
{"name": "usgs_quake", "enabled": False, "cadence_s": 300, "settings": {}},
|
||||
])
|
||||
mock_conn.__aenter__ = AsyncMock(return_value=mock_conn)
|
||||
mock_conn.__aexit__ = AsyncMock()
|
||||
mock_pool.acquire = MagicMock(return_value=mock_conn)
|
||||
|
||||
mock_templates = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_templates.TemplateResponse.return_value = mock_response
|
||||
|
||||
with patch("central.gui.routes._get_templates", return_value=mock_templates):
|
||||
with patch("central.gui.routes.get_pool", return_value=mock_pool):
|
||||
with patch("central.gui.routes.get_settings") as mock_settings:
|
||||
mock_settings.return_value.csrf_secret = "testsecret12345678901234567890ab"
|
||||
with patch("central.gui.routes.validate_pre_auth_csrf", return_value=True):
|
||||
with patch("central.gui.wizard.get_wizard_state", return_value=mock_state):
|
||||
with patch("central.gui.routes.reuse_or_generate_pre_auth_csrf", return_value=("csrf", None)):
|
||||
result = await setup_adapters_submit(mock_request)
|
||||
|
||||
assert result.status_code == 200
|
||||
|
||||
call_args = mock_templates.TemplateResponse.call_args
|
||||
context = call_args.kwargs.get("context", call_args[1].get("context"))
|
||||
|
||||
assert context["errors"] is not None
|
||||
assert "nws_contact_email" in context["errors"]
|
||||
# Error should be from Pydantic pattern validation
|
||||
error_msg = context["errors"]["nws_contact_email"].lower()
|
||||
assert "pattern" in error_msg or "string" in error_msg or "match" in error_msg
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_api_key_alias_generic(self):
|
||||
"""POST /setup/adapters with FIRMS api_key_alias='bogus' shows generic error."""
|
||||
from central.gui.routes import setup_adapters_submit
|
||||
|
||||
mock_request = MagicMock()
|
||||
mock_request.cookies = {}
|
||||
mock_request.state = MagicMock()
|
||||
|
||||
mock_form = MagicMock()
|
||||
mock_form.get.side_effect = lambda k, d="": {
|
||||
"csrf_token": "test_csrf_token",
|
||||
"nws_cadence_s": "300",
|
||||
"nws_contact_email": "test@example.com",
|
||||
"nws_region_north": "49.0",
|
||||
"nws_region_south": "31.0",
|
||||
"nws_region_east": "-102.0",
|
||||
"nws_region_west": "-124.0",
|
||||
"firms_cadence_s": "300",
|
||||
"firms_api_key_alias": "bogus-alias-not-in-state", # Invalid alias
|
||||
"firms_region_north": "49.0",
|
||||
"firms_region_south": "31.0",
|
||||
"firms_region_east": "-102.0",
|
||||
"firms_region_west": "-124.0",
|
||||
"usgs_quake_cadence_s": "300",
|
||||
"usgs_quake_feed": "all_hour",
|
||||
"usgs_quake_region_north": "49.0",
|
||||
"usgs_quake_region_south": "31.0",
|
||||
"usgs_quake_region_east": "-102.0",
|
||||
"usgs_quake_region_west": "-124.0",
|
||||
}.get(k, d)
|
||||
mock_form.getlist.side_effect = lambda k: {
|
||||
"firms_satellites": ["VIIRS_SNPP_NRT"],
|
||||
}.get(k, [])
|
||||
mock_form.__contains__ = lambda self, k: False
|
||||
|
||||
mock_request.form = AsyncMock(return_value=mock_form)
|
||||
|
||||
mock_state = MagicMock()
|
||||
mock_state.operator = {"username": "test", "password_hash": "hash"}
|
||||
mock_state.api_keys = [{"alias": "valid_key"}] # Only valid_key exists
|
||||
mock_state.adapters = None
|
||||
mock_state.system = None
|
||||
|
||||
mock_pool = MagicMock()
|
||||
mock_conn = MagicMock()
|
||||
mock_conn.fetch = AsyncMock(return_value=[
|
||||
{"name": "nws", "enabled": False, "cadence_s": 300, "settings": {}},
|
||||
{"name": "firms", "enabled": False, "cadence_s": 300, "settings": {}},
|
||||
{"name": "usgs_quake", "enabled": False, "cadence_s": 300, "settings": {}},
|
||||
])
|
||||
mock_conn.__aenter__ = AsyncMock(return_value=mock_conn)
|
||||
mock_conn.__aexit__ = AsyncMock()
|
||||
mock_pool.acquire = MagicMock(return_value=mock_conn)
|
||||
|
||||
mock_templates = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_templates.TemplateResponse.return_value = mock_response
|
||||
|
||||
with patch("central.gui.routes._get_templates", return_value=mock_templates):
|
||||
with patch("central.gui.routes.get_pool", return_value=mock_pool):
|
||||
with patch("central.gui.routes.get_settings") as mock_settings:
|
||||
mock_settings.return_value.csrf_secret = "testsecret12345678901234567890ab"
|
||||
with patch("central.gui.routes.validate_pre_auth_csrf", return_value=True):
|
||||
with patch("central.gui.wizard.get_wizard_state", return_value=mock_state):
|
||||
with patch("central.gui.routes.reuse_or_generate_pre_auth_csrf", return_value=("csrf", None)):
|
||||
result = await setup_adapters_submit(mock_request)
|
||||
|
||||
assert result.status_code == 200
|
||||
|
||||
call_args = mock_templates.TemplateResponse.call_args
|
||||
context = call_args.kwargs.get("context", call_args[1].get("context"))
|
||||
|
||||
assert context["errors"] is not None
|
||||
assert "firms_api_key_alias" in context["errors"]
|
||||
assert "API key alias does not exist" in context["errors"]["firms_api_key_alias"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_key_field_none_no_check(self):
|
||||
"""Adapters with api_key_field=None do not trigger the api_key check."""
|
||||
# Verify that NWSAdapter has api_key_field=None
|
||||
from central.adapters.nws import NWSAdapter
|
||||
from central.adapters.firms import FIRMSAdapter
|
||||
from central.adapters.usgs_quake import USGSQuakeAdapter
|
||||
|
||||
# NWS and USGS should have api_key_field=None
|
||||
assert NWSAdapter.api_key_field is None
|
||||
assert USGSQuakeAdapter.api_key_field is None
|
||||
|
||||
# FIRMS should have api_key_field set
|
||||
assert FIRMSAdapter.api_key_field == "api_key_alias"
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue