mirror of
https://github.com/zvx-echo6/central.git
synced 2026-06-10 11:54:37 +02:00
Compare commits
No commits in common. "37a778468d25ccaf9ab59e2f83718ef866c98465" and "87f46e8b35e4de61c9538d9d2300920c3ad28206" have entirely different histories.
37a778468d
...
87f46e8b35
42 changed files with 574 additions and 5759 deletions
|
|
@ -1,6 +0,0 @@
|
||||||
-- Migration: 015_add_adapters_last_error
|
|
||||||
-- Adds last_error column for adapter-side error reporting.
|
|
||||||
-- Populated by supervisor when an adapter fails to start or apply config.
|
|
||||||
|
|
||||||
ALTER TABLE config.adapters
|
|
||||||
ADD COLUMN IF NOT EXISTS last_error TEXT;
|
|
||||||
|
|
@ -1,37 +0,0 @@
|
||||||
-- Migration: 016_add_wfigs_adapters
|
|
||||||
-- Add WFIGS incident and perimeter adapters to config.adapters
|
|
||||||
-- Idempotent: uses ON CONFLICT DO NOTHING
|
|
||||||
|
|
||||||
-- WFIGS Incidents adapter
|
|
||||||
INSERT INTO config.adapters (name, enabled, cadence_s, settings)
|
|
||||||
VALUES (
|
|
||||||
'wfigs_incidents',
|
|
||||||
false, -- Ships disabled; operator enables via GUI
|
|
||||||
300,
|
|
||||||
jsonb_build_object(
|
|
||||||
'region', jsonb_build_object(
|
|
||||||
'north', 49.0,
|
|
||||||
'south', 31.0,
|
|
||||||
'east', -102.0,
|
|
||||||
'west', -124.0
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
ON CONFLICT (name) DO NOTHING;
|
|
||||||
|
|
||||||
-- WFIGS Perimeters adapter
|
|
||||||
INSERT INTO config.adapters (name, enabled, cadence_s, settings)
|
|
||||||
VALUES (
|
|
||||||
'wfigs_perimeters',
|
|
||||||
false, -- Ships disabled; operator enables via GUI
|
|
||||||
300,
|
|
||||||
jsonb_build_object(
|
|
||||||
'region', jsonb_build_object(
|
|
||||||
'north', 49.0,
|
|
||||||
'south', 31.0,
|
|
||||||
'east', -102.0,
|
|
||||||
'west', -124.0
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
ON CONFLICT (name) DO NOTHING;
|
|
||||||
|
|
@ -1,19 +0,0 @@
|
||||||
-- Migration: 017_add_inciweb_adapter
|
|
||||||
-- Add InciWeb adapter to config.adapters
|
|
||||||
-- Idempotent: uses ON CONFLICT DO NOTHING
|
|
||||||
|
|
||||||
INSERT INTO config.adapters (name, enabled, cadence_s, settings)
|
|
||||||
VALUES (
|
|
||||||
'inciweb',
|
|
||||||
false, -- Ships disabled; operator enables via GUI
|
|
||||||
600,
|
|
||||||
jsonb_build_object(
|
|
||||||
'region', jsonb_build_object(
|
|
||||||
'north', 49.0,
|
|
||||||
'south', 31.0,
|
|
||||||
'east', -102.0,
|
|
||||||
'west', -124.0
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
ON CONFLICT (name) DO NOTHING;
|
|
||||||
|
|
@ -1,11 +0,0 @@
|
||||||
-- Migration: 018_add_swpc_adapters
|
|
||||||
-- Add NOAA SWPC space weather adapters to config.adapters.
|
|
||||||
-- All three ship disabled; operator enables individually via GUI.
|
|
||||||
-- Idempotent: uses ON CONFLICT DO NOTHING.
|
|
||||||
|
|
||||||
INSERT INTO config.adapters (name, enabled, cadence_s, settings)
|
|
||||||
VALUES
|
|
||||||
('swpc_alerts', false, 300, '{}'::jsonb),
|
|
||||||
('swpc_kindex', false, 600, '{}'::jsonb),
|
|
||||||
('swpc_protons', false, 600, '{}'::jsonb)
|
|
||||||
ON CONFLICT (name) DO NOTHING;
|
|
||||||
|
|
@ -1,8 +0,0 @@
|
||||||
-- Migration: 019_add_central_space_stream
|
|
||||||
-- Seeds the CENTRAL_SPACE JetStream stream row for central.space.> subjects.
|
|
||||||
-- 7-day retention, 1 GiB max_bytes (clamped by supervisor recompute) -- mirrors CENTRAL_FIRE / CENTRAL_QUAKE.
|
|
||||||
-- Idempotent: uses ON CONFLICT DO NOTHING.
|
|
||||||
|
|
||||||
INSERT INTO config.streams (name, max_age_s, max_bytes)
|
|
||||||
VALUES ('CENTRAL_SPACE', 604800, 1073741824)
|
|
||||||
ON CONFLICT (name) DO NOTHING;
|
|
||||||
|
|
@ -34,10 +34,6 @@ class SourceAdapter(ABC):
|
||||||
description: str
|
description: str
|
||||||
settings_schema: type[BaseModel]
|
settings_schema: type[BaseModel]
|
||||||
requires_api_key: str | None = None
|
requires_api_key: str | None = None
|
||||||
api_key_field: str | None = None
|
|
||||||
"""Names the settings_schema field that holds an api_key alias reference, if any.
|
|
||||||
The GUI renders this field as a select populated from config.api_keys;
|
|
||||||
the wizard validates it against staged api_keys state."""
|
|
||||||
wizard_order: int | None = None
|
wizard_order: int | None = None
|
||||||
default_cadence_s: int
|
default_cadence_s: int
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,34 +0,0 @@
|
||||||
"""Adapter discovery utilities."""
|
|
||||||
|
|
||||||
import importlib
|
|
||||||
import logging
|
|
||||||
import pkgutil
|
|
||||||
|
|
||||||
import central.adapters
|
|
||||||
from central.adapter import SourceAdapter
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
def discover_adapters() -> dict[str, type[SourceAdapter]]:
|
|
||||||
"""Auto-discover adapter classes from central.adapters package."""
|
|
||||||
registry: dict[str, type[SourceAdapter]] = {}
|
|
||||||
for module_info in pkgutil.iter_modules(central.adapters.__path__):
|
|
||||||
try:
|
|
||||||
module = importlib.import_module(f"central.adapters.{module_info.name}")
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(
|
|
||||||
"Failed to import adapter module",
|
|
||||||
extra={"module": module_info.name, "error": str(e)},
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
for attr_name in dir(module):
|
|
||||||
attr = getattr(module, attr_name)
|
|
||||||
if (
|
|
||||||
isinstance(attr, type)
|
|
||||||
and issubclass(attr, SourceAdapter)
|
|
||||||
and attr is not SourceAdapter
|
|
||||||
and hasattr(attr, "name")
|
|
||||||
):
|
|
||||||
registry[attr.name] = attr
|
|
||||||
return registry
|
|
||||||
|
|
@ -7,7 +7,7 @@ from collections.abc import AsyncIterator
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from io import StringIO
|
from io import StringIO
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Literal
|
from typing import Any
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
from tenacity import (
|
from tenacity import (
|
||||||
|
|
@ -54,7 +54,7 @@ SEVERITY_MAP = {
|
||||||
class FIRMSSettings(BaseModel):
|
class FIRMSSettings(BaseModel):
|
||||||
"""Settings schema for FIRMS adapter."""
|
"""Settings schema for FIRMS adapter."""
|
||||||
api_key_alias: str = "firms"
|
api_key_alias: str = "firms"
|
||||||
satellites: list[Literal["VIIRS_SNPP_NRT", "VIIRS_NOAA20_NRT", "VIIRS_NOAA21_NRT"]] = ["VIIRS_SNPP_NRT", "VIIRS_NOAA20_NRT"]
|
satellites: list[str] = ["VIIRS_SNPP_NRT", "VIIRS_NOAA20_NRT"]
|
||||||
region: RegionConfig | None = None
|
region: RegionConfig | None = None
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -66,7 +66,6 @@ class FIRMSAdapter(SourceAdapter):
|
||||||
description = "Near-real-time satellite-detected fire hotspots from NASA FIRMS."
|
description = "Near-real-time satellite-detected fire hotspots from NASA FIRMS."
|
||||||
settings_schema = FIRMSSettings
|
settings_schema = FIRMSSettings
|
||||||
requires_api_key = "firms"
|
requires_api_key = "firms"
|
||||||
api_key_field = "api_key_alias"
|
|
||||||
wizard_order = 2
|
wizard_order = 2
|
||||||
default_cadence_s = 300
|
default_cadence_s = 300
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,477 +0,0 @@
|
||||||
"""InciWeb adapter for wildfire narrative updates."""
|
|
||||||
|
|
||||||
import html
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
import sqlite3
|
|
||||||
from collections.abc import AsyncIterator
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
from email.utils import parsedate_to_datetime
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
from xml.etree import ElementTree as ET
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from pydantic import BaseModel
|
|
||||||
from tenacity import (
|
|
||||||
retry,
|
|
||||||
retry_if_exception_type,
|
|
||||||
stop_after_attempt,
|
|
||||||
wait_exponential_jitter,
|
|
||||||
)
|
|
||||||
|
|
||||||
from central.adapter import SourceAdapter
|
|
||||||
from central.config_models import AdapterConfig, RegionConfig
|
|
||||||
from central.config_store import ConfigStore
|
|
||||||
from central.models import Event, Geo
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
# InciWeb RSS feed URL
|
|
||||||
INCIWEB_RSS_URL = "https://inciweb.wildfire.gov/incidents/rss.xml"
|
|
||||||
|
|
||||||
# State name to 2-letter code mapping
|
|
||||||
STATE_NAME_TO_CODE = {
|
|
||||||
"alabama": "AL", "alaska": "AK", "arizona": "AZ", "arkansas": "AR",
|
|
||||||
"california": "CA", "colorado": "CO", "connecticut": "CT", "delaware": "DE",
|
|
||||||
"florida": "FL", "georgia": "GA", "hawaii": "HI", "idaho": "ID",
|
|
||||||
"illinois": "IL", "indiana": "IN", "iowa": "IA", "kansas": "KS",
|
|
||||||
"kentucky": "KY", "louisiana": "LA", "maine": "ME", "maryland": "MD",
|
|
||||||
"massachusetts": "MA", "michigan": "MI", "minnesota": "MN", "mississippi": "MS",
|
|
||||||
"missouri": "MO", "montana": "MT", "nebraska": "NE", "nevada": "NV",
|
|
||||||
"new hampshire": "NH", "new jersey": "NJ", "new mexico": "NM", "new york": "NY",
|
|
||||||
"north carolina": "NC", "north dakota": "ND", "ohio": "OH", "oklahoma": "OK",
|
|
||||||
"oregon": "OR", "pennsylvania": "PA", "rhode island": "RI", "south carolina": "SC",
|
|
||||||
"south dakota": "SD", "tennessee": "TN", "texas": "TX", "utah": "UT",
|
|
||||||
"vermont": "VT", "virginia": "VA", "washington": "WA", "west virginia": "WV",
|
|
||||||
"wisconsin": "WI", "wyoming": "WY", "district of columbia": "DC",
|
|
||||||
"puerto rico": "PR", "guam": "GU", "virgin islands": "VI",
|
|
||||||
"american samoa": "AS", "northern mariana islands": "MP",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def parse_coordinates_from_description(description: str) -> tuple[float, float] | None:
|
|
||||||
"""
|
|
||||||
Parse latitude/longitude from InciWeb description text.
|
|
||||||
|
|
||||||
Format: "Latitude: 47° 3 17 Longitude: 91° 38 6"
|
|
||||||
InciWeb uses unsigned values for US coordinates (west longitude implied).
|
|
||||||
Returns (lon, lat) tuple or None if not found.
|
|
||||||
"""
|
|
||||||
# Pattern for degree/minute/second format
|
|
||||||
lat_pattern = r"Latitude:\s*(-?\d+)°\s*(\d+)\s*(\d+(?:\.\d+)?)"
|
|
||||||
lon_pattern = r"Longitude:\s*(-?\d+)°\s*(\d+)\s*(\d+(?:\.\d+)?)"
|
|
||||||
|
|
||||||
lat_match = re.search(lat_pattern, description)
|
|
||||||
lon_match = re.search(lon_pattern, description)
|
|
||||||
|
|
||||||
if not lat_match or not lon_match:
|
|
||||||
return None
|
|
||||||
|
|
||||||
try:
|
|
||||||
lat_deg = int(lat_match.group(1))
|
|
||||||
lat_min = int(lat_match.group(2))
|
|
||||||
lat_sec = float(lat_match.group(3))
|
|
||||||
|
|
||||||
lon_deg = int(lon_match.group(1))
|
|
||||||
lon_min = int(lon_match.group(2))
|
|
||||||
lon_sec = float(lon_match.group(3))
|
|
||||||
|
|
||||||
# Convert to decimal degrees
|
|
||||||
# Latitude: positive in northern hemisphere
|
|
||||||
if lat_deg >= 0:
|
|
||||||
lat = lat_deg + lat_min / 60 + lat_sec / 3600
|
|
||||||
else:
|
|
||||||
lat = lat_deg - lat_min / 60 - lat_sec / 3600
|
|
||||||
|
|
||||||
# Longitude: InciWeb gives unsigned values for US west longitudes
|
|
||||||
# Make negative for western hemisphere (US coordinates)
|
|
||||||
lon = lon_deg + lon_min / 60 + lon_sec / 3600
|
|
||||||
if lon > 0:
|
|
||||||
lon = -lon # US longitudes are west (negative)
|
|
||||||
|
|
||||||
return (lon, lat)
|
|
||||||
except (ValueError, TypeError):
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def parse_state_from_description(description: str) -> str | None:
|
|
||||||
"""
|
|
||||||
Parse state name from InciWeb description text.
|
|
||||||
|
|
||||||
Format: "State: Minnesota" or "State: New Mexico"
|
|
||||||
Returns 2-letter state code or None if not found.
|
|
||||||
|
|
||||||
Design note: State is parsed from the description rather than the title
|
|
||||||
because InciWeb titles use unit code prefixes (e.g., "MNMNS Stewart Trail",
|
|
||||||
"CACNP Santa Rosa Island Fire") which are not reliable state indicators.
|
|
||||||
The description has a structured "State: <name>" field that reliably
|
|
||||||
identifies the state for all incidents.
|
|
||||||
"""
|
|
||||||
pattern = r"State:\s*([A-Za-z\s]+?)(?:\n|---|$)"
|
|
||||||
match = re.search(pattern, description)
|
|
||||||
|
|
||||||
if not match:
|
|
||||||
return None
|
|
||||||
|
|
||||||
state_name = match.group(1).strip().lower()
|
|
||||||
return STATE_NAME_TO_CODE.get(state_name)
|
|
||||||
|
|
||||||
|
|
||||||
def strip_html(html_text: str) -> str:
|
|
||||||
"""
|
|
||||||
Strip HTML tags and decode entities to plain text.
|
|
||||||
"""
|
|
||||||
# Decode HTML entities (handles & < > etc.)
|
|
||||||
text = html.unescape(html_text)
|
|
||||||
|
|
||||||
# Handle specifically (not a standard Python html entity)
|
|
||||||
text = text.replace(" ", " ")
|
|
||||||
text = text.replace("\xa0", " ") # Non-breaking space character
|
|
||||||
|
|
||||||
# Remove HTML tags
|
|
||||||
text = re.sub(r"<[^>]+>", "", text)
|
|
||||||
|
|
||||||
# Normalize whitespace
|
|
||||||
text = re.sub(r"\s+", " ", text)
|
|
||||||
|
|
||||||
return text.strip()
|
|
||||||
|
|
||||||
|
|
||||||
def point_in_bbox(
|
|
||||||
lon: float,
|
|
||||||
lat: float,
|
|
||||||
west: float,
|
|
||||||
south: float,
|
|
||||||
east: float,
|
|
||||||
north: float,
|
|
||||||
) -> bool:
|
|
||||||
"""Check if a point is within a bounding box."""
|
|
||||||
return west <= lon <= east and south <= lat <= north
|
|
||||||
|
|
||||||
|
|
||||||
class InciWebSettings(BaseModel):
|
|
||||||
"""Settings schema for InciWeb adapter."""
|
|
||||||
|
|
||||||
region: RegionConfig | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class InciWebAdapter(SourceAdapter):
|
|
||||||
"""NIFC InciWeb wildfire narrative adapter."""
|
|
||||||
|
|
||||||
name = "inciweb"
|
|
||||||
display_name = "NIFC InciWeb — Wildfire Narrative"
|
|
||||||
description = (
|
|
||||||
"Narrative wildfire updates from InciWeb. Editorial; lower precision "
|
|
||||||
"than WFIGS. Use as supplementary context."
|
|
||||||
)
|
|
||||||
settings_schema = InciWebSettings
|
|
||||||
requires_api_key = None
|
|
||||||
api_key_field = None
|
|
||||||
wizard_order = None # Ships disabled
|
|
||||||
default_cadence_s = 600
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
config: AdapterConfig,
|
|
||||||
config_store: ConfigStore,
|
|
||||||
cursor_db_path: Path,
|
|
||||||
) -> None:
|
|
||||||
self._config_store = config_store
|
|
||||||
self._cursor_db_path = cursor_db_path
|
|
||||||
self._session: aiohttp.ClientSession | None = None
|
|
||||||
self._db: sqlite3.Connection | None = None
|
|
||||||
|
|
||||||
# Conditional fetch state
|
|
||||||
self._last_modified: str | None = None
|
|
||||||
self._etag: str | None = None
|
|
||||||
|
|
||||||
# Parse region from settings
|
|
||||||
region_dict = config.settings.get("region")
|
|
||||||
if region_dict:
|
|
||||||
self.region: RegionConfig | None = RegionConfig(**region_dict)
|
|
||||||
else:
|
|
||||||
self.region = None
|
|
||||||
|
|
||||||
async def startup(self) -> None:
|
|
||||||
"""Initialize HTTP session and SQLite connection."""
|
|
||||||
self._session = aiohttp.ClientSession(
|
|
||||||
timeout=aiohttp.ClientTimeout(total=60),
|
|
||||||
)
|
|
||||||
self._db = sqlite3.connect(self._cursor_db_path)
|
|
||||||
|
|
||||||
# Create table for dedup tracking
|
|
||||||
self._db.execute("""
|
|
||||||
CREATE TABLE IF NOT EXISTS published_ids (
|
|
||||||
adapter TEXT NOT NULL,
|
|
||||||
event_id TEXT NOT NULL,
|
|
||||||
first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
PRIMARY KEY (adapter, event_id)
|
|
||||||
)
|
|
||||||
""")
|
|
||||||
self._db.execute("""
|
|
||||||
CREATE INDEX IF NOT EXISTS published_ids_last_seen
|
|
||||||
ON published_ids (last_seen)
|
|
||||||
""")
|
|
||||||
self._db.commit()
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
"InciWeb adapter started",
|
|
||||||
extra={"region": self.region.model_dump() if self.region else None},
|
|
||||||
)
|
|
||||||
|
|
||||||
async def shutdown(self) -> None:
|
|
||||||
"""Close HTTP session and SQLite connection."""
|
|
||||||
if self._session:
|
|
||||||
await self._session.close()
|
|
||||||
self._session = None
|
|
||||||
if self._db:
|
|
||||||
self._db.close()
|
|
||||||
self._db = None
|
|
||||||
logger.info("InciWeb adapter shut down")
|
|
||||||
|
|
||||||
async def apply_config(self, new_config: AdapterConfig) -> None:
|
|
||||||
"""Apply new configuration from hot-reload."""
|
|
||||||
region_dict = new_config.settings.get("region")
|
|
||||||
if region_dict:
|
|
||||||
self.region = RegionConfig(**region_dict)
|
|
||||||
else:
|
|
||||||
self.region = None
|
|
||||||
logger.info(
|
|
||||||
"InciWeb config updated",
|
|
||||||
extra={"region": self.region.model_dump() if self.region else None},
|
|
||||||
)
|
|
||||||
|
|
||||||
def is_published(self, event_id: str) -> bool:
|
|
||||||
"""Check if an event has already been published."""
|
|
||||||
if not self._db:
|
|
||||||
return False
|
|
||||||
cur = self._db.execute(
|
|
||||||
"SELECT 1 FROM published_ids WHERE adapter = ? AND event_id = ?",
|
|
||||||
(self.name, event_id),
|
|
||||||
)
|
|
||||||
return cur.fetchone() is not None
|
|
||||||
|
|
||||||
def mark_published(self, event_id: str) -> None:
|
|
||||||
"""Mark an event as published."""
|
|
||||||
if not self._db:
|
|
||||||
return
|
|
||||||
self._db.execute(
|
|
||||||
"""
|
|
||||||
INSERT INTO published_ids (adapter, event_id, first_seen, last_seen)
|
|
||||||
VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
||||||
ON CONFLICT (adapter, event_id) DO UPDATE SET
|
|
||||||
last_seen = CURRENT_TIMESTAMP
|
|
||||||
""",
|
|
||||||
(self.name, event_id),
|
|
||||||
)
|
|
||||||
self._db.commit()
|
|
||||||
|
|
||||||
def bump_last_seen(self, event_id: str) -> None:
|
|
||||||
"""Bump the last_seen timestamp for an event."""
|
|
||||||
if not self._db:
|
|
||||||
return
|
|
||||||
self._db.execute(
|
|
||||||
"UPDATE published_ids SET last_seen = CURRENT_TIMESTAMP WHERE adapter = ? AND event_id = ?",
|
|
||||||
(self.name, event_id),
|
|
||||||
)
|
|
||||||
self._db.commit()
|
|
||||||
|
|
||||||
def sweep_old_ids(self) -> int:
|
|
||||||
"""Remove published_ids older than 14 days. Returns count deleted."""
|
|
||||||
if not self._db:
|
|
||||||
return 0
|
|
||||||
cur = self._db.execute(
|
|
||||||
"DELETE FROM published_ids WHERE adapter = ? AND last_seen < datetime('now', '-14 days')",
|
|
||||||
(self.name,),
|
|
||||||
)
|
|
||||||
self._db.commit()
|
|
||||||
count = cur.rowcount
|
|
||||||
if count > 0:
|
|
||||||
logger.info("InciWeb swept old dedup entries", extra={"count": count})
|
|
||||||
return count
|
|
||||||
|
|
||||||
def subject_for(self, event: Event) -> str:
|
|
||||||
"""Compute NATS subject for an event."""
|
|
||||||
state = event.geo.primary_region
|
|
||||||
if state and state.startswith("US-") and len(state) == 5:
|
|
||||||
state_code = state[3:].lower()
|
|
||||||
return f"central.fire.narrative.inciweb.{state_code}"
|
|
||||||
return "central.fire.narrative.inciweb.unknown"
|
|
||||||
|
|
||||||
@retry(
|
|
||||||
stop=stop_after_attempt(3),
|
|
||||||
wait=wait_exponential_jitter(initial=1, max=30),
|
|
||||||
retry=retry_if_exception_type((aiohttp.ClientError, TimeoutError)),
|
|
||||||
)
|
|
||||||
async def _fetch_rss(self) -> list[dict[str, Any]]:
|
|
||||||
"""Fetch and parse RSS feed from InciWeb."""
|
|
||||||
if not self._session:
|
|
||||||
raise RuntimeError("Session not initialized")
|
|
||||||
|
|
||||||
# Build request headers with conditional fetch support
|
|
||||||
headers = {"User-Agent": "Central/0.4"}
|
|
||||||
if self._last_modified:
|
|
||||||
headers["If-Modified-Since"] = self._last_modified
|
|
||||||
if self._etag:
|
|
||||||
headers["If-None-Match"] = self._etag
|
|
||||||
|
|
||||||
async with self._session.get(INCIWEB_RSS_URL, headers=headers) as resp:
|
|
||||||
# Handle 304 Not Modified
|
|
||||||
if resp.status == 304:
|
|
||||||
logger.info("InciWeb not modified")
|
|
||||||
return []
|
|
||||||
|
|
||||||
resp.raise_for_status()
|
|
||||||
|
|
||||||
# Capture conditional fetch headers for next request
|
|
||||||
self._last_modified = resp.headers.get("Last-Modified")
|
|
||||||
self._etag = resp.headers.get("ETag")
|
|
||||||
|
|
||||||
content = await resp.text()
|
|
||||||
|
|
||||||
# Parse RSS XML
|
|
||||||
items = []
|
|
||||||
try:
|
|
||||||
root = ET.fromstring(content)
|
|
||||||
channel = root.find("channel")
|
|
||||||
if channel is None:
|
|
||||||
return []
|
|
||||||
|
|
||||||
for item_elem in channel.findall("item"):
|
|
||||||
item: dict[str, Any] = {}
|
|
||||||
|
|
||||||
title = item_elem.find("title")
|
|
||||||
item["title"] = title.text if title is not None and title.text else ""
|
|
||||||
|
|
||||||
link = item_elem.find("link")
|
|
||||||
item["link"] = link.text if link is not None and link.text else ""
|
|
||||||
|
|
||||||
description = item_elem.find("description")
|
|
||||||
item["description"] = description.text if description is not None and description.text else ""
|
|
||||||
|
|
||||||
pub_date = item_elem.find("pubDate")
|
|
||||||
item["pubDate"] = pub_date.text if pub_date is not None and pub_date.text else ""
|
|
||||||
|
|
||||||
guid = item_elem.find("guid")
|
|
||||||
item["guid"] = guid.text if guid is not None and guid.text else ""
|
|
||||||
|
|
||||||
# Check for dc:creator
|
|
||||||
creator = item_elem.find("{http://purl.org/dc/elements/1.1/}creator")
|
|
||||||
item["creator"] = creator.text if creator is not None and creator.text else ""
|
|
||||||
|
|
||||||
items.append(item)
|
|
||||||
|
|
||||||
except ET.ParseError as e:
|
|
||||||
logger.error("InciWeb RSS parse error", extra={"error": str(e)})
|
|
||||||
raise
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
"InciWeb fetch completed",
|
|
||||||
extra={"item_count": len(items)},
|
|
||||||
)
|
|
||||||
return items
|
|
||||||
|
|
||||||
async def poll(self) -> AsyncIterator[Event]:
|
|
||||||
"""Poll InciWeb for narrative updates."""
|
|
||||||
if not self._db:
|
|
||||||
raise RuntimeError("Database not initialized")
|
|
||||||
|
|
||||||
# Fetch RSS feed
|
|
||||||
try:
|
|
||||||
items = await self._fetch_rss()
|
|
||||||
except Exception as e:
|
|
||||||
logger.error("InciWeb fetch failed", extra={"error": str(e)})
|
|
||||||
raise
|
|
||||||
|
|
||||||
events_yielded = 0
|
|
||||||
|
|
||||||
for item in items:
|
|
||||||
guid = item.get("guid", "")
|
|
||||||
if not guid:
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Dedup: skip if already published
|
|
||||||
if self.is_published(guid):
|
|
||||||
self.bump_last_seen(guid)
|
|
||||||
continue
|
|
||||||
|
|
||||||
description_html = item.get("description", "")
|
|
||||||
|
|
||||||
# Parse coordinates from description
|
|
||||||
centroid = parse_coordinates_from_description(description_html)
|
|
||||||
|
|
||||||
# Post-filter: skip if point outside region bbox
|
|
||||||
if self.region and centroid:
|
|
||||||
lon, lat = centroid
|
|
||||||
if not point_in_bbox(
|
|
||||||
lon, lat,
|
|
||||||
self.region.west, self.region.south,
|
|
||||||
self.region.east, self.region.north,
|
|
||||||
):
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Parse state from description
|
|
||||||
state_code = parse_state_from_description(description_html)
|
|
||||||
|
|
||||||
# Build regions
|
|
||||||
if state_code:
|
|
||||||
regions = [f"US-{state_code}"]
|
|
||||||
primary_region = f"US-{state_code}"
|
|
||||||
else:
|
|
||||||
regions = []
|
|
||||||
primary_region = None
|
|
||||||
|
|
||||||
# Parse pubDate (RFC 822 format)
|
|
||||||
pub_date_str = item.get("pubDate", "")
|
|
||||||
try:
|
|
||||||
event_time = parsedate_to_datetime(pub_date_str)
|
|
||||||
# Ensure UTC
|
|
||||||
if event_time.tzinfo is None:
|
|
||||||
event_time = event_time.replace(tzinfo=timezone.utc)
|
|
||||||
else:
|
|
||||||
event_time = event_time.astimezone(timezone.utc)
|
|
||||||
except (ValueError, TypeError):
|
|
||||||
event_time = datetime.now(timezone.utc)
|
|
||||||
|
|
||||||
# Build geo
|
|
||||||
geo = Geo(
|
|
||||||
centroid=centroid,
|
|
||||||
bbox=(centroid[0], centroid[1], centroid[0], centroid[1]) if centroid else None,
|
|
||||||
regions=regions,
|
|
||||||
primary_region=primary_region,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Strip HTML from description
|
|
||||||
description_plain = strip_html(description_html)
|
|
||||||
|
|
||||||
# Build event
|
|
||||||
event = Event(
|
|
||||||
id=guid,
|
|
||||||
adapter=self.name,
|
|
||||||
category="fire.narrative.inciweb",
|
|
||||||
time=event_time,
|
|
||||||
severity=0, # Narrative; not authoritative
|
|
||||||
geo=geo,
|
|
||||||
data={
|
|
||||||
"title": item.get("title", ""),
|
|
||||||
"description": description_plain,
|
|
||||||
"description_html": description_html,
|
|
||||||
"url": item.get("link", ""),
|
|
||||||
"guid": guid,
|
|
||||||
"raw": item,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
yield event
|
|
||||||
self.mark_published(guid)
|
|
||||||
events_yielded += 1
|
|
||||||
|
|
||||||
# Periodic cleanup of old entries
|
|
||||||
self.sweep_old_ids()
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
"InciWeb poll completed",
|
|
||||||
extra={"events_yielded": events_yielded},
|
|
||||||
)
|
|
||||||
|
|
@ -19,7 +19,7 @@ from tenacity import (
|
||||||
|
|
||||||
from central import __version__
|
from central import __version__
|
||||||
from central.adapter import SourceAdapter
|
from central.adapter import SourceAdapter
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel
|
||||||
|
|
||||||
from central.config_models import AdapterConfig, RegionConfig
|
from central.config_models import AdapterConfig, RegionConfig
|
||||||
from central.config_store import ConfigStore
|
from central.config_store import ConfigStore
|
||||||
|
|
@ -193,11 +193,7 @@ def _build_regions(same_codes: list[str], ugc_codes: list[str]) -> list[str]:
|
||||||
|
|
||||||
class NWSSettings(BaseModel):
|
class NWSSettings(BaseModel):
|
||||||
"""Settings schema for NWS adapter."""
|
"""Settings schema for NWS adapter."""
|
||||||
contact_email: str = Field(
|
contact_email: str = ""
|
||||||
default="",
|
|
||||||
pattern=r"^$|^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$",
|
|
||||||
description="Contact email for NWS API User-Agent header",
|
|
||||||
)
|
|
||||||
region: RegionConfig | None = None
|
region: RegionConfig | None = None
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,186 +0,0 @@
|
||||||
"""NOAA SWPC space weather alerts adapter."""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import sqlite3
|
|
||||||
from collections.abc import AsyncIterator
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from tenacity import (
|
|
||||||
retry,
|
|
||||||
retry_if_exception_type,
|
|
||||||
stop_after_attempt,
|
|
||||||
wait_exponential_jitter,
|
|
||||||
)
|
|
||||||
|
|
||||||
from central.adapter import SourceAdapter
|
|
||||||
from central.adapters.swpc_common import (
|
|
||||||
SWPC_ALERTS_URL,
|
|
||||||
SWPCSettings,
|
|
||||||
parse_swpc_timestamp,
|
|
||||||
severity_from_alert_product_id,
|
|
||||||
)
|
|
||||||
from central.config_models import AdapterConfig
|
|
||||||
from central.config_store import ConfigStore
|
|
||||||
from central.models import Event, Geo
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
class SWPCAlertsAdapter(SourceAdapter):
|
|
||||||
"""NOAA SWPC space weather alerts adapter."""
|
|
||||||
|
|
||||||
name = "swpc_alerts"
|
|
||||||
display_name = "NOAA SWPC — Space Weather Alerts"
|
|
||||||
description = "Active NOAA SWPC space weather alerts, watches, warnings, and summaries."
|
|
||||||
settings_schema = SWPCSettings
|
|
||||||
requires_api_key = None
|
|
||||||
api_key_field = None
|
|
||||||
wizard_order = None
|
|
||||||
default_cadence_s = 300
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
config: AdapterConfig,
|
|
||||||
config_store: ConfigStore,
|
|
||||||
cursor_db_path: Path,
|
|
||||||
) -> None:
|
|
||||||
self._config_store = config_store
|
|
||||||
self._cursor_db_path = cursor_db_path
|
|
||||||
self._session: aiohttp.ClientSession | None = None
|
|
||||||
self._db: sqlite3.Connection | None = None
|
|
||||||
|
|
||||||
async def startup(self) -> None:
|
|
||||||
self._session = aiohttp.ClientSession(
|
|
||||||
timeout=aiohttp.ClientTimeout(total=60),
|
|
||||||
)
|
|
||||||
self._db = sqlite3.connect(self._cursor_db_path)
|
|
||||||
self._db.execute("""
|
|
||||||
CREATE TABLE IF NOT EXISTS published_ids (
|
|
||||||
adapter TEXT NOT NULL,
|
|
||||||
event_id TEXT NOT NULL,
|
|
||||||
first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
PRIMARY KEY (adapter, event_id)
|
|
||||||
)
|
|
||||||
""")
|
|
||||||
self._db.execute("""
|
|
||||||
CREATE INDEX IF NOT EXISTS published_ids_last_seen
|
|
||||||
ON published_ids (last_seen)
|
|
||||||
""")
|
|
||||||
self._db.commit()
|
|
||||||
logger.info("SWPC alerts adapter started")
|
|
||||||
|
|
||||||
async def shutdown(self) -> None:
|
|
||||||
if self._session:
|
|
||||||
await self._session.close()
|
|
||||||
self._session = None
|
|
||||||
if self._db:
|
|
||||||
self._db.close()
|
|
||||||
self._db = None
|
|
||||||
logger.info("SWPC alerts adapter shut down")
|
|
||||||
|
|
||||||
async def apply_config(self, new_config: AdapterConfig) -> None:
|
|
||||||
logger.info("SWPC alerts config updated")
|
|
||||||
|
|
||||||
def is_published(self, event_id: str) -> bool:
|
|
||||||
if not self._db:
|
|
||||||
return False
|
|
||||||
cur = self._db.execute(
|
|
||||||
"SELECT 1 FROM published_ids WHERE adapter = ? AND event_id = ?",
|
|
||||||
(self.name, event_id),
|
|
||||||
)
|
|
||||||
return cur.fetchone() is not None
|
|
||||||
|
|
||||||
def mark_published(self, event_id: str) -> None:
|
|
||||||
if not self._db:
|
|
||||||
return
|
|
||||||
self._db.execute(
|
|
||||||
"""
|
|
||||||
INSERT INTO published_ids (adapter, event_id, first_seen, last_seen)
|
|
||||||
VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
||||||
ON CONFLICT (adapter, event_id) DO UPDATE SET
|
|
||||||
last_seen = CURRENT_TIMESTAMP
|
|
||||||
""",
|
|
||||||
(self.name, event_id),
|
|
||||||
)
|
|
||||||
self._db.commit()
|
|
||||||
|
|
||||||
def sweep_old_ids(self) -> int:
|
|
||||||
if not self._db:
|
|
||||||
return 0
|
|
||||||
cur = self._db.execute(
|
|
||||||
"DELETE FROM published_ids WHERE adapter = ? AND last_seen < datetime('now', '-14 days')",
|
|
||||||
(self.name,),
|
|
||||||
)
|
|
||||||
self._db.commit()
|
|
||||||
count = cur.rowcount
|
|
||||||
if count > 0:
|
|
||||||
logger.info("SWPC alerts swept old dedup entries", extra={"count": count})
|
|
||||||
return count
|
|
||||||
|
|
||||||
def subject_for(self, event: Event) -> str:
|
|
||||||
product_id = event.data.get("product_id") or "unknown"
|
|
||||||
return f"central.space.alert.{product_id.lower()}"
|
|
||||||
|
|
||||||
@retry(
|
|
||||||
stop=stop_after_attempt(3),
|
|
||||||
wait=wait_exponential_jitter(initial=1, max=30),
|
|
||||||
retry=retry_if_exception_type((aiohttp.ClientError, TimeoutError)),
|
|
||||||
)
|
|
||||||
async def _fetch(self) -> list[dict[str, Any]]:
|
|
||||||
if not self._session:
|
|
||||||
raise RuntimeError("Session not initialized")
|
|
||||||
async with self._session.get(
|
|
||||||
SWPC_ALERTS_URL, headers={"User-Agent": "Central/0.4"}
|
|
||||||
) as resp:
|
|
||||||
resp.raise_for_status()
|
|
||||||
data = await resp.json()
|
|
||||||
logger.info("SWPC alerts fetch completed", extra={"item_count": len(data)})
|
|
||||||
return data
|
|
||||||
|
|
||||||
async def poll(self) -> AsyncIterator[Event]:
|
|
||||||
if not self._db:
|
|
||||||
raise RuntimeError("Database not initialized")
|
|
||||||
|
|
||||||
try:
|
|
||||||
items = await self._fetch()
|
|
||||||
except Exception as e:
|
|
||||||
logger.error("SWPC alerts fetch failed", extra={"error": str(e)})
|
|
||||||
raise
|
|
||||||
|
|
||||||
events_yielded = 0
|
|
||||||
for item in items:
|
|
||||||
product_id = item.get("product_id")
|
|
||||||
issue_dt_raw = item.get("issue_datetime")
|
|
||||||
if not product_id or not issue_dt_raw:
|
|
||||||
continue
|
|
||||||
|
|
||||||
event_id = f"{product_id}|{issue_dt_raw}"
|
|
||||||
if self.is_published(event_id):
|
|
||||||
continue
|
|
||||||
|
|
||||||
issue_dt = parse_swpc_timestamp(issue_dt_raw, "alerts") or datetime.now(timezone.utc)
|
|
||||||
|
|
||||||
event = Event(
|
|
||||||
id=event_id,
|
|
||||||
adapter=self.name,
|
|
||||||
category="space.alert",
|
|
||||||
time=issue_dt,
|
|
||||||
severity=severity_from_alert_product_id(product_id),
|
|
||||||
geo=Geo(),
|
|
||||||
data={
|
|
||||||
"product_id": product_id,
|
|
||||||
"issue_datetime": issue_dt_raw,
|
|
||||||
"message": item.get("message", ""),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
yield event
|
|
||||||
self.mark_published(event_id)
|
|
||||||
events_yielded += 1
|
|
||||||
|
|
||||||
self.sweep_old_ids()
|
|
||||||
logger.info("SWPC alerts poll completed", extra={"events_yielded": events_yielded})
|
|
||||||
|
|
@ -1,81 +0,0 @@
|
||||||
"""Shared utilities for NOAA SWPC space weather adapters."""
|
|
||||||
|
|
||||||
import re
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
|
|
||||||
from pydantic import BaseModel
|
|
||||||
|
|
||||||
SWPC_ALERTS_URL = "https://services.swpc.noaa.gov/products/alerts.json"
|
|
||||||
SWPC_KINDEX_URL = "https://services.swpc.noaa.gov/products/noaa-planetary-k-index.json"
|
|
||||||
SWPC_PROTONS_URL = "https://services.swpc.noaa.gov/json/goes/primary/integral-protons-1-day.json"
|
|
||||||
|
|
||||||
|
|
||||||
class SWPCSettings(BaseModel):
|
|
||||||
"""Settings schema for SWPC adapters. No operator-tunable knobs today."""
|
|
||||||
|
|
||||||
|
|
||||||
def parse_swpc_timestamp(raw: str | None, endpoint_kind: str) -> datetime | None:
|
|
||||||
"""Normalize SWPC timestamp strings to UTC datetime.
|
|
||||||
|
|
||||||
endpoint_kind shapes:
|
|
||||||
alerts -> "2026-05-19 05:14:59.780" (space-separated, no TZ; UTC per message body)
|
|
||||||
kindex -> "2026-05-12T00:00:00" (ISO without TZ; UTC by convention)
|
|
||||||
protons -> "2026-05-18T05:35:00Z" (ISO with Z)
|
|
||||||
"""
|
|
||||||
if not raw:
|
|
||||||
return None
|
|
||||||
if endpoint_kind == "alerts":
|
|
||||||
try:
|
|
||||||
dt = datetime.strptime(raw, "%Y-%m-%d %H:%M:%S.%f")
|
|
||||||
except ValueError:
|
|
||||||
dt = datetime.strptime(raw, "%Y-%m-%d %H:%M:%S")
|
|
||||||
return dt.replace(tzinfo=timezone.utc)
|
|
||||||
if endpoint_kind == "kindex":
|
|
||||||
dt = datetime.fromisoformat(raw)
|
|
||||||
if dt.tzinfo is None:
|
|
||||||
dt = dt.replace(tzinfo=timezone.utc)
|
|
||||||
return dt.astimezone(timezone.utc)
|
|
||||||
if endpoint_kind == "protons":
|
|
||||||
raw_norm = raw[:-1] + "+00:00" if raw.endswith("Z") else raw
|
|
||||||
dt = datetime.fromisoformat(raw_norm)
|
|
||||||
if dt.tzinfo is None:
|
|
||||||
dt = dt.replace(tzinfo=timezone.utc)
|
|
||||||
return dt.astimezone(timezone.utc)
|
|
||||||
raise ValueError(f"unknown endpoint_kind: {endpoint_kind!r}")
|
|
||||||
|
|
||||||
|
|
||||||
def severity_from_kp(kp: float | int | None) -> int:
|
|
||||||
"""Map planetary K-index value (0-9) to severity 0-4 via the G-scale.
|
|
||||||
|
|
||||||
Kp 5 = G1 = severity 1, Kp 6 = G2 = severity 2, Kp 7 = G3 = severity 3,
|
|
||||||
Kp 8 = G4 = severity 4, Kp 9 = G5 = severity 4 (capped).
|
|
||||||
"""
|
|
||||||
if kp is None:
|
|
||||||
return 0
|
|
||||||
if kp < 5:
|
|
||||||
return 0
|
|
||||||
if kp < 6:
|
|
||||||
return 1
|
|
||||||
if kp < 7:
|
|
||||||
return 2
|
|
||||||
if kp < 8:
|
|
||||||
return 3
|
|
||||||
return 4
|
|
||||||
|
|
||||||
|
|
||||||
_ALERT_KP_PATTERN = re.compile(r"^K0([5-9])[AW]$")
|
|
||||||
|
|
||||||
|
|
||||||
def severity_from_alert_product_id(product_id: str | None) -> int:
|
|
||||||
"""Best-effort severity for an alert from its product_id G-scale.
|
|
||||||
|
|
||||||
Product IDs of form K0[5-9][AW] identify Kp-based geomagnetic storm
|
|
||||||
alerts and warnings (K05A=G1, K06A=G2, K07A=G3, K08A=G4, K09A=G5).
|
|
||||||
All other product IDs return 0.
|
|
||||||
"""
|
|
||||||
if not product_id:
|
|
||||||
return 0
|
|
||||||
m = _ALERT_KP_PATTERN.match(product_id.upper())
|
|
||||||
if not m:
|
|
||||||
return 0
|
|
||||||
return severity_from_kp(int(m.group(1)))
|
|
||||||
|
|
@ -1,186 +0,0 @@
|
||||||
"""NOAA SWPC Planetary K-Index adapter."""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import sqlite3
|
|
||||||
from collections.abc import AsyncIterator
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from tenacity import (
|
|
||||||
retry,
|
|
||||||
retry_if_exception_type,
|
|
||||||
stop_after_attempt,
|
|
||||||
wait_exponential_jitter,
|
|
||||||
)
|
|
||||||
|
|
||||||
from central.adapter import SourceAdapter
|
|
||||||
from central.adapters.swpc_common import (
|
|
||||||
SWPC_KINDEX_URL,
|
|
||||||
SWPCSettings,
|
|
||||||
parse_swpc_timestamp,
|
|
||||||
severity_from_kp,
|
|
||||||
)
|
|
||||||
from central.config_models import AdapterConfig
|
|
||||||
from central.config_store import ConfigStore
|
|
||||||
from central.models import Event, Geo
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
class SWPCKindexAdapter(SourceAdapter):
|
|
||||||
"""NOAA SWPC planetary K-index adapter."""
|
|
||||||
|
|
||||||
name = "swpc_kindex"
|
|
||||||
display_name = "NOAA SWPC — Planetary K-Index"
|
|
||||||
description = "Planetary K-index measurements at 3-hour cadence from NOAA SWPC."
|
|
||||||
settings_schema = SWPCSettings
|
|
||||||
requires_api_key = None
|
|
||||||
api_key_field = None
|
|
||||||
wizard_order = None
|
|
||||||
default_cadence_s = 600
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
config: AdapterConfig,
|
|
||||||
config_store: ConfigStore,
|
|
||||||
cursor_db_path: Path,
|
|
||||||
) -> None:
|
|
||||||
self._config_store = config_store
|
|
||||||
self._cursor_db_path = cursor_db_path
|
|
||||||
self._session: aiohttp.ClientSession | None = None
|
|
||||||
self._db: sqlite3.Connection | None = None
|
|
||||||
|
|
||||||
async def startup(self) -> None:
|
|
||||||
self._session = aiohttp.ClientSession(
|
|
||||||
timeout=aiohttp.ClientTimeout(total=60),
|
|
||||||
)
|
|
||||||
self._db = sqlite3.connect(self._cursor_db_path)
|
|
||||||
self._db.execute("""
|
|
||||||
CREATE TABLE IF NOT EXISTS published_ids (
|
|
||||||
adapter TEXT NOT NULL,
|
|
||||||
event_id TEXT NOT NULL,
|
|
||||||
first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
PRIMARY KEY (adapter, event_id)
|
|
||||||
)
|
|
||||||
""")
|
|
||||||
self._db.execute("""
|
|
||||||
CREATE INDEX IF NOT EXISTS published_ids_last_seen
|
|
||||||
ON published_ids (last_seen)
|
|
||||||
""")
|
|
||||||
self._db.commit()
|
|
||||||
logger.info("SWPC kindex adapter started")
|
|
||||||
|
|
||||||
async def shutdown(self) -> None:
|
|
||||||
if self._session:
|
|
||||||
await self._session.close()
|
|
||||||
self._session = None
|
|
||||||
if self._db:
|
|
||||||
self._db.close()
|
|
||||||
self._db = None
|
|
||||||
logger.info("SWPC kindex adapter shut down")
|
|
||||||
|
|
||||||
async def apply_config(self, new_config: AdapterConfig) -> None:
|
|
||||||
logger.info("SWPC kindex config updated")
|
|
||||||
|
|
||||||
def is_published(self, event_id: str) -> bool:
|
|
||||||
if not self._db:
|
|
||||||
return False
|
|
||||||
cur = self._db.execute(
|
|
||||||
"SELECT 1 FROM published_ids WHERE adapter = ? AND event_id = ?",
|
|
||||||
(self.name, event_id),
|
|
||||||
)
|
|
||||||
return cur.fetchone() is not None
|
|
||||||
|
|
||||||
def mark_published(self, event_id: str) -> None:
|
|
||||||
if not self._db:
|
|
||||||
return
|
|
||||||
self._db.execute(
|
|
||||||
"""
|
|
||||||
INSERT INTO published_ids (adapter, event_id, first_seen, last_seen)
|
|
||||||
VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
||||||
ON CONFLICT (adapter, event_id) DO UPDATE SET
|
|
||||||
last_seen = CURRENT_TIMESTAMP
|
|
||||||
""",
|
|
||||||
(self.name, event_id),
|
|
||||||
)
|
|
||||||
self._db.commit()
|
|
||||||
|
|
||||||
def sweep_old_ids(self) -> int:
|
|
||||||
if not self._db:
|
|
||||||
return 0
|
|
||||||
cur = self._db.execute(
|
|
||||||
"DELETE FROM published_ids WHERE adapter = ? AND last_seen < datetime('now', '-14 days')",
|
|
||||||
(self.name,),
|
|
||||||
)
|
|
||||||
self._db.commit()
|
|
||||||
count = cur.rowcount
|
|
||||||
if count > 0:
|
|
||||||
logger.info("SWPC kindex swept old dedup entries", extra={"count": count})
|
|
||||||
return count
|
|
||||||
|
|
||||||
def subject_for(self, event: Event) -> str:
|
|
||||||
return "central.space.kindex"
|
|
||||||
|
|
||||||
@retry(
|
|
||||||
stop=stop_after_attempt(3),
|
|
||||||
wait=wait_exponential_jitter(initial=1, max=30),
|
|
||||||
retry=retry_if_exception_type((aiohttp.ClientError, TimeoutError)),
|
|
||||||
)
|
|
||||||
async def _fetch(self) -> list[dict[str, Any]]:
|
|
||||||
if not self._session:
|
|
||||||
raise RuntimeError("Session not initialized")
|
|
||||||
async with self._session.get(
|
|
||||||
SWPC_KINDEX_URL, headers={"User-Agent": "Central/0.4"}
|
|
||||||
) as resp:
|
|
||||||
resp.raise_for_status()
|
|
||||||
data = await resp.json()
|
|
||||||
logger.info("SWPC kindex fetch completed", extra={"item_count": len(data)})
|
|
||||||
return data
|
|
||||||
|
|
||||||
async def poll(self) -> AsyncIterator[Event]:
|
|
||||||
if not self._db:
|
|
||||||
raise RuntimeError("Database not initialized")
|
|
||||||
|
|
||||||
try:
|
|
||||||
items = await self._fetch()
|
|
||||||
except Exception as e:
|
|
||||||
logger.error("SWPC kindex fetch failed", extra={"error": str(e)})
|
|
||||||
raise
|
|
||||||
|
|
||||||
events_yielded = 0
|
|
||||||
for item in items:
|
|
||||||
time_tag = item.get("time_tag")
|
|
||||||
kp = item.get("Kp")
|
|
||||||
if not time_tag or kp is None:
|
|
||||||
continue
|
|
||||||
|
|
||||||
event_id = time_tag
|
|
||||||
if self.is_published(event_id):
|
|
||||||
continue
|
|
||||||
|
|
||||||
event_time = parse_swpc_timestamp(time_tag, "kindex") or datetime.now(timezone.utc)
|
|
||||||
|
|
||||||
event = Event(
|
|
||||||
id=event_id,
|
|
||||||
adapter=self.name,
|
|
||||||
category="space.kindex",
|
|
||||||
time=event_time,
|
|
||||||
severity=severity_from_kp(kp),
|
|
||||||
geo=Geo(),
|
|
||||||
data={
|
|
||||||
"time_tag": time_tag,
|
|
||||||
"Kp": kp,
|
|
||||||
"a_running": item.get("a_running"),
|
|
||||||
"station_count": item.get("station_count"),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
yield event
|
|
||||||
self.mark_published(event_id)
|
|
||||||
events_yielded += 1
|
|
||||||
|
|
||||||
self.sweep_old_ids()
|
|
||||||
logger.info("SWPC kindex poll completed", extra={"events_yielded": events_yielded})
|
|
||||||
|
|
@ -1,185 +0,0 @@
|
||||||
"""NOAA SWPC GOES integral proton flux adapter."""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import sqlite3
|
|
||||||
from collections.abc import AsyncIterator
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from tenacity import (
|
|
||||||
retry,
|
|
||||||
retry_if_exception_type,
|
|
||||||
stop_after_attempt,
|
|
||||||
wait_exponential_jitter,
|
|
||||||
)
|
|
||||||
|
|
||||||
from central.adapter import SourceAdapter
|
|
||||||
from central.adapters.swpc_common import (
|
|
||||||
SWPC_PROTONS_URL,
|
|
||||||
SWPCSettings,
|
|
||||||
parse_swpc_timestamp,
|
|
||||||
)
|
|
||||||
from central.config_models import AdapterConfig
|
|
||||||
from central.config_store import ConfigStore
|
|
||||||
from central.models import Event, Geo
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
class SWPCProtonsAdapter(SourceAdapter):
|
|
||||||
"""NOAA SWPC GOES integral proton flux adapter."""
|
|
||||||
|
|
||||||
name = "swpc_protons"
|
|
||||||
display_name = "NOAA SWPC — GOES Proton Flux"
|
|
||||||
description = "GOES primary satellite integral proton flux measurements (1-day window) from NOAA SWPC."
|
|
||||||
settings_schema = SWPCSettings
|
|
||||||
requires_api_key = None
|
|
||||||
api_key_field = None
|
|
||||||
wizard_order = None
|
|
||||||
default_cadence_s = 600
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
config: AdapterConfig,
|
|
||||||
config_store: ConfigStore,
|
|
||||||
cursor_db_path: Path,
|
|
||||||
) -> None:
|
|
||||||
self._config_store = config_store
|
|
||||||
self._cursor_db_path = cursor_db_path
|
|
||||||
self._session: aiohttp.ClientSession | None = None
|
|
||||||
self._db: sqlite3.Connection | None = None
|
|
||||||
|
|
||||||
async def startup(self) -> None:
|
|
||||||
self._session = aiohttp.ClientSession(
|
|
||||||
timeout=aiohttp.ClientTimeout(total=60),
|
|
||||||
)
|
|
||||||
self._db = sqlite3.connect(self._cursor_db_path)
|
|
||||||
self._db.execute("""
|
|
||||||
CREATE TABLE IF NOT EXISTS published_ids (
|
|
||||||
adapter TEXT NOT NULL,
|
|
||||||
event_id TEXT NOT NULL,
|
|
||||||
first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
PRIMARY KEY (adapter, event_id)
|
|
||||||
)
|
|
||||||
""")
|
|
||||||
self._db.execute("""
|
|
||||||
CREATE INDEX IF NOT EXISTS published_ids_last_seen
|
|
||||||
ON published_ids (last_seen)
|
|
||||||
""")
|
|
||||||
self._db.commit()
|
|
||||||
logger.info("SWPC protons adapter started")
|
|
||||||
|
|
||||||
async def shutdown(self) -> None:
|
|
||||||
if self._session:
|
|
||||||
await self._session.close()
|
|
||||||
self._session = None
|
|
||||||
if self._db:
|
|
||||||
self._db.close()
|
|
||||||
self._db = None
|
|
||||||
logger.info("SWPC protons adapter shut down")
|
|
||||||
|
|
||||||
async def apply_config(self, new_config: AdapterConfig) -> None:
|
|
||||||
logger.info("SWPC protons config updated")
|
|
||||||
|
|
||||||
def is_published(self, event_id: str) -> bool:
|
|
||||||
if not self._db:
|
|
||||||
return False
|
|
||||||
cur = self._db.execute(
|
|
||||||
"SELECT 1 FROM published_ids WHERE adapter = ? AND event_id = ?",
|
|
||||||
(self.name, event_id),
|
|
||||||
)
|
|
||||||
return cur.fetchone() is not None
|
|
||||||
|
|
||||||
def mark_published(self, event_id: str) -> None:
|
|
||||||
if not self._db:
|
|
||||||
return
|
|
||||||
self._db.execute(
|
|
||||||
"""
|
|
||||||
INSERT INTO published_ids (adapter, event_id, first_seen, last_seen)
|
|
||||||
VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
||||||
ON CONFLICT (adapter, event_id) DO UPDATE SET
|
|
||||||
last_seen = CURRENT_TIMESTAMP
|
|
||||||
""",
|
|
||||||
(self.name, event_id),
|
|
||||||
)
|
|
||||||
self._db.commit()
|
|
||||||
|
|
||||||
def sweep_old_ids(self) -> int:
|
|
||||||
if not self._db:
|
|
||||||
return 0
|
|
||||||
cur = self._db.execute(
|
|
||||||
"DELETE FROM published_ids WHERE adapter = ? AND last_seen < datetime('now', '-14 days')",
|
|
||||||
(self.name,),
|
|
||||||
)
|
|
||||||
self._db.commit()
|
|
||||||
count = cur.rowcount
|
|
||||||
if count > 0:
|
|
||||||
logger.info("SWPC protons swept old dedup entries", extra={"count": count})
|
|
||||||
return count
|
|
||||||
|
|
||||||
def subject_for(self, event: Event) -> str:
|
|
||||||
return "central.space.proton_flux"
|
|
||||||
|
|
||||||
@retry(
|
|
||||||
stop=stop_after_attempt(3),
|
|
||||||
wait=wait_exponential_jitter(initial=1, max=30),
|
|
||||||
retry=retry_if_exception_type((aiohttp.ClientError, TimeoutError)),
|
|
||||||
)
|
|
||||||
async def _fetch(self) -> list[dict[str, Any]]:
|
|
||||||
if not self._session:
|
|
||||||
raise RuntimeError("Session not initialized")
|
|
||||||
async with self._session.get(
|
|
||||||
SWPC_PROTONS_URL, headers={"User-Agent": "Central/0.4"}
|
|
||||||
) as resp:
|
|
||||||
resp.raise_for_status()
|
|
||||||
data = await resp.json()
|
|
||||||
logger.info("SWPC protons fetch completed", extra={"item_count": len(data)})
|
|
||||||
return data
|
|
||||||
|
|
||||||
async def poll(self) -> AsyncIterator[Event]:
|
|
||||||
if not self._db:
|
|
||||||
raise RuntimeError("Database not initialized")
|
|
||||||
|
|
||||||
try:
|
|
||||||
items = await self._fetch()
|
|
||||||
except Exception as e:
|
|
||||||
logger.error("SWPC protons fetch failed", extra={"error": str(e)})
|
|
||||||
raise
|
|
||||||
|
|
||||||
events_yielded = 0
|
|
||||||
for item in items:
|
|
||||||
time_tag = item.get("time_tag")
|
|
||||||
energy = item.get("energy")
|
|
||||||
if not time_tag or not energy:
|
|
||||||
continue
|
|
||||||
|
|
||||||
event_id = f"{time_tag}|{energy}"
|
|
||||||
if self.is_published(event_id):
|
|
||||||
continue
|
|
||||||
|
|
||||||
event_time = parse_swpc_timestamp(time_tag, "protons") or datetime.now(timezone.utc)
|
|
||||||
|
|
||||||
event = Event(
|
|
||||||
id=event_id,
|
|
||||||
adapter=self.name,
|
|
||||||
category="space.proton_flux",
|
|
||||||
time=event_time,
|
|
||||||
severity=0,
|
|
||||||
geo=Geo(),
|
|
||||||
data={
|
|
||||||
"time_tag": time_tag,
|
|
||||||
"satellite": item.get("satellite"),
|
|
||||||
"flux": item.get("flux"),
|
|
||||||
"energy": energy,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
yield event
|
|
||||||
self.mark_published(event_id)
|
|
||||||
events_yielded += 1
|
|
||||||
|
|
||||||
self.sweep_old_ids()
|
|
||||||
logger.info("SWPC protons poll completed", extra={"events_yielded": events_yielded})
|
|
||||||
|
|
@ -5,7 +5,7 @@ import sqlite3
|
||||||
from collections.abc import AsyncIterator
|
from collections.abc import AsyncIterator
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Literal
|
from typing import Any
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
from shapely.geometry import Point, box as shapely_box
|
from shapely.geometry import Point, box as shapely_box
|
||||||
|
|
@ -64,7 +64,7 @@ def magnitude_to_severity(mag: float) -> int:
|
||||||
|
|
||||||
class USGSQuakeSettings(BaseModel):
|
class USGSQuakeSettings(BaseModel):
|
||||||
"""Settings schema for USGS quake adapter."""
|
"""Settings schema for USGS quake adapter."""
|
||||||
feed: Literal["all_hour", "all_day", "all_week", "all_month"] = "all_hour"
|
feed: str = "all_hour"
|
||||||
region: RegionConfig | None = None
|
region: RegionConfig | None = None
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,242 +0,0 @@
|
||||||
"""Shared utilities for WFIGS (Wildland Fire Interagency Geospatial Services) adapters."""
|
|
||||||
|
|
||||||
import sqlite3
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
# WFIGS FeatureServer endpoints
|
|
||||||
WFIGS_INCIDENTS_URL = (
|
|
||||||
"https://services3.arcgis.com/T4QMspbfLg3qTGWY/ArcGIS/rest/services/"
|
|
||||||
"WFIGS_Incident_Locations_Current/FeatureServer/0/query"
|
|
||||||
)
|
|
||||||
WFIGS_PERIMETERS_URL = (
|
|
||||||
"https://services3.arcgis.com/T4QMspbfLg3qTGWY/ArcGIS/rest/services/"
|
|
||||||
"WFIGS_Interagency_Perimeters_Current/FeatureServer/0/query"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Fall-off sweep window: 14 days (matches WFIGS's longest fall-off: large fires)
|
|
||||||
FALLOFF_WINDOW_DAYS = 14
|
|
||||||
|
|
||||||
# Incident type code mappings (WFIGS uses 2-letter codes)
|
|
||||||
INCIDENT_TYPE_MAP = {
|
|
||||||
"WF": "wildfire",
|
|
||||||
"RX": "prescribed_fire",
|
|
||||||
"CX": "complex",
|
|
||||||
"FA": "false_alarm",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def normalize_state(state: str | None) -> str | None:
|
|
||||||
"""Strip 'US-' prefix from POOState (ISO 3166-2 -> 2-letter)."""
|
|
||||||
if not state:
|
|
||||||
return None
|
|
||||||
if state.startswith("US-") and len(state) == 5:
|
|
||||||
return state[3:]
|
|
||||||
if len(state) == 2:
|
|
||||||
return state
|
|
||||||
return state # unknown shape, pass through
|
|
||||||
|
|
||||||
|
|
||||||
def normalize_incident_type(code: str | None) -> str:
|
|
||||||
"""Map IncidentTypeCategory code to a readable name."""
|
|
||||||
if not code:
|
|
||||||
return "unknown"
|
|
||||||
upper = code.upper()
|
|
||||||
if upper in INCIDENT_TYPE_MAP:
|
|
||||||
return INCIDENT_TYPE_MAP[upper]
|
|
||||||
return code.lower()
|
|
||||||
|
|
||||||
|
|
||||||
def severity_from_acres(acres: float | None) -> int:
|
|
||||||
"""Map DailyAcres to severity level 0-4."""
|
|
||||||
if acres is None or acres == 0:
|
|
||||||
return 0
|
|
||||||
if acres < 10:
|
|
||||||
return 1
|
|
||||||
if acres < 100:
|
|
||||||
return 2
|
|
||||||
if acres < 1000:
|
|
||||||
return 3
|
|
||||||
return 4
|
|
||||||
|
|
||||||
|
|
||||||
def parse_wfigs_timestamp(epoch_ms: int | None) -> datetime | None:
|
|
||||||
"""Parse WFIGS epoch milliseconds to UTC datetime."""
|
|
||||||
if epoch_ms is None:
|
|
||||||
return None
|
|
||||||
return datetime.fromtimestamp(epoch_ms / 1000, tz=timezone.utc)
|
|
||||||
|
|
||||||
|
|
||||||
def build_regions(state: str | None, county: str | None) -> tuple[list[str], str | None]:
|
|
||||||
"""
|
|
||||||
Build geo.regions list and primary_region from POOState and POOCounty.
|
|
||||||
|
|
||||||
Expects normalized 2-letter state codes (e.g., "MT" not "US-MT").
|
|
||||||
Returns (regions, primary_region).
|
|
||||||
"""
|
|
||||||
if not state:
|
|
||||||
return [], None
|
|
||||||
|
|
||||||
state_upper = state.upper()
|
|
||||||
if county:
|
|
||||||
# Normalize county: remove spaces, uppercase
|
|
||||||
county_normalized = county.replace(" ", "_").upper()
|
|
||||||
region = f"US-{state_upper}-{county_normalized}"
|
|
||||||
return [region], region
|
|
||||||
else:
|
|
||||||
region = f"US-{state_upper}"
|
|
||||||
return [region], region
|
|
||||||
|
|
||||||
|
|
||||||
def subject_suffix(state: str | None, county: str | None) -> str:
|
|
||||||
"""
|
|
||||||
Build subject suffix from state and county.
|
|
||||||
|
|
||||||
Expects normalized 2-letter state codes.
|
|
||||||
Returns lowercase state.county (county with spaces→underscores).
|
|
||||||
Falls back to "unknown" if state is not available.
|
|
||||||
"""
|
|
||||||
if not state:
|
|
||||||
return "unknown"
|
|
||||||
|
|
||||||
state_lower = state.lower()
|
|
||||||
if county:
|
|
||||||
county_lower = county.lower().replace(" ", "_")
|
|
||||||
return f"{state_lower}.{county_lower}"
|
|
||||||
return state_lower
|
|
||||||
|
|
||||||
|
|
||||||
def init_observed_table(db: sqlite3.Connection) -> None:
|
|
||||||
"""Create the wfigs_observed table if it doesn't exist."""
|
|
||||||
db.execute("""
|
|
||||||
CREATE TABLE IF NOT EXISTS wfigs_observed (
|
|
||||||
layer TEXT NOT NULL,
|
|
||||||
irwin_id TEXT NOT NULL,
|
|
||||||
last_observed_at TEXT NOT NULL,
|
|
||||||
state TEXT,
|
|
||||||
county TEXT,
|
|
||||||
PRIMARY KEY (layer, irwin_id)
|
|
||||||
)
|
|
||||||
""")
|
|
||||||
db.commit()
|
|
||||||
|
|
||||||
|
|
||||||
def get_observed_guids(db: sqlite3.Connection, layer: str) -> dict[str, tuple[str, str | None, str | None]]:
|
|
||||||
"""
|
|
||||||
Get all observed IRWIN GUIDs for a layer.
|
|
||||||
|
|
||||||
Returns dict mapping irwin_id -> (last_observed_at, state, county).
|
|
||||||
"""
|
|
||||||
cursor = db.execute(
|
|
||||||
"SELECT irwin_id, last_observed_at, state, county FROM wfigs_observed WHERE layer = ?",
|
|
||||||
(layer,),
|
|
||||||
)
|
|
||||||
return {row[0]: (row[1], row[2], row[3]) for row in cursor.fetchall()}
|
|
||||||
|
|
||||||
|
|
||||||
def update_observed(
|
|
||||||
db: sqlite3.Connection,
|
|
||||||
layer: str,
|
|
||||||
current_guids: dict[str, tuple[str | None, str | None]],
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
Update the observed table with current poll's GUIDs.
|
|
||||||
|
|
||||||
current_guids: dict mapping irwin_id -> (state, county)
|
|
||||||
"""
|
|
||||||
now_iso = datetime.now(timezone.utc).isoformat()
|
|
||||||
|
|
||||||
# Use INSERT OR REPLACE to upsert
|
|
||||||
for irwin_id, (state, county) in current_guids.items():
|
|
||||||
db.execute(
|
|
||||||
"""
|
|
||||||
INSERT OR REPLACE INTO wfigs_observed (layer, irwin_id, last_observed_at, state, county)
|
|
||||||
VALUES (?, ?, ?, ?, ?)
|
|
||||||
""",
|
|
||||||
(layer, irwin_id, now_iso, state, county),
|
|
||||||
)
|
|
||||||
db.commit()
|
|
||||||
|
|
||||||
|
|
||||||
def delete_observed(db: sqlite3.Connection, layer: str, irwin_ids: set[str]) -> None:
|
|
||||||
"""Delete fallen-off GUIDs from the observed table."""
|
|
||||||
for irwin_id in irwin_ids:
|
|
||||||
db.execute(
|
|
||||||
"DELETE FROM wfigs_observed WHERE layer = ? AND irwin_id = ?",
|
|
||||||
(layer, irwin_id),
|
|
||||||
)
|
|
||||||
db.commit()
|
|
||||||
|
|
||||||
|
|
||||||
def cleanup_old_observed(db: sqlite3.Connection, layer: str, days: int = FALLOFF_WINDOW_DAYS) -> None:
|
|
||||||
"""Remove observed entries older than the sweep window."""
|
|
||||||
cutoff = datetime.now(timezone.utc).isoformat()
|
|
||||||
db.execute(
|
|
||||||
f"""
|
|
||||||
DELETE FROM wfigs_observed
|
|
||||||
WHERE layer = ?
|
|
||||||
AND datetime(last_observed_at) < datetime(?, '-{days} days')
|
|
||||||
""",
|
|
||||||
(layer, cutoff),
|
|
||||||
)
|
|
||||||
db.commit()
|
|
||||||
|
|
||||||
|
|
||||||
def point_in_bbox(
|
|
||||||
lon: float,
|
|
||||||
lat: float,
|
|
||||||
west: float,
|
|
||||||
south: float,
|
|
||||||
east: float,
|
|
||||||
north: float,
|
|
||||||
) -> bool:
|
|
||||||
"""Check if a point is within a bounding box."""
|
|
||||||
return west <= lon <= east and south <= lat <= north
|
|
||||||
|
|
||||||
|
|
||||||
def polygon_intersects_bbox(
|
|
||||||
geometry: dict[str, Any],
|
|
||||||
west: float,
|
|
||||||
south: float,
|
|
||||||
east: float,
|
|
||||||
north: float,
|
|
||||||
) -> bool:
|
|
||||||
"""
|
|
||||||
Check if a GeoJSON geometry intersects a bounding box.
|
|
||||||
|
|
||||||
Uses shapely for accurate polygon intersection.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
from shapely.geometry import box, shape
|
|
||||||
|
|
||||||
bbox_polygon = box(west, south, east, north)
|
|
||||||
geom = shape(geometry)
|
|
||||||
return bbox_polygon.intersects(geom)
|
|
||||||
except Exception:
|
|
||||||
# If shapely fails, fall back to centroid check
|
|
||||||
if geometry.get("type") == "Point":
|
|
||||||
coords = geometry.get("coordinates", [])
|
|
||||||
if len(coords) >= 2:
|
|
||||||
return point_in_bbox(coords[0], coords[1], west, south, east, north)
|
|
||||||
return True # Include if we can't determine
|
|
||||||
|
|
||||||
|
|
||||||
def extract_centroid(geometry: dict[str, Any]) -> tuple[float, float] | None:
|
|
||||||
"""Extract centroid from GeoJSON geometry."""
|
|
||||||
if not geometry:
|
|
||||||
return None
|
|
||||||
|
|
||||||
geom_type = geometry.get("type")
|
|
||||||
coords = geometry.get("coordinates")
|
|
||||||
|
|
||||||
if geom_type == "Point" and coords and len(coords) >= 2:
|
|
||||||
return (coords[0], coords[1])
|
|
||||||
|
|
||||||
# For polygons, use shapely to compute centroid
|
|
||||||
try:
|
|
||||||
from shapely.geometry import shape
|
|
||||||
geom = shape(geometry)
|
|
||||||
centroid = geom.centroid
|
|
||||||
return (centroid.x, centroid.y)
|
|
||||||
except Exception:
|
|
||||||
return None
|
|
||||||
|
|
@ -1,383 +0,0 @@
|
||||||
"""WFIGS Incidents adapter for wildfire incident locations."""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import sqlite3
|
|
||||||
from collections.abc import AsyncIterator
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from pydantic import BaseModel
|
|
||||||
from tenacity import (
|
|
||||||
retry,
|
|
||||||
retry_if_exception_type,
|
|
||||||
stop_after_attempt,
|
|
||||||
wait_exponential_jitter,
|
|
||||||
)
|
|
||||||
|
|
||||||
from central.adapter import SourceAdapter
|
|
||||||
from central.adapters.wfigs_common import (
|
|
||||||
WFIGS_INCIDENTS_URL,
|
|
||||||
build_regions,
|
|
||||||
cleanup_old_observed,
|
|
||||||
delete_observed,
|
|
||||||
extract_centroid,
|
|
||||||
get_observed_guids,
|
|
||||||
init_observed_table,
|
|
||||||
normalize_incident_type,
|
|
||||||
normalize_state,
|
|
||||||
parse_wfigs_timestamp,
|
|
||||||
point_in_bbox,
|
|
||||||
severity_from_acres,
|
|
||||||
subject_suffix,
|
|
||||||
update_observed,
|
|
||||||
)
|
|
||||||
from central.config_models import AdapterConfig, RegionConfig
|
|
||||||
from central.config_store import ConfigStore
|
|
||||||
from central.models import Event, Geo
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
LAYER_NAME = "incidents"
|
|
||||||
|
|
||||||
|
|
||||||
class WFIGSIncidentsSettings(BaseModel):
|
|
||||||
"""Settings schema for WFIGS Incidents adapter."""
|
|
||||||
|
|
||||||
region: RegionConfig | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class WFIGSIncidentsAdapter(SourceAdapter):
|
|
||||||
"""NIFC WFIGS wildfire incidents adapter."""
|
|
||||||
|
|
||||||
name = "wfigs_incidents"
|
|
||||||
display_name = "NIFC WFIGS — Wildfire Incidents"
|
|
||||||
description = "Active wildfire incident locations from NIFC WFIGS."
|
|
||||||
settings_schema = WFIGSIncidentsSettings
|
|
||||||
requires_api_key = None
|
|
||||||
api_key_field = None
|
|
||||||
wizard_order = None # Not in setup wizard
|
|
||||||
default_cadence_s = 300
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
config: AdapterConfig,
|
|
||||||
config_store: ConfigStore,
|
|
||||||
cursor_db_path: Path,
|
|
||||||
) -> None:
|
|
||||||
self._config_store = config_store
|
|
||||||
self._cursor_db_path = cursor_db_path
|
|
||||||
self._session: aiohttp.ClientSession | None = None
|
|
||||||
self._db: sqlite3.Connection | None = None
|
|
||||||
self._last_poll_time: datetime | None = None
|
|
||||||
|
|
||||||
# Parse region from settings
|
|
||||||
region_dict = config.settings.get("region")
|
|
||||||
if region_dict:
|
|
||||||
self.region: RegionConfig | None = RegionConfig(**region_dict)
|
|
||||||
else:
|
|
||||||
self.region = None
|
|
||||||
|
|
||||||
async def startup(self) -> None:
|
|
||||||
"""Initialize HTTP session and SQLite connection."""
|
|
||||||
self._session = aiohttp.ClientSession(
|
|
||||||
timeout=aiohttp.ClientTimeout(total=60),
|
|
||||||
)
|
|
||||||
self._db = sqlite3.connect(self._cursor_db_path)
|
|
||||||
|
|
||||||
# Create tables for dedup and fall-off tracking
|
|
||||||
self._db.execute("""
|
|
||||||
CREATE TABLE IF NOT EXISTS published_ids (
|
|
||||||
adapter TEXT NOT NULL,
|
|
||||||
event_id TEXT NOT NULL,
|
|
||||||
first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
PRIMARY KEY (adapter, event_id)
|
|
||||||
)
|
|
||||||
""")
|
|
||||||
self._db.execute("""
|
|
||||||
CREATE INDEX IF NOT EXISTS published_ids_last_seen
|
|
||||||
ON published_ids (last_seen)
|
|
||||||
""")
|
|
||||||
init_observed_table(self._db)
|
|
||||||
self._db.commit()
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
"WFIGS incidents adapter started",
|
|
||||||
extra={"region": self.region.model_dump() if self.region else None},
|
|
||||||
)
|
|
||||||
|
|
||||||
async def shutdown(self) -> None:
|
|
||||||
"""Close HTTP session and SQLite connection."""
|
|
||||||
if self._session:
|
|
||||||
await self._session.close()
|
|
||||||
self._session = None
|
|
||||||
if self._db:
|
|
||||||
self._db.close()
|
|
||||||
self._db = None
|
|
||||||
logger.info("WFIGS incidents adapter shut down")
|
|
||||||
|
|
||||||
async def apply_config(self, new_config: AdapterConfig) -> None:
|
|
||||||
"""Apply new configuration from hot-reload."""
|
|
||||||
region_dict = new_config.settings.get("region")
|
|
||||||
if region_dict:
|
|
||||||
self.region = RegionConfig(**region_dict)
|
|
||||||
else:
|
|
||||||
self.region = None
|
|
||||||
logger.info(
|
|
||||||
"WFIGS incidents config updated",
|
|
||||||
extra={"region": self.region.model_dump() if self.region else None},
|
|
||||||
)
|
|
||||||
|
|
||||||
def is_published(self, event_id: str) -> bool:
|
|
||||||
"""Check if an event has already been published."""
|
|
||||||
if not self._db:
|
|
||||||
return False
|
|
||||||
cur = self._db.execute(
|
|
||||||
"SELECT 1 FROM published_ids WHERE adapter = ? AND event_id = ?",
|
|
||||||
(self.name, event_id),
|
|
||||||
)
|
|
||||||
return cur.fetchone() is not None
|
|
||||||
|
|
||||||
def mark_published(self, event_id: str) -> None:
|
|
||||||
"""Mark an event as published."""
|
|
||||||
if not self._db:
|
|
||||||
return
|
|
||||||
self._db.execute(
|
|
||||||
"""
|
|
||||||
INSERT INTO published_ids (adapter, event_id, first_seen, last_seen)
|
|
||||||
VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
||||||
ON CONFLICT (adapter, event_id) DO UPDATE SET
|
|
||||||
last_seen = CURRENT_TIMESTAMP
|
|
||||||
""",
|
|
||||||
(self.name, event_id),
|
|
||||||
)
|
|
||||||
self._db.commit()
|
|
||||||
|
|
||||||
def bump_last_seen(self, event_id: str) -> None:
|
|
||||||
"""Bump the last_seen timestamp for an event."""
|
|
||||||
if not self._db:
|
|
||||||
return
|
|
||||||
self._db.execute(
|
|
||||||
"UPDATE published_ids SET last_seen = CURRENT_TIMESTAMP WHERE adapter = ? AND event_id = ?",
|
|
||||||
(self.name, event_id),
|
|
||||||
)
|
|
||||||
self._db.commit()
|
|
||||||
|
|
||||||
def sweep_old_ids(self) -> int:
|
|
||||||
"""Remove published_ids older than 14 days. Returns count deleted."""
|
|
||||||
if not self._db:
|
|
||||||
return 0
|
|
||||||
cur = self._db.execute(
|
|
||||||
"DELETE FROM published_ids WHERE adapter = ? AND last_seen < datetime('now', '-14 days')",
|
|
||||||
(self.name,),
|
|
||||||
)
|
|
||||||
self._db.commit()
|
|
||||||
count = cur.rowcount
|
|
||||||
if count > 0:
|
|
||||||
logger.info("WFIGS incidents swept old dedup entries", extra={"count": count})
|
|
||||||
return count
|
|
||||||
|
|
||||||
def subject_for(self, event: Event) -> str:
|
|
||||||
"""Compute NATS subject for an event."""
|
|
||||||
# Removal events have a different subject pattern
|
|
||||||
if event.category.startswith("fire.incident.removed"):
|
|
||||||
state = event.data.get("state", "").lower() or "unknown"
|
|
||||||
return f"central.fire.incident.removed.{state}"
|
|
||||||
|
|
||||||
# Regular incidents: central.fire.incident.<state>.<county>
|
|
||||||
# POOState is already normalized (2-letter code)
|
|
||||||
state = event.data.get("POOState")
|
|
||||||
county = event.data.get("POOCounty")
|
|
||||||
suffix = subject_suffix(state, county)
|
|
||||||
return f"central.fire.incident.{suffix}"
|
|
||||||
|
|
||||||
@retry(
|
|
||||||
stop=stop_after_attempt(3),
|
|
||||||
wait=wait_exponential_jitter(initial=1, max=30),
|
|
||||||
retry=retry_if_exception_type((aiohttp.ClientError, TimeoutError)),
|
|
||||||
)
|
|
||||||
async def _fetch_features(self) -> list[dict[str, Any]]:
|
|
||||||
"""Fetch features from WFIGS FeatureServer."""
|
|
||||||
if not self._session:
|
|
||||||
raise RuntimeError("Session not initialized")
|
|
||||||
|
|
||||||
# Build query params
|
|
||||||
params: dict[str, str] = {
|
|
||||||
"outFields": "*",
|
|
||||||
"returnGeometry": "true",
|
|
||||||
"f": "geojson",
|
|
||||||
}
|
|
||||||
|
|
||||||
# Time filter: only fetch modified since last poll
|
|
||||||
if self._last_poll_time:
|
|
||||||
iso_time = self._last_poll_time.strftime("%Y-%m-%d %H:%M:%S")
|
|
||||||
params["where"] = f"ModifiedOnDateTime > timestamp '{iso_time}'"
|
|
||||||
else:
|
|
||||||
params["where"] = "1=1"
|
|
||||||
|
|
||||||
# Bbox filter if region configured
|
|
||||||
if self.region:
|
|
||||||
bbox = f"{self.region.west},{self.region.south},{self.region.east},{self.region.north}"
|
|
||||||
params["geometry"] = bbox
|
|
||||||
params["geometryType"] = "esriGeometryEnvelope"
|
|
||||||
params["spatialRel"] = "esriSpatialRelIntersects"
|
|
||||||
params["inSR"] = "4326"
|
|
||||||
|
|
||||||
async with self._session.get(WFIGS_INCIDENTS_URL, params=params) as resp:
|
|
||||||
resp.raise_for_status()
|
|
||||||
data = await resp.json()
|
|
||||||
|
|
||||||
features = data.get("features", [])
|
|
||||||
logger.info(
|
|
||||||
"WFIGS incidents fetch completed",
|
|
||||||
extra={"feature_count": len(features)},
|
|
||||||
)
|
|
||||||
return features
|
|
||||||
|
|
||||||
async def poll(self) -> AsyncIterator[Event]:
|
|
||||||
"""Poll WFIGS for incident updates."""
|
|
||||||
if not self._db:
|
|
||||||
raise RuntimeError("Database not initialized")
|
|
||||||
|
|
||||||
# Fetch features from upstream
|
|
||||||
try:
|
|
||||||
features = await self._fetch_features()
|
|
||||||
except Exception as e:
|
|
||||||
logger.error("WFIGS incidents fetch failed", extra={"error": str(e)})
|
|
||||||
raise
|
|
||||||
|
|
||||||
# Get previous poll's observed GUIDs for fall-off detection
|
|
||||||
observed_before = get_observed_guids(self._db, LAYER_NAME)
|
|
||||||
|
|
||||||
# Process features and track current GUIDs
|
|
||||||
current_guids: dict[str, tuple[str | None, str | None]] = {}
|
|
||||||
events_yielded = 0
|
|
||||||
|
|
||||||
for feature in features:
|
|
||||||
props = feature.get("properties", {})
|
|
||||||
geometry = feature.get("geometry")
|
|
||||||
|
|
||||||
irwin_id = props.get("IrwinID")
|
|
||||||
if not irwin_id:
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Extract location
|
|
||||||
centroid = extract_centroid(geometry)
|
|
||||||
|
|
||||||
# Post-filter: skip if outside region bbox
|
|
||||||
if self.region and centroid:
|
|
||||||
lon, lat = centroid
|
|
||||||
if not point_in_bbox(
|
|
||||||
lon, lat,
|
|
||||||
self.region.west, self.region.south,
|
|
||||||
self.region.east, self.region.north,
|
|
||||||
):
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Normalize at parse boundary
|
|
||||||
state_raw = props.get("POOState")
|
|
||||||
state = normalize_state(state_raw)
|
|
||||||
county = props.get("POOCounty")
|
|
||||||
incident_type_raw = props.get("IncidentTypeCategory")
|
|
||||||
incident_type = normalize_incident_type(incident_type_raw)
|
|
||||||
|
|
||||||
# Track this GUID as observed (for fall-off detection)
|
|
||||||
# Store normalized state for consistency
|
|
||||||
current_guids[irwin_id] = (state, county)
|
|
||||||
|
|
||||||
# Parse fields
|
|
||||||
discovery_time = parse_wfigs_timestamp(props.get("FireDiscoveryDateTime"))
|
|
||||||
daily_acres = props.get("DailyAcres")
|
|
||||||
|
|
||||||
# Build regions (expects normalized 2-letter state code)
|
|
||||||
regions, primary_region = build_regions(state, county)
|
|
||||||
|
|
||||||
# Build geo
|
|
||||||
if centroid:
|
|
||||||
geo = Geo(
|
|
||||||
centroid=centroid,
|
|
||||||
bbox=(centroid[0], centroid[1], centroid[0], centroid[1]),
|
|
||||||
regions=regions,
|
|
||||||
primary_region=primary_region,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
geo = Geo(regions=regions, primary_region=primary_region)
|
|
||||||
|
|
||||||
# Build event with normalized values in data
|
|
||||||
event = Event(
|
|
||||||
id=irwin_id,
|
|
||||||
adapter=self.name,
|
|
||||||
category=f"fire.incident.{incident_type}",
|
|
||||||
time=discovery_time or datetime.now(timezone.utc),
|
|
||||||
severity=severity_from_acres(daily_acres),
|
|
||||||
geo=geo,
|
|
||||||
data={
|
|
||||||
"IrwinID": irwin_id,
|
|
||||||
"IncidentName": props.get("IncidentName"),
|
|
||||||
"IncidentTypeCategory": incident_type,
|
|
||||||
"IncidentTypeCategory_raw": incident_type_raw,
|
|
||||||
"DailyAcres": daily_acres,
|
|
||||||
"PercentContained": props.get("PercentContained"),
|
|
||||||
"FireDiscoveryDateTime": props.get("FireDiscoveryDateTime"),
|
|
||||||
"ModifiedOnDateTime": props.get("ModifiedOnDateTime"),
|
|
||||||
"POOState": state,
|
|
||||||
"POOState_raw": state_raw,
|
|
||||||
"POOCounty": county,
|
|
||||||
"raw": props,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
yield event
|
|
||||||
events_yielded += 1
|
|
||||||
|
|
||||||
# Detect fall-offs: GUIDs in previous but not current
|
|
||||||
fallen_off = set(observed_before.keys()) - set(current_guids.keys())
|
|
||||||
|
|
||||||
for irwin_id in fallen_off:
|
|
||||||
last_observed, state, county = observed_before[irwin_id]
|
|
||||||
now = datetime.now(timezone.utc)
|
|
||||||
|
|
||||||
removal_event = Event(
|
|
||||||
id=f"{irwin_id}:removed:{now.isoformat()}",
|
|
||||||
adapter=self.name,
|
|
||||||
category="fire.incident.removed",
|
|
||||||
time=now,
|
|
||||||
severity=0,
|
|
||||||
geo=Geo(),
|
|
||||||
data={
|
|
||||||
"irwin_id": irwin_id,
|
|
||||||
"last_observed_at": last_observed,
|
|
||||||
"state": state,
|
|
||||||
"county": county,
|
|
||||||
"reason": "fallen_off_current_service",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
yield removal_event
|
|
||||||
events_yielded += 1
|
|
||||||
logger.info(
|
|
||||||
"WFIGS incident fall-off detected",
|
|
||||||
extra={"irwin_id": irwin_id, "state": state},
|
|
||||||
)
|
|
||||||
|
|
||||||
# Update observed table
|
|
||||||
update_observed(self._db, LAYER_NAME, current_guids)
|
|
||||||
delete_observed(self._db, LAYER_NAME, fallen_off)
|
|
||||||
|
|
||||||
# Periodic cleanup of old entries
|
|
||||||
cleanup_old_observed(self._db, LAYER_NAME)
|
|
||||||
self.sweep_old_ids()
|
|
||||||
|
|
||||||
# Update last poll time
|
|
||||||
self._last_poll_time = datetime.now(timezone.utc)
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
"WFIGS incidents poll completed",
|
|
||||||
extra={
|
|
||||||
"events_yielded": events_yielded,
|
|
||||||
"current_observed": len(current_guids),
|
|
||||||
"fallen_off": len(fallen_off),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
@ -1,397 +0,0 @@
|
||||||
"""WFIGS Perimeters adapter for wildfire perimeter polygons."""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import sqlite3
|
|
||||||
from collections.abc import AsyncIterator
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import aiohttp
|
|
||||||
from pydantic import BaseModel
|
|
||||||
from tenacity import (
|
|
||||||
retry,
|
|
||||||
retry_if_exception_type,
|
|
||||||
stop_after_attempt,
|
|
||||||
wait_exponential_jitter,
|
|
||||||
)
|
|
||||||
|
|
||||||
from central.adapter import SourceAdapter
|
|
||||||
from central.adapters.wfigs_common import (
|
|
||||||
WFIGS_PERIMETERS_URL,
|
|
||||||
build_regions,
|
|
||||||
cleanup_old_observed,
|
|
||||||
delete_observed,
|
|
||||||
extract_centroid,
|
|
||||||
get_observed_guids,
|
|
||||||
init_observed_table,
|
|
||||||
normalize_incident_type,
|
|
||||||
normalize_state,
|
|
||||||
parse_wfigs_timestamp,
|
|
||||||
polygon_intersects_bbox,
|
|
||||||
severity_from_acres,
|
|
||||||
subject_suffix,
|
|
||||||
update_observed,
|
|
||||||
)
|
|
||||||
from central.config_models import AdapterConfig, RegionConfig
|
|
||||||
from central.config_store import ConfigStore
|
|
||||||
from central.models import Event, Geo
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
LAYER_NAME = "perimeters"
|
|
||||||
|
|
||||||
|
|
||||||
class WFIGSPerimetersSettings(BaseModel):
|
|
||||||
"""Settings schema for WFIGS Perimeters adapter."""
|
|
||||||
|
|
||||||
region: RegionConfig | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class WFIGSPerimetersAdapter(SourceAdapter):
|
|
||||||
"""NIFC WFIGS wildfire perimeters adapter."""
|
|
||||||
|
|
||||||
name = "wfigs_perimeters"
|
|
||||||
display_name = "NIFC WFIGS — Wildfire Perimeters"
|
|
||||||
description = "Active wildfire perimeter polygons from NIFC WFIGS."
|
|
||||||
settings_schema = WFIGSPerimetersSettings
|
|
||||||
requires_api_key = None
|
|
||||||
api_key_field = None
|
|
||||||
wizard_order = None # Not in setup wizard
|
|
||||||
default_cadence_s = 300
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
config: AdapterConfig,
|
|
||||||
config_store: ConfigStore,
|
|
||||||
cursor_db_path: Path,
|
|
||||||
) -> None:
|
|
||||||
self._config_store = config_store
|
|
||||||
self._cursor_db_path = cursor_db_path
|
|
||||||
self._session: aiohttp.ClientSession | None = None
|
|
||||||
self._db: sqlite3.Connection | None = None
|
|
||||||
self._last_poll_time: datetime | None = None
|
|
||||||
|
|
||||||
# Parse region from settings
|
|
||||||
region_dict = config.settings.get("region")
|
|
||||||
if region_dict:
|
|
||||||
self.region: RegionConfig | None = RegionConfig(**region_dict)
|
|
||||||
else:
|
|
||||||
self.region = None
|
|
||||||
|
|
||||||
async def startup(self) -> None:
|
|
||||||
"""Initialize HTTP session and SQLite connection."""
|
|
||||||
self._session = aiohttp.ClientSession(
|
|
||||||
timeout=aiohttp.ClientTimeout(total=120), # Longer timeout for large polygons
|
|
||||||
)
|
|
||||||
self._db = sqlite3.connect(self._cursor_db_path)
|
|
||||||
|
|
||||||
# Create tables for dedup and fall-off tracking
|
|
||||||
self._db.execute("""
|
|
||||||
CREATE TABLE IF NOT EXISTS published_ids (
|
|
||||||
adapter TEXT NOT NULL,
|
|
||||||
event_id TEXT NOT NULL,
|
|
||||||
first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
PRIMARY KEY (adapter, event_id)
|
|
||||||
)
|
|
||||||
""")
|
|
||||||
self._db.execute("""
|
|
||||||
CREATE INDEX IF NOT EXISTS published_ids_last_seen
|
|
||||||
ON published_ids (last_seen)
|
|
||||||
""")
|
|
||||||
init_observed_table(self._db)
|
|
||||||
self._db.commit()
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
"WFIGS perimeters adapter started",
|
|
||||||
extra={"region": self.region.model_dump() if self.region else None},
|
|
||||||
)
|
|
||||||
|
|
||||||
async def shutdown(self) -> None:
|
|
||||||
"""Close HTTP session and SQLite connection."""
|
|
||||||
if self._session:
|
|
||||||
await self._session.close()
|
|
||||||
self._session = None
|
|
||||||
if self._db:
|
|
||||||
self._db.close()
|
|
||||||
self._db = None
|
|
||||||
logger.info("WFIGS perimeters adapter shut down")
|
|
||||||
|
|
||||||
async def apply_config(self, new_config: AdapterConfig) -> None:
|
|
||||||
"""Apply new configuration from hot-reload."""
|
|
||||||
region_dict = new_config.settings.get("region")
|
|
||||||
if region_dict:
|
|
||||||
self.region = RegionConfig(**region_dict)
|
|
||||||
else:
|
|
||||||
self.region = None
|
|
||||||
logger.info(
|
|
||||||
"WFIGS perimeters config updated",
|
|
||||||
extra={"region": self.region.model_dump() if self.region else None},
|
|
||||||
)
|
|
||||||
|
|
||||||
def is_published(self, event_id: str) -> bool:
|
|
||||||
"""Check if an event has already been published."""
|
|
||||||
if not self._db:
|
|
||||||
return False
|
|
||||||
cur = self._db.execute(
|
|
||||||
"SELECT 1 FROM published_ids WHERE adapter = ? AND event_id = ?",
|
|
||||||
(self.name, event_id),
|
|
||||||
)
|
|
||||||
return cur.fetchone() is not None
|
|
||||||
|
|
||||||
def mark_published(self, event_id: str) -> None:
|
|
||||||
"""Mark an event as published."""
|
|
||||||
if not self._db:
|
|
||||||
return
|
|
||||||
self._db.execute(
|
|
||||||
"""
|
|
||||||
INSERT INTO published_ids (adapter, event_id, first_seen, last_seen)
|
|
||||||
VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
||||||
ON CONFLICT (adapter, event_id) DO UPDATE SET
|
|
||||||
last_seen = CURRENT_TIMESTAMP
|
|
||||||
""",
|
|
||||||
(self.name, event_id),
|
|
||||||
)
|
|
||||||
self._db.commit()
|
|
||||||
|
|
||||||
def bump_last_seen(self, event_id: str) -> None:
|
|
||||||
"""Bump the last_seen timestamp for an event."""
|
|
||||||
if not self._db:
|
|
||||||
return
|
|
||||||
self._db.execute(
|
|
||||||
"UPDATE published_ids SET last_seen = CURRENT_TIMESTAMP WHERE adapter = ? AND event_id = ?",
|
|
||||||
(self.name, event_id),
|
|
||||||
)
|
|
||||||
self._db.commit()
|
|
||||||
|
|
||||||
def sweep_old_ids(self) -> int:
|
|
||||||
"""Remove published_ids older than 14 days. Returns count deleted."""
|
|
||||||
if not self._db:
|
|
||||||
return 0
|
|
||||||
cur = self._db.execute(
|
|
||||||
"DELETE FROM published_ids WHERE adapter = ? AND last_seen < datetime('now', '-14 days')",
|
|
||||||
(self.name,),
|
|
||||||
)
|
|
||||||
self._db.commit()
|
|
||||||
count = cur.rowcount
|
|
||||||
if count > 0:
|
|
||||||
logger.info("WFIGS perimeters swept old dedup entries", extra={"count": count})
|
|
||||||
return count
|
|
||||||
|
|
||||||
def subject_for(self, event: Event) -> str:
|
|
||||||
"""Compute NATS subject for an event."""
|
|
||||||
# Removal events have a different subject pattern
|
|
||||||
if event.category.startswith("fire.perimeter.removed"):
|
|
||||||
state = event.data.get("state", "").lower() or "unknown"
|
|
||||||
return f"central.fire.perimeter.removed.{state}"
|
|
||||||
|
|
||||||
# Regular perimeters: central.fire.perimeter.<state>.<county>
|
|
||||||
# POOState is already normalized (2-letter code)
|
|
||||||
state = event.data.get("POOState")
|
|
||||||
county = event.data.get("POOCounty")
|
|
||||||
suffix = subject_suffix(state, county)
|
|
||||||
return f"central.fire.perimeter.{suffix}"
|
|
||||||
|
|
||||||
@retry(
|
|
||||||
stop=stop_after_attempt(3),
|
|
||||||
wait=wait_exponential_jitter(initial=1, max=30),
|
|
||||||
retry=retry_if_exception_type((aiohttp.ClientError, TimeoutError)),
|
|
||||||
)
|
|
||||||
async def _fetch_features(self) -> list[dict[str, Any]]:
|
|
||||||
"""Fetch features from WFIGS FeatureServer."""
|
|
||||||
if not self._session:
|
|
||||||
raise RuntimeError("Session not initialized")
|
|
||||||
|
|
||||||
# Build query params
|
|
||||||
params: dict[str, str] = {
|
|
||||||
"outFields": "*",
|
|
||||||
"returnGeometry": "true",
|
|
||||||
"f": "geojson",
|
|
||||||
}
|
|
||||||
|
|
||||||
# Time filter: only fetch modified since last poll
|
|
||||||
# Note: perimeters use attr_ModifiedOnDateTime_dt field
|
|
||||||
if self._last_poll_time:
|
|
||||||
iso_time = self._last_poll_time.strftime("%Y-%m-%d %H:%M:%S")
|
|
||||||
params["where"] = f"attr_ModifiedOnDateTime_dt > timestamp '{iso_time}'"
|
|
||||||
else:
|
|
||||||
params["where"] = "1=1"
|
|
||||||
|
|
||||||
# Bbox filter if region configured
|
|
||||||
if self.region:
|
|
||||||
bbox = f"{self.region.west},{self.region.south},{self.region.east},{self.region.north}"
|
|
||||||
params["geometry"] = bbox
|
|
||||||
params["geometryType"] = "esriGeometryEnvelope"
|
|
||||||
params["spatialRel"] = "esriSpatialRelIntersects"
|
|
||||||
params["inSR"] = "4326"
|
|
||||||
|
|
||||||
async with self._session.get(WFIGS_PERIMETERS_URL, params=params) as resp:
|
|
||||||
resp.raise_for_status()
|
|
||||||
data = await resp.json()
|
|
||||||
|
|
||||||
features = data.get("features", [])
|
|
||||||
logger.info(
|
|
||||||
"WFIGS perimeters fetch completed",
|
|
||||||
extra={"feature_count": len(features)},
|
|
||||||
)
|
|
||||||
return features
|
|
||||||
|
|
||||||
async def poll(self) -> AsyncIterator[Event]:
|
|
||||||
"""Poll WFIGS for perimeter updates."""
|
|
||||||
if not self._db:
|
|
||||||
raise RuntimeError("Database not initialized")
|
|
||||||
|
|
||||||
# Fetch features from upstream
|
|
||||||
try:
|
|
||||||
features = await self._fetch_features()
|
|
||||||
except Exception as e:
|
|
||||||
logger.error("WFIGS perimeters fetch failed", extra={"error": str(e)})
|
|
||||||
raise
|
|
||||||
|
|
||||||
# Get previous poll's observed GUIDs for fall-off detection
|
|
||||||
observed_before = get_observed_guids(self._db, LAYER_NAME)
|
|
||||||
|
|
||||||
# Process features and track current GUIDs
|
|
||||||
current_guids: dict[str, tuple[str | None, str | None]] = {}
|
|
||||||
events_yielded = 0
|
|
||||||
|
|
||||||
for feature in features:
|
|
||||||
props = feature.get("properties", {})
|
|
||||||
geometry = feature.get("geometry")
|
|
||||||
|
|
||||||
# WFIGS Perimeters use prefixed field names (attr_*, poly_*)
|
|
||||||
irwin_id = props.get("attr_IrwinID") or props.get("poly_IRWINID")
|
|
||||||
if not irwin_id:
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Post-filter: skip if geometry doesn't intersect region bbox
|
|
||||||
if self.region and geometry:
|
|
||||||
if not polygon_intersects_bbox(
|
|
||||||
geometry,
|
|
||||||
self.region.west, self.region.south,
|
|
||||||
self.region.east, self.region.north,
|
|
||||||
):
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Normalize at parse boundary
|
|
||||||
state_raw = props.get("attr_POOState")
|
|
||||||
state = normalize_state(state_raw)
|
|
||||||
county = props.get("attr_POOCounty")
|
|
||||||
incident_type_raw = props.get("attr_IncidentTypeCategory")
|
|
||||||
incident_type = normalize_incident_type(incident_type_raw)
|
|
||||||
|
|
||||||
# Track this GUID as observed (for fall-off detection)
|
|
||||||
# Store normalized state for consistency
|
|
||||||
current_guids[irwin_id] = (state, county)
|
|
||||||
|
|
||||||
# Parse fields using prefixed names
|
|
||||||
discovery_time = parse_wfigs_timestamp(props.get("attr_FireDiscoveryDateTime"))
|
|
||||||
# Use poly_GISAcres or attr_IncidentSize for acreage
|
|
||||||
daily_acres = props.get("attr_IncidentSize") or props.get("poly_GISAcres")
|
|
||||||
|
|
||||||
# Build regions (expects normalized 2-letter state code)
|
|
||||||
regions, primary_region = build_regions(state, county)
|
|
||||||
|
|
||||||
# Extract centroid for geo
|
|
||||||
centroid = extract_centroid(geometry)
|
|
||||||
|
|
||||||
# Build bbox from geometry if available
|
|
||||||
bbox = None
|
|
||||||
if geometry:
|
|
||||||
try:
|
|
||||||
from shapely.geometry import shape
|
|
||||||
geom = shape(geometry)
|
|
||||||
bounds = geom.bounds # (minx, miny, maxx, maxy)
|
|
||||||
bbox = (bounds[0], bounds[1], bounds[2], bounds[3])
|
|
||||||
except Exception:
|
|
||||||
if centroid:
|
|
||||||
bbox = (centroid[0], centroid[1], centroid[0], centroid[1])
|
|
||||||
|
|
||||||
# Build geo
|
|
||||||
geo = Geo(
|
|
||||||
centroid=centroid,
|
|
||||||
bbox=bbox,
|
|
||||||
regions=regions,
|
|
||||||
primary_region=primary_region,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Build event with geometry in data
|
|
||||||
# Use normalized field names in event data for consistency
|
|
||||||
event = Event(
|
|
||||||
id=irwin_id,
|
|
||||||
adapter=self.name,
|
|
||||||
category=f"fire.perimeter.{incident_type}",
|
|
||||||
time=discovery_time or datetime.now(timezone.utc),
|
|
||||||
severity=severity_from_acres(daily_acres),
|
|
||||||
geo=geo,
|
|
||||||
data={
|
|
||||||
"IrwinID": irwin_id,
|
|
||||||
"IncidentName": props.get("attr_IncidentName") or props.get("poly_IncidentName"),
|
|
||||||
"IncidentTypeCategory": incident_type,
|
|
||||||
"IncidentTypeCategory_raw": incident_type_raw,
|
|
||||||
"DailyAcres": props.get("attr_IncidentSize"),
|
|
||||||
"GISAcres": props.get("poly_GISAcres"),
|
|
||||||
"PercentContained": props.get("attr_PercentContained"),
|
|
||||||
"FireDiscoveryDateTime": props.get("attr_FireDiscoveryDateTime"),
|
|
||||||
"ModifiedOnDateTime": props.get("attr_ModifiedOnDateTime_dt"),
|
|
||||||
"POOState": state,
|
|
||||||
"POOState_raw": state_raw,
|
|
||||||
"POOCounty": county,
|
|
||||||
"geometry": geometry, # Full GeoJSON polygon
|
|
||||||
"raw": props,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
yield event
|
|
||||||
events_yielded += 1
|
|
||||||
|
|
||||||
# Detect fall-offs: GUIDs in previous but not current
|
|
||||||
fallen_off = set(observed_before.keys()) - set(current_guids.keys())
|
|
||||||
|
|
||||||
for irwin_id in fallen_off:
|
|
||||||
last_observed, state, county = observed_before[irwin_id]
|
|
||||||
now = datetime.now(timezone.utc)
|
|
||||||
|
|
||||||
removal_event = Event(
|
|
||||||
id=f"{irwin_id}:removed:{now.isoformat()}",
|
|
||||||
adapter=self.name,
|
|
||||||
category="fire.perimeter.removed",
|
|
||||||
time=now,
|
|
||||||
severity=0,
|
|
||||||
geo=Geo(),
|
|
||||||
data={
|
|
||||||
"irwin_id": irwin_id,
|
|
||||||
"last_observed_at": last_observed,
|
|
||||||
"state": state,
|
|
||||||
"county": county,
|
|
||||||
"reason": "fallen_off_current_service",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
yield removal_event
|
|
||||||
events_yielded += 1
|
|
||||||
logger.info(
|
|
||||||
"WFIGS perimeter fall-off detected",
|
|
||||||
extra={"irwin_id": irwin_id, "state": state},
|
|
||||||
)
|
|
||||||
|
|
||||||
# Update observed table
|
|
||||||
update_observed(self._db, LAYER_NAME, current_guids)
|
|
||||||
delete_observed(self._db, LAYER_NAME, fallen_off)
|
|
||||||
|
|
||||||
# Periodic cleanup of old entries
|
|
||||||
cleanup_old_observed(self._db, LAYER_NAME)
|
|
||||||
self.sweep_old_ids()
|
|
||||||
|
|
||||||
# Update last poll time
|
|
||||||
self._last_poll_time = datetime.now(timezone.utc)
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
"WFIGS perimeters poll completed",
|
|
||||||
extra={
|
|
||||||
"events_yielded": events_yielded,
|
|
||||||
"current_observed": len(current_guids),
|
|
||||||
"fallen_off": len(fallen_off),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
@ -25,7 +25,6 @@ STREAMS = [
|
||||||
("CENTRAL_WX", "central.wx.>"),
|
("CENTRAL_WX", "central.wx.>"),
|
||||||
("CENTRAL_FIRE", "central.fire.>"),
|
("CENTRAL_FIRE", "central.fire.>"),
|
||||||
("CENTRAL_QUAKE", "central.quake.>"),
|
("CENTRAL_QUAKE", "central.quake.>"),
|
||||||
("CENTRAL_SPACE", "central.space.>"),
|
|
||||||
]
|
]
|
||||||
|
|
||||||
BATCH_SIZE = 100
|
BATCH_SIZE = 100
|
||||||
|
|
|
||||||
|
|
@ -32,7 +32,7 @@ class AdapterConfig(BaseModel):
|
||||||
|
|
||||||
name: str = Field(description="Unique adapter identifier")
|
name: str = Field(description="Unique adapter identifier")
|
||||||
enabled: bool = Field(default=True, description="Whether adapter is active")
|
enabled: bool = Field(default=True, description="Whether adapter is active")
|
||||||
cadence_s: int = Field(ge=10, description="Poll interval in seconds")
|
cadence_s: int = Field(description="Poll interval in seconds")
|
||||||
settings: dict[str, Any] = Field(
|
settings: dict[str, Any] = Field(
|
||||||
default_factory=dict, description="Adapter-specific settings"
|
default_factory=dict, description="Adapter-specific settings"
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -241,14 +241,6 @@ class ConfigStore:
|
||||||
)
|
)
|
||||||
return result == "DELETE 1"
|
return result == "DELETE 1"
|
||||||
|
|
||||||
async def set_adapter_last_error(self, name: str, error: str | None) -> None:
|
|
||||||
"""Set or clear the last_error field on an adapter row."""
|
|
||||||
async with self._pool.acquire() as conn:
|
|
||||||
await conn.execute(
|
|
||||||
"UPDATE config.adapters SET last_error = $1 WHERE name = $2",
|
|
||||||
error, name,
|
|
||||||
)
|
|
||||||
|
|
||||||
# -------------------------------------------------------------------------
|
# -------------------------------------------------------------------------
|
||||||
# Change notifications
|
# Change notifications
|
||||||
# -------------------------------------------------------------------------
|
# -------------------------------------------------------------------------
|
||||||
|
|
|
||||||
|
|
@ -247,37 +247,18 @@ def _create_app() -> FastAPI:
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
# Add field descriptors to adapters
|
# Import helper functions for valid values
|
||||||
from central.gui.routes import _adapter_classes
|
from central.gui.routes import _get_valid_satellites, _get_valid_feeds
|
||||||
from central.gui.form_descriptors import describe_fields
|
|
||||||
adapter_classes = _adapter_classes()
|
|
||||||
wizard_adapters = sorted(
|
|
||||||
[(name, cls) for name, cls in adapter_classes.items() if cls.wizard_order is not None],
|
|
||||||
key=lambda nc: nc[1].wizard_order
|
|
||||||
)
|
|
||||||
# Rebuild adapters with fields
|
|
||||||
enriched_adapters = []
|
|
||||||
for name, cls in wizard_adapters:
|
|
||||||
adapter_data = next((a for a in adapters if a["name"] == name), None)
|
|
||||||
if adapter_data:
|
|
||||||
settings_dict = adapter_data.get("settings", {})
|
|
||||||
fields = describe_fields(cls.settings_schema, settings_dict)
|
|
||||||
enriched_adapters.append({
|
|
||||||
"name": name,
|
|
||||||
"display_name": cls.display_name,
|
|
||||||
"enabled": adapter_data.get("enabled", False),
|
|
||||||
"cadence_s": adapter_data.get("cadence_s", 300),
|
|
||||||
"settings": settings_dict,
|
|
||||||
"fields": fields,
|
|
||||||
})
|
|
||||||
|
|
||||||
response = templates.TemplateResponse(
|
response = templates.TemplateResponse(
|
||||||
request=request,
|
request=request,
|
||||||
name="setup_adapters.html",
|
name="setup_adapters.html",
|
||||||
context={
|
context={
|
||||||
"csrf_token": csrf_token,
|
"csrf_token": csrf_token,
|
||||||
"adapters": enriched_adapters,
|
"adapters": adapters,
|
||||||
"api_keys": api_keys,
|
"api_keys": api_keys,
|
||||||
|
"valid_satellites": _get_valid_satellites(),
|
||||||
|
"valid_feeds": sorted(_get_valid_feeds()),
|
||||||
"tile_url": tile_url,
|
"tile_url": tile_url,
|
||||||
"tile_attribution": tile_attribution,
|
"tile_attribution": tile_attribution,
|
||||||
"error": error_msg,
|
"error": error_msg,
|
||||||
|
|
|
||||||
|
|
@ -1,163 +0,0 @@
|
||||||
"""Form field descriptors for adapter settings.
|
|
||||||
|
|
||||||
If a second nested settings type beyond RegionConfig appears,
|
|
||||||
refactor this helper to recurse over nested models.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from dataclasses import dataclass, field
|
|
||||||
from typing import Any, Literal, Union, get_args, get_origin
|
|
||||||
|
|
||||||
from pydantic import BaseModel
|
|
||||||
from pydantic.fields import FieldInfo
|
|
||||||
from pydantic_core import PydanticUndefined
|
|
||||||
|
|
||||||
from central.config_models import RegionConfig
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class FieldDescriptor:
|
|
||||||
"""Describes a form field for rendering."""
|
|
||||||
name: str
|
|
||||||
label: str
|
|
||||||
widget: str # "text", "number", "checkbox", "csv", "select", "checkboxes", "region"
|
|
||||||
current_value: Any
|
|
||||||
default: Any
|
|
||||||
description: str
|
|
||||||
required: bool
|
|
||||||
options: list[str] | None = None # For select/checkboxes widgets
|
|
||||||
|
|
||||||
|
|
||||||
def _is_literal(tp: type) -> bool:
|
|
||||||
"""Check if a type is a Literal type."""
|
|
||||||
return get_origin(tp) is Literal
|
|
||||||
|
|
||||||
|
|
||||||
def _get_literal_values(tp: type) -> list[str]:
|
|
||||||
"""Extract the literal values from a Literal type."""
|
|
||||||
return list(get_args(tp))
|
|
||||||
|
|
||||||
|
|
||||||
def _type_to_widget_and_options(field_name: str, field_type: type) -> tuple[str, list[str] | None]:
|
|
||||||
"""Map a Python type to a widget type and optional options list.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Tuple of (widget_type, options_list_or_none)
|
|
||||||
"""
|
|
||||||
# Handle Optional/Union types
|
|
||||||
origin = get_origin(field_type)
|
|
||||||
args = get_args(field_type)
|
|
||||||
|
|
||||||
# Check for Optional[X] (Union[X, None])
|
|
||||||
if origin is Union or (origin is not None and type(None) in args):
|
|
||||||
# Get the non-None type
|
|
||||||
non_none_args = [a for a in args if a is not type(None)]
|
|
||||||
if non_none_args:
|
|
||||||
inner_type = non_none_args[0]
|
|
||||||
# Recursively determine widget for the inner type
|
|
||||||
return _type_to_widget_and_options(field_name, inner_type)
|
|
||||||
|
|
||||||
# Check for Literal type (single select)
|
|
||||||
if _is_literal(field_type):
|
|
||||||
options = _get_literal_values(field_type)
|
|
||||||
return "select", [str(o) for o in options]
|
|
||||||
|
|
||||||
# Direct type checks
|
|
||||||
if field_type is str:
|
|
||||||
return "text", None
|
|
||||||
if field_type is int:
|
|
||||||
return "number", None
|
|
||||||
if field_type is bool:
|
|
||||||
return "checkbox", None
|
|
||||||
if field_type is RegionConfig:
|
|
||||||
return "region", None
|
|
||||||
|
|
||||||
# Check for list types
|
|
||||||
if origin is list:
|
|
||||||
inner_type = args[0] if args else None
|
|
||||||
|
|
||||||
# list[Literal[...]] -> checkboxes
|
|
||||||
if inner_type is not None and _is_literal(inner_type):
|
|
||||||
options = _get_literal_values(inner_type)
|
|
||||||
return "checkboxes", [str(o) for o in options]
|
|
||||||
|
|
||||||
# list[str] -> csv
|
|
||||||
if inner_type is str:
|
|
||||||
return "csv", None
|
|
||||||
|
|
||||||
raise NotImplementedError(
|
|
||||||
f"Field '{field_name}' has unsupported list type: list[{inner_type.__name__ if inner_type else '?'}]"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Check if it's a BaseModel subclass (nested model other than RegionConfig)
|
|
||||||
if isinstance(field_type, type) and issubclass(field_type, BaseModel):
|
|
||||||
raise NotImplementedError(
|
|
||||||
f"Field '{field_name}' has unsupported nested type: {field_type.__name__}. "
|
|
||||||
f"If a second nested type beyond RegionConfig is needed, "
|
|
||||||
f"refactor describe_fields to recurse over nested models."
|
|
||||||
)
|
|
||||||
|
|
||||||
raise NotImplementedError(
|
|
||||||
f"Field '{field_name}' has unsupported type: {field_type}"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _name_to_label(name: str) -> str:
|
|
||||||
"""Convert field name to human-readable label."""
|
|
||||||
return name.replace("_", " ").title()
|
|
||||||
|
|
||||||
|
|
||||||
def _is_undefined(value: Any) -> bool:
|
|
||||||
"""Check if a value is Pydantic's undefined sentinel."""
|
|
||||||
return value is PydanticUndefined
|
|
||||||
|
|
||||||
|
|
||||||
def describe_fields(model_cls: type[BaseModel], current: dict) -> list[FieldDescriptor]:
|
|
||||||
"""Generate field descriptors for a Pydantic model.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
model_cls: The Pydantic model class (e.g., NWSSettings)
|
|
||||||
current: Current settings values from the database
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
List of FieldDescriptor objects for rendering the form
|
|
||||||
"""
|
|
||||||
descriptors = []
|
|
||||||
|
|
||||||
for field_name, field_info in model_cls.model_fields.items():
|
|
||||||
# Get the field type
|
|
||||||
field_type = field_info.annotation
|
|
||||||
|
|
||||||
# Determine widget and options
|
|
||||||
widget, options = _type_to_widget_and_options(field_name, field_type)
|
|
||||||
|
|
||||||
# Get current value, falling back to default
|
|
||||||
if field_name in current:
|
|
||||||
current_value = current[field_name]
|
|
||||||
elif not _is_undefined(field_info.default):
|
|
||||||
current_value = field_info.default
|
|
||||||
else:
|
|
||||||
current_value = None
|
|
||||||
|
|
||||||
# Get default
|
|
||||||
default = field_info.default if not _is_undefined(field_info.default) else None
|
|
||||||
|
|
||||||
# Get description
|
|
||||||
description = ""
|
|
||||||
if field_info.description:
|
|
||||||
description = field_info.description
|
|
||||||
|
|
||||||
# Determine if required (no default and not Optional)
|
|
||||||
required = _is_undefined(field_info.default) and field_info.is_required()
|
|
||||||
|
|
||||||
descriptors.append(FieldDescriptor(
|
|
||||||
name=field_name,
|
|
||||||
label=_name_to_label(field_name),
|
|
||||||
widget=widget,
|
|
||||||
current_value=current_value,
|
|
||||||
default=default,
|
|
||||||
description=description,
|
|
||||||
required=required,
|
|
||||||
options=options,
|
|
||||||
))
|
|
||||||
|
|
||||||
return descriptors
|
|
||||||
|
|
@ -9,7 +9,6 @@ from typing import Any
|
||||||
|
|
||||||
logger = logging.getLogger("central.gui.routes")
|
logger = logging.getLogger("central.gui.routes")
|
||||||
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, Form, Request
|
from fastapi import APIRouter, Depends, Form, Request
|
||||||
from fastapi.responses import HTMLResponse, RedirectResponse, Response
|
from fastapi.responses import HTMLResponse, RedirectResponse, Response
|
||||||
from central.bootstrap_config import get_settings
|
from central.bootstrap_config import get_settings
|
||||||
|
|
@ -44,27 +43,12 @@ from central.gui.audit import (
|
||||||
SYSTEM_UPDATE,
|
SYSTEM_UPDATE,
|
||||||
write_audit,
|
write_audit,
|
||||||
)
|
)
|
||||||
from functools import cache
|
|
||||||
|
|
||||||
from central.gui.db import get_pool
|
from central.gui.db import get_pool
|
||||||
from central.gui.form_descriptors import describe_fields, FieldDescriptor
|
|
||||||
from central.adapter_discovery import discover_adapters
|
|
||||||
from pydantic import ValidationError
|
|
||||||
|
|
||||||
@cache
|
|
||||||
def _adapter_classes() -> dict:
|
|
||||||
"""Cached adapter class discovery.
|
|
||||||
|
|
||||||
GUI is a separate process from supervisor; walks pkgutil itself.
|
|
||||||
Python's import cache makes subsequent calls free.
|
|
||||||
"""
|
|
||||||
return discover_adapters()
|
|
||||||
|
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
# Streams to display on dashboard
|
# Streams to display on dashboard
|
||||||
DASHBOARD_STREAMS = ["CENTRAL_WX", "CENTRAL_FIRE", "CENTRAL_QUAKE", "CENTRAL_SPACE", "CENTRAL_META"]
|
DASHBOARD_STREAMS = ["CENTRAL_WX", "CENTRAL_FIRE", "CENTRAL_QUAKE", "CENTRAL_META"]
|
||||||
|
|
||||||
# Email validation regex (simple but effective)
|
# Email validation regex (simple but effective)
|
||||||
ALIAS_REGEX = re.compile(r"^[a-zA-Z0-9_]+$")
|
ALIAS_REGEX = re.compile(r"^[a-zA-Z0-9_]+$")
|
||||||
|
|
@ -73,6 +57,18 @@ ALIAS_REGEX = re.compile(r"^[a-zA-Z0-9_]+$")
|
||||||
EMAIL_REGEX = re.compile(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$")
|
EMAIL_REGEX = re.compile(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$")
|
||||||
|
|
||||||
|
|
||||||
|
def _get_valid_satellites() -> list[str]:
|
||||||
|
"""Get valid satellite identifiers from firms adapter."""
|
||||||
|
from central.adapters.firms import SATELLITE_SHORT
|
||||||
|
return list(SATELLITE_SHORT.keys())
|
||||||
|
|
||||||
|
|
||||||
|
def _get_valid_feeds() -> set[str]:
|
||||||
|
"""Get valid feed values from usgs_quake adapter."""
|
||||||
|
from central.adapters.usgs_quake import VALID_FEEDS
|
||||||
|
return VALID_FEEDS
|
||||||
|
|
||||||
|
|
||||||
def _get_templates():
|
def _get_templates():
|
||||||
"""Get templates instance (deferred import to avoid circular)."""
|
"""Get templates instance (deferred import to avoid circular)."""
|
||||||
from central.gui import templates
|
from central.gui import templates
|
||||||
|
|
@ -635,36 +631,18 @@ async def setup_adapters_form(request: Request) -> HTMLResponse:
|
||||||
templates = _get_templates()
|
templates = _get_templates()
|
||||||
pool = get_pool()
|
pool = get_pool()
|
||||||
|
|
||||||
# Get wizard adapters (filtered by wizard_order)
|
|
||||||
adapter_classes = _adapter_classes()
|
|
||||||
wizard_adapters = sorted(
|
|
||||||
[(name, cls) for name, cls in adapter_classes.items() if cls.wizard_order is not None],
|
|
||||||
key=lambda nc: nc[1].wizard_order
|
|
||||||
)
|
|
||||||
|
|
||||||
# Pre-fill from cookie state or DB defaults
|
# Pre-fill from cookie state or DB defaults
|
||||||
if state.adapters:
|
if state.adapters:
|
||||||
adapters = []
|
adapters = []
|
||||||
for name, cls in wizard_adapters:
|
for name in ["firms", "nws", "usgs_quake"]:
|
||||||
if name in state.adapters:
|
if name in state.adapters:
|
||||||
a = state.adapters[name]
|
a = state.adapters[name]
|
||||||
settings_dict = a["settings"]
|
adapters.append({
|
||||||
else:
|
"name": name,
|
||||||
settings_dict = {}
|
"enabled": a["enabled"],
|
||||||
fields = describe_fields(cls.settings_schema, settings_dict)
|
"cadence_s": a["cadence_s"],
|
||||||
# Swap widget for api_key_field to api_key_select
|
"settings": a["settings"],
|
||||||
if cls.api_key_field is not None:
|
})
|
||||||
for f in fields:
|
|
||||||
if f.name == cls.api_key_field:
|
|
||||||
f.widget = "api_key_select"
|
|
||||||
adapters.append({
|
|
||||||
"name": name,
|
|
||||||
"display_name": cls.display_name,
|
|
||||||
"enabled": a["enabled"] if name in state.adapters else False,
|
|
||||||
"cadence_s": a["cadence_s"] if name in state.adapters else 300,
|
|
||||||
"settings": settings_dict,
|
|
||||||
"fields": fields,
|
|
||||||
})
|
|
||||||
else:
|
else:
|
||||||
async with pool.acquire() as conn:
|
async with pool.acquire() as conn:
|
||||||
rows = await conn.fetch(
|
rows = await conn.fetch(
|
||||||
|
|
@ -674,33 +652,15 @@ async def setup_adapters_form(request: Request) -> HTMLResponse:
|
||||||
ORDER BY name
|
ORDER BY name
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
db_adapters = {row["name"]: row for row in rows}
|
adapters = []
|
||||||
|
for row in rows:
|
||||||
adapters = []
|
settings_data = row["settings"] or {}
|
||||||
for name, cls in wizard_adapters:
|
adapters.append({
|
||||||
if name in db_adapters:
|
"name": row["name"],
|
||||||
row = db_adapters[name]
|
"enabled": row["enabled"],
|
||||||
settings_dict = row["settings"] or {}
|
"cadence_s": row["cadence_s"],
|
||||||
enabled = row["enabled"]
|
"settings": settings_data,
|
||||||
cadence_s = row["cadence_s"]
|
})
|
||||||
else:
|
|
||||||
settings_dict = {}
|
|
||||||
enabled = False
|
|
||||||
cadence_s = 300
|
|
||||||
fields = describe_fields(cls.settings_schema, settings_dict)
|
|
||||||
# Swap widget for api_key_field to api_key_select
|
|
||||||
if cls.api_key_field is not None:
|
|
||||||
for f in fields:
|
|
||||||
if f.name == cls.api_key_field:
|
|
||||||
f.widget = "api_key_select"
|
|
||||||
adapters.append({
|
|
||||||
"name": name,
|
|
||||||
"display_name": cls.display_name,
|
|
||||||
"enabled": enabled,
|
|
||||||
"cadence_s": cadence_s,
|
|
||||||
"settings": settings_dict,
|
|
||||||
"fields": fields,
|
|
||||||
})
|
|
||||||
|
|
||||||
# Get API keys from wizard state (not DB)
|
# Get API keys from wizard state (not DB)
|
||||||
api_keys = [{"alias": k["alias"]} for k in state.api_keys]
|
api_keys = [{"alias": k["alias"]} for k in state.api_keys]
|
||||||
|
|
@ -725,6 +685,8 @@ async def setup_adapters_form(request: Request) -> HTMLResponse:
|
||||||
"csrf_token": csrf_token,
|
"csrf_token": csrf_token,
|
||||||
"adapters": adapters,
|
"adapters": adapters,
|
||||||
"api_keys": api_keys,
|
"api_keys": api_keys,
|
||||||
|
"valid_satellites": _get_valid_satellites(),
|
||||||
|
"valid_feeds": sorted(_get_valid_feeds()),
|
||||||
"tile_url": tile_url,
|
"tile_url": tile_url,
|
||||||
"tile_attribution": tile_attribution,
|
"tile_attribution": tile_attribution,
|
||||||
"error": None,
|
"error": None,
|
||||||
|
|
@ -777,14 +739,7 @@ async def setup_adapters_submit(request: Request) -> Response:
|
||||||
"settings": row["settings"] or {},
|
"settings": row["settings"] or {},
|
||||||
}
|
}
|
||||||
|
|
||||||
# Get wizard adapters (filtered by wizard_order)
|
for adapter_name in ["firms", "nws", "usgs_quake"]:
|
||||||
adapter_classes = _adapter_classes()
|
|
||||||
wizard_adapters = sorted(
|
|
||||||
[(name, cls) for name, cls in adapter_classes.items() if cls.wizard_order is not None],
|
|
||||||
key=lambda nc: nc[1].wizard_order
|
|
||||||
)
|
|
||||||
|
|
||||||
for adapter_name, adapter_cls in wizard_adapters:
|
|
||||||
current = current_adapters.get(adapter_name, {"enabled": False, "cadence_s": 300, "settings": {}})
|
current = current_adapters.get(adapter_name, {"enabled": False, "cadence_s": 300, "settings": {}})
|
||||||
current_settings = current.get("settings", {})
|
current_settings = current.get("settings", {})
|
||||||
new_settings = dict(current_settings)
|
new_settings = dict(current_settings)
|
||||||
|
|
@ -792,108 +747,83 @@ async def setup_adapters_submit(request: Request) -> Response:
|
||||||
# Parse enabled
|
# Parse enabled
|
||||||
enabled = f"{adapter_name}_enabled" in form
|
enabled = f"{adapter_name}_enabled" in form
|
||||||
|
|
||||||
# Parse cadence using AdapterConfig field constraint
|
# Parse cadence
|
||||||
cadence_str = form.get(f"{adapter_name}_cadence_s", "")
|
cadence_str = form.get(f"{adapter_name}_cadence_s", "")
|
||||||
try:
|
try:
|
||||||
cadence_s = int(cadence_str)
|
cadence_s = int(cadence_str)
|
||||||
from central.config_models import AdapterConfig
|
if cadence_s < 60 or cadence_s > 3600:
|
||||||
min_cadence = AdapterConfig.model_fields["cadence_s"].metadata[0].ge
|
errors[f"{adapter_name}_cadence_s"] = "Cadence must be between 60 and 3600 seconds"
|
||||||
if cadence_s < min_cadence:
|
|
||||||
errors[f"{adapter_name}_cadence_s"] = (
|
|
||||||
f"Input should be greater than or equal to {min_cadence}"
|
|
||||||
)
|
|
||||||
except ValueError:
|
except ValueError:
|
||||||
errors[f"{adapter_name}_cadence_s"] = "Cadence must be a valid integer"
|
errors[f"{adapter_name}_cadence_s"] = "Cadence must be a valid integer"
|
||||||
cadence_s = current.get("cadence_s", 300)
|
cadence_s = current.get("cadence_s", 300)
|
||||||
|
|
||||||
# Generic field parsing using describe_fields
|
# Adapter-specific validation
|
||||||
fields = describe_fields(adapter_cls.settings_schema, current_settings)
|
if adapter_name == "nws":
|
||||||
for field in fields:
|
contact_email = form.get(f"{adapter_name}_contact_email", "").strip()
|
||||||
form_key = f"{adapter_name}_{field.name}"
|
if enabled:
|
||||||
|
if not contact_email:
|
||||||
if field.widget == "text":
|
errors[f"{adapter_name}_contact_email"] = "Contact email is required when enabled"
|
||||||
value = form.get(form_key, "").strip()
|
elif not EMAIL_REGEX.match(contact_email):
|
||||||
new_settings[field.name] = value if value else current_settings.get(field.name)
|
errors[f"{adapter_name}_contact_email"] = "Invalid email format"
|
||||||
|
|
||||||
elif field.widget == "api_key_select":
|
|
||||||
# API key alias field - stored as text, validated post-loop
|
|
||||||
value = form.get(form_key, "").strip()
|
|
||||||
new_settings[field.name] = value if value else None
|
|
||||||
|
|
||||||
elif field.widget == "number":
|
|
||||||
value_str = form.get(form_key, "").strip()
|
|
||||||
if value_str:
|
|
||||||
try:
|
|
||||||
new_settings[field.name] = int(value_str)
|
|
||||||
except ValueError:
|
|
||||||
errors[form_key] = f"{field.label} must be a valid number"
|
|
||||||
else:
|
else:
|
||||||
new_settings[field.name] = current_settings.get(field.name)
|
new_settings["contact_email"] = contact_email
|
||||||
|
else:
|
||||||
|
new_settings["contact_email"] = contact_email if contact_email else current_settings.get("contact_email")
|
||||||
|
|
||||||
elif field.widget == "checkbox":
|
elif adapter_name == "firms":
|
||||||
new_settings[field.name] = form_key in form
|
api_key_alias = form.get(f"{adapter_name}_api_key_alias", "").strip()
|
||||||
|
satellites = form.getlist(f"{adapter_name}_satellites")
|
||||||
|
|
||||||
elif field.widget == "csv":
|
if api_key_alias:
|
||||||
value = form.get(form_key, "").strip()
|
# Validate against wizard state keys
|
||||||
if value:
|
if not any(k["alias"] == api_key_alias for k in state.api_keys):
|
||||||
new_settings[field.name] = [v.strip() for v in value.split(",") if v.strip()]
|
errors[f"{adapter_name}_api_key_alias"] = f"API key alias does not exist"
|
||||||
else:
|
else:
|
||||||
new_settings[field.name] = []
|
new_settings["api_key_alias"] = api_key_alias
|
||||||
|
else:
|
||||||
|
new_settings["api_key_alias"] = None
|
||||||
|
|
||||||
elif field.widget == "select":
|
# Validate satellites
|
||||||
value = form.get(form_key, "").strip()
|
valid_sats = set(_get_valid_satellites())
|
||||||
if value and field.options and value not in field.options:
|
invalid_sats = [s for s in satellites if s not in valid_sats]
|
||||||
errors[form_key] = f"Invalid {field.label.lower()}"
|
if invalid_sats:
|
||||||
else:
|
errors[f"{adapter_name}_satellites"] = f"Invalid satellites: " + ", ".join(invalid_sats)
|
||||||
new_settings[field.name] = value
|
else:
|
||||||
|
new_settings["satellites"] = satellites
|
||||||
|
|
||||||
elif field.widget == "checkboxes":
|
elif adapter_name == "usgs_quake":
|
||||||
# Use getlist for checkbox groups - absence means empty list
|
feed = form.get(f"{adapter_name}_feed", "").strip()
|
||||||
values = form.getlist(form_key)
|
valid_feeds = _get_valid_feeds()
|
||||||
if field.options:
|
if feed not in valid_feeds:
|
||||||
invalid = [v for v in values if v not in field.options]
|
errors[f"{adapter_name}_feed"] = "Invalid feed"
|
||||||
if invalid:
|
else:
|
||||||
errors[form_key] = f"Invalid values: {', '.join(invalid)}"
|
new_settings["feed"] = feed
|
||||||
else:
|
|
||||||
new_settings[field.name] = values
|
|
||||||
else:
|
|
||||||
new_settings[field.name] = values
|
|
||||||
|
|
||||||
elif field.widget == "region":
|
# Region validation (all adapters)
|
||||||
# Region validation via RegionConfig model
|
region_north_str = form.get(f"{adapter_name}_region_north", "").strip()
|
||||||
from central.config_models import RegionConfig
|
region_south_str = form.get(f"{adapter_name}_region_south", "").strip()
|
||||||
region_north_str = form.get(f"{adapter_name}_{field.name}_north", "").strip()
|
region_east_str = form.get(f"{adapter_name}_region_east", "").strip()
|
||||||
region_south_str = form.get(f"{adapter_name}_{field.name}_south", "").strip()
|
region_west_str = form.get(f"{adapter_name}_region_west", "").strip()
|
||||||
region_east_str = form.get(f"{adapter_name}_{field.name}_east", "").strip()
|
|
||||||
region_west_str = form.get(f"{adapter_name}_{field.name}_west", "").strip()
|
|
||||||
|
|
||||||
try:
|
|
||||||
region_model = RegionConfig(
|
|
||||||
north=float(region_north_str),
|
|
||||||
south=float(region_south_str),
|
|
||||||
east=float(region_east_str),
|
|
||||||
west=float(region_west_str),
|
|
||||||
)
|
|
||||||
new_settings[field.name] = region_model.model_dump()
|
|
||||||
except (ValueError, ValidationError) as e:
|
|
||||||
errors[f"{adapter_name}_{field.name}"] = str(e)
|
|
||||||
|
|
||||||
# Run Pydantic validation on assembled settings to catch Literal violations etc.
|
|
||||||
try:
|
try:
|
||||||
adapter_cls.settings_schema(**new_settings)
|
region_north = float(region_north_str)
|
||||||
except ValidationError as e:
|
region_south = float(region_south_str)
|
||||||
for err in e.errors():
|
region_east = float(region_east_str)
|
||||||
loc = err["loc"][0] if err["loc"] else "unknown"
|
region_west = float(region_west_str)
|
||||||
errors[f"{adapter_name}_{loc}"] = err["msg"]
|
|
||||||
|
|
||||||
# Generic api_key_field validation against wizard state
|
if not (-90 <= region_south < region_north <= 90):
|
||||||
if adapter_cls.api_key_field is not None:
|
errors[f"{adapter_name}_region"] = "Invalid latitude: south < north, both -90 to 90"
|
||||||
field_value = new_settings.get(adapter_cls.api_key_field)
|
elif not (-180 <= region_west < region_east <= 180):
|
||||||
if field_value:
|
errors[f"{adapter_name}_region"] = "Invalid longitude: west < east, both -180 to 180"
|
||||||
if not any(k["alias"] == field_value for k in state.api_keys):
|
else:
|
||||||
errors[f"{adapter_name}_{adapter_cls.api_key_field}"] = (
|
new_settings["region"] = {
|
||||||
"API key alias does not exist"
|
"north": region_north,
|
||||||
)
|
"south": region_south,
|
||||||
|
"east": region_east,
|
||||||
|
"west": region_west,
|
||||||
|
}
|
||||||
|
except ValueError:
|
||||||
|
errors[f"{adapter_name}_region"] = "Region coordinates must be valid numbers"
|
||||||
|
|
||||||
new_adapters[adapter_name] = {
|
new_adapters[adapter_name] = {
|
||||||
"enabled": enabled,
|
"enabled": enabled,
|
||||||
|
|
@ -903,23 +833,12 @@ async def setup_adapters_submit(request: Request) -> Response:
|
||||||
|
|
||||||
# If errors, re-render
|
# If errors, re-render
|
||||||
if errors:
|
if errors:
|
||||||
adapters = []
|
adapters = [
|
||||||
for name, cls in wizard_adapters:
|
{"name": name, "enabled": new_adapters[name]["enabled"],
|
||||||
settings_dict = new_adapters[name]["settings"]
|
"cadence_s": new_adapters[name]["cadence_s"],
|
||||||
fields = describe_fields(cls.settings_schema, settings_dict)
|
"settings": new_adapters[name]["settings"]}
|
||||||
# Swap widget for api_key_field to api_key_select
|
for name in ["firms", "nws", "usgs_quake"]
|
||||||
if cls.api_key_field is not None:
|
]
|
||||||
for f in fields:
|
|
||||||
if f.name == cls.api_key_field:
|
|
||||||
f.widget = "api_key_select"
|
|
||||||
adapters.append({
|
|
||||||
"name": name,
|
|
||||||
"display_name": cls.display_name,
|
|
||||||
"enabled": new_adapters[name]["enabled"],
|
|
||||||
"cadence_s": new_adapters[name]["cadence_s"],
|
|
||||||
"settings": settings_dict,
|
|
||||||
"fields": fields,
|
|
||||||
})
|
|
||||||
api_keys = [{"alias": k["alias"]} for k in state.api_keys]
|
api_keys = [{"alias": k["alias"]} for k in state.api_keys]
|
||||||
|
|
||||||
if state.system:
|
if state.system:
|
||||||
|
|
@ -937,6 +856,8 @@ async def setup_adapters_submit(request: Request) -> Response:
|
||||||
"csrf_token": csrf_token,
|
"csrf_token": csrf_token,
|
||||||
"adapters": adapters,
|
"adapters": adapters,
|
||||||
"api_keys": api_keys,
|
"api_keys": api_keys,
|
||||||
|
"valid_satellites": _get_valid_satellites(),
|
||||||
|
"valid_feeds": sorted(_get_valid_feeds()),
|
||||||
"tile_url": tile_url,
|
"tile_url": tile_url,
|
||||||
"tile_attribution": tile_attribution,
|
"tile_attribution": tile_attribution,
|
||||||
"error": "Please fix the errors below.",
|
"error": "Please fix the errors below.",
|
||||||
|
|
@ -977,20 +898,10 @@ async def setup_finish_form(request: Request) -> HTMLResponse:
|
||||||
|
|
||||||
adapters = []
|
adapters = []
|
||||||
if state.adapters:
|
if state.adapters:
|
||||||
adapter_classes = _adapter_classes()
|
for name in ["firms", "nws", "usgs_quake"]:
|
||||||
wizard_adapters = sorted(
|
|
||||||
[(name, cls) for name, cls in adapter_classes.items() if cls.wizard_order is not None],
|
|
||||||
key=lambda nc: nc[1].wizard_order
|
|
||||||
)
|
|
||||||
for name, cls in wizard_adapters:
|
|
||||||
if name in state.adapters:
|
if name in state.adapters:
|
||||||
a = state.adapters[name]
|
a = state.adapters[name]
|
||||||
adapters.append({
|
adapters.append({"name": name, "enabled": a["enabled"], "cadence_s": a["cadence_s"]})
|
||||||
"name": name,
|
|
||||||
"display_name": cls.display_name,
|
|
||||||
"enabled": a["enabled"],
|
|
||||||
"cadence_s": a["cadence_s"],
|
|
||||||
})
|
|
||||||
|
|
||||||
csrf_token, signed_token = reuse_or_generate_pre_auth_csrf(request, settings.csrf_secret)
|
csrf_token, signed_token = reuse_or_generate_pre_auth_csrf(request, settings.csrf_secret)
|
||||||
response = templates.TemplateResponse(
|
response = templates.TemplateResponse(
|
||||||
|
|
@ -1318,45 +1229,27 @@ async def adapters_list(
|
||||||
templates = _get_templates()
|
templates = _get_templates()
|
||||||
pool = get_pool()
|
pool = get_pool()
|
||||||
operator = request.state.operator
|
operator = request.state.operator
|
||||||
adapter_classes = _adapter_classes()
|
|
||||||
|
|
||||||
async with pool.acquire() as conn:
|
async with pool.acquire() as conn:
|
||||||
rows = await conn.fetch(
|
rows = await conn.fetch(
|
||||||
"""
|
"""
|
||||||
SELECT name, enabled, cadence_s, settings, paused_at, updated_at, last_error
|
SELECT name, enabled, cadence_s, settings, paused_at, updated_at
|
||||||
FROM config.adapters
|
FROM config.adapters
|
||||||
ORDER BY name
|
ORDER BY name
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
|
|
||||||
adapters = []
|
adapters = []
|
||||||
for row in rows:
|
for row in rows:
|
||||||
settings = row["settings"] or {}
|
settings = row["settings"] or {}
|
||||||
adapter_cls = adapter_classes.get(row["name"])
|
adapters.append({
|
||||||
|
"name": row["name"],
|
||||||
# Check if required API key is missing
|
"enabled": row["enabled"],
|
||||||
api_key_missing = False
|
"cadence_s": row["cadence_s"],
|
||||||
requires_api_key_alias = None
|
"settings": settings,
|
||||||
if adapter_cls and adapter_cls.requires_api_key is not None:
|
"paused_at": row["paused_at"],
|
||||||
requires_api_key_alias = adapter_cls.requires_api_key
|
"updated_at": row["updated_at"],
|
||||||
has_key = await conn.fetchval(
|
})
|
||||||
"SELECT 1 FROM config.api_keys WHERE alias = $1",
|
|
||||||
requires_api_key_alias,
|
|
||||||
)
|
|
||||||
api_key_missing = not has_key
|
|
||||||
|
|
||||||
adapters.append({
|
|
||||||
"name": row["name"],
|
|
||||||
"display_name": getattr(adapter_cls, "display_name", row["name"]) if adapter_cls else row["name"],
|
|
||||||
"enabled": row["enabled"],
|
|
||||||
"cadence_s": row["cadence_s"],
|
|
||||||
"settings": settings,
|
|
||||||
"paused_at": row["paused_at"],
|
|
||||||
"updated_at": row["updated_at"],
|
|
||||||
"last_error": row["last_error"],
|
|
||||||
"api_key_missing": api_key_missing,
|
|
||||||
"requires_api_key_alias": requires_api_key_alias,
|
|
||||||
})
|
|
||||||
|
|
||||||
csrf_token = request.state.csrf_token
|
csrf_token = request.state.csrf_token
|
||||||
response = templates.TemplateResponse(
|
response = templates.TemplateResponse(
|
||||||
|
|
@ -1382,14 +1275,10 @@ async def adapters_edit_form(
|
||||||
pool = get_pool()
|
pool = get_pool()
|
||||||
operator = request.state.operator
|
operator = request.state.operator
|
||||||
|
|
||||||
# Look up the adapter class
|
|
||||||
adapter_classes = _adapter_classes()
|
|
||||||
adapter_cls = adapter_classes.get(name)
|
|
||||||
|
|
||||||
async with pool.acquire() as conn:
|
async with pool.acquire() as conn:
|
||||||
row = await conn.fetchrow(
|
row = await conn.fetchrow(
|
||||||
"""
|
"""
|
||||||
SELECT name, enabled, cadence_s, settings, paused_at, updated_at, last_error
|
SELECT name, enabled, cadence_s, settings, paused_at, updated_at
|
||||||
FROM config.adapters
|
FROM config.adapters
|
||||||
WHERE name = $1
|
WHERE name = $1
|
||||||
""",
|
""",
|
||||||
|
|
@ -1399,6 +1288,11 @@ async def adapters_edit_form(
|
||||||
if row is None:
|
if row is None:
|
||||||
return Response(status_code=404, content="Adapter not found")
|
return Response(status_code=404, content="Adapter not found")
|
||||||
|
|
||||||
|
# Get API keys for firms dropdown
|
||||||
|
api_keys = await conn.fetch(
|
||||||
|
"SELECT alias FROM config.api_keys ORDER BY alias"
|
||||||
|
)
|
||||||
|
|
||||||
# Get map tile settings from config.system
|
# Get map tile settings from config.system
|
||||||
sys_row = await conn.fetchrow(
|
sys_row = await conn.fetchrow(
|
||||||
"SELECT map_tile_url, map_attribution FROM config.system WHERE id = true"
|
"SELECT map_tile_url, map_attribution FROM config.system WHERE id = true"
|
||||||
|
|
@ -1407,48 +1301,15 @@ async def adapters_edit_form(
|
||||||
tile_attribution = sys_row["map_attribution"] if sys_row else "© OpenStreetMap contributors"
|
tile_attribution = sys_row["map_attribution"] if sys_row else "© OpenStreetMap contributors"
|
||||||
|
|
||||||
settings = row["settings"] or {}
|
settings = row["settings"] or {}
|
||||||
|
|
||||||
# Build adapter dict with class metadata
|
|
||||||
adapter = {
|
adapter = {
|
||||||
"name": row["name"],
|
"name": row["name"],
|
||||||
"display_name": getattr(adapter_cls, "display_name", row["name"]) if adapter_cls else row["name"],
|
|
||||||
"description": getattr(adapter_cls, "description", "") if adapter_cls else "",
|
|
||||||
"enabled": row["enabled"],
|
"enabled": row["enabled"],
|
||||||
"cadence_s": row["cadence_s"],
|
"cadence_s": row["cadence_s"],
|
||||||
"settings": settings,
|
"settings": settings,
|
||||||
"paused_at": row["paused_at"],
|
"paused_at": row["paused_at"],
|
||||||
"updated_at": row["updated_at"],
|
"updated_at": row["updated_at"],
|
||||||
"last_error": row["last_error"],
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# Generate field descriptors if we have the adapter class
|
|
||||||
fields = []
|
|
||||||
if adapter_cls and hasattr(adapter_cls, "settings_schema"):
|
|
||||||
fields = describe_fields(adapter_cls.settings_schema, settings)
|
|
||||||
# Swap widget for api_key_field to api_key_select
|
|
||||||
if adapter_cls.api_key_field is not None:
|
|
||||||
for f in fields:
|
|
||||||
if f.name == adapter_cls.api_key_field:
|
|
||||||
f.widget = "api_key_select"
|
|
||||||
|
|
||||||
# Fetch API keys for api_key_select widget
|
|
||||||
api_keys = []
|
|
||||||
async with pool.acquire() as conn:
|
|
||||||
api_key_rows = await conn.fetch("SELECT alias FROM config.api_keys ORDER BY alias")
|
|
||||||
api_keys = [{"alias": r["alias"]} for r in api_key_rows]
|
|
||||||
|
|
||||||
# Check if required API key is missing
|
|
||||||
api_key_missing = False
|
|
||||||
requires_api_key_alias = None
|
|
||||||
if adapter_cls and adapter_cls.requires_api_key is not None:
|
|
||||||
requires_api_key_alias = adapter_cls.requires_api_key
|
|
||||||
async with pool.acquire() as conn:
|
|
||||||
has_key = await conn.fetchval(
|
|
||||||
"SELECT 1 FROM config.api_keys WHERE alias = $1",
|
|
||||||
requires_api_key_alias,
|
|
||||||
)
|
|
||||||
api_key_missing = not has_key
|
|
||||||
|
|
||||||
csrf_token = request.state.csrf_token
|
csrf_token = request.state.csrf_token
|
||||||
response = templates.TemplateResponse(
|
response = templates.TemplateResponse(
|
||||||
request=request,
|
request=request,
|
||||||
|
|
@ -1457,14 +1318,13 @@ async def adapters_edit_form(
|
||||||
"operator": operator,
|
"operator": operator,
|
||||||
"csrf_token": csrf_token,
|
"csrf_token": csrf_token,
|
||||||
"adapter": adapter,
|
"adapter": adapter,
|
||||||
"fields": fields,
|
|
||||||
"api_keys": api_keys,
|
|
||||||
"errors": None,
|
"errors": None,
|
||||||
"form_data": None,
|
"form_data": None,
|
||||||
|
"api_keys": [{"alias": k["alias"]} for k in api_keys],
|
||||||
|
"valid_satellites": _get_valid_satellites(),
|
||||||
|
"valid_feeds": sorted(_get_valid_feeds()),
|
||||||
"tile_url": tile_url,
|
"tile_url": tile_url,
|
||||||
"tile_attribution": tile_attribution,
|
"tile_attribution": tile_attribution,
|
||||||
"api_key_missing": api_key_missing,
|
|
||||||
"requires_api_key_alias": requires_api_key_alias,
|
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
return response
|
return response
|
||||||
|
|
@ -1487,27 +1347,24 @@ async def adapters_edit_submit(
|
||||||
if not form_csrf or form_csrf != request.state.csrf_token:
|
if not form_csrf or form_csrf != request.state.csrf_token:
|
||||||
raise CsrfValidationError("Invalid CSRF token")
|
raise CsrfValidationError("Invalid CSRF token")
|
||||||
|
|
||||||
# Look up the adapter class
|
# Parse form data
|
||||||
adapter_classes = _adapter_classes()
|
form = await request.form()
|
||||||
adapter_cls = adapter_classes.get(name)
|
|
||||||
|
|
||||||
# Parse common form fields
|
|
||||||
enabled = "enabled" in form
|
enabled = "enabled" in form
|
||||||
cadence_s_str = form.get("cadence_s", "")
|
cadence_s_str = form.get("cadence_s", "")
|
||||||
|
|
||||||
errors: dict[str, str] = {}
|
# Build form_data for re-render on error
|
||||||
form_data: dict[str, Any] = {
|
form_data: dict[str, Any] = {
|
||||||
"enabled": enabled,
|
"enabled": enabled,
|
||||||
"cadence_s": cadence_s_str,
|
"cadence_s": cadence_s_str,
|
||||||
}
|
}
|
||||||
|
|
||||||
# Validate cadence_s using AdapterConfig field constraint (ge=10)
|
errors: dict[str, str] = {}
|
||||||
|
|
||||||
|
# Validate cadence_s
|
||||||
try:
|
try:
|
||||||
cadence_s = int(cadence_s_str)
|
cadence_s = int(cadence_s_str)
|
||||||
from central.config_models import AdapterConfig
|
if cadence_s < 60 or cadence_s > 3600:
|
||||||
min_cadence = AdapterConfig.model_fields["cadence_s"].metadata[0].ge
|
errors["cadence_s"] = "Cadence must be between 60 and 3600 seconds"
|
||||||
if cadence_s < min_cadence:
|
|
||||||
errors["cadence_s"] = f"Input should be greater than or equal to {min_cadence}"
|
|
||||||
except ValueError:
|
except ValueError:
|
||||||
errors["cadence_s"] = "Cadence must be a valid integer"
|
errors["cadence_s"] = "Cadence must be a valid integer"
|
||||||
cadence_s = 0
|
cadence_s = 0
|
||||||
|
|
@ -1516,7 +1373,7 @@ async def adapters_edit_submit(
|
||||||
# Get current adapter state
|
# Get current adapter state
|
||||||
row = await conn.fetchrow(
|
row = await conn.fetchrow(
|
||||||
"""
|
"""
|
||||||
SELECT name, enabled, cadence_s, settings, paused_at, updated_at, last_error
|
SELECT name, enabled, cadence_s, settings, paused_at, updated_at
|
||||||
FROM config.adapters
|
FROM config.adapters
|
||||||
WHERE name = $1
|
WHERE name = $1
|
||||||
""",
|
""",
|
||||||
|
|
@ -1527,113 +1384,103 @@ async def adapters_edit_submit(
|
||||||
return Response(status_code=404, content="Adapter not found")
|
return Response(status_code=404, content="Adapter not found")
|
||||||
|
|
||||||
current_settings = row["settings"] or {}
|
current_settings = row["settings"] or {}
|
||||||
|
new_settings = dict(current_settings)
|
||||||
|
|
||||||
# Parse and validate settings via Pydantic if we have the adapter class
|
# Adapter-specific validation and settings update
|
||||||
new_settings = {}
|
if name == "nws":
|
||||||
if adapter_cls and hasattr(adapter_cls, "settings_schema"):
|
contact_email = form.get("contact_email", "").strip()
|
||||||
schema = adapter_cls.settings_schema
|
form_data["contact_email"] = contact_email
|
||||||
fields = describe_fields(schema, current_settings)
|
if not contact_email:
|
||||||
|
errors["contact_email"] = "Contact email is required"
|
||||||
# Parse form values based on widget type
|
elif not EMAIL_REGEX.match(contact_email):
|
||||||
parsed_values = {}
|
errors["contact_email"] = "Invalid email format"
|
||||||
for field in fields:
|
|
||||||
raw = form.get(field.name, "")
|
|
||||||
form_data[field.name] = raw
|
|
||||||
|
|
||||||
if field.widget == "text":
|
|
||||||
parsed_values[field.name] = raw.strip() if raw else None
|
|
||||||
elif field.widget == "number":
|
|
||||||
try:
|
|
||||||
parsed_values[field.name] = int(raw) if raw else None
|
|
||||||
except ValueError:
|
|
||||||
errors[field.name] = f"{field.label} must be a number"
|
|
||||||
elif field.widget == "checkbox":
|
|
||||||
parsed_values[field.name] = field.name in form
|
|
||||||
elif field.widget == "csv":
|
|
||||||
if raw.strip():
|
|
||||||
parsed_values[field.name] = [v.strip() for v in raw.split(",") if v.strip()]
|
|
||||||
else:
|
|
||||||
parsed_values[field.name] = []
|
|
||||||
elif field.widget == "select":
|
|
||||||
value = raw.strip() if raw else None
|
|
||||||
if value and field.options and value not in field.options:
|
|
||||||
errors[field.name] = f"Invalid {field.label.lower()}"
|
|
||||||
else:
|
|
||||||
parsed_values[field.name] = value
|
|
||||||
elif field.widget == "checkboxes":
|
|
||||||
# Use getlist for checkbox groups
|
|
||||||
values = form.getlist(field.name)
|
|
||||||
form_data[field.name] = values # Override raw value
|
|
||||||
if field.options:
|
|
||||||
invalid = [v for v in values if v not in field.options]
|
|
||||||
if invalid:
|
|
||||||
errors[field.name] = f"Invalid values: {', '.join(invalid)}"
|
|
||||||
else:
|
|
||||||
parsed_values[field.name] = values
|
|
||||||
else:
|
|
||||||
parsed_values[field.name] = values
|
|
||||||
elif field.widget == "api_key_select":
|
|
||||||
# API key select - validate against existing keys
|
|
||||||
value = raw.strip() if raw else None
|
|
||||||
parsed_values[field.name] = value
|
|
||||||
elif field.widget == "region":
|
|
||||||
# Region handled separately below
|
|
||||||
pass
|
|
||||||
|
|
||||||
# Handle region fields (common pattern)
|
|
||||||
region_north_str = form.get("region_north", "").strip()
|
|
||||||
region_south_str = form.get("region_south", "").strip()
|
|
||||||
region_east_str = form.get("region_east", "").strip()
|
|
||||||
region_west_str = form.get("region_west", "").strip()
|
|
||||||
|
|
||||||
form_data["region_north"] = region_north_str
|
|
||||||
form_data["region_south"] = region_south_str
|
|
||||||
form_data["region_east"] = region_east_str
|
|
||||||
form_data["region_west"] = region_west_str
|
|
||||||
|
|
||||||
# Check if any region field has a value
|
|
||||||
has_region = any([region_north_str, region_south_str, region_east_str, region_west_str])
|
|
||||||
|
|
||||||
if has_region:
|
|
||||||
try:
|
|
||||||
region_north = float(region_north_str)
|
|
||||||
region_south = float(region_south_str)
|
|
||||||
region_east = float(region_east_str)
|
|
||||||
region_west = float(region_west_str)
|
|
||||||
|
|
||||||
if not (-90 <= region_south < region_north <= 90):
|
|
||||||
errors["region"] = "Invalid latitude: south must be less than north, both between -90 and 90"
|
|
||||||
elif not (-180 <= region_west < region_east <= 180):
|
|
||||||
errors["region"] = "Invalid longitude: west must be less than east, both between -180 and 180"
|
|
||||||
else:
|
|
||||||
parsed_values["region"] = {
|
|
||||||
"north": region_north,
|
|
||||||
"south": region_south,
|
|
||||||
"east": region_east,
|
|
||||||
"west": region_west,
|
|
||||||
}
|
|
||||||
except ValueError:
|
|
||||||
errors["region"] = "Region coordinates must be valid numbers"
|
|
||||||
else:
|
else:
|
||||||
parsed_values["region"] = None
|
new_settings["contact_email"] = contact_email
|
||||||
|
|
||||||
# Only validate with Pydantic if no parse errors
|
elif name == "firms":
|
||||||
if not errors:
|
api_key_alias = form.get("api_key_alias", "").strip()
|
||||||
try:
|
satellites = form.getlist("satellites")
|
||||||
# Filter out None values for optional fields without defaults
|
form_data["api_key_alias"] = api_key_alias
|
||||||
validated_data = {k: v for k, v in parsed_values.items() if v is not None}
|
form_data["satellites"] = satellites
|
||||||
validated = schema(**validated_data)
|
|
||||||
new_settings = validated.model_dump(mode="json")
|
# Validate api_key_alias if set
|
||||||
except ValidationError as e:
|
if api_key_alias:
|
||||||
for err in e.errors():
|
key_exists = await conn.fetchrow(
|
||||||
field_name = err["loc"][0] if err["loc"] else "unknown"
|
"SELECT 1 FROM config.api_keys WHERE alias = $1",
|
||||||
errors[str(field_name)] = err["msg"]
|
api_key_alias,
|
||||||
else:
|
)
|
||||||
# No schema - just preserve existing settings
|
if not key_exists:
|
||||||
new_settings = dict(current_settings)
|
errors["api_key_alias"] = f"API key alias '{api_key_alias}' does not exist"
|
||||||
|
else:
|
||||||
|
new_settings["api_key_alias"] = api_key_alias
|
||||||
|
else:
|
||||||
|
new_settings["api_key_alias"] = None
|
||||||
|
|
||||||
|
# Validate satellites
|
||||||
|
valid_sats = set(_get_valid_satellites())
|
||||||
|
invalid_sats = [s for s in satellites if s not in valid_sats]
|
||||||
|
if invalid_sats:
|
||||||
|
errors["satellites"] = f"Invalid satellites: {', '.join(invalid_sats)}"
|
||||||
|
else:
|
||||||
|
new_settings["satellites"] = satellites
|
||||||
|
|
||||||
|
elif name == "usgs_quake":
|
||||||
|
feed = form.get("feed", "").strip()
|
||||||
|
form_data["feed"] = feed
|
||||||
|
valid_feeds = _get_valid_feeds()
|
||||||
|
if feed not in valid_feeds:
|
||||||
|
errors["feed"] = f"Invalid feed. Must be one of: {', '.join(sorted(valid_feeds))}"
|
||||||
|
else:
|
||||||
|
new_settings["feed"] = feed
|
||||||
|
|
||||||
|
# Region validation (applies to all adapters)
|
||||||
|
region_north_str = form.get("region_north", "").strip()
|
||||||
|
region_south_str = form.get("region_south", "").strip()
|
||||||
|
region_east_str = form.get("region_east", "").strip()
|
||||||
|
region_west_str = form.get("region_west", "").strip()
|
||||||
|
|
||||||
|
form_data["region_north"] = region_north_str
|
||||||
|
form_data["region_south"] = region_south_str
|
||||||
|
form_data["region_east"] = region_east_str
|
||||||
|
form_data["region_west"] = region_west_str
|
||||||
|
|
||||||
|
try:
|
||||||
|
region_north = float(region_north_str)
|
||||||
|
region_south = float(region_south_str)
|
||||||
|
region_east = float(region_east_str)
|
||||||
|
region_west = float(region_west_str)
|
||||||
|
|
||||||
|
# Validate latitude bounds
|
||||||
|
if not (-90 <= region_south < region_north <= 90):
|
||||||
|
errors["region"] = "Invalid latitude: south must be less than north, both between -90 and 90"
|
||||||
|
# Validate longitude bounds
|
||||||
|
elif not (-180 <= region_west < region_east <= 180):
|
||||||
|
errors["region"] = "Invalid longitude: west must be less than east, both between -180 and 180"
|
||||||
|
else:
|
||||||
|
new_settings["region"] = {
|
||||||
|
"north": region_north,
|
||||||
|
"south": region_south,
|
||||||
|
"east": region_east,
|
||||||
|
"west": region_west,
|
||||||
|
}
|
||||||
|
except ValueError:
|
||||||
|
errors["region"] = "Region coordinates must be valid numbers"
|
||||||
|
|
||||||
# If there are errors, re-render the form
|
# If there are errors, re-render the form
|
||||||
if errors:
|
if errors:
|
||||||
|
adapter = {
|
||||||
|
"name": row["name"],
|
||||||
|
"enabled": row["enabled"],
|
||||||
|
"cadence_s": row["cadence_s"],
|
||||||
|
"settings": current_settings,
|
||||||
|
"paused_at": row["paused_at"],
|
||||||
|
"updated_at": row["updated_at"],
|
||||||
|
}
|
||||||
|
|
||||||
|
api_keys = await conn.fetch(
|
||||||
|
"SELECT alias FROM config.api_keys ORDER BY alias"
|
||||||
|
)
|
||||||
|
|
||||||
# Get map tile settings for re-render
|
# Get map tile settings for re-render
|
||||||
sys_row = await conn.fetchrow(
|
sys_row = await conn.fetchrow(
|
||||||
"SELECT map_tile_url, map_attribution FROM config.system WHERE id = true"
|
"SELECT map_tile_url, map_attribution FROM config.system WHERE id = true"
|
||||||
|
|
@ -1641,42 +1488,6 @@ async def adapters_edit_submit(
|
||||||
tile_url = sys_row["map_tile_url"] if sys_row else "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png"
|
tile_url = sys_row["map_tile_url"] if sys_row else "https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png"
|
||||||
tile_attribution = sys_row["map_attribution"] if sys_row else "© OpenStreetMap contributors"
|
tile_attribution = sys_row["map_attribution"] if sys_row else "© OpenStreetMap contributors"
|
||||||
|
|
||||||
adapter = {
|
|
||||||
"name": row["name"],
|
|
||||||
"display_name": getattr(adapter_cls, "display_name", row["name"]) if adapter_cls else row["name"],
|
|
||||||
"description": getattr(adapter_cls, "description", "") if adapter_cls else "",
|
|
||||||
"enabled": row["enabled"],
|
|
||||||
"cadence_s": row["cadence_s"],
|
|
||||||
"settings": current_settings,
|
|
||||||
"paused_at": row["paused_at"],
|
|
||||||
"updated_at": row["updated_at"],
|
|
||||||
"last_error": row["last_error"],
|
|
||||||
}
|
|
||||||
|
|
||||||
fields = []
|
|
||||||
if adapter_cls and hasattr(adapter_cls, "settings_schema"):
|
|
||||||
fields = describe_fields(adapter_cls.settings_schema, current_settings)
|
|
||||||
# Swap widget for api_key_field to api_key_select
|
|
||||||
if adapter_cls.api_key_field is not None:
|
|
||||||
for f in fields:
|
|
||||||
if f.name == adapter_cls.api_key_field:
|
|
||||||
f.widget = "api_key_select"
|
|
||||||
|
|
||||||
# Fetch API keys for api_key_select widget
|
|
||||||
api_key_rows = await conn.fetch("SELECT alias FROM config.api_keys ORDER BY alias")
|
|
||||||
api_keys = [{"alias": r["alias"]} for r in api_key_rows]
|
|
||||||
|
|
||||||
# Check if required API key is missing
|
|
||||||
api_key_missing = False
|
|
||||||
requires_api_key_alias = None
|
|
||||||
if adapter_cls and adapter_cls.requires_api_key is not None:
|
|
||||||
requires_api_key_alias = adapter_cls.requires_api_key
|
|
||||||
has_key = await conn.fetchval(
|
|
||||||
"SELECT 1 FROM config.api_keys WHERE alias = $1",
|
|
||||||
requires_api_key_alias,
|
|
||||||
)
|
|
||||||
api_key_missing = not has_key
|
|
||||||
|
|
||||||
csrf_token = request.state.csrf_token
|
csrf_token = request.state.csrf_token
|
||||||
response = templates.TemplateResponse(
|
response = templates.TemplateResponse(
|
||||||
request=request,
|
request=request,
|
||||||
|
|
@ -1685,14 +1496,13 @@ async def adapters_edit_submit(
|
||||||
"operator": operator,
|
"operator": operator,
|
||||||
"csrf_token": csrf_token,
|
"csrf_token": csrf_token,
|
||||||
"adapter": adapter,
|
"adapter": adapter,
|
||||||
"fields": fields,
|
|
||||||
"api_keys": api_keys,
|
|
||||||
"errors": errors,
|
"errors": errors,
|
||||||
"form_data": form_data,
|
"form_data": form_data,
|
||||||
|
"api_keys": [{"alias": k["alias"]} for k in api_keys],
|
||||||
|
"valid_satellites": _get_valid_satellites(),
|
||||||
|
"valid_feeds": sorted(_get_valid_feeds()),
|
||||||
"tile_url": tile_url,
|
"tile_url": tile_url,
|
||||||
"tile_attribution": tile_attribution,
|
"tile_attribution": tile_attribution,
|
||||||
"api_key_missing": api_key_missing,
|
|
||||||
"requires_api_key_alias": requires_api_key_alias,
|
|
||||||
},
|
},
|
||||||
status_code=200,
|
status_code=200,
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
{% extends "base.html" %}
|
{% extends "base.html" %}
|
||||||
|
|
||||||
{% block title %}Central — Edit {{ adapter.display_name }}{% endblock %}
|
{% block title %}Central — Edit {{ adapter.name }}{% endblock %}
|
||||||
|
|
||||||
{% block head %}
|
{% block head %}
|
||||||
<link rel="stylesheet" href="https://unpkg.com/leaflet@1.9.4/dist/leaflet.css" integrity="sha256-p4NxAoJBhIIN+hmNHrzRCf9tD/miZyoHS5obTRR9BMY=" crossorigin="">
|
<link rel="stylesheet" href="https://unpkg.com/leaflet@1.9.4/dist/leaflet.css" integrity="sha256-p4NxAoJBhIIN+hmNHrzRCf9tD/miZyoHS5obTRR9BMY=" crossorigin="">
|
||||||
|
|
@ -10,173 +10,35 @@
|
||||||
{% endblock %}
|
{% endblock %}
|
||||||
|
|
||||||
{% block content %}
|
{% block content %}
|
||||||
<h1>{{ adapter.display_name }}</h1>
|
<h1>Edit Adapter: {{ adapter.name }}</h1>
|
||||||
<p class="secondary">{{ adapter.description }}</p>
|
|
||||||
|
|
||||||
{% if adapter.paused_at %}
|
|
||||||
<article aria-label="Adapter Paused" style="background-color: var(--pico-mark-background-color); margin-bottom: 1rem;">
|
|
||||||
<strong>⏸️ Paused</strong> since {{ adapter.paused_at }}
|
|
||||||
</article>
|
|
||||||
{% endif %}
|
|
||||||
|
|
||||||
{% if adapter.last_error %}
|
|
||||||
<article aria-label="Last Error" style="background-color: var(--pico-del-color); margin-bottom: 1rem;">
|
|
||||||
<strong>Last Error:</strong> {{ adapter.last_error }}
|
|
||||||
</article>
|
|
||||||
{% endif %}
|
|
||||||
|
|
||||||
{% if api_key_missing %}
|
|
||||||
<article aria-label="API Key Required" style="background-color: var(--pico-mark-background-color); margin-bottom: 1rem;">
|
|
||||||
<strong>⚠️ API Key Required:</strong> This adapter requires the <code>{{ requires_api_key_alias }}</code> API key to be configured before it can be enabled.
|
|
||||||
<a href="/api-keys">Configure API Keys</a>
|
|
||||||
</article>
|
|
||||||
{% endif %}
|
|
||||||
|
|
||||||
<form method="post" action="/adapters/{{ adapter.name }}">
|
<form method="post" action="/adapters/{{ adapter.name }}">
|
||||||
<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
|
<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
|
||||||
|
|
||||||
<fieldset>
|
<fieldset>
|
||||||
<legend>Core Settings</legend>
|
<legend>Universal Settings</legend>
|
||||||
|
|
||||||
<label>
|
<label>
|
||||||
<input type="checkbox" name="enabled" {% if form_data %}{% if form_data.enabled %}checked{% endif %}{% elif adapter.enabled %}checked{% endif %}{% if api_key_missing %} disabled{% endif %}>
|
<input type="checkbox" name="enabled" {% if adapter.enabled %}checked{% endif %}>
|
||||||
Enabled{% if api_key_missing %} <small>(requires API key)</small>{% endif %}
|
Enabled
|
||||||
</label>
|
</label>
|
||||||
|
|
||||||
<label for="cadence_s">Cadence (seconds)</label>
|
<label for="cadence_s">Cadence (seconds)</label>
|
||||||
<input type="number" id="cadence_s" name="cadence_s"
|
<input type="number" id="cadence_s" name="cadence_s" value="{{ form_data.cadence_s if form_data else adapter.cadence_s }}" min="60" max="3600" required>
|
||||||
value="{{ form_data.cadence_s if form_data else adapter.cadence_s }}"
|
|
||||||
required>
|
|
||||||
{% if errors and errors.cadence_s %}
|
{% if errors and errors.cadence_s %}
|
||||||
<small style="color: var(--pico-color-red-500);">{{ errors.cadence_s }}</small>
|
<small style="color: var(--pico-color-red-500);">{{ errors.cadence_s }}</small>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</fieldset>
|
</fieldset>
|
||||||
|
|
||||||
{% if fields %}
|
|
||||||
<fieldset>
|
<fieldset>
|
||||||
<legend>Adapter Settings</legend>
|
<legend>Adapter-Specific Settings</legend>
|
||||||
|
{% include "adapters_edit_" + adapter.name + ".html" %}
|
||||||
{% for field in fields %}
|
|
||||||
{% if field.widget == "region" %}
|
|
||||||
{# Region is rendered in a separate fieldset below #}
|
|
||||||
{% elif field.widget == "text" %}
|
|
||||||
<label for="{{ field.name }}">{{ field.label }}</label>
|
|
||||||
<input type="text" id="{{ field.name }}" name="{{ field.name }}"
|
|
||||||
value="{{ form_data[field.name] if form_data and field.name in form_data else field.current_value or '' }}"
|
|
||||||
{% if field.required %}required{% endif %}>
|
|
||||||
{% if field.description %}
|
|
||||||
<small>{{ field.description }}</small>
|
|
||||||
{% endif %}
|
|
||||||
{% if errors and errors[field.name] %}
|
|
||||||
<small style="color: var(--pico-color-red-500);">{{ errors[field.name] }}</small>
|
|
||||||
{% endif %}
|
|
||||||
|
|
||||||
{% elif field.widget == "number" %}
|
|
||||||
<label for="{{ field.name }}">{{ field.label }}</label>
|
|
||||||
<input type="number" id="{{ field.name }}" name="{{ field.name }}"
|
|
||||||
value="{{ form_data[field.name] if form_data and field.name in form_data else field.current_value or '' }}"
|
|
||||||
{% if field.required %}required{% endif %}>
|
|
||||||
{% if field.description %}
|
|
||||||
<small>{{ field.description }}</small>
|
|
||||||
{% endif %}
|
|
||||||
{% if errors and errors[field.name] %}
|
|
||||||
<small style="color: var(--pico-color-red-500);">{{ errors[field.name] }}</small>
|
|
||||||
{% endif %}
|
|
||||||
|
|
||||||
{% elif field.widget == "checkbox" %}
|
|
||||||
<label>
|
|
||||||
<input type="checkbox" name="{{ field.name }}"
|
|
||||||
{% if form_data and field.name in form_data %}
|
|
||||||
{% if form_data[field.name] %}checked{% endif %}
|
|
||||||
{% elif field.current_value %}checked{% endif %}>
|
|
||||||
{{ field.label }}
|
|
||||||
</label>
|
|
||||||
{% if field.description %}
|
|
||||||
<small>{{ field.description }}</small>
|
|
||||||
{% endif %}
|
|
||||||
{% if errors and errors[field.name] %}
|
|
||||||
<small style="color: var(--pico-color-red-500);">{{ errors[field.name] }}</small>
|
|
||||||
{% endif %}
|
|
||||||
|
|
||||||
{% elif field.widget == "csv" %}
|
|
||||||
<label for="{{ field.name }}">{{ field.label }}</label>
|
|
||||||
<input type="text" id="{{ field.name }}" name="{{ field.name }}"
|
|
||||||
value="{{ form_data[field.name] if form_data and field.name in form_data else (field.current_value | join(',') if field.current_value else '') }}"
|
|
||||||
{% if field.required %}required{% endif %}>
|
|
||||||
<small>Comma-separated values{% if field.description %} — {{ field.description }}{% endif %}</small>
|
|
||||||
{% if errors and errors[field.name] %}
|
|
||||||
<small style="color: var(--pico-color-red-500);">{{ errors[field.name] }}</small>
|
|
||||||
{% endif %}
|
|
||||||
|
|
||||||
{% elif field.widget == "select" %}
|
|
||||||
<label for="{{ field.name }}">{{ field.label }}</label>
|
|
||||||
<select id="{{ field.name }}" name="{{ field.name }}">
|
|
||||||
{% for opt in field.options %}
|
|
||||||
<option value="{{ opt }}"
|
|
||||||
{% if (form_data[field.name] if form_data and field.name in form_data else field.current_value) == opt %}selected{% endif %}>
|
|
||||||
{{ opt }}
|
|
||||||
</option>
|
|
||||||
{% endfor %}
|
|
||||||
</select>
|
|
||||||
{% if field.description %}
|
|
||||||
<small>{{ field.description }}</small>
|
|
||||||
{% endif %}
|
|
||||||
{% if errors and errors[field.name] %}
|
|
||||||
<small style="color: var(--pico-color-red-500);">{{ errors[field.name] }}</small>
|
|
||||||
{% endif %}
|
|
||||||
|
|
||||||
{% elif field.widget == "checkboxes" %}
|
|
||||||
<label>{{ field.label }}</label>
|
|
||||||
{% set current_values = form_data.getlist(field.name) if form_data and form_data.getlist else (field.current_value or []) %}
|
|
||||||
{% for opt in field.options %}
|
|
||||||
<label style="display: inline-block; margin-right: 1rem;">
|
|
||||||
<input type="checkbox" name="{{ field.name }}" value="{{ opt }}"
|
|
||||||
{% if opt in current_values %}checked{% endif %}>
|
|
||||||
{{ opt }}
|
|
||||||
</label>
|
|
||||||
{% endfor %}
|
|
||||||
{% if field.description %}
|
|
||||||
<small style="display: block;">{{ field.description }}</small>
|
|
||||||
{% endif %}
|
|
||||||
{% if errors and errors[field.name] %}
|
|
||||||
<small style="color: var(--pico-color-red-500); display: block;">{{ errors[field.name] }}</small>
|
|
||||||
{% endif %}
|
|
||||||
|
|
||||||
{% elif field.widget == "api_key_select" %}
|
|
||||||
<label for="{{ field.name }}">{{ field.label }}</label>
|
|
||||||
<select id="{{ field.name }}" name="{{ field.name }}">
|
|
||||||
<option value="">(none)</option>
|
|
||||||
{% for key in api_keys %}
|
|
||||||
<option value="{{ key.alias }}"
|
|
||||||
{% if (form_data[field.name] if form_data and field.name in form_data else field.current_value) == key.alias %}selected{% endif %}>
|
|
||||||
{{ key.alias }}
|
|
||||||
</option>
|
|
||||||
{% endfor %}
|
|
||||||
</select>
|
|
||||||
{% if field.description %}
|
|
||||||
<small>{{ field.description }}</small>
|
|
||||||
{% endif %}
|
|
||||||
{% if errors and errors[field.name] %}
|
|
||||||
<small style="color: var(--pico-color-red-500);">{{ errors[field.name] }}</small>
|
|
||||||
{% endif %}
|
|
||||||
{% endif %}
|
|
||||||
{% endfor %}
|
|
||||||
</fieldset>
|
</fieldset>
|
||||||
{% endif %}
|
|
||||||
|
|
||||||
{% set has_region = namespace(value=false) %}
|
|
||||||
{% for field in fields %}
|
|
||||||
{% if field.widget == "region" %}
|
|
||||||
{% set has_region.value = true %}
|
|
||||||
{% endif %}
|
|
||||||
{% endfor %}
|
|
||||||
|
|
||||||
{% if has_region.value %}
|
|
||||||
<fieldset>
|
<fieldset>
|
||||||
<legend>Region</legend>
|
<legend>Region</legend>
|
||||||
{% include "_region_picker.html" %}
|
{% include "_region_picker.html" %}
|
||||||
</fieldset>
|
</fieldset>
|
||||||
{% endif %}
|
|
||||||
|
|
||||||
<button type="submit">Save Changes</button>
|
<button type="submit">Save Changes</button>
|
||||||
<a href="/adapters" role="button" class="outline">Cancel</a>
|
<a href="/adapters" role="button" class="outline">Cancel</a>
|
||||||
|
|
|
||||||
21
src/central/gui/templates/adapters_edit_firms.html
Normal file
21
src/central/gui/templates/adapters_edit_firms.html
Normal file
|
|
@ -0,0 +1,21 @@
|
||||||
|
<label for="api_key_alias">API Key Alias</label>
|
||||||
|
<select id="api_key_alias" name="api_key_alias">
|
||||||
|
<option value="" {% if not (form_data.api_key_alias if form_data else adapter.settings.api_key_alias) %}selected{% endif %}>(none)</option>
|
||||||
|
{% for key in api_keys %}
|
||||||
|
<option value="{{ key.alias }}" {% if (form_data.api_key_alias if form_data else adapter.settings.api_key_alias) == key.alias %}selected{% endif %}>{{ key.alias }}</option>
|
||||||
|
{% endfor %}
|
||||||
|
</select>
|
||||||
|
{% if errors and errors.api_key_alias %}
|
||||||
|
<small style="color: var(--pico-color-red-500);">{{ errors.api_key_alias }}</small>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
<label>Satellites</label>
|
||||||
|
{% for sat in valid_satellites %}
|
||||||
|
<label>
|
||||||
|
<input type="checkbox" name="satellites" value="{{ sat }}" {% if sat in (form_data.satellites if form_data else adapter.settings.satellites or []) %}checked{% endif %}>
|
||||||
|
{{ sat }}
|
||||||
|
</label>
|
||||||
|
{% endfor %}
|
||||||
|
{% if errors and errors.satellites %}
|
||||||
|
<small style="color: var(--pico-color-red-500);">{{ errors.satellites }}</small>
|
||||||
|
{% endif %}
|
||||||
5
src/central/gui/templates/adapters_edit_nws.html
Normal file
5
src/central/gui/templates/adapters_edit_nws.html
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
<label for="contact_email">Contact Email</label>
|
||||||
|
<input type="email" id="contact_email" name="contact_email" value="{{ form_data.contact_email if form_data else adapter.settings.contact_email }}" required>
|
||||||
|
{% if errors and errors.contact_email %}
|
||||||
|
<small style="color: var(--pico-color-red-500);">{{ errors.contact_email }}</small>
|
||||||
|
{% endif %}
|
||||||
9
src/central/gui/templates/adapters_edit_usgs_quake.html
Normal file
9
src/central/gui/templates/adapters_edit_usgs_quake.html
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
<label for="feed">Feed</label>
|
||||||
|
<select id="feed" name="feed" required>
|
||||||
|
{% for f in valid_feeds %}
|
||||||
|
<option value="{{ f }}" {% if (form_data.feed if form_data else adapter.settings.feed) == f %}selected{% endif %}>{{ f }}</option>
|
||||||
|
{% endfor %}
|
||||||
|
</select>
|
||||||
|
{% if errors and errors.feed %}
|
||||||
|
<small style="color: var(--pico-color-red-500);">{{ errors.feed }}</small>
|
||||||
|
{% endif %}
|
||||||
|
|
@ -17,12 +17,7 @@
|
||||||
<tbody>
|
<tbody>
|
||||||
{% for adapter in adapters %}
|
{% for adapter in adapters %}
|
||||||
<tr>
|
<tr>
|
||||||
<td>
|
<td>{{ adapter.name }}</td>
|
||||||
{{ adapter.display_name or adapter.name }}
|
|
||||||
{% if adapter.api_key_missing %}
|
|
||||||
<span style="color: var(--pico-color-orange-500); margin-left: 0.5rem;" title="Missing API key: {{ adapter.requires_api_key_alias }}">⚠️ API Key Missing</span>
|
|
||||||
{% endif %}
|
|
||||||
</td>
|
|
||||||
<td>{% if adapter.enabled %}Yes{% else %}No{% endif %}</td>
|
<td>{% if adapter.enabled %}Yes{% else %}No{% endif %}</td>
|
||||||
<td>{{ adapter.cadence_s }}s</td>
|
<td>{{ adapter.cadence_s }}s</td>
|
||||||
<td>{{ adapter.updated_at.strftime('%Y-%m-%d %H:%M') if adapter.updated_at else '—' }}</td>
|
<td>{{ adapter.updated_at.strftime('%Y-%m-%d %H:%M') if adapter.updated_at else '—' }}</td>
|
||||||
|
|
|
||||||
|
|
@ -29,7 +29,7 @@
|
||||||
|
|
||||||
{% for adapter in adapters %}
|
{% for adapter in adapters %}
|
||||||
<details open style="margin-bottom: 2rem;">
|
<details open style="margin-bottom: 2rem;">
|
||||||
<summary><strong>{{ adapter.display_name or adapter.name }}</strong></summary>
|
<summary><strong>{{ adapter.name }}</strong></summary>
|
||||||
|
|
||||||
<div style="padding: 1rem; border-left: 3px solid var(--pico-primary);">
|
<div style="padding: 1rem; border-left: 3px solid var(--pico-primary);">
|
||||||
<label>
|
<label>
|
||||||
|
|
@ -44,158 +44,100 @@
|
||||||
|
|
||||||
<label for="{{ adapter.name }}_cadence_s">Cadence (seconds)</label>
|
<label for="{{ adapter.name }}_cadence_s">Cadence (seconds)</label>
|
||||||
<input type="number" id="{{ adapter.name }}_cadence_s" name="{{ adapter.name }}_cadence_s"
|
<input type="number" id="{{ adapter.name }}_cadence_s" name="{{ adapter.name }}_cadence_s"
|
||||||
value="{{ form_data.get(adapter.name + '_cadence_s') if form_data else adapter.cadence_s }}">
|
value="{{ form_data.get(adapter.name + '_cadence_s') if form_data else adapter.cadence_s }}"
|
||||||
|
min="60" max="3600">
|
||||||
{% if errors and errors.get(adapter.name + '_cadence_s') %}
|
{% if errors and errors.get(adapter.name + '_cadence_s') %}
|
||||||
<small style="color: var(--pico-color-red-500);">{{ errors[adapter.name + '_cadence_s'] }}</small>
|
<small style="color: var(--pico-color-red-500);">{{ errors[adapter.name + '_cadence_s'] }}</small>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
{% for field in adapter.fields %}
|
{% if adapter.name == 'nws' %}
|
||||||
{% set form_key = adapter.name + '_' + field.name %}
|
<label for="{{ adapter.name }}_contact_email">Contact Email</label>
|
||||||
|
<input type="email" id="{{ adapter.name }}_contact_email" name="{{ adapter.name }}_contact_email"
|
||||||
|
value="{{ form_data.get(adapter.name + '_contact_email') if form_data else adapter.settings.contact_email }}">
|
||||||
|
{% if errors and errors.get(adapter.name + '_contact_email') %}
|
||||||
|
<small style="color: var(--pico-color-red-500);">{{ errors[adapter.name + '_contact_email'] }}</small>
|
||||||
|
{% endif %}
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
{% if field.widget == "text" %}
|
{% if adapter.name == 'firms' %}
|
||||||
<label for="{{ form_key }}">{{ field.label }}</label>
|
<label for="{{ adapter.name }}_api_key_alias">API Key Alias</label>
|
||||||
<input type="text" id="{{ form_key }}" name="{{ form_key }}"
|
<select id="{{ adapter.name }}_api_key_alias" name="{{ adapter.name }}_api_key_alias">
|
||||||
value="{{ form_data.get(form_key) if form_data else field.current_value or '' }}"
|
<option value="">(none)</option>
|
||||||
{% if field.required %}required{% endif %}>
|
{% for key in api_keys %}
|
||||||
{% if field.description %}
|
<option value="{{ key.alias }}"
|
||||||
<small>{{ field.description }}</small>
|
{% if (form_data.get(adapter.name + '_api_key_alias') if form_data else adapter.settings.api_key_alias) == key.alias %}selected{% endif %}>
|
||||||
{% endif %}
|
{{ key.alias }}
|
||||||
{% if errors and errors.get(form_key) %}
|
</option>
|
||||||
<small style="color: var(--pico-color-red-500);">{{ errors[form_key] }}</small>
|
{% endfor %}
|
||||||
{% endif %}
|
</select>
|
||||||
|
{% if errors and errors.get(adapter.name + '_api_key_alias') %}
|
||||||
|
<small style="color: var(--pico-color-red-500);">{{ errors[adapter.name + '_api_key_alias'] }}</small>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
{% elif field.widget == "api_key_select" %}
|
<label>Satellites</label>
|
||||||
<label for="{{ form_key }}">{{ field.label }}</label>
|
{% for sat in valid_satellites %}
|
||||||
<select id="{{ form_key }}" name="{{ form_key }}">
|
<label style="display: inline-block; margin-right: 1rem;">
|
||||||
<option value="">(none)</option>
|
<input type="checkbox" name="{{ adapter.name }}_satellites" value="{{ sat }}"
|
||||||
{% for key in api_keys %}
|
{% if sat in (form_data.getlist(adapter.name + '_satellites') if form_data else adapter.settings.satellites or []) %}checked{% endif %}>
|
||||||
<option value="{{ key.alias }}"
|
{{ sat }}
|
||||||
{% if (form_data.get(form_key) if form_data else field.current_value) == key.alias %}selected{% endif %}>
|
</label>
|
||||||
{{ key.alias }}
|
|
||||||
</option>
|
|
||||||
{% endfor %}
|
|
||||||
</select>
|
|
||||||
{% if field.description %}
|
|
||||||
<small>{{ field.description }}</small>
|
|
||||||
{% endif %}
|
|
||||||
{% if errors and errors.get(form_key) %}
|
|
||||||
<small style="color: var(--pico-color-red-500);">{{ errors[form_key] }}</small>
|
|
||||||
{% endif %}
|
|
||||||
|
|
||||||
{% elif field.widget == "number" %}
|
|
||||||
<label for="{{ form_key }}">{{ field.label }}</label>
|
|
||||||
<input type="number" id="{{ form_key }}" name="{{ form_key }}"
|
|
||||||
value="{{ form_data.get(form_key) if form_data else field.current_value or '' }}"
|
|
||||||
{% if field.required %}required{% endif %}>
|
|
||||||
{% if field.description %}
|
|
||||||
<small>{{ field.description }}</small>
|
|
||||||
{% endif %}
|
|
||||||
{% if errors and errors.get(form_key) %}
|
|
||||||
<small style="color: var(--pico-color-red-500);">{{ errors[form_key] }}</small>
|
|
||||||
{% endif %}
|
|
||||||
|
|
||||||
{% elif field.widget == "checkbox" %}
|
|
||||||
<label>
|
|
||||||
<input type="checkbox" name="{{ form_key }}"
|
|
||||||
{% if form_data and form_data.get(form_key) %}checked
|
|
||||||
{% elif not form_data and field.current_value %}checked{% endif %}>
|
|
||||||
{{ field.label }}
|
|
||||||
</label>
|
|
||||||
{% if field.description %}
|
|
||||||
<small>{{ field.description }}</small>
|
|
||||||
{% endif %}
|
|
||||||
{% if errors and errors.get(form_key) %}
|
|
||||||
<small style="color: var(--pico-color-red-500);">{{ errors[form_key] }}</small>
|
|
||||||
{% endif %}
|
|
||||||
|
|
||||||
{% elif field.widget == "csv" %}
|
|
||||||
<label for="{{ form_key }}">{{ field.label }}</label>
|
|
||||||
<input type="text" id="{{ form_key }}" name="{{ form_key }}"
|
|
||||||
value="{{ form_data.get(form_key) if form_data else (field.current_value | join(',') if field.current_value else '') }}"
|
|
||||||
{% if field.required %}required{% endif %}>
|
|
||||||
<small>Comma-separated values{% if field.description %} — {{ field.description }}{% endif %}</small>
|
|
||||||
{% if errors and errors.get(form_key) %}
|
|
||||||
<small style="color: var(--pico-color-red-500);">{{ errors[form_key] }}</small>
|
|
||||||
{% endif %}
|
|
||||||
|
|
||||||
{% elif field.widget == "select" %}
|
|
||||||
<label for="{{ form_key }}">{{ field.label }}</label>
|
|
||||||
<select id="{{ form_key }}" name="{{ form_key }}">
|
|
||||||
{% for opt in field.options %}
|
|
||||||
<option value="{{ opt }}"
|
|
||||||
{% if (form_data.get(form_key) if form_data else field.current_value) == opt %}selected{% endif %}>
|
|
||||||
{{ opt }}
|
|
||||||
</option>
|
|
||||||
{% endfor %}
|
|
||||||
</select>
|
|
||||||
{% if field.description %}
|
|
||||||
<small>{{ field.description }}</small>
|
|
||||||
{% endif %}
|
|
||||||
{% if errors and errors.get(form_key) %}
|
|
||||||
<small style="color: var(--pico-color-red-500);">{{ errors[form_key] }}</small>
|
|
||||||
{% endif %}
|
|
||||||
|
|
||||||
{% elif field.widget == "checkboxes" %}
|
|
||||||
<label>{{ field.label }}</label>
|
|
||||||
{% set current_values = form_data.getlist(form_key) if form_data else (field.current_value or []) %}
|
|
||||||
{% for opt in field.options %}
|
|
||||||
<label style="display: inline-block; margin-right: 1rem;">
|
|
||||||
<input type="checkbox" name="{{ form_key }}" value="{{ opt }}"
|
|
||||||
{% if opt in current_values %}checked{% endif %}>
|
|
||||||
{{ opt }}
|
|
||||||
</label>
|
|
||||||
{% endfor %}
|
|
||||||
{% if field.description %}
|
|
||||||
<small style="display: block;">{{ field.description }}</small>
|
|
||||||
{% endif %}
|
|
||||||
{% if errors and errors.get(form_key) %}
|
|
||||||
<small style="color: var(--pico-color-red-500); display: block;">{{ errors[form_key] }}</small>
|
|
||||||
{% endif %}
|
|
||||||
|
|
||||||
{% elif field.widget == "region" %}
|
|
||||||
<h4>Region</h4>
|
|
||||||
{% set region_key = adapter.name + '_' + field.name %}
|
|
||||||
{% set region = field.current_value or {} %}
|
|
||||||
<div id="region-picker-{{ adapter.name }}"
|
|
||||||
data-adapter="{{ adapter.name }}"
|
|
||||||
data-field="{{ field.name }}"
|
|
||||||
data-north="{{ form_data.get(region_key + '_north') if form_data else (region.north if region else 49.5) }}"
|
|
||||||
data-south="{{ form_data.get(region_key + '_south') if form_data else (region.south if region else 31.0) }}"
|
|
||||||
data-east="{{ form_data.get(region_key + '_east') if form_data else (region.east if region else -102.0) }}"
|
|
||||||
data-west="{{ form_data.get(region_key + '_west') if form_data else (region.west if region else -124.5) }}"
|
|
||||||
data-tile-url="{{ tile_url }}"
|
|
||||||
data-tile-attr="{{ tile_attribution }}">
|
|
||||||
|
|
||||||
<div id="region-map-{{ adapter.name }}" style="height: 300px; margin-bottom: 1rem;"></div>
|
|
||||||
|
|
||||||
<div class="grid">
|
|
||||||
<div>
|
|
||||||
<label>North</label>
|
|
||||||
<input type="number" name="{{ region_key }}_north" step="0.0001" min="-90" max="90" readonly
|
|
||||||
value="{{ form_data.get(region_key + '_north') if form_data else (region.north if region else 49.5) }}">
|
|
||||||
</div>
|
|
||||||
<div>
|
|
||||||
<label>South</label>
|
|
||||||
<input type="number" name="{{ region_key }}_south" step="0.0001" min="-90" max="90" readonly
|
|
||||||
value="{{ form_data.get(region_key + '_south') if form_data else (region.south if region else 31.0) }}">
|
|
||||||
</div>
|
|
||||||
<div>
|
|
||||||
<label>East</label>
|
|
||||||
<input type="number" name="{{ region_key }}_east" step="0.0001" min="-180" max="180" readonly
|
|
||||||
value="{{ form_data.get(region_key + '_east') if form_data else (region.east if region else -102.0) }}">
|
|
||||||
</div>
|
|
||||||
<div>
|
|
||||||
<label>West</label>
|
|
||||||
<input type="number" name="{{ region_key }}_west" step="0.0001" min="-180" max="180" readonly
|
|
||||||
value="{{ form_data.get(region_key + '_west') if form_data else (region.west if region else -124.5) }}">
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
{% if errors and errors.get(region_key) %}
|
|
||||||
<small style="color: var(--pico-color-red-500);">{{ errors[region_key] }}</small>
|
|
||||||
{% endif %}
|
|
||||||
</div>
|
|
||||||
{% endif %}
|
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{% if adapter.name == 'usgs_quake' %}
|
||||||
|
<label for="{{ adapter.name }}_feed">Feed</label>
|
||||||
|
<select id="{{ adapter.name }}_feed" name="{{ adapter.name }}_feed">
|
||||||
|
{% for f in valid_feeds %}
|
||||||
|
<option value="{{ f }}"
|
||||||
|
{% if (form_data.get(adapter.name + '_feed') if form_data else adapter.settings.feed) == f %}selected{% endif %}>
|
||||||
|
{{ f }}
|
||||||
|
</option>
|
||||||
|
{% endfor %}
|
||||||
|
</select>
|
||||||
|
{% if errors and errors.get(adapter.name + '_feed') %}
|
||||||
|
<small style="color: var(--pico-color-red-500);">{{ errors[adapter.name + '_feed'] }}</small>
|
||||||
|
{% endif %}
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
<h4>Region</h4>
|
||||||
|
{% set region = form_data if form_data else adapter.settings.region %}
|
||||||
|
<div id="region-picker-{{ adapter.name }}"
|
||||||
|
data-adapter="{{ adapter.name }}"
|
||||||
|
data-north="{{ form_data.get(adapter.name + '_region_north') if form_data else (adapter.settings.region.north if adapter.settings.region else 49.5) }}"
|
||||||
|
data-south="{{ form_data.get(adapter.name + '_region_south') if form_data else (adapter.settings.region.south if adapter.settings.region else 31.0) }}"
|
||||||
|
data-east="{{ form_data.get(adapter.name + '_region_east') if form_data else (adapter.settings.region.east if adapter.settings.region else -102.0) }}"
|
||||||
|
data-west="{{ form_data.get(adapter.name + '_region_west') if form_data else (adapter.settings.region.west if adapter.settings.region else -124.5) }}"
|
||||||
|
data-tile-url="{{ tile_url }}"
|
||||||
|
data-tile-attr="{{ tile_attribution }}">
|
||||||
|
|
||||||
|
<div id="region-map-{{ adapter.name }}" style="height: 300px; margin-bottom: 1rem;"></div>
|
||||||
|
|
||||||
|
<div class="grid">
|
||||||
|
<div>
|
||||||
|
<label>North</label>
|
||||||
|
<input type="number" name="{{ adapter.name }}_region_north" step="0.0001" min="-90" max="90" readonly
|
||||||
|
value="{{ form_data.get(adapter.name + '_region_north') if form_data else (adapter.settings.region.north if adapter.settings.region else 49.5) }}">
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label>South</label>
|
||||||
|
<input type="number" name="{{ adapter.name }}_region_south" step="0.0001" min="-90" max="90" readonly
|
||||||
|
value="{{ form_data.get(adapter.name + '_region_south') if form_data else (adapter.settings.region.south if adapter.settings.region else 31.0) }}">
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label>East</label>
|
||||||
|
<input type="number" name="{{ adapter.name }}_region_east" step="0.0001" min="-180" max="180" readonly
|
||||||
|
value="{{ form_data.get(adapter.name + '_region_east') if form_data else (adapter.settings.region.east if adapter.settings.region else -102.0) }}">
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label>West</label>
|
||||||
|
<input type="number" name="{{ adapter.name }}_region_west" step="0.0001" min="-180" max="180" readonly
|
||||||
|
value="{{ form_data.get(adapter.name + '_region_west') if form_data else (adapter.settings.region.west if adapter.settings.region else -124.5) }}">
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% if errors and errors.get(adapter.name + '_region') %}
|
||||||
|
<small style="color: var(--pico-color-red-500);">{{ errors[adapter.name + '_region'] }}</small>
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</details>
|
</details>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
|
|
@ -209,12 +151,11 @@
|
||||||
|
|
||||||
<script>
|
<script>
|
||||||
document.addEventListener('DOMContentLoaded', function() {
|
document.addEventListener('DOMContentLoaded', function() {
|
||||||
// Find all region pickers dynamically
|
const adapters = ['nws', 'firms', 'usgs_quake'];
|
||||||
const regionPickers = document.querySelectorAll('[id^="region-picker-"]');
|
|
||||||
|
|
||||||
regionPickers.forEach(function(container) {
|
adapters.forEach(function(adapterName) {
|
||||||
const adapterName = container.dataset.adapter;
|
const container = document.getElementById('region-picker-' + adapterName);
|
||||||
const fieldName = container.dataset.field || 'region';
|
if (!container) return;
|
||||||
|
|
||||||
const savedNorth = parseFloat(container.dataset.north);
|
const savedNorth = parseFloat(container.dataset.north);
|
||||||
const savedSouth = parseFloat(container.dataset.south);
|
const savedSouth = parseFloat(container.dataset.south);
|
||||||
|
|
@ -274,11 +215,10 @@ document.addEventListener('DOMContentLoaded', function() {
|
||||||
|
|
||||||
rectangle.editing.enable();
|
rectangle.editing.enable();
|
||||||
|
|
||||||
const inputPrefix = adapterName + '_' + fieldName;
|
const northInput = container.querySelector('input[name="' + adapterName + '_region_north"]');
|
||||||
const northInput = container.querySelector('input[name="' + inputPrefix + '_north"]');
|
const southInput = container.querySelector('input[name="' + adapterName + '_region_south"]');
|
||||||
const southInput = container.querySelector('input[name="' + inputPrefix + '_south"]');
|
const eastInput = container.querySelector('input[name="' + adapterName + '_region_east"]');
|
||||||
const eastInput = container.querySelector('input[name="' + inputPrefix + '_east"]');
|
const westInput = container.querySelector('input[name="' + adapterName + '_region_west"]');
|
||||||
const westInput = container.querySelector('input[name="' + inputPrefix + '_west"]');
|
|
||||||
|
|
||||||
function updateInputs() {
|
function updateInputs() {
|
||||||
const b = rectangle.getBounds();
|
const b = rectangle.getBounds();
|
||||||
|
|
|
||||||
|
|
@ -13,14 +13,41 @@ from typing import Any
|
||||||
import nats
|
import nats
|
||||||
from nats.js import JetStreamContext
|
from nats.js import JetStreamContext
|
||||||
|
|
||||||
|
import importlib
|
||||||
|
import pkgutil
|
||||||
|
|
||||||
from central.adapter import SourceAdapter
|
from central.adapter import SourceAdapter
|
||||||
from central.adapter_discovery import discover_adapters
|
|
||||||
from central.cloudevents_wire import wrap_event
|
from central.cloudevents_wire import wrap_event
|
||||||
from central.config_models import AdapterConfig
|
from central.config_models import AdapterConfig
|
||||||
from central.config_source import ConfigSource, DbConfigSource
|
from central.config_source import ConfigSource, DbConfigSource
|
||||||
from central.config_store import ConfigStore
|
from central.config_store import ConfigStore
|
||||||
from central.bootstrap_config import get_settings
|
from central.bootstrap_config import get_settings
|
||||||
from central.stream_manager import StreamManager
|
from central.stream_manager import StreamManager
|
||||||
|
import central.adapters
|
||||||
|
|
||||||
|
def discover_adapters() -> dict[str, type[SourceAdapter]]:
|
||||||
|
"""Auto-discover adapter classes from central.adapters package."""
|
||||||
|
registry: dict[str, type[SourceAdapter]] = {}
|
||||||
|
for module_info in pkgutil.iter_modules(central.adapters.__path__):
|
||||||
|
try:
|
||||||
|
module = importlib.import_module(f"central.adapters.{module_info.name}")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(
|
||||||
|
"Failed to import adapter module",
|
||||||
|
extra={"module": module_info.name, "error": str(e)},
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
for attr_name in dir(module):
|
||||||
|
attr = getattr(module, attr_name)
|
||||||
|
if (
|
||||||
|
isinstance(attr, type)
|
||||||
|
and issubclass(attr, SourceAdapter)
|
||||||
|
and attr is not SourceAdapter
|
||||||
|
and hasattr(attr, "name")
|
||||||
|
):
|
||||||
|
registry[attr.name] = attr
|
||||||
|
return registry
|
||||||
|
|
||||||
CURSOR_DB_PATH = Path("/var/lib/central/cursors.db")
|
CURSOR_DB_PATH = Path("/var/lib/central/cursors.db")
|
||||||
|
|
||||||
# Stream subject mappings
|
# Stream subject mappings
|
||||||
|
|
@ -29,7 +56,6 @@ STREAM_SUBJECTS = {
|
||||||
"CENTRAL_META": ["central.meta.>"],
|
"CENTRAL_META": ["central.meta.>"],
|
||||||
"CENTRAL_FIRE": ["central.fire.>"],
|
"CENTRAL_FIRE": ["central.fire.>"],
|
||||||
"CENTRAL_QUAKE": ["central.quake.>"],
|
"CENTRAL_QUAKE": ["central.quake.>"],
|
||||||
"CENTRAL_SPACE": ["central.space.>"],
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# Recompute interval for stream max_bytes (1 hour)
|
# Recompute interval for stream max_bytes (1 hour)
|
||||||
|
|
@ -267,23 +293,6 @@ class Supervisor:
|
||||||
If the adapter was previously stopped (state exists but task is not running),
|
If the adapter was previously stopped (state exists but task is not running),
|
||||||
reuses the existing state to preserve last_completed_poll for rate limiting.
|
reuses the existing state to preserve last_completed_poll for rate limiting.
|
||||||
"""
|
"""
|
||||||
# API key precondition
|
|
||||||
adapter_cls = self._adapters.get(config.name)
|
|
||||||
if adapter_cls is not None and adapter_cls.requires_api_key is not None:
|
|
||||||
alias = adapter_cls.requires_api_key
|
|
||||||
key_value = await self._config_store.get_api_key(alias)
|
|
||||||
if not key_value:
|
|
||||||
error_msg = f"missing api key: {alias}"
|
|
||||||
logger.warning(
|
|
||||||
"Adapter cannot start - api key missing",
|
|
||||||
extra={"adapter": config.name, "alias": alias},
|
|
||||||
)
|
|
||||||
await self._config_store.set_adapter_last_error(config.name, error_msg)
|
|
||||||
return
|
|
||||||
|
|
||||||
# Clear any stale last_error before proceeding
|
|
||||||
await self._config_store.set_adapter_last_error(config.name, None)
|
|
||||||
|
|
||||||
existing_state = self._adapter_states.get(config.name)
|
existing_state = self._adapter_states.get(config.name)
|
||||||
|
|
||||||
if existing_state is not None:
|
if existing_state is not None:
|
||||||
|
|
|
||||||
|
|
@ -42,9 +42,9 @@ class TestAdaptersListAuthenticated:
|
||||||
|
|
||||||
mock_conn = AsyncMock()
|
mock_conn = AsyncMock()
|
||||||
mock_conn.fetch.return_value = [
|
mock_conn.fetch.return_value = [
|
||||||
{"name": "firms", "enabled": True, "cadence_s": 300, "settings": {"api_key_alias": "firms"}, "paused_at": None, "updated_at": None, "last_error": None},
|
{"name": "firms", "enabled": True, "cadence_s": 300, "settings": {"api_key_alias": "firms"}, "paused_at": None, "updated_at": None},
|
||||||
{"name": "nws", "enabled": True, "cadence_s": 60, "settings": {"contact_email": "test@test.com"}, "paused_at": None, "updated_at": None, "last_error": None},
|
{"name": "nws", "enabled": True, "cadence_s": 60, "settings": {"contact_email": "test@test.com"}, "paused_at": None, "updated_at": None},
|
||||||
{"name": "usgs_quake", "enabled": True, "cadence_s": 120, "settings": {"feed": "all_hour"}, "paused_at": None, "updated_at": None, "last_error": None},
|
{"name": "usgs_quake", "enabled": True, "cadence_s": 120, "settings": {"feed": "all_hour"}, "paused_at": None, "updated_at": None},
|
||||||
]
|
]
|
||||||
|
|
||||||
mock_pool = MagicMock()
|
mock_pool = MagicMock()
|
||||||
|
|
@ -55,22 +55,9 @@ class TestAdaptersListAuthenticated:
|
||||||
mock_response = MagicMock()
|
mock_response = MagicMock()
|
||||||
mock_templates.TemplateResponse.return_value = mock_response
|
mock_templates.TemplateResponse.return_value = mock_response
|
||||||
|
|
||||||
# Mock adapter classes
|
|
||||||
mock_firms_cls = MagicMock()
|
|
||||||
mock_firms_cls.requires_api_key = "firms"
|
|
||||||
mock_firms_cls.display_name = "FIRMS"
|
|
||||||
mock_nws_cls = MagicMock()
|
|
||||||
mock_nws_cls.requires_api_key = None
|
|
||||||
mock_nws_cls.display_name = "NWS"
|
|
||||||
mock_usgs_cls = MagicMock()
|
|
||||||
mock_usgs_cls.requires_api_key = None
|
|
||||||
mock_usgs_cls.display_name = "USGS Quake"
|
|
||||||
mock_adapter_classes = {"firms": mock_firms_cls, "nws": mock_nws_cls, "usgs_quake": mock_usgs_cls}
|
|
||||||
|
|
||||||
with patch("central.gui.routes._get_templates", return_value=mock_templates):
|
with patch("central.gui.routes._get_templates", return_value=mock_templates):
|
||||||
with patch("central.gui.routes.get_pool", return_value=mock_pool):
|
with patch("central.gui.routes.get_pool", return_value=mock_pool):
|
||||||
with patch("central.gui.routes._adapter_classes", return_value=mock_adapter_classes):
|
result = await adapters_list(mock_request)
|
||||||
result = await adapters_list(mock_request)
|
|
||||||
|
|
||||||
# Verify template was called with adapters
|
# Verify template was called with adapters
|
||||||
call_args = mock_templates.TemplateResponse.call_args
|
call_args = mock_templates.TemplateResponse.call_args
|
||||||
|
|
@ -91,7 +78,6 @@ class TestAdaptersEditForm:
|
||||||
|
|
||||||
mock_request = MagicMock()
|
mock_request = MagicMock()
|
||||||
mock_request.state.operator = MagicMock(id=1, username="testop")
|
mock_request.state.operator = MagicMock(id=1, username="testop")
|
||||||
mock_request.state.csrf_token = "test_csrf"
|
|
||||||
|
|
||||||
mock_conn = AsyncMock()
|
mock_conn = AsyncMock()
|
||||||
mock_conn.fetchrow.side_effect = [
|
mock_conn.fetchrow.side_effect = [
|
||||||
|
|
@ -102,10 +88,10 @@ class TestAdaptersEditForm:
|
||||||
"settings": {"contact_email": "test@example.com", "region": {"north": 49, "south": 24, "east": -66, "west": -125}},
|
"settings": {"contact_email": "test@example.com", "region": {"north": 49, "south": 24, "east": -66, "west": -125}},
|
||||||
"paused_at": None,
|
"paused_at": None,
|
||||||
"updated_at": None,
|
"updated_at": None,
|
||||||
"last_error": None,
|
|
||||||
},
|
},
|
||||||
{"map_tile_url": "https://tile.example.com/{z}/{x}/{y}.png", "map_attribution": "Test"},
|
{"map_tile_url": "https://tile.example.com/{z}/{x}/{y}.png", "map_attribution": "Test"},
|
||||||
]
|
]
|
||||||
|
mock_conn.fetch.return_value = [] # No API keys
|
||||||
|
|
||||||
mock_pool = MagicMock()
|
mock_pool = MagicMock()
|
||||||
mock_pool.acquire.return_value.__aenter__ = AsyncMock(return_value=mock_conn)
|
mock_pool.acquire.return_value.__aenter__ = AsyncMock(return_value=mock_conn)
|
||||||
|
|
@ -123,8 +109,6 @@ class TestAdaptersEditForm:
|
||||||
context = call_args.kwargs.get("context", call_args[1].get("context"))
|
context = call_args.kwargs.get("context", call_args[1].get("context"))
|
||||||
assert context["adapter"]["name"] == "nws"
|
assert context["adapter"]["name"] == "nws"
|
||||||
assert context["adapter"]["settings"]["contact_email"] == "test@example.com"
|
assert context["adapter"]["settings"]["contact_email"] == "test@example.com"
|
||||||
# Verify fields are generated
|
|
||||||
assert "fields" in context
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_adapters_edit_nonexistent_returns_404(self):
|
async def test_adapters_edit_nonexistent_returns_404(self):
|
||||||
|
|
@ -183,7 +167,6 @@ class TestAdaptersEditSubmit:
|
||||||
"settings": {"contact_email": "old@example.com", "region": {"north": 49, "south": 24, "east": -66, "west": -125}},
|
"settings": {"contact_email": "old@example.com", "region": {"north": 49, "south": 24, "east": -66, "west": -125}},
|
||||||
"paused_at": None,
|
"paused_at": None,
|
||||||
"updated_at": None,
|
"updated_at": None,
|
||||||
"last_error": None,
|
|
||||||
}
|
}
|
||||||
mock_conn.execute = AsyncMock()
|
mock_conn.execute = AsyncMock()
|
||||||
|
|
||||||
|
|
@ -202,17 +185,17 @@ class TestAdaptersEditSubmit:
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_adapters_edit_invalid_cadence_shows_error(self):
|
async def test_adapters_edit_invalid_cadence_shows_error(self):
|
||||||
"""POST /adapters/nws with cadence_s=5 shows error, no DB update."""
|
"""POST /adapters/nws with cadence_s=30 shows error, no DB update."""
|
||||||
from central.gui.routes import adapters_edit_submit
|
from central.gui.routes import adapters_edit_submit
|
||||||
|
|
||||||
mock_request = MagicMock()
|
mock_request = MagicMock()
|
||||||
mock_request.state.operator = MagicMock(id=1, username="testop")
|
mock_request.state.operator = MagicMock(id=1, username="testop")
|
||||||
mock_request.state.csrf_token = "test_csrf_token"
|
|
||||||
|
|
||||||
mock_form = MagicMock()
|
mock_form = MagicMock()
|
||||||
|
mock_request.state.csrf_token = "test_csrf_token"
|
||||||
mock_form.get.side_effect = lambda k, d="": {
|
mock_form.get.side_effect = lambda k, d="": {
|
||||||
"csrf_token": "test_csrf_token",
|
"csrf_token": "test_csrf_token",
|
||||||
"cadence_s": "5",
|
"cadence_s": "30",
|
||||||
"contact_email": "test@example.com",
|
"contact_email": "test@example.com",
|
||||||
"region_north": "49.0",
|
"region_north": "49.0",
|
||||||
"region_south": "24.0",
|
"region_south": "24.0",
|
||||||
|
|
@ -232,10 +215,10 @@ class TestAdaptersEditSubmit:
|
||||||
"settings": {"contact_email": "test@example.com", "region": {"north": 49, "south": 24, "east": -66, "west": -125}},
|
"settings": {"contact_email": "test@example.com", "region": {"north": 49, "south": 24, "east": -66, "west": -125}},
|
||||||
"paused_at": None,
|
"paused_at": None,
|
||||||
"updated_at": None,
|
"updated_at": None,
|
||||||
"last_error": None,
|
|
||||||
},
|
},
|
||||||
{"map_tile_url": None, "map_attribution": None}, # system settings for re-render
|
{"map_tile_url": None, "map_attribution": None}, # system settings for re-render
|
||||||
]
|
]
|
||||||
|
mock_conn.fetch.return_value = []
|
||||||
|
|
||||||
mock_pool = MagicMock()
|
mock_pool = MagicMock()
|
||||||
mock_pool.acquire.return_value.__aenter__ = AsyncMock(return_value=mock_conn)
|
mock_pool.acquire.return_value.__aenter__ = AsyncMock(return_value=mock_conn)
|
||||||
|
|
@ -254,7 +237,117 @@ class TestAdaptersEditSubmit:
|
||||||
call_args = mock_templates.TemplateResponse.call_args
|
call_args = mock_templates.TemplateResponse.call_args
|
||||||
context = call_args.kwargs.get("context", call_args[1].get("context"))
|
context = call_args.kwargs.get("context", call_args[1].get("context"))
|
||||||
assert "cadence_s" in context["errors"]
|
assert "cadence_s" in context["errors"]
|
||||||
assert "10" in context["errors"]["cadence_s"]
|
assert "60" in context["errors"]["cadence_s"] or "3600" in context["errors"]["cadence_s"]
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_adapters_edit_firms_unknown_api_key_shows_error(self):
|
||||||
|
"""POST /adapters/firms with unknown api_key_alias shows error."""
|
||||||
|
from central.gui.routes import adapters_edit_submit
|
||||||
|
|
||||||
|
mock_request = MagicMock()
|
||||||
|
mock_request.state.operator = MagicMock(id=1, username="testop")
|
||||||
|
|
||||||
|
mock_form = MagicMock()
|
||||||
|
mock_request.state.csrf_token = "test_csrf_token"
|
||||||
|
mock_form.get.side_effect = lambda k, d="": {
|
||||||
|
"csrf_token": "test_csrf_token",
|
||||||
|
"cadence_s": "300",
|
||||||
|
"api_key_alias": "nonexistent_key",
|
||||||
|
"region_north": "49.5",
|
||||||
|
"region_south": "31.0",
|
||||||
|
"region_east": "-102.0",
|
||||||
|
"region_west": "-124.5",
|
||||||
|
}.get(k, d)
|
||||||
|
mock_form.getlist.return_value = ["VIIRS_SNPP_NRT"]
|
||||||
|
mock_form.__contains__ = lambda self, k: k == "enabled"
|
||||||
|
mock_request.form = AsyncMock(return_value=mock_form)
|
||||||
|
|
||||||
|
mock_conn = AsyncMock()
|
||||||
|
mock_conn.fetchrow.side_effect = [
|
||||||
|
{ # First call: get adapter
|
||||||
|
"name": "firms",
|
||||||
|
"enabled": True,
|
||||||
|
"cadence_s": 300,
|
||||||
|
"settings": {"api_key_alias": "firms", "satellites": ["VIIRS_SNPP_NRT"], "region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}},
|
||||||
|
"paused_at": None,
|
||||||
|
"updated_at": None,
|
||||||
|
},
|
||||||
|
None, # Second call: check api_key exists - returns None
|
||||||
|
{"map_tile_url": None, "map_attribution": None}, # system settings for re-render
|
||||||
|
]
|
||||||
|
mock_conn.fetch.return_value = []
|
||||||
|
|
||||||
|
mock_pool = MagicMock()
|
||||||
|
mock_pool.acquire.return_value.__aenter__ = AsyncMock(return_value=mock_conn)
|
||||||
|
mock_pool.acquire.return_value.__aexit__ = AsyncMock(return_value=None)
|
||||||
|
|
||||||
|
mock_templates = MagicMock()
|
||||||
|
mock_response = MagicMock()
|
||||||
|
mock_response.status_code = 200
|
||||||
|
mock_templates.TemplateResponse.return_value = mock_response
|
||||||
|
|
||||||
|
with patch("central.gui.routes._get_templates", return_value=mock_templates):
|
||||||
|
with patch("central.gui.routes.get_pool", return_value=mock_pool):
|
||||||
|
result = await adapters_edit_submit(mock_request, "firms")
|
||||||
|
|
||||||
|
call_args = mock_templates.TemplateResponse.call_args
|
||||||
|
context = call_args.kwargs.get("context", call_args[1].get("context"))
|
||||||
|
assert "api_key_alias" in context["errors"]
|
||||||
|
assert "nonexistent_key" in context["errors"]["api_key_alias"]
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_adapters_edit_usgs_unknown_feed_shows_error(self):
|
||||||
|
"""POST /adapters/usgs_quake with unknown feed shows error."""
|
||||||
|
from central.gui.routes import adapters_edit_submit
|
||||||
|
|
||||||
|
mock_request = MagicMock()
|
||||||
|
mock_request.state.operator = MagicMock(id=1, username="testop")
|
||||||
|
|
||||||
|
mock_form = MagicMock()
|
||||||
|
mock_request.state.csrf_token = "test_csrf_token"
|
||||||
|
mock_form.get.side_effect = lambda k, d="": {
|
||||||
|
"csrf_token": "test_csrf_token",
|
||||||
|
"cadence_s": "120",
|
||||||
|
"feed": "invalid_feed",
|
||||||
|
"region_north": "49.0",
|
||||||
|
"region_south": "24.0",
|
||||||
|
"region_east": "-66.0",
|
||||||
|
"region_west": "-125.0",
|
||||||
|
}.get(k, d)
|
||||||
|
mock_form.getlist.return_value = []
|
||||||
|
mock_form.__contains__ = lambda self, k: k == "enabled"
|
||||||
|
mock_request.form = AsyncMock(return_value=mock_form)
|
||||||
|
|
||||||
|
mock_conn = AsyncMock()
|
||||||
|
mock_conn.fetchrow.side_effect = [
|
||||||
|
{
|
||||||
|
"name": "usgs_quake",
|
||||||
|
"enabled": True,
|
||||||
|
"cadence_s": 120,
|
||||||
|
"settings": {"feed": "all_hour", "region": {"north": 49, "south": 24, "east": -66, "west": -125}},
|
||||||
|
"paused_at": None,
|
||||||
|
"updated_at": None,
|
||||||
|
},
|
||||||
|
{"map_tile_url": None, "map_attribution": None}, # system settings for re-render
|
||||||
|
]
|
||||||
|
mock_conn.fetch.return_value = []
|
||||||
|
|
||||||
|
mock_pool = MagicMock()
|
||||||
|
mock_pool.acquire.return_value.__aenter__ = AsyncMock(return_value=mock_conn)
|
||||||
|
mock_pool.acquire.return_value.__aexit__ = AsyncMock(return_value=None)
|
||||||
|
|
||||||
|
mock_templates = MagicMock()
|
||||||
|
mock_response = MagicMock()
|
||||||
|
mock_response.status_code = 200
|
||||||
|
mock_templates.TemplateResponse.return_value = mock_response
|
||||||
|
|
||||||
|
with patch("central.gui.routes._get_templates", return_value=mock_templates):
|
||||||
|
with patch("central.gui.routes.get_pool", return_value=mock_pool):
|
||||||
|
result = await adapters_edit_submit(mock_request, "usgs_quake")
|
||||||
|
|
||||||
|
call_args = mock_templates.TemplateResponse.call_args
|
||||||
|
context = call_args.kwargs.get("context", call_args[1].get("context"))
|
||||||
|
assert "feed" in context["errors"]
|
||||||
|
|
||||||
|
|
||||||
class TestAdaptersAudit:
|
class TestAdaptersAudit:
|
||||||
|
|
@ -291,7 +384,6 @@ class TestAdaptersAudit:
|
||||||
"settings": {"contact_email": "old@example.com", "region": {"north": 49, "south": 24, "east": -66, "west": -125}},
|
"settings": {"contact_email": "old@example.com", "region": {"north": 49, "south": 24, "east": -66, "west": -125}},
|
||||||
"paused_at": None,
|
"paused_at": None,
|
||||||
"updated_at": None,
|
"updated_at": None,
|
||||||
"last_error": None,
|
|
||||||
}
|
}
|
||||||
mock_conn.execute = AsyncMock()
|
mock_conn.execute = AsyncMock()
|
||||||
|
|
||||||
|
|
@ -315,6 +407,8 @@ class TestAdaptersAudit:
|
||||||
assert captured_audit["target"] == "nws"
|
assert captured_audit["target"] == "nws"
|
||||||
assert captured_audit["before"]["cadence_s"] == 60
|
assert captured_audit["before"]["cadence_s"] == 60
|
||||||
assert captured_audit["after"]["cadence_s"] == 120
|
assert captured_audit["after"]["cadence_s"] == 120
|
||||||
|
assert captured_audit["before"]["settings"]["contact_email"] == "old@example.com"
|
||||||
|
assert captured_audit["after"]["settings"]["contact_email"] == "new@example.com"
|
||||||
|
|
||||||
|
|
||||||
class TestAdaptersJsonbRegression:
|
class TestAdaptersJsonbRegression:
|
||||||
|
|
@ -355,7 +449,6 @@ class TestAdaptersJsonbRegression:
|
||||||
"settings": {"contact_email": "old@example.com", "region": {"north": 49, "south": 24, "east": -66, "west": -125}}, # dict, as asyncpg returns
|
"settings": {"contact_email": "old@example.com", "region": {"north": 49, "south": 24, "east": -66, "west": -125}}, # dict, as asyncpg returns
|
||||||
"paused_at": None,
|
"paused_at": None,
|
||||||
"updated_at": None,
|
"updated_at": None,
|
||||||
"last_error": None,
|
|
||||||
}
|
}
|
||||||
mock_conn.execute = AsyncMock()
|
mock_conn.execute = AsyncMock()
|
||||||
|
|
||||||
|
|
@ -375,6 +468,7 @@ class TestAdaptersJsonbRegression:
|
||||||
# CRITICAL: settings must be a dict, NOT a string
|
# CRITICAL: settings must be a dict, NOT a string
|
||||||
# If json.dumps() was called, this would be a str like {contact_email: ...}
|
# If json.dumps() was called, this would be a str like {contact_email: ...}
|
||||||
assert isinstance(settings_arg, dict), f"settings should be dict, got {type(settings_arg)}: {settings_arg}"
|
assert isinstance(settings_arg, dict), f"settings should be dict, got {type(settings_arg)}: {settings_arg}"
|
||||||
|
assert settings_arg["contact_email"] == "test@example.com"
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_audit_before_after_passed_as_dict(self):
|
async def test_audit_before_after_passed_as_dict(self):
|
||||||
|
|
@ -407,7 +501,6 @@ class TestAdaptersJsonbRegression:
|
||||||
"settings": {"contact_email": "old@example.com", "region": {"north": 49, "south": 24, "east": -66, "west": -125}}, # dict
|
"settings": {"contact_email": "old@example.com", "region": {"north": 49, "south": 24, "east": -66, "west": -125}}, # dict
|
||||||
"paused_at": None,
|
"paused_at": None,
|
||||||
"updated_at": None,
|
"updated_at": None,
|
||||||
"last_error": None,
|
|
||||||
}
|
}
|
||||||
mock_conn.execute = AsyncMock()
|
mock_conn.execute = AsyncMock()
|
||||||
|
|
||||||
|
|
@ -430,47 +523,3 @@ class TestAdaptersJsonbRegression:
|
||||||
assert isinstance(captured_audit["after"], dict), f"after should be dict, got {type(captured_audit['after'])}"
|
assert isinstance(captured_audit["after"], dict), f"after should be dict, got {type(captured_audit['after'])}"
|
||||||
assert isinstance(captured_audit["before"]["settings"], dict), "before.settings should be dict"
|
assert isinstance(captured_audit["before"]["settings"], dict), "before.settings should be dict"
|
||||||
assert isinstance(captured_audit["after"]["settings"], dict), "after.settings should be dict"
|
assert isinstance(captured_audit["after"]["settings"], dict), "after.settings should be dict"
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_adapters_edit_fetches_api_keys_into_context(self):
|
|
||||||
"""GET /adapters/firms includes api_keys from database in context."""
|
|
||||||
from central.gui.routes import adapters_edit_form
|
|
||||||
|
|
||||||
mock_request = MagicMock()
|
|
||||||
mock_request.state.operator = MagicMock(id=1, username="testop")
|
|
||||||
mock_request.state.csrf_token = "test_csrf_token"
|
|
||||||
|
|
||||||
mock_conn = MagicMock()
|
|
||||||
mock_conn.fetchrow = AsyncMock(side_effect=[
|
|
||||||
# Adapter row
|
|
||||||
{"name": "firms", "enabled": True, "cadence_s": 300, "settings": {},
|
|
||||||
"paused_at": None, "updated_at": None, "last_error": None},
|
|
||||||
# System row
|
|
||||||
{"map_tile_url": "https://tile.example.com", "map_attribution": "Test"},
|
|
||||||
])
|
|
||||||
mock_conn.fetch = AsyncMock(return_value=[
|
|
||||||
{"alias": "firms_key"},
|
|
||||||
{"alias": "other_key"},
|
|
||||||
])
|
|
||||||
mock_conn.__aenter__ = AsyncMock(return_value=mock_conn)
|
|
||||||
mock_conn.__aexit__ = AsyncMock()
|
|
||||||
|
|
||||||
mock_pool = MagicMock()
|
|
||||||
mock_pool.acquire = MagicMock(return_value=mock_conn)
|
|
||||||
|
|
||||||
mock_templates = MagicMock()
|
|
||||||
mock_response = MagicMock()
|
|
||||||
mock_templates.TemplateResponse.return_value = mock_response
|
|
||||||
|
|
||||||
with patch("central.gui.routes._get_templates", return_value=mock_templates):
|
|
||||||
with patch("central.gui.routes.get_pool", return_value=mock_pool):
|
|
||||||
result = await adapters_edit_form(mock_request, "firms")
|
|
||||||
|
|
||||||
call_args = mock_templates.TemplateResponse.call_args
|
|
||||||
context = call_args.kwargs.get("context", call_args[1].get("context"))
|
|
||||||
|
|
||||||
assert "api_keys" in context
|
|
||||||
assert len(context["api_keys"]) == 2
|
|
||||||
assert context["api_keys"][0]["alias"] == "firms_key"
|
|
||||||
assert context["api_keys"][1]["alias"] == "other_key"
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -29,9 +29,9 @@ class TestConsumerNaming:
|
||||||
class TestStreamsConfiguration:
|
class TestStreamsConfiguration:
|
||||||
"""Test streams configuration."""
|
"""Test streams configuration."""
|
||||||
|
|
||||||
def test_streams_list_has_four_entries(self):
|
def test_streams_list_has_three_entries(self):
|
||||||
"""STREAMS list has four event-bearing streams."""
|
"""STREAMS list has three event-bearing streams."""
|
||||||
assert len(STREAMS) == 4
|
assert len(STREAMS) == 3
|
||||||
|
|
||||||
def test_streams_contains_central_wx(self):
|
def test_streams_contains_central_wx(self):
|
||||||
"""STREAMS contains CENTRAL_WX with correct filter."""
|
"""STREAMS contains CENTRAL_WX with correct filter."""
|
||||||
|
|
@ -45,10 +45,6 @@ class TestStreamsConfiguration:
|
||||||
"""STREAMS contains CENTRAL_QUAKE with correct filter."""
|
"""STREAMS contains CENTRAL_QUAKE with correct filter."""
|
||||||
assert ("CENTRAL_QUAKE", "central.quake.>") in STREAMS
|
assert ("CENTRAL_QUAKE", "central.quake.>") in STREAMS
|
||||||
|
|
||||||
def test_streams_contains_central_space(self):
|
|
||||||
"""STREAMS contains CENTRAL_SPACE with correct filter."""
|
|
||||||
assert ("CENTRAL_SPACE", "central.space.>") in STREAMS
|
|
||||||
|
|
||||||
def test_streams_excludes_central_meta(self):
|
def test_streams_excludes_central_meta(self):
|
||||||
"""STREAMS does not contain CENTRAL_META (status messages only)."""
|
"""STREAMS does not contain CENTRAL_META (status messages only)."""
|
||||||
stream_names = [s[0] for s in STREAMS]
|
stream_names = [s[0] for s in STREAMS]
|
||||||
|
|
|
||||||
|
|
@ -205,7 +205,7 @@ class TestDashboardStreamsIsolation:
|
||||||
call_args = mock_templates.TemplateResponse.call_args
|
call_args = mock_templates.TemplateResponse.call_args
|
||||||
context = call_args.kwargs.get("context", call_args[1].get("context"))
|
context = call_args.kwargs.get("context", call_args[1].get("context"))
|
||||||
streams = context["streams"]
|
streams = context["streams"]
|
||||||
assert len(streams) == 5
|
assert len(streams) == 4
|
||||||
fire_stream = next(s for s in streams if s["name"] == "CENTRAL_FIRE")
|
fire_stream = next(s for s in streams if s["name"] == "CENTRAL_FIRE")
|
||||||
assert fire_stream.get("error") == "unavailable"
|
assert fire_stream.get("error") == "unavailable"
|
||||||
wx_stream = next(s for s in streams if s["name"] == "CENTRAL_WX")
|
wx_stream = next(s for s in streams if s["name"] == "CENTRAL_WX")
|
||||||
|
|
|
||||||
|
|
@ -1,238 +0,0 @@
|
||||||
"""Tests for form_descriptors module."""
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
from pydantic import BaseModel
|
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
from central.gui.form_descriptors import describe_fields, FieldDescriptor, _type_to_widget_and_options
|
|
||||||
from central.config_models import RegionConfig
|
|
||||||
|
|
||||||
|
|
||||||
class SimpleSettings(BaseModel):
|
|
||||||
"""Simple settings model for testing."""
|
|
||||||
name: str
|
|
||||||
count: int
|
|
||||||
enabled: bool
|
|
||||||
|
|
||||||
|
|
||||||
class SettingsWithOptional(BaseModel):
|
|
||||||
"""Settings with optional fields."""
|
|
||||||
required_field: str
|
|
||||||
optional_field: Optional[str] = None
|
|
||||||
with_default: str = "default_value"
|
|
||||||
|
|
||||||
|
|
||||||
class SettingsWithList(BaseModel):
|
|
||||||
"""Settings with list field."""
|
|
||||||
tags: list[str]
|
|
||||||
|
|
||||||
|
|
||||||
class SettingsWithRegion(BaseModel):
|
|
||||||
"""Settings with region config."""
|
|
||||||
region: Optional[RegionConfig] = None
|
|
||||||
|
|
||||||
|
|
||||||
class TestTypeToWidget:
|
|
||||||
"""Tests for _type_to_widget_and_options function."""
|
|
||||||
|
|
||||||
def test_str_maps_to_text(self):
|
|
||||||
assert _type_to_widget_and_options("field", str) == ("text", None)
|
|
||||||
|
|
||||||
def test_int_maps_to_number(self):
|
|
||||||
assert _type_to_widget_and_options("field", int) == ("number", None)
|
|
||||||
|
|
||||||
def test_bool_maps_to_checkbox(self):
|
|
||||||
assert _type_to_widget_and_options("field", bool) == ("checkbox", None)
|
|
||||||
|
|
||||||
def test_list_str_maps_to_csv(self):
|
|
||||||
assert _type_to_widget_and_options("field", list[str]) == ("csv", None)
|
|
||||||
|
|
||||||
def test_region_config_maps_to_region(self):
|
|
||||||
assert _type_to_widget_and_options("field", RegionConfig) == ("region", None)
|
|
||||||
|
|
||||||
def test_optional_region_maps_to_region(self):
|
|
||||||
assert _type_to_widget_and_options("field", Optional[RegionConfig]) == ("region", None)
|
|
||||||
|
|
||||||
def test_optional_str_maps_to_text(self):
|
|
||||||
"""Optional[str] should map to text widget."""
|
|
||||||
assert _type_to_widget_and_options("field", Optional[str]) == ("text", None)
|
|
||||||
|
|
||||||
def test_optional_int_maps_to_number(self):
|
|
||||||
"""Optional[int] should map to number widget."""
|
|
||||||
assert _type_to_widget_and_options("field", Optional[int]) == ("number", None)
|
|
||||||
|
|
||||||
def test_unsupported_type_raises(self):
|
|
||||||
class CustomType:
|
|
||||||
pass
|
|
||||||
with pytest.raises(NotImplementedError):
|
|
||||||
_type_to_widget_and_options("field", CustomType)
|
|
||||||
|
|
||||||
|
|
||||||
class TestDescribeFields:
|
|
||||||
"""Tests for describe_fields function."""
|
|
||||||
|
|
||||||
def test_simple_model_fields(self):
|
|
||||||
"""describe_fields returns correct descriptors for simple model."""
|
|
||||||
fields = describe_fields(SimpleSettings, {"name": "test", "count": 5, "enabled": True})
|
|
||||||
|
|
||||||
assert len(fields) == 3
|
|
||||||
|
|
||||||
name_field = next(f for f in fields if f.name == "name")
|
|
||||||
assert name_field.label == "Name"
|
|
||||||
assert name_field.widget == "text"
|
|
||||||
assert name_field.current_value == "test"
|
|
||||||
|
|
||||||
count_field = next(f for f in fields if f.name == "count")
|
|
||||||
assert count_field.label == "Count"
|
|
||||||
assert count_field.widget == "number"
|
|
||||||
assert count_field.current_value == 5
|
|
||||||
|
|
||||||
enabled_field = next(f for f in fields if f.name == "enabled")
|
|
||||||
assert enabled_field.label == "Enabled"
|
|
||||||
assert enabled_field.widget == "checkbox"
|
|
||||||
assert enabled_field.current_value is True
|
|
||||||
|
|
||||||
def test_uses_current_values(self):
|
|
||||||
"""Current values from dict are used."""
|
|
||||||
fields = describe_fields(SimpleSettings, {"name": "current_name", "count": 42, "enabled": False})
|
|
||||||
|
|
||||||
name_field = next(f for f in fields if f.name == "name")
|
|
||||||
assert name_field.current_value == "current_name"
|
|
||||||
|
|
||||||
count_field = next(f for f in fields if f.name == "count")
|
|
||||||
assert count_field.current_value == 42
|
|
||||||
|
|
||||||
def test_missing_values_use_defaults(self):
|
|
||||||
"""Missing values fall back to model defaults."""
|
|
||||||
fields = describe_fields(SettingsWithOptional, {"required_field": "value"})
|
|
||||||
|
|
||||||
optional_field = next(f for f in fields if f.name == "optional_field")
|
|
||||||
assert optional_field.current_value is None
|
|
||||||
assert optional_field.widget == "text" # Optional[str] -> text
|
|
||||||
|
|
||||||
default_field = next(f for f in fields if f.name == "with_default")
|
|
||||||
assert default_field.current_value == "default_value"
|
|
||||||
|
|
||||||
def test_list_field_returns_csv_widget(self):
|
|
||||||
"""List[str] fields get csv widget."""
|
|
||||||
fields = describe_fields(SettingsWithList, {"tags": ["a", "b", "c"]})
|
|
||||||
|
|
||||||
tags_field = next(f for f in fields if f.name == "tags")
|
|
||||||
assert tags_field.widget == "csv"
|
|
||||||
assert tags_field.current_value == ["a", "b", "c"]
|
|
||||||
|
|
||||||
def test_region_field_returns_region_widget(self):
|
|
||||||
"""RegionConfig fields get region widget."""
|
|
||||||
fields = describe_fields(SettingsWithRegion, {
|
|
||||||
"region": {"north": 50.0, "south": 40.0, "east": -100.0, "west": -120.0}
|
|
||||||
})
|
|
||||||
|
|
||||||
region_field = next(f for f in fields if f.name == "region")
|
|
||||||
assert region_field.widget == "region"
|
|
||||||
|
|
||||||
def test_empty_current_dict(self):
|
|
||||||
"""Works with empty current values dict."""
|
|
||||||
fields = describe_fields(SettingsWithOptional, {})
|
|
||||||
|
|
||||||
required_field = next(f for f in fields if f.name == "required_field")
|
|
||||||
assert required_field.current_value is None
|
|
||||||
assert required_field.widget == "text"
|
|
||||||
|
|
||||||
def test_field_descriptor_attributes(self):
|
|
||||||
"""FieldDescriptor has all expected attributes."""
|
|
||||||
fields = describe_fields(SimpleSettings, {"name": "test", "count": 1, "enabled": True})
|
|
||||||
field = fields[0]
|
|
||||||
|
|
||||||
assert hasattr(field, "name")
|
|
||||||
assert hasattr(field, "label")
|
|
||||||
assert hasattr(field, "widget")
|
|
||||||
assert hasattr(field, "current_value")
|
|
||||||
assert hasattr(field, "default")
|
|
||||||
assert hasattr(field, "description")
|
|
||||||
assert hasattr(field, "required")
|
|
||||||
|
|
||||||
|
|
||||||
class TestRealAdapterSchemas:
|
|
||||||
"""Test with actual adapter settings schemas."""
|
|
||||||
|
|
||||||
def test_nws_settings(self):
|
|
||||||
"""NWSSettings generates correct field descriptors."""
|
|
||||||
from central.adapters.nws import NWSSettings
|
|
||||||
|
|
||||||
fields = describe_fields(NWSSettings, {"contact_email": "test@example.com"})
|
|
||||||
|
|
||||||
assert len(fields) >= 1
|
|
||||||
email_field = next(f for f in fields if f.name == "contact_email")
|
|
||||||
assert email_field.widget == "text"
|
|
||||||
assert email_field.current_value == "test@example.com"
|
|
||||||
|
|
||||||
def test_firms_settings(self):
|
|
||||||
"""FIRMSSettings generates correct field descriptors."""
|
|
||||||
from central.adapters.firms import FIRMSSettings
|
|
||||||
|
|
||||||
fields = describe_fields(FIRMSSettings, {
|
|
||||||
"api_key_alias": "firms_key",
|
|
||||||
"satellites": ["VIIRS_SNPP_NRT"]
|
|
||||||
})
|
|
||||||
|
|
||||||
key_field = next(f for f in fields if f.name == "api_key_alias")
|
|
||||||
assert key_field.widget == "text"
|
|
||||||
|
|
||||||
sat_field = next(f for f in fields if f.name == "satellites")
|
|
||||||
assert sat_field.widget == "checkboxes"
|
|
||||||
assert sat_field.current_value == ["VIIRS_SNPP_NRT"]
|
|
||||||
assert sat_field.options is not None
|
|
||||||
assert "VIIRS_SNPP_NRT" in sat_field.options
|
|
||||||
|
|
||||||
def test_usgs_quake_settings(self):
|
|
||||||
"""USGSQuakeSettings generates correct field descriptors."""
|
|
||||||
from central.adapters.usgs_quake import USGSQuakeSettings
|
|
||||||
|
|
||||||
fields = describe_fields(USGSQuakeSettings, {"feed": "all_hour"})
|
|
||||||
|
|
||||||
feed_field = next(f for f in fields if f.name == "feed")
|
|
||||||
assert feed_field.widget == "select"
|
|
||||||
assert feed_field.current_value == "all_hour"
|
|
||||||
assert feed_field.options is not None
|
|
||||||
assert "all_hour" in feed_field.options
|
|
||||||
assert "all_day" in feed_field.options
|
|
||||||
|
|
||||||
def test_all_adapters_have_region_field(self):
|
|
||||||
"""All adapter settings schemas include region field."""
|
|
||||||
from central.adapters.nws import NWSSettings
|
|
||||||
from central.adapters.firms import FIRMSSettings
|
|
||||||
from central.adapters.usgs_quake import USGSQuakeSettings
|
|
||||||
|
|
||||||
for schema in [NWSSettings, FIRMSSettings, USGSQuakeSettings]:
|
|
||||||
fields = describe_fields(schema, {})
|
|
||||||
region_field = next((f for f in fields if f.name == "region"), None)
|
|
||||||
assert region_field is not None, f"{schema.__name__} should have region field"
|
|
||||||
assert region_field.widget == "region"
|
|
||||||
|
|
||||||
class TestLiteralTypes:
|
|
||||||
"""Tests for Literal type support."""
|
|
||||||
|
|
||||||
def test_literal_maps_to_select(self):
|
|
||||||
"""Literal type maps to select widget with options."""
|
|
||||||
from typing import Literal
|
|
||||||
|
|
||||||
widget, options = _type_to_widget_and_options("field", Literal["a", "b", "c"])
|
|
||||||
assert widget == "select"
|
|
||||||
assert options == ["a", "b", "c"]
|
|
||||||
|
|
||||||
def test_list_literal_maps_to_checkboxes(self):
|
|
||||||
"""list[Literal] maps to checkboxes widget with options."""
|
|
||||||
from typing import Literal
|
|
||||||
|
|
||||||
widget, options = _type_to_widget_and_options("field", list[Literal["x", "y", "z"]])
|
|
||||||
assert widget == "checkboxes"
|
|
||||||
assert options == ["x", "y", "z"]
|
|
||||||
|
|
||||||
def test_optional_literal_maps_to_select(self):
|
|
||||||
"""Optional[Literal] maps to select widget."""
|
|
||||||
from typing import Literal, Optional
|
|
||||||
|
|
||||||
widget, options = _type_to_widget_and_options("field", Optional[Literal["one", "two"]])
|
|
||||||
assert widget == "select"
|
|
||||||
assert options == ["one", "two"]
|
|
||||||
|
|
||||||
|
|
@ -1,599 +0,0 @@
|
||||||
"""Tests for InciWeb adapter."""
|
|
||||||
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
from pathlib import Path
|
|
||||||
from unittest.mock import AsyncMock, MagicMock, patch
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from central.config_models import AdapterConfig
|
|
||||||
from central.models import Event, Geo
|
|
||||||
|
|
||||||
|
|
||||||
# Real RSS snippet from InciWeb (frozen fixture)
|
|
||||||
SAMPLE_RSS_CONTENT = """<?xml version="1.0" encoding="utf-8"?>
|
|
||||||
<rss xmlns:dc="http://purl.org/dc/elements/1.1/" version="2.0" xml:base="http://inciweb.wildfire.gov/">
|
|
||||||
<channel>
|
|
||||||
<title>InciWeb</title>
|
|
||||||
<link>http://inciweb.wildfire.gov/</link>
|
|
||||||
<description>Inciweb Fire Incidents</description>
|
|
||||||
<language>en</language>
|
|
||||||
<item>
|
|
||||||
<title>MNMNS Stewart Trail</title>
|
|
||||||
<link>http://inciweb.wildfire.gov/incident-information/mnmns-stewart-trail</link>
|
|
||||||
<description>Last updated: 2026-05-18
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
The type of incident is Wildfire and involves the following unit(s) Minnesota Department of Natural Resources.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
State: Minnesota
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
Coordinates:
|
|
||||||
|
|
||||||
Latitude: 47° 3 17 Longitude: 91° 38 6
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
NOTE: All fire perimeters and points are approximations.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
Incident Overview: The Stewart Trail Fire was detected during the afternoon hours on Friday, May 15, 2026.&nbsp;A temporary flight restriction (TFR) is in place.</description>
|
|
||||||
<pubDate>Fri, 15 May 2026 08:48:11 EDT</pubDate>
|
|
||||||
<dc:creator>llangeberg</dc:creator>
|
|
||||||
<guid isPermaLink="false">327828</guid>
|
|
||||||
</item>
|
|
||||||
<item>
|
|
||||||
<title>CACNP Santa Rosa Island Fire</title>
|
|
||||||
<link>http://inciweb.wildfire.gov/incident-information/cacnp-santa-rosa-island-fire</link>
|
|
||||||
<description>Last updated: 2026-05-18
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
The type of incident is Wildfire and involves the following unit(s) Channel Islands National Park.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
State: California
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
Coordinates:
|
|
||||||
|
|
||||||
Latitude: 33° 55 2 Longitude: 120° 5 10
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
NOTE: All fire perimeters and points are approximations.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
Incident Overview: On Friday, May 15, 2026, an aircraft flying over Santa Rosa Island in Channel Islands National Park reported a wildfire.<br><p>This is a <strong>full-suppression</strong> human-caused wildfire and is under investigation.</p>&nbsp;</description>
|
|
||||||
<pubDate>Sat, 16 May 2026 12:09:07 EDT</pubDate>
|
|
||||||
<dc:creator>mtheune</dc:creator>
|
|
||||||
<guid isPermaLink="false">327838</guid>
|
|
||||||
</item>
|
|
||||||
<item>
|
|
||||||
<title>Some Fire Without Coordinates</title>
|
|
||||||
<link>http://inciweb.wildfire.gov/incident-information/no-coords-fire</link>
|
|
||||||
<description>Last updated: 2026-05-18
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
The type of incident is Wildfire.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
State: Unknown State
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
Incident Overview: This is a test incident without coordinates.</description>
|
|
||||||
<pubDate>Mon, 18 May 2026 09:00:00 EDT</pubDate>
|
|
||||||
<dc:creator>test</dc:creator>
|
|
||||||
<guid isPermaLink="false">999999</guid>
|
|
||||||
</item>
|
|
||||||
<item>
|
|
||||||
<title>Florida Fire Outside Bbox</title>
|
|
||||||
<link>http://inciweb.wildfire.gov/incident-information/florida-fire</link>
|
|
||||||
<description>Last updated: 2026-05-18
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
State: Florida
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
Coordinates:
|
|
||||||
|
|
||||||
Latitude: 26° 0 0 Longitude: 80° 0 0
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
Incident Overview: This fire is in Florida, outside the CONUS west bbox.</description>
|
|
||||||
<pubDate>Mon, 18 May 2026 10:00:00 EDT</pubDate>
|
|
||||||
<dc:creator>test</dc:creator>
|
|
||||||
<guid isPermaLink="false">888888</guid>
|
|
||||||
</item>
|
|
||||||
</channel>
|
|
||||||
</rss>"""
|
|
||||||
|
|
||||||
|
|
||||||
class TestInciWebHelpers:
|
|
||||||
"""Tests for InciWeb helper functions."""
|
|
||||||
|
|
||||||
def test_parse_coordinates_from_description(self):
|
|
||||||
"""Parse coordinates from description text."""
|
|
||||||
from central.adapters.inciweb import parse_coordinates_from_description
|
|
||||||
|
|
||||||
description = """Coordinates:
|
|
||||||
|
|
||||||
Latitude: 47° 3 17 Longitude: 91° 38 6"""
|
|
||||||
|
|
||||||
result = parse_coordinates_from_description(description)
|
|
||||||
assert result is not None
|
|
||||||
lon, lat = result
|
|
||||||
# 47° 3' 17" = 47.054722...
|
|
||||||
assert 47.0 < lat < 47.1
|
|
||||||
# 91° 38' 6" = -91.635 (west longitude)
|
|
||||||
assert -92.0 < lon < -91.0
|
|
||||||
|
|
||||||
def test_parse_coordinates_no_match(self):
|
|
||||||
"""No coordinates in description returns None."""
|
|
||||||
from central.adapters.inciweb import parse_coordinates_from_description
|
|
||||||
|
|
||||||
result = parse_coordinates_from_description("No coordinates here")
|
|
||||||
assert result is None
|
|
||||||
|
|
||||||
def test_parse_state_from_description(self):
|
|
||||||
"""Parse state name and return 2-letter code."""
|
|
||||||
from central.adapters.inciweb import parse_state_from_description
|
|
||||||
|
|
||||||
description = """---
|
|
||||||
|
|
||||||
State: Minnesota
|
|
||||||
|
|
||||||
---"""
|
|
||||||
assert parse_state_from_description(description) == "MN"
|
|
||||||
|
|
||||||
def test_parse_state_from_description_new_mexico(self):
|
|
||||||
"""Parse multi-word state name."""
|
|
||||||
from central.adapters.inciweb import parse_state_from_description
|
|
||||||
|
|
||||||
description = """State: New Mexico
|
|
||||||
|
|
||||||
---"""
|
|
||||||
assert parse_state_from_description(description) == "NM"
|
|
||||||
|
|
||||||
def test_parse_state_from_description_no_match(self):
|
|
||||||
"""Unknown state name returns None."""
|
|
||||||
from central.adapters.inciweb import parse_state_from_description
|
|
||||||
|
|
||||||
description = """State: Unknown State
|
|
||||||
|
|
||||||
---"""
|
|
||||||
assert parse_state_from_description(description) is None
|
|
||||||
|
|
||||||
def test_strip_html(self):
|
|
||||||
"""HTML tags are stripped, entities decoded."""
|
|
||||||
from central.adapters.inciweb import strip_html
|
|
||||||
|
|
||||||
html = "This is &nbsp;a <strong>test</strong> with <br>line breaks."
|
|
||||||
result = strip_html(html)
|
|
||||||
assert "<" not in result
|
|
||||||
assert ">" not in result
|
|
||||||
assert " " not in result
|
|
||||||
assert "&" not in result
|
|
||||||
assert "test" in result
|
|
||||||
|
|
||||||
|
|
||||||
class TestInciWebAdapter:
|
|
||||||
"""Tests for InciWeb adapter."""
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def mock_config(self) -> AdapterConfig:
|
|
||||||
return AdapterConfig(
|
|
||||||
name="inciweb",
|
|
||||||
enabled=True,
|
|
||||||
cadence_s=600,
|
|
||||||
settings={
|
|
||||||
"region": {"north": 49.0, "south": 31.0, "east": -102.0, "west": -124.0}
|
|
||||||
},
|
|
||||||
updated_at=datetime.now(timezone.utc),
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def mock_config_no_region(self) -> AdapterConfig:
|
|
||||||
return AdapterConfig(
|
|
||||||
name="inciweb",
|
|
||||||
enabled=True,
|
|
||||||
cadence_s=600,
|
|
||||||
settings={},
|
|
||||||
updated_at=datetime.now(timezone.utc),
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def mock_config_store(self) -> MagicMock:
|
|
||||||
return MagicMock()
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def cursor_db_path(self, tmp_path: Path) -> Path:
|
|
||||||
return tmp_path / "cursors.db"
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_normalization_with_georss_point(
|
|
||||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
|
||||||
):
|
|
||||||
"""Items with coordinates are correctly normalized."""
|
|
||||||
from central.adapters.inciweb import InciWebAdapter
|
|
||||||
|
|
||||||
adapter = InciWebAdapter(mock_config, mock_config_store, cursor_db_path)
|
|
||||||
await adapter.startup()
|
|
||||||
|
|
||||||
mock_response = AsyncMock()
|
|
||||||
mock_response.raise_for_status = MagicMock()
|
|
||||||
mock_response.text = AsyncMock(return_value=SAMPLE_RSS_CONTENT)
|
|
||||||
|
|
||||||
with patch.object(adapter._session, "get", return_value=AsyncMock(__aenter__=AsyncMock(return_value=mock_response), __aexit__=AsyncMock())):
|
|
||||||
events = [e async for e in adapter.poll()]
|
|
||||||
|
|
||||||
await adapter.shutdown()
|
|
||||||
|
|
||||||
# Bbox is west=-124, east=-102 (CONUS west)
|
|
||||||
# Minnesota at -91 longitude is OUTSIDE bbox (east of -102)
|
|
||||||
# California at -120 longitude is INSIDE bbox
|
|
||||||
# Florida at -80 longitude is OUTSIDE bbox
|
|
||||||
# Unknown state without coords passes through
|
|
||||||
assert len(events) == 2
|
|
||||||
|
|
||||||
# Check California event
|
|
||||||
ca_event = next(e for e in events if e.data["guid"] == "327838")
|
|
||||||
assert ca_event.id == "327838"
|
|
||||||
assert ca_event.adapter == "inciweb"
|
|
||||||
assert ca_event.category == "fire.narrative.inciweb"
|
|
||||||
assert ca_event.severity == 0
|
|
||||||
assert ca_event.geo.primary_region == "US-CA"
|
|
||||||
assert ca_event.geo.centroid is not None
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_normalization_without_georss_point(
|
|
||||||
self, mock_config_no_region: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
|
||||||
):
|
|
||||||
"""Items without coordinates have centroid=None."""
|
|
||||||
from central.adapters.inciweb import InciWebAdapter
|
|
||||||
|
|
||||||
adapter = InciWebAdapter(mock_config_no_region, mock_config_store, cursor_db_path)
|
|
||||||
await adapter.startup()
|
|
||||||
|
|
||||||
mock_response = AsyncMock()
|
|
||||||
mock_response.raise_for_status = MagicMock()
|
|
||||||
mock_response.text = AsyncMock(return_value=SAMPLE_RSS_CONTENT)
|
|
||||||
|
|
||||||
with patch.object(adapter._session, "get", return_value=AsyncMock(__aenter__=AsyncMock(return_value=mock_response), __aexit__=AsyncMock())):
|
|
||||||
events = [e async for e in adapter.poll()]
|
|
||||||
|
|
||||||
await adapter.shutdown()
|
|
||||||
|
|
||||||
# All 4 items pass (no region filter)
|
|
||||||
assert len(events) == 4
|
|
||||||
|
|
||||||
# Check item without coords
|
|
||||||
no_coords_event = next(e for e in events if e.data["guid"] == "999999")
|
|
||||||
assert no_coords_event.geo.centroid is None
|
|
||||||
assert no_coords_event.geo.regions == []
|
|
||||||
assert no_coords_event.geo.primary_region is None
|
|
||||||
|
|
||||||
def test_state_parse_from_title(
|
|
||||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
|
||||||
):
|
|
||||||
"""State parsing from description produces correct region."""
|
|
||||||
from central.adapters.inciweb import parse_state_from_description
|
|
||||||
|
|
||||||
# Test California
|
|
||||||
assert parse_state_from_description("State: California\n") == "CA"
|
|
||||||
# Test Minnesota
|
|
||||||
assert parse_state_from_description("State: Minnesota\n---") == "MN"
|
|
||||||
# Test multi-word
|
|
||||||
assert parse_state_from_description("State: New York\n") == "NY"
|
|
||||||
# Test unknown
|
|
||||||
assert parse_state_from_description("State: Narnia\n") is None
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_html_stripping(
|
|
||||||
self, mock_config_no_region: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
|
||||||
):
|
|
||||||
"""HTML is stripped from description, raw preserved in description_html."""
|
|
||||||
from central.adapters.inciweb import InciWebAdapter
|
|
||||||
|
|
||||||
adapter = InciWebAdapter(mock_config_no_region, mock_config_store, cursor_db_path)
|
|
||||||
await adapter.startup()
|
|
||||||
|
|
||||||
mock_response = AsyncMock()
|
|
||||||
mock_response.raise_for_status = MagicMock()
|
|
||||||
mock_response.text = AsyncMock(return_value=SAMPLE_RSS_CONTENT)
|
|
||||||
|
|
||||||
with patch.object(adapter._session, "get", return_value=AsyncMock(__aenter__=AsyncMock(return_value=mock_response), __aexit__=AsyncMock())):
|
|
||||||
events = [e async for e in adapter.poll()]
|
|
||||||
|
|
||||||
await adapter.shutdown()
|
|
||||||
|
|
||||||
# California item has HTML tags in description
|
|
||||||
ca_event = next(e for e in events if e.data["guid"] == "327838")
|
|
||||||
|
|
||||||
# Plain text should not have HTML tags
|
|
||||||
assert "<br>" not in ca_event.data["description"]
|
|
||||||
assert "<p>" not in ca_event.data["description"]
|
|
||||||
assert "<strong>" not in ca_event.data["description"]
|
|
||||||
assert " " not in ca_event.data["description"]
|
|
||||||
|
|
||||||
# Raw HTML should be preserved
|
|
||||||
assert "<br>" in ca_event.data["description_html"] or "<br>" in ca_event.data["description_html"]
|
|
||||||
|
|
||||||
def test_subject_for_with_state(
|
|
||||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
|
||||||
):
|
|
||||||
"""subject_for returns correct subject with state."""
|
|
||||||
from central.adapters.inciweb import InciWebAdapter
|
|
||||||
|
|
||||||
adapter = InciWebAdapter(mock_config, mock_config_store, cursor_db_path)
|
|
||||||
|
|
||||||
event = Event(
|
|
||||||
id="test-id",
|
|
||||||
adapter="inciweb",
|
|
||||||
category="fire.narrative.inciweb",
|
|
||||||
time=datetime.now(timezone.utc),
|
|
||||||
severity=0,
|
|
||||||
geo=Geo(primary_region="US-CA"),
|
|
||||||
data={"title": "Test Fire", "description": "Test"},
|
|
||||||
)
|
|
||||||
|
|
||||||
subject = adapter.subject_for(event)
|
|
||||||
assert subject == "central.fire.narrative.inciweb.ca"
|
|
||||||
|
|
||||||
def test_subject_for_without_state(
|
|
||||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
|
||||||
):
|
|
||||||
"""subject_for returns unknown when no state."""
|
|
||||||
from central.adapters.inciweb import InciWebAdapter
|
|
||||||
|
|
||||||
adapter = InciWebAdapter(mock_config, mock_config_store, cursor_db_path)
|
|
||||||
|
|
||||||
event = Event(
|
|
||||||
id="test-id",
|
|
||||||
adapter="inciweb",
|
|
||||||
category="fire.narrative.inciweb",
|
|
||||||
time=datetime.now(timezone.utc),
|
|
||||||
severity=0,
|
|
||||||
geo=Geo(),
|
|
||||||
data={"title": "Test Fire", "description": "Test"},
|
|
||||||
)
|
|
||||||
|
|
||||||
subject = adapter.subject_for(event)
|
|
||||||
assert subject == "central.fire.narrative.inciweb.unknown"
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_dedup_same_guid(
|
|
||||||
self, mock_config_no_region: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
|
||||||
):
|
|
||||||
"""is_published/mark_published provides dedup functionality."""
|
|
||||||
from central.adapters.inciweb import InciWebAdapter
|
|
||||||
|
|
||||||
adapter = InciWebAdapter(mock_config_no_region, mock_config_store, cursor_db_path)
|
|
||||||
await adapter.startup()
|
|
||||||
|
|
||||||
# Initially not published
|
|
||||||
assert adapter.is_published("327828") is False
|
|
||||||
|
|
||||||
# Mark as published
|
|
||||||
adapter.mark_published("327828")
|
|
||||||
|
|
||||||
# Now it should be published
|
|
||||||
assert adapter.is_published("327828") is True
|
|
||||||
|
|
||||||
await adapter.shutdown()
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_bbox_filters_point_outside(
|
|
||||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
|
||||||
):
|
|
||||||
"""Items with coords outside bbox are filtered; items without coords pass."""
|
|
||||||
from central.adapters.inciweb import InciWebAdapter
|
|
||||||
|
|
||||||
adapter = InciWebAdapter(mock_config, mock_config_store, cursor_db_path)
|
|
||||||
await adapter.startup()
|
|
||||||
|
|
||||||
mock_response = AsyncMock()
|
|
||||||
mock_response.raise_for_status = MagicMock()
|
|
||||||
mock_response.text = AsyncMock(return_value=SAMPLE_RSS_CONTENT)
|
|
||||||
|
|
||||||
with patch.object(adapter._session, "get", return_value=AsyncMock(__aenter__=AsyncMock(return_value=mock_response), __aexit__=AsyncMock())):
|
|
||||||
events = [e async for e in adapter.poll()]
|
|
||||||
|
|
||||||
await adapter.shutdown()
|
|
||||||
|
|
||||||
# Florida (-80 longitude) should be filtered out
|
|
||||||
guids = {e.data["guid"] for e in events}
|
|
||||||
assert "888888" not in guids # Florida, outside bbox
|
|
||||||
|
|
||||||
# Item without coords should pass through
|
|
||||||
assert "999999" in guids
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_apply_config_region_change(
|
|
||||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
|
||||||
):
|
|
||||||
"""apply_config updates region."""
|
|
||||||
from central.adapters.inciweb import InciWebAdapter
|
|
||||||
|
|
||||||
adapter = InciWebAdapter(mock_config, mock_config_store, cursor_db_path)
|
|
||||||
|
|
||||||
assert adapter.region is not None
|
|
||||||
assert adapter.region.north == 49.0
|
|
||||||
|
|
||||||
new_config = AdapterConfig(
|
|
||||||
name="inciweb",
|
|
||||||
enabled=True,
|
|
||||||
cadence_s=600,
|
|
||||||
settings={
|
|
||||||
"region": {"north": 50.0, "south": 35.0, "east": -100.0, "west": -120.0}
|
|
||||||
},
|
|
||||||
updated_at=datetime.now(timezone.utc),
|
|
||||||
)
|
|
||||||
await adapter.apply_config(new_config)
|
|
||||||
|
|
||||||
assert adapter.region.north == 50.0
|
|
||||||
assert adapter.region.south == 35.0
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_dedup_in_poll_loop(
|
|
||||||
self, mock_config_no_region: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
|
||||||
):
|
|
||||||
"""Dedup integration: second poll with same items yields zero events."""
|
|
||||||
from central.adapters.inciweb import InciWebAdapter
|
|
||||||
|
|
||||||
adapter = InciWebAdapter(mock_config_no_region, mock_config_store, cursor_db_path)
|
|
||||||
await adapter.startup()
|
|
||||||
|
|
||||||
# Single-item RSS for clarity
|
|
||||||
single_item_rss = """<?xml version="1.0" encoding="utf-8"?>
|
|
||||||
<rss xmlns:dc="http://purl.org/dc/elements/1.1/" version="2.0">
|
|
||||||
<channel>
|
|
||||||
<title>InciWeb</title>
|
|
||||||
<item>
|
|
||||||
<title>Test Fire</title>
|
|
||||||
<link>http://inciweb.wildfire.gov/test</link>
|
|
||||||
<description>State: California</description>
|
|
||||||
<pubDate>Mon, 18 May 2026 09:00:00 EDT</pubDate>
|
|
||||||
<guid isPermaLink="false">DEDUP-TEST-001</guid>
|
|
||||||
</item>
|
|
||||||
</channel>
|
|
||||||
</rss>"""
|
|
||||||
|
|
||||||
def make_mock_response():
|
|
||||||
mock_response = AsyncMock()
|
|
||||||
mock_response.status = 200
|
|
||||||
mock_response.raise_for_status = MagicMock()
|
|
||||||
mock_response.text = AsyncMock(return_value=single_item_rss)
|
|
||||||
mock_response.headers = {"Last-Modified": None, "ETag": None}
|
|
||||||
return mock_response
|
|
||||||
|
|
||||||
# First poll: should yield 1 event
|
|
||||||
with patch.object(
|
|
||||||
adapter._session, "get",
|
|
||||||
return_value=AsyncMock(
|
|
||||||
__aenter__=AsyncMock(return_value=make_mock_response()),
|
|
||||||
__aexit__=AsyncMock()
|
|
||||||
)
|
|
||||||
):
|
|
||||||
events_first = [e async for e in adapter.poll()]
|
|
||||||
|
|
||||||
assert len(events_first) == 1
|
|
||||||
assert events_first[0].data["guid"] == "DEDUP-TEST-001"
|
|
||||||
|
|
||||||
# Verify mark_published was called
|
|
||||||
assert adapter.is_published("DEDUP-TEST-001") is True
|
|
||||||
|
|
||||||
# Second poll: same item should be skipped (dedup)
|
|
||||||
with patch.object(
|
|
||||||
adapter._session, "get",
|
|
||||||
return_value=AsyncMock(
|
|
||||||
__aenter__=AsyncMock(return_value=make_mock_response()),
|
|
||||||
__aexit__=AsyncMock()
|
|
||||||
)
|
|
||||||
):
|
|
||||||
events_second = [e async for e in adapter.poll()]
|
|
||||||
|
|
||||||
assert len(events_second) == 0 # Dedup prevents re-yield
|
|
||||||
|
|
||||||
await adapter.shutdown()
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_conditional_304_yields_zero(
|
|
||||||
self, mock_config_no_region: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
|
||||||
):
|
|
||||||
"""HTTP 304 Not Modified returns empty list and yields zero events."""
|
|
||||||
from central.adapters.inciweb import InciWebAdapter
|
|
||||||
|
|
||||||
adapter = InciWebAdapter(mock_config_no_region, mock_config_store, cursor_db_path)
|
|
||||||
await adapter.startup()
|
|
||||||
|
|
||||||
# Mock 304 response
|
|
||||||
mock_response = AsyncMock()
|
|
||||||
mock_response.status = 304
|
|
||||||
mock_response.raise_for_status = MagicMock()
|
|
||||||
|
|
||||||
with patch.object(
|
|
||||||
adapter._session, "get",
|
|
||||||
return_value=AsyncMock(
|
|
||||||
__aenter__=AsyncMock(return_value=mock_response),
|
|
||||||
__aexit__=AsyncMock()
|
|
||||||
)
|
|
||||||
):
|
|
||||||
events = [e async for e in adapter.poll()]
|
|
||||||
|
|
||||||
assert len(events) == 0
|
|
||||||
|
|
||||||
await adapter.shutdown()
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_conditional_headers_sent_after_first_poll(
|
|
||||||
self, mock_config_no_region: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
|
||||||
):
|
|
||||||
"""Conditional fetch headers sent on second poll after first captures them."""
|
|
||||||
from central.adapters.inciweb import InciWebAdapter
|
|
||||||
|
|
||||||
adapter = InciWebAdapter(mock_config_no_region, mock_config_store, cursor_db_path)
|
|
||||||
await adapter.startup()
|
|
||||||
|
|
||||||
# First response with Last-Modified and ETag
|
|
||||||
first_response = AsyncMock()
|
|
||||||
first_response.status = 200
|
|
||||||
first_response.raise_for_status = MagicMock()
|
|
||||||
first_response.text = AsyncMock(return_value="""<?xml version="1.0"?>
|
|
||||||
<rss version="2.0"><channel><title>Test</title></channel></rss>""")
|
|
||||||
first_response.headers = {
|
|
||||||
"Last-Modified": "Tue, 19 May 2026 03:00:00 GMT",
|
|
||||||
"ETag": "\"abc123\"",
|
|
||||||
}
|
|
||||||
|
|
||||||
# Track headers sent on second request
|
|
||||||
captured_headers = {}
|
|
||||||
|
|
||||||
def capture_get(*args, **kwargs):
|
|
||||||
captured_headers.update(kwargs.get("headers", {}))
|
|
||||||
second_response = AsyncMock()
|
|
||||||
second_response.status = 304
|
|
||||||
second_response.raise_for_status = MagicMock()
|
|
||||||
return AsyncMock(
|
|
||||||
__aenter__=AsyncMock(return_value=second_response),
|
|
||||||
__aexit__=AsyncMock()
|
|
||||||
)
|
|
||||||
|
|
||||||
# First poll
|
|
||||||
with patch.object(
|
|
||||||
adapter._session, "get",
|
|
||||||
return_value=AsyncMock(
|
|
||||||
__aenter__=AsyncMock(return_value=first_response),
|
|
||||||
__aexit__=AsyncMock()
|
|
||||||
)
|
|
||||||
):
|
|
||||||
[e async for e in adapter.poll()]
|
|
||||||
|
|
||||||
# Verify adapter captured the headers
|
|
||||||
assert adapter._last_modified == "Tue, 19 May 2026 03:00:00 GMT"
|
|
||||||
assert adapter._etag == "\"abc123\""
|
|
||||||
|
|
||||||
# Second poll with header capture
|
|
||||||
with patch.object(adapter._session, "get", side_effect=capture_get):
|
|
||||||
[e async for e in adapter.poll()]
|
|
||||||
|
|
||||||
# Verify conditional headers were sent
|
|
||||||
assert captured_headers.get("If-Modified-Since") == "Tue, 19 May 2026 03:00:00 GMT"
|
|
||||||
assert captured_headers.get("If-None-Match") == "\"abc123\""
|
|
||||||
|
|
||||||
await adapter.shutdown()
|
|
||||||
|
|
@ -21,7 +21,6 @@ class TestRegionPickerInTemplate:
|
||||||
|
|
||||||
mock_request = MagicMock()
|
mock_request = MagicMock()
|
||||||
mock_request.state.operator = MagicMock(id=1, username="testop")
|
mock_request.state.operator = MagicMock(id=1, username="testop")
|
||||||
mock_request.state.csrf_token = "test_csrf"
|
|
||||||
|
|
||||||
mock_conn = AsyncMock()
|
mock_conn = AsyncMock()
|
||||||
mock_conn.fetchrow.side_effect = [
|
mock_conn.fetchrow.side_effect = [
|
||||||
|
|
@ -36,13 +35,13 @@ class TestRegionPickerInTemplate:
|
||||||
},
|
},
|
||||||
"paused_at": None,
|
"paused_at": None,
|
||||||
"updated_at": None,
|
"updated_at": None,
|
||||||
"last_error": None,
|
|
||||||
},
|
},
|
||||||
{ # System settings row
|
{ # System settings row
|
||||||
"map_tile_url": "https://tile.example.com/{z}/{x}/{y}.png",
|
"map_tile_url": "https://tile.example.com/{z}/{x}/{y}.png",
|
||||||
"map_attribution": "Test Attribution",
|
"map_attribution": "Test Attribution",
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
|
mock_conn.fetch.return_value = [{"alias": "firms"}]
|
||||||
|
|
||||||
mock_pool = MagicMock()
|
mock_pool = MagicMock()
|
||||||
mock_pool.acquire.return_value.__aenter__ = AsyncMock(return_value=mock_conn)
|
mock_pool.acquire.return_value.__aenter__ = AsyncMock(return_value=mock_conn)
|
||||||
|
|
@ -81,26 +80,27 @@ class TestRegionValidation:
|
||||||
"csrf_token": "test_csrf_token",
|
"csrf_token": "test_csrf_token",
|
||||||
"cadence_s": "300",
|
"cadence_s": "300",
|
||||||
"api_key_alias": "firms",
|
"api_key_alias": "firms",
|
||||||
"satellites": "VIIRS_SNPP_NRT",
|
|
||||||
"region_north": "45.0",
|
"region_north": "45.0",
|
||||||
"region_south": "35.0",
|
"region_south": "35.0",
|
||||||
"region_east": "-100.0",
|
"region_east": "-100.0",
|
||||||
"region_west": "-120.0",
|
"region_west": "-120.0",
|
||||||
}.get(k, d)
|
}.get(k, d)
|
||||||
mock_form.getlist.return_value = []
|
mock_form.getlist.return_value = ["VIIRS_SNPP_NRT"]
|
||||||
mock_form.__contains__ = lambda self, k: k == "enabled"
|
mock_form.__contains__ = lambda self, k: k == "enabled"
|
||||||
mock_request.form = AsyncMock(return_value=mock_form)
|
mock_request.form = AsyncMock(return_value=mock_form)
|
||||||
|
|
||||||
mock_conn = AsyncMock()
|
mock_conn = AsyncMock()
|
||||||
mock_conn.fetchrow.return_value = {
|
mock_conn.fetchrow.side_effect = [
|
||||||
"name": "firms",
|
{ # Adapter row
|
||||||
"enabled": True,
|
"name": "firms",
|
||||||
"cadence_s": 300,
|
"enabled": True,
|
||||||
"settings": {"api_key_alias": "firms", "satellites": ["VIIRS_SNPP_NRT"], "region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}},
|
"cadence_s": 300,
|
||||||
"paused_at": None,
|
"settings": {"api_key_alias": "firms", "region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}},
|
||||||
"updated_at": None,
|
"paused_at": None,
|
||||||
"last_error": None,
|
"updated_at": None,
|
||||||
}
|
},
|
||||||
|
{"id": 1}, # api_key exists check
|
||||||
|
]
|
||||||
mock_conn.execute = AsyncMock()
|
mock_conn.execute = AsyncMock()
|
||||||
|
|
||||||
mock_pool = MagicMock()
|
mock_pool = MagicMock()
|
||||||
|
|
@ -139,13 +139,12 @@ class TestRegionValidation:
|
||||||
"csrf_token": "test_csrf_token",
|
"csrf_token": "test_csrf_token",
|
||||||
"cadence_s": "300",
|
"cadence_s": "300",
|
||||||
"api_key_alias": "firms",
|
"api_key_alias": "firms",
|
||||||
"satellites": "VIIRS_SNPP_NRT",
|
|
||||||
"region_north": "30.0", # Less than south!
|
"region_north": "30.0", # Less than south!
|
||||||
"region_south": "35.0",
|
"region_south": "35.0",
|
||||||
"region_east": "-100.0",
|
"region_east": "-100.0",
|
||||||
"region_west": "-120.0",
|
"region_west": "-120.0",
|
||||||
}.get(k, d)
|
}.get(k, d)
|
||||||
mock_form.getlist.return_value = []
|
mock_form.getlist.return_value = ["VIIRS_SNPP_NRT"]
|
||||||
mock_form.__contains__ = lambda self, k: k == "enabled"
|
mock_form.__contains__ = lambda self, k: k == "enabled"
|
||||||
mock_request.form = AsyncMock(return_value=mock_form)
|
mock_request.form = AsyncMock(return_value=mock_form)
|
||||||
|
|
||||||
|
|
@ -155,13 +154,14 @@ class TestRegionValidation:
|
||||||
"name": "firms",
|
"name": "firms",
|
||||||
"enabled": True,
|
"enabled": True,
|
||||||
"cadence_s": 300,
|
"cadence_s": 300,
|
||||||
"settings": {"api_key_alias": "firms", "satellites": ["VIIRS_SNPP_NRT"], "region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}},
|
"settings": {"api_key_alias": "firms", "region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}},
|
||||||
"paused_at": None,
|
"paused_at": None,
|
||||||
"updated_at": None,
|
"updated_at": None,
|
||||||
"last_error": None,
|
|
||||||
},
|
},
|
||||||
|
{"id": 1}, # api_key exists
|
||||||
{"map_tile_url": None, "map_attribution": None}, # system settings for re-render
|
{"map_tile_url": None, "map_attribution": None}, # system settings for re-render
|
||||||
]
|
]
|
||||||
|
mock_conn.fetch.return_value = [{"alias": "firms"}]
|
||||||
|
|
||||||
mock_pool = MagicMock()
|
mock_pool = MagicMock()
|
||||||
mock_pool.acquire.return_value.__aenter__ = AsyncMock(return_value=mock_conn)
|
mock_pool.acquire.return_value.__aenter__ = AsyncMock(return_value=mock_conn)
|
||||||
|
|
@ -195,13 +195,12 @@ class TestRegionValidation:
|
||||||
"csrf_token": "test_csrf_token",
|
"csrf_token": "test_csrf_token",
|
||||||
"cadence_s": "300",
|
"cadence_s": "300",
|
||||||
"api_key_alias": "firms",
|
"api_key_alias": "firms",
|
||||||
"satellites": "VIIRS_SNPP_NRT",
|
|
||||||
"region_north": "45.0",
|
"region_north": "45.0",
|
||||||
"region_south": "35.0",
|
"region_south": "35.0",
|
||||||
"region_east": "-130.0", # Less than west!
|
"region_east": "-130.0", # Less than west!
|
||||||
"region_west": "-120.0",
|
"region_west": "-120.0",
|
||||||
}.get(k, d)
|
}.get(k, d)
|
||||||
mock_form.getlist.return_value = []
|
mock_form.getlist.return_value = ["VIIRS_SNPP_NRT"]
|
||||||
mock_form.__contains__ = lambda self, k: k == "enabled"
|
mock_form.__contains__ = lambda self, k: k == "enabled"
|
||||||
mock_request.form = AsyncMock(return_value=mock_form)
|
mock_request.form = AsyncMock(return_value=mock_form)
|
||||||
|
|
||||||
|
|
@ -211,13 +210,14 @@ class TestRegionValidation:
|
||||||
"name": "firms",
|
"name": "firms",
|
||||||
"enabled": True,
|
"enabled": True,
|
||||||
"cadence_s": 300,
|
"cadence_s": 300,
|
||||||
"settings": {"api_key_alias": "firms", "satellites": ["VIIRS_SNPP_NRT"], "region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}},
|
"settings": {"api_key_alias": "firms", "region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}},
|
||||||
"paused_at": None,
|
"paused_at": None,
|
||||||
"updated_at": None,
|
"updated_at": None,
|
||||||
"last_error": None,
|
|
||||||
},
|
},
|
||||||
|
{"id": 1},
|
||||||
{"map_tile_url": None, "map_attribution": None},
|
{"map_tile_url": None, "map_attribution": None},
|
||||||
]
|
]
|
||||||
|
mock_conn.fetch.return_value = [{"alias": "firms"}]
|
||||||
|
|
||||||
mock_pool = MagicMock()
|
mock_pool = MagicMock()
|
||||||
mock_pool.acquire.return_value.__aenter__ = AsyncMock(return_value=mock_conn)
|
mock_pool.acquire.return_value.__aenter__ = AsyncMock(return_value=mock_conn)
|
||||||
|
|
@ -251,13 +251,12 @@ class TestRegionValidation:
|
||||||
"csrf_token": "test_csrf_token",
|
"csrf_token": "test_csrf_token",
|
||||||
"cadence_s": "300",
|
"cadence_s": "300",
|
||||||
"api_key_alias": "firms",
|
"api_key_alias": "firms",
|
||||||
"satellites": "VIIRS_SNPP_NRT",
|
|
||||||
"region_north": "95.0", # > 90!
|
"region_north": "95.0", # > 90!
|
||||||
"region_south": "35.0",
|
"region_south": "35.0",
|
||||||
"region_east": "-100.0",
|
"region_east": "-100.0",
|
||||||
"region_west": "-120.0",
|
"region_west": "-120.0",
|
||||||
}.get(k, d)
|
}.get(k, d)
|
||||||
mock_form.getlist.return_value = []
|
mock_form.getlist.return_value = ["VIIRS_SNPP_NRT"]
|
||||||
mock_form.__contains__ = lambda self, k: k == "enabled"
|
mock_form.__contains__ = lambda self, k: k == "enabled"
|
||||||
mock_request.form = AsyncMock(return_value=mock_form)
|
mock_request.form = AsyncMock(return_value=mock_form)
|
||||||
|
|
||||||
|
|
@ -267,13 +266,14 @@ class TestRegionValidation:
|
||||||
"name": "firms",
|
"name": "firms",
|
||||||
"enabled": True,
|
"enabled": True,
|
||||||
"cadence_s": 300,
|
"cadence_s": 300,
|
||||||
"settings": {"api_key_alias": "firms", "satellites": ["VIIRS_SNPP_NRT"], "region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}},
|
"settings": {"api_key_alias": "firms", "region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}},
|
||||||
"paused_at": None,
|
"paused_at": None,
|
||||||
"updated_at": None,
|
"updated_at": None,
|
||||||
"last_error": None,
|
|
||||||
},
|
},
|
||||||
|
{"id": 1},
|
||||||
{"map_tile_url": None, "map_attribution": None},
|
{"map_tile_url": None, "map_attribution": None},
|
||||||
]
|
]
|
||||||
|
mock_conn.fetch.return_value = [{"alias": "firms"}]
|
||||||
|
|
||||||
mock_pool = MagicMock()
|
mock_pool = MagicMock()
|
||||||
mock_pool.acquire.return_value.__aenter__ = AsyncMock(return_value=mock_conn)
|
mock_pool.acquire.return_value.__aenter__ = AsyncMock(return_value=mock_conn)
|
||||||
|
|
@ -310,30 +310,30 @@ class TestRegionAuditLog:
|
||||||
"csrf_token": "test_csrf_token",
|
"csrf_token": "test_csrf_token",
|
||||||
"cadence_s": "300",
|
"cadence_s": "300",
|
||||||
"api_key_alias": "firms",
|
"api_key_alias": "firms",
|
||||||
"satellites": "VIIRS_SNPP_NRT",
|
|
||||||
"region_north": "45.0",
|
"region_north": "45.0",
|
||||||
"region_south": "35.0",
|
"region_south": "35.0",
|
||||||
"region_east": "-100.0",
|
"region_east": "-100.0",
|
||||||
"region_west": "-120.0",
|
"region_west": "-120.0",
|
||||||
}.get(k, d)
|
}.get(k, d)
|
||||||
mock_form.getlist.return_value = []
|
mock_form.getlist.return_value = ["VIIRS_SNPP_NRT"]
|
||||||
mock_form.__contains__ = lambda self, k: k == "enabled"
|
mock_form.__contains__ = lambda self, k: k == "enabled"
|
||||||
mock_request.form = AsyncMock(return_value=mock_form)
|
mock_request.form = AsyncMock(return_value=mock_form)
|
||||||
|
|
||||||
mock_conn = AsyncMock()
|
mock_conn = AsyncMock()
|
||||||
mock_conn.fetchrow.return_value = {
|
mock_conn.fetchrow.side_effect = [
|
||||||
"name": "firms",
|
{
|
||||||
"enabled": True,
|
"name": "firms",
|
||||||
"cadence_s": 300,
|
"enabled": True,
|
||||||
"settings": {
|
"cadence_s": 300,
|
||||||
"api_key_alias": "firms",
|
"settings": {
|
||||||
"satellites": ["VIIRS_SNPP_NRT"],
|
"api_key_alias": "firms",
|
||||||
"region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}
|
"region": {"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}
|
||||||
|
},
|
||||||
|
"paused_at": None,
|
||||||
|
"updated_at": None,
|
||||||
},
|
},
|
||||||
"paused_at": None,
|
{"id": 1},
|
||||||
"updated_at": None,
|
]
|
||||||
"last_error": None,
|
|
||||||
}
|
|
||||||
mock_conn.execute = AsyncMock()
|
mock_conn.execute = AsyncMock()
|
||||||
|
|
||||||
mock_pool = MagicMock()
|
mock_pool = MagicMock()
|
||||||
|
|
|
||||||
|
|
@ -1,361 +0,0 @@
|
||||||
"""Tests for requires_api_key enforcement."""
|
|
||||||
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
from pathlib import Path
|
|
||||||
from unittest.mock import MagicMock, AsyncMock, patch
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from central.config_models import AdapterConfig
|
|
||||||
|
|
||||||
|
|
||||||
class TestConfigStoreSetAdapterLastError:
|
|
||||||
"""Tests for ConfigStore.set_adapter_last_error method."""
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_set_adapter_last_error_updates_row(self):
|
|
||||||
"""set_adapter_last_error should update the last_error column."""
|
|
||||||
from central.config_store import ConfigStore
|
|
||||||
|
|
||||||
mock_pool = MagicMock()
|
|
||||||
mock_conn = MagicMock()
|
|
||||||
mock_conn.execute = AsyncMock()
|
|
||||||
mock_conn.__aenter__ = AsyncMock(return_value=mock_conn)
|
|
||||||
mock_conn.__aexit__ = AsyncMock()
|
|
||||||
mock_pool.acquire = MagicMock(return_value=mock_conn)
|
|
||||||
|
|
||||||
config_store = ConfigStore.__new__(ConfigStore)
|
|
||||||
config_store._pool = mock_pool
|
|
||||||
|
|
||||||
await config_store.set_adapter_last_error("firms", "missing api key: firms")
|
|
||||||
|
|
||||||
mock_conn.execute.assert_called_once()
|
|
||||||
call_args = mock_conn.execute.call_args[0]
|
|
||||||
assert "UPDATE config.adapters SET last_error" in call_args[0]
|
|
||||||
assert call_args[1] == "missing api key: firms"
|
|
||||||
assert call_args[2] == "firms"
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_clear_adapter_last_error(self):
|
|
||||||
"""set_adapter_last_error with None should clear the error."""
|
|
||||||
from central.config_store import ConfigStore
|
|
||||||
|
|
||||||
mock_pool = MagicMock()
|
|
||||||
mock_conn = MagicMock()
|
|
||||||
mock_conn.execute = AsyncMock()
|
|
||||||
mock_conn.__aenter__ = AsyncMock(return_value=mock_conn)
|
|
||||||
mock_conn.__aexit__ = AsyncMock()
|
|
||||||
mock_pool.acquire = MagicMock(return_value=mock_conn)
|
|
||||||
|
|
||||||
config_store = ConfigStore.__new__(ConfigStore)
|
|
||||||
config_store._pool = mock_pool
|
|
||||||
|
|
||||||
await config_store.set_adapter_last_error("firms", None)
|
|
||||||
|
|
||||||
mock_conn.execute.assert_called_once()
|
|
||||||
call_args = mock_conn.execute.call_args[0]
|
|
||||||
assert call_args[1] is None
|
|
||||||
assert call_args[2] == "firms"
|
|
||||||
|
|
||||||
|
|
||||||
class TestRoutesApiKeyMissing:
|
|
||||||
"""Tests for routes api_key_missing computation."""
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_adapters_list_includes_api_key_missing_flag(self):
|
|
||||||
"""adapters_list should compute api_key_missing for each adapter."""
|
|
||||||
from central.gui.routes import adapters_list
|
|
||||||
|
|
||||||
mock_request = MagicMock()
|
|
||||||
mock_request.state = MagicMock()
|
|
||||||
mock_request.state.operator = {"username": "test"}
|
|
||||||
mock_request.state.csrf_token = "test_token"
|
|
||||||
|
|
||||||
mock_pool = MagicMock()
|
|
||||||
mock_conn = MagicMock()
|
|
||||||
mock_conn.fetch = AsyncMock(return_value=[
|
|
||||||
{"name": "firms", "enabled": False, "cadence_s": 300, "settings": {}, "paused_at": None, "updated_at": None, "last_error": None},
|
|
||||||
])
|
|
||||||
mock_conn.fetchval = AsyncMock(return_value=None) # No API key exists
|
|
||||||
mock_conn.__aenter__ = AsyncMock(return_value=mock_conn)
|
|
||||||
mock_conn.__aexit__ = AsyncMock()
|
|
||||||
mock_pool.acquire = MagicMock(return_value=mock_conn)
|
|
||||||
|
|
||||||
# Mock adapter class with requires_api_key
|
|
||||||
mock_firms_cls = MagicMock()
|
|
||||||
mock_firms_cls.requires_api_key = "firms"
|
|
||||||
mock_firms_cls.display_name = "FIRMS"
|
|
||||||
|
|
||||||
with patch("central.gui.routes._get_templates") as mock_templates:
|
|
||||||
with patch("central.gui.routes.get_pool", return_value=mock_pool):
|
|
||||||
with patch("central.gui.routes._adapter_classes", return_value={"firms": mock_firms_cls}):
|
|
||||||
mock_template_response = MagicMock()
|
|
||||||
mock_templates.return_value.TemplateResponse = MagicMock(return_value=mock_template_response)
|
|
||||||
|
|
||||||
await adapters_list(mock_request)
|
|
||||||
|
|
||||||
# Check the context passed to template
|
|
||||||
call_kwargs = mock_templates.return_value.TemplateResponse.call_args[1]
|
|
||||||
adapters = call_kwargs["context"]["adapters"]
|
|
||||||
|
|
||||||
assert len(adapters) == 1
|
|
||||||
assert adapters[0]["api_key_missing"] is True
|
|
||||||
assert adapters[0]["requires_api_key_alias"] == "firms"
|
|
||||||
|
|
||||||
|
|
||||||
class TestAdapterClassRequiresApiKey:
|
|
||||||
"""Tests for adapter class requires_api_key attribute."""
|
|
||||||
|
|
||||||
def test_firms_adapter_requires_api_key(self):
|
|
||||||
"""FIRMS adapter should declare requires_api_key."""
|
|
||||||
from central.adapters.firms import FIRMSAdapter
|
|
||||||
assert FIRMSAdapter.requires_api_key == "firms"
|
|
||||||
|
|
||||||
def test_nws_adapter_no_requires_api_key(self):
|
|
||||||
"""NWS adapter should not require an API key."""
|
|
||||||
from central.adapters.nws import NWSAdapter
|
|
||||||
assert NWSAdapter.requires_api_key is None
|
|
||||||
|
|
||||||
def test_usgs_quake_adapter_no_requires_api_key(self):
|
|
||||||
"""USGS Quake adapter should not require an API key."""
|
|
||||||
from central.adapters.usgs_quake import USGSQuakeAdapter
|
|
||||||
assert USGSQuakeAdapter.requires_api_key is None
|
|
||||||
|
|
||||||
|
|
||||||
class TestSupervisorApiKeyPrecondition:
|
|
||||||
"""Tests for supervisor API key precondition check in _start_adapter."""
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_start_adapter_refuses_when_required_key_missing(self, tmp_path: Path):
|
|
||||||
"""Adapter with requires_api_key but missing key should not start."""
|
|
||||||
from central.supervisor import Supervisor
|
|
||||||
from central.adapters.firms import FIRMSAdapter
|
|
||||||
|
|
||||||
# Create mock config store
|
|
||||||
mock_config_store = MagicMock()
|
|
||||||
mock_config_store.get_api_key = AsyncMock(return_value=None) # Key missing
|
|
||||||
mock_config_store.set_adapter_last_error = AsyncMock()
|
|
||||||
|
|
||||||
# Create mock NATS
|
|
||||||
mock_nats = MagicMock()
|
|
||||||
mock_nats.publish = AsyncMock()
|
|
||||||
|
|
||||||
# Build supervisor with FIRMS adapter
|
|
||||||
supervisor = Supervisor.__new__(Supervisor)
|
|
||||||
supervisor._config_store = mock_config_store
|
|
||||||
supervisor._adapters = {"firms": FIRMSAdapter}
|
|
||||||
supervisor._adapter_states = {}
|
|
||||||
supervisor._nats = mock_nats
|
|
||||||
supervisor._cursor_db_path = tmp_path / "cursors.db"
|
|
||||||
supervisor._log = MagicMock()
|
|
||||||
|
|
||||||
config = AdapterConfig(
|
|
||||||
name="firms",
|
|
||||||
enabled=True,
|
|
||||||
cadence_s=300,
|
|
||||||
settings={"api_key_alias": "firms", "satellites": ["VIIRS_SNPP_NRT"]},
|
|
||||||
updated_at=datetime.now(timezone.utc),
|
|
||||||
)
|
|
||||||
|
|
||||||
await supervisor._start_adapter(config)
|
|
||||||
|
|
||||||
# Should have checked for key
|
|
||||||
mock_config_store.get_api_key.assert_called_once_with("firms")
|
|
||||||
|
|
||||||
# Should have set error
|
|
||||||
mock_config_store.set_adapter_last_error.assert_called_once()
|
|
||||||
args = mock_config_store.set_adapter_last_error.call_args[0]
|
|
||||||
assert args[0] == "firms"
|
|
||||||
assert "missing api key" in args[1].lower()
|
|
||||||
|
|
||||||
# Should NOT have created adapter state (adapter did not start)
|
|
||||||
assert "firms" not in supervisor._adapter_states
|
|
||||||
|
|
||||||
# Should NOT have published to NATS
|
|
||||||
mock_nats.publish.assert_not_called()
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_start_adapter_succeeds_after_key_added_and_clears_last_error(self, tmp_path: Path):
|
|
||||||
"""Adapter with requires_api_key and key present should start and clear last_error."""
|
|
||||||
from central.supervisor import Supervisor
|
|
||||||
from central.adapters.firms import FIRMSAdapter
|
|
||||||
|
|
||||||
# Create mock config store with key present
|
|
||||||
mock_config_store = MagicMock()
|
|
||||||
mock_config_store.get_api_key = AsyncMock(return_value="encrypted-firms-key")
|
|
||||||
mock_config_store.set_adapter_last_error = AsyncMock()
|
|
||||||
|
|
||||||
# Create mock NATS
|
|
||||||
mock_nats = MagicMock()
|
|
||||||
mock_nats.publish = AsyncMock()
|
|
||||||
|
|
||||||
# Build supervisor with FIRMS adapter
|
|
||||||
supervisor = Supervisor.__new__(Supervisor)
|
|
||||||
supervisor._config_store = mock_config_store
|
|
||||||
supervisor._adapters = {"firms": FIRMSAdapter}
|
|
||||||
supervisor._adapter_states = {}
|
|
||||||
supervisor._nats = mock_nats
|
|
||||||
supervisor._cursor_db_path = tmp_path / "cursors.db"
|
|
||||||
supervisor._log = MagicMock()
|
|
||||||
|
|
||||||
config = AdapterConfig(
|
|
||||||
name="firms",
|
|
||||||
enabled=True,
|
|
||||||
cadence_s=300,
|
|
||||||
settings={"api_key_alias": "firms", "satellites": ["VIIRS_SNPP_NRT"]},
|
|
||||||
updated_at=datetime.now(timezone.utc),
|
|
||||||
)
|
|
||||||
|
|
||||||
# Mock the adapter instantiation to avoid actual HTTP calls
|
|
||||||
with patch.object(FIRMSAdapter, "__init__", return_value=None):
|
|
||||||
with patch.object(FIRMSAdapter, "startup", new_callable=AsyncMock):
|
|
||||||
await supervisor._start_adapter(config)
|
|
||||||
|
|
||||||
# Should have checked for key
|
|
||||||
mock_config_store.get_api_key.assert_called_once_with("firms")
|
|
||||||
|
|
||||||
# Should have cleared any stale error (called with None)
|
|
||||||
mock_config_store.set_adapter_last_error.assert_called_once_with("firms", None)
|
|
||||||
|
|
||||||
# Should have created adapter state
|
|
||||||
assert "firms" in supervisor._adapter_states
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_start_adapter_does_not_check_when_no_requires_api_key(self, tmp_path: Path):
|
|
||||||
"""Adapter without requires_api_key should skip the API key check."""
|
|
||||||
from central.supervisor import Supervisor
|
|
||||||
from central.adapters.nws import NWSAdapter
|
|
||||||
|
|
||||||
# Create mock config store
|
|
||||||
mock_config_store = MagicMock()
|
|
||||||
mock_config_store.get_api_key = AsyncMock()
|
|
||||||
mock_config_store.set_adapter_last_error = AsyncMock()
|
|
||||||
|
|
||||||
# Create mock NATS
|
|
||||||
mock_nats = MagicMock()
|
|
||||||
mock_nats.publish = AsyncMock()
|
|
||||||
|
|
||||||
# Build supervisor with NWS adapter (no requires_api_key)
|
|
||||||
supervisor = Supervisor.__new__(Supervisor)
|
|
||||||
supervisor._config_store = mock_config_store
|
|
||||||
supervisor._adapters = {"nws": NWSAdapter}
|
|
||||||
supervisor._adapter_states = {}
|
|
||||||
supervisor._nats = mock_nats
|
|
||||||
supervisor._cursor_db_path = tmp_path / "cursors.db"
|
|
||||||
supervisor._log = MagicMock()
|
|
||||||
|
|
||||||
config = AdapterConfig(
|
|
||||||
name="nws",
|
|
||||||
enabled=True,
|
|
||||||
cadence_s=300,
|
|
||||||
settings={"contact_email": "test@example.com"},
|
|
||||||
updated_at=datetime.now(timezone.utc),
|
|
||||||
)
|
|
||||||
|
|
||||||
# Mock the adapter instantiation to avoid actual HTTP calls
|
|
||||||
with patch.object(NWSAdapter, "__init__", return_value=None):
|
|
||||||
with patch.object(NWSAdapter, "startup", new_callable=AsyncMock):
|
|
||||||
await supervisor._start_adapter(config)
|
|
||||||
|
|
||||||
# Should NOT have called get_api_key (no requires_api_key)
|
|
||||||
mock_config_store.get_api_key.assert_not_called()
|
|
||||||
|
|
||||||
# Should have cleared stale error (routine clear)
|
|
||||||
mock_config_store.set_adapter_last_error.assert_called_once_with("nws", None)
|
|
||||||
|
|
||||||
# Should have created adapter state
|
|
||||||
assert "nws" in supervisor._adapter_states
|
|
||||||
|
|
||||||
|
|
||||||
class TestAdaptersEditSubmitErrorRerender:
|
|
||||||
"""Tests for adapters_edit_submit error re-render including api_key_missing."""
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_adapters_edit_submit_error_rerender_includes_api_key_missing(self):
|
|
||||||
"""Error re-render on /adapters/firms should include api_key_missing in context."""
|
|
||||||
from central.gui.routes import adapters_edit_submit
|
|
||||||
from pydantic import BaseModel
|
|
||||||
from typing import Literal
|
|
||||||
|
|
||||||
mock_request = MagicMock()
|
|
||||||
mock_request.state = MagicMock()
|
|
||||||
mock_request.state.operator = {"username": "test"}
|
|
||||||
mock_request.state.csrf_token = "test_token"
|
|
||||||
|
|
||||||
# Mock form with invalid cadence (below minimum of 10)
|
|
||||||
mock_form = MagicMock()
|
|
||||||
def form_get(k, d=""):
|
|
||||||
values = {
|
|
||||||
"csrf_token": "test_token",
|
|
||||||
"cadence_s": "5", # Invalid - below minimum
|
|
||||||
"api_key_alias": "firms",
|
|
||||||
"satellites": "",
|
|
||||||
"region_north": "",
|
|
||||||
"region_south": "",
|
|
||||||
"region_east": "",
|
|
||||||
"region_west": "",
|
|
||||||
}
|
|
||||||
return values.get(k, d)
|
|
||||||
mock_form.get = MagicMock(side_effect=form_get)
|
|
||||||
mock_form.getlist = MagicMock(return_value=["VIIRS_SNPP_NRT"])
|
|
||||||
mock_form.__contains__ = lambda self, k: k == "enabled"
|
|
||||||
mock_request.form = AsyncMock(return_value=mock_form)
|
|
||||||
|
|
||||||
mock_pool = MagicMock()
|
|
||||||
mock_conn = MagicMock()
|
|
||||||
mock_conn.fetchrow = AsyncMock(side_effect=[
|
|
||||||
# First call: adapter row
|
|
||||||
{
|
|
||||||
"name": "firms",
|
|
||||||
"enabled": False,
|
|
||||||
"cadence_s": 300,
|
|
||||||
"settings": {"api_key_alias": "firms", "satellites": ["VIIRS_SNPP_NRT"]},
|
|
||||||
"paused_at": None,
|
|
||||||
"updated_at": datetime.now(timezone.utc),
|
|
||||||
"last_error": None,
|
|
||||||
},
|
|
||||||
# Second call: system row for map tiles
|
|
||||||
{"map_tile_url": "https://tile.example.com/{z}/{x}/{y}.png", "map_attribution": "Test"},
|
|
||||||
])
|
|
||||||
mock_conn.fetchval = AsyncMock(return_value=None) # No API key exists
|
|
||||||
mock_conn.fetch = AsyncMock(return_value=[]) # No API keys
|
|
||||||
mock_conn.__aenter__ = AsyncMock(return_value=mock_conn)
|
|
||||||
mock_conn.__aexit__ = AsyncMock()
|
|
||||||
mock_pool.acquire = MagicMock(return_value=mock_conn)
|
|
||||||
|
|
||||||
# Mock FIRMS adapter class
|
|
||||||
class MockFIRMSSettings(BaseModel):
|
|
||||||
api_key_alias: str = ""
|
|
||||||
satellites: list[Literal["VIIRS_SNPP_NRT", "VIIRS_NOAA20_NRT"]] = []
|
|
||||||
|
|
||||||
mock_firms_cls = MagicMock()
|
|
||||||
mock_firms_cls.requires_api_key = "firms"
|
|
||||||
mock_firms_cls.api_key_field = "api_key_alias"
|
|
||||||
mock_firms_cls.display_name = "FIRMS"
|
|
||||||
mock_firms_cls.description = "Fire detection"
|
|
||||||
mock_firms_cls.settings_schema = MockFIRMSSettings
|
|
||||||
|
|
||||||
with patch("central.gui.routes._get_templates") as mock_templates:
|
|
||||||
with patch("central.gui.routes.get_pool", return_value=mock_pool):
|
|
||||||
with patch("central.gui.routes._adapter_classes", return_value={"firms": mock_firms_cls}):
|
|
||||||
with patch("central.gui.routes.describe_fields", return_value=[]):
|
|
||||||
mock_template_response = MagicMock()
|
|
||||||
mock_template_response.status_code = 200
|
|
||||||
mock_templates.return_value.TemplateResponse = MagicMock(return_value=mock_template_response)
|
|
||||||
|
|
||||||
result = await adapters_edit_submit(mock_request, "firms")
|
|
||||||
|
|
||||||
# Verify TemplateResponse was called (error re-render)
|
|
||||||
assert mock_templates.return_value.TemplateResponse.called
|
|
||||||
|
|
||||||
# Check the context passed to template
|
|
||||||
call_kwargs = mock_templates.return_value.TemplateResponse.call_args[1]
|
|
||||||
context = call_kwargs["context"]
|
|
||||||
|
|
||||||
# Should have errors (invalid cadence)
|
|
||||||
assert context.get("errors") is not None
|
|
||||||
assert "cadence_s" in context["errors"]
|
|
||||||
|
|
||||||
# Should include api_key_missing
|
|
||||||
assert context["api_key_missing"] is True
|
|
||||||
assert context["requires_api_key_alias"] == "firms"
|
|
||||||
|
|
@ -94,8 +94,6 @@ class MockConfigSource:
|
||||||
class MockNWSAdapter:
|
class MockNWSAdapter:
|
||||||
"""Mock NWSAdapter that tracks poll calls and allows control."""
|
"""Mock NWSAdapter that tracks poll calls and allows control."""
|
||||||
|
|
||||||
requires_api_key = None # Mock adapters don't require API keys
|
|
||||||
|
|
||||||
def __init__(self, config, config_store, cursor_db_path) -> None:
|
def __init__(self, config, config_store, cursor_db_path) -> None:
|
||||||
self.config = config
|
self.config = config
|
||||||
self._config_store = config_store
|
self._config_store = config_store
|
||||||
|
|
@ -154,8 +152,6 @@ def mock_config_store():
|
||||||
store = MagicMock()
|
store = MagicMock()
|
||||||
store.list_streams = AsyncMock(return_value=[])
|
store.list_streams = AsyncMock(return_value=[])
|
||||||
store.get_stream = AsyncMock(return_value=None)
|
store.get_stream = AsyncMock(return_value=None)
|
||||||
store.set_adapter_last_error = AsyncMock()
|
|
||||||
store.get_api_key = AsyncMock(return_value=None)
|
|
||||||
return store
|
return store
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,339 +0,0 @@
|
||||||
"""Tests for NOAA SWPC space weather adapters."""
|
|
||||||
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
from pathlib import Path
|
|
||||||
from unittest.mock import AsyncMock, MagicMock
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from central.config_models import AdapterConfig
|
|
||||||
from central.models import Event
|
|
||||||
|
|
||||||
|
|
||||||
# Frozen fixtures captured from upstream feeds; real shapes.
|
|
||||||
SAMPLE_ALERTS = [
|
|
||||||
{
|
|
||||||
"product_id": "EF3A",
|
|
||||||
"issue_datetime": "2026-05-19 05:14:59.780",
|
|
||||||
"message": (
|
|
||||||
"Space Weather Message Code: ALTEF3\r\nSerial Number: 3689\r\n"
|
|
||||||
"Issue Time: 2026 May 19 0514 UTC\r\n\r\n"
|
|
||||||
"ALERT: Electron 2MeV Integral Flux exceeded 1000pfu \n"
|
|
||||||
"Threshold Reached: 2026 May 16 1740 UTC\n"
|
|
||||||
"Station: GOES-19\n"
|
|
||||||
),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"product_id": "K05A",
|
|
||||||
"issue_datetime": "2026-05-15 14:30:00.000",
|
|
||||||
"message": (
|
|
||||||
"Space Weather Message Code: ALTK05\r\nSerial Number: 100\r\n"
|
|
||||||
"Issue Time: 2026 May 15 1430 UTC\r\n\r\n"
|
|
||||||
"ALERT: Geomagnetic K-index of 5\n"
|
|
||||||
),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"product_id": "K07A",
|
|
||||||
"issue_datetime": "2026-05-15 18:00:00.000",
|
|
||||||
"message": "Space Weather Message Code: ALTK07\r\nSerial Number: 101\r\n",
|
|
||||||
},
|
|
||||||
]
|
|
||||||
|
|
||||||
SAMPLE_KINDEX = [
|
|
||||||
{"time_tag": "2026-05-12T00:00:00", "Kp": 0.67, "a_running": 3, "station_count": 8},
|
|
||||||
{"time_tag": "2026-05-12T03:00:00", "Kp": 5.33, "a_running": 30, "station_count": 8},
|
|
||||||
{"time_tag": "2026-05-12T06:00:00", "Kp": 8.0, "a_running": 100, "station_count": 8},
|
|
||||||
]
|
|
||||||
|
|
||||||
SAMPLE_PROTONS = [
|
|
||||||
{"time_tag": "2026-05-18T05:35:00Z", "satellite": 19, "flux": 7.09, "energy": ">=1 MeV"},
|
|
||||||
{"time_tag": "2026-05-18T05:35:00Z", "satellite": 19, "flux": 0.21, "energy": ">=10 MeV"},
|
|
||||||
{"time_tag": "2026-05-18T05:40:00Z", "satellite": 19, "flux": 7.10, "energy": ">=1 MeV"},
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def _config(name: str, cadence: int) -> AdapterConfig:
|
|
||||||
return AdapterConfig(
|
|
||||||
name=name,
|
|
||||||
enabled=True,
|
|
||||||
cadence_s=cadence,
|
|
||||||
settings={},
|
|
||||||
updated_at=datetime.now(timezone.utc),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class TestSWPCCommon:
|
|
||||||
"""Tests for swpc_common helpers."""
|
|
||||||
|
|
||||||
def test_parse_swpc_timestamp_alerts(self):
|
|
||||||
from central.adapters.swpc_common import parse_swpc_timestamp
|
|
||||||
|
|
||||||
dt = parse_swpc_timestamp("2026-05-19 05:14:59.780", "alerts")
|
|
||||||
assert dt == datetime(2026, 5, 19, 5, 14, 59, 780000, tzinfo=timezone.utc)
|
|
||||||
|
|
||||||
def test_parse_swpc_timestamp_alerts_no_fraction(self):
|
|
||||||
from central.adapters.swpc_common import parse_swpc_timestamp
|
|
||||||
|
|
||||||
dt = parse_swpc_timestamp("2026-05-19 05:14:59", "alerts")
|
|
||||||
assert dt == datetime(2026, 5, 19, 5, 14, 59, tzinfo=timezone.utc)
|
|
||||||
|
|
||||||
def test_parse_swpc_timestamp_kindex(self):
|
|
||||||
from central.adapters.swpc_common import parse_swpc_timestamp
|
|
||||||
|
|
||||||
dt = parse_swpc_timestamp("2026-05-12T03:00:00", "kindex")
|
|
||||||
assert dt == datetime(2026, 5, 12, 3, 0, 0, tzinfo=timezone.utc)
|
|
||||||
|
|
||||||
def test_parse_swpc_timestamp_protons(self):
|
|
||||||
from central.adapters.swpc_common import parse_swpc_timestamp
|
|
||||||
|
|
||||||
dt = parse_swpc_timestamp("2026-05-18T05:35:00Z", "protons")
|
|
||||||
assert dt == datetime(2026, 5, 18, 5, 35, 0, tzinfo=timezone.utc)
|
|
||||||
|
|
||||||
def test_parse_swpc_timestamp_empty(self):
|
|
||||||
from central.adapters.swpc_common import parse_swpc_timestamp
|
|
||||||
|
|
||||||
assert parse_swpc_timestamp("", "alerts") is None
|
|
||||||
assert parse_swpc_timestamp(None, "alerts") is None
|
|
||||||
|
|
||||||
def test_severity_from_kp_boundaries(self):
|
|
||||||
from central.adapters.swpc_common import severity_from_kp
|
|
||||||
|
|
||||||
assert severity_from_kp(None) == 0
|
|
||||||
assert severity_from_kp(0) == 0
|
|
||||||
assert severity_from_kp(4.5) == 0
|
|
||||||
assert severity_from_kp(4.9) == 0
|
|
||||||
assert severity_from_kp(5.0) == 1
|
|
||||||
assert severity_from_kp(5.99) == 1
|
|
||||||
assert severity_from_kp(6.0) == 2
|
|
||||||
assert severity_from_kp(6.99) == 2
|
|
||||||
assert severity_from_kp(7.0) == 3
|
|
||||||
assert severity_from_kp(7.99) == 3
|
|
||||||
assert severity_from_kp(8.0) == 4
|
|
||||||
assert severity_from_kp(9.0) == 4
|
|
||||||
|
|
||||||
def test_severity_from_alert_product_id(self):
|
|
||||||
from central.adapters.swpc_common import severity_from_alert_product_id
|
|
||||||
|
|
||||||
assert severity_from_alert_product_id(None) == 0
|
|
||||||
assert severity_from_alert_product_id("") == 0
|
|
||||||
assert severity_from_alert_product_id("EF3A") == 0
|
|
||||||
assert severity_from_alert_product_id("BHIS") == 0
|
|
||||||
assert severity_from_alert_product_id("K04A") == 0
|
|
||||||
assert severity_from_alert_product_id("K05A") == 1
|
|
||||||
assert severity_from_alert_product_id("K05W") == 1
|
|
||||||
assert severity_from_alert_product_id("K06A") == 2
|
|
||||||
assert severity_from_alert_product_id("K07A") == 3
|
|
||||||
assert severity_from_alert_product_id("K08A") == 4
|
|
||||||
assert severity_from_alert_product_id("K09A") == 4
|
|
||||||
|
|
||||||
|
|
||||||
class TestSWPCAlertsAdapter:
|
|
||||||
"""Tests for SWPCAlertsAdapter."""
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_alerts_normalization(self, tmp_path: Path):
|
|
||||||
from central.adapters.swpc_alerts import SWPCAlertsAdapter
|
|
||||||
|
|
||||||
adapter = SWPCAlertsAdapter(
|
|
||||||
_config("swpc_alerts", 300), MagicMock(), tmp_path / "cursors.db"
|
|
||||||
)
|
|
||||||
adapter._fetch = AsyncMock(return_value=SAMPLE_ALERTS)
|
|
||||||
|
|
||||||
await adapter.startup()
|
|
||||||
events: list[Event] = [e async for e in adapter.poll()]
|
|
||||||
await adapter.shutdown()
|
|
||||||
|
|
||||||
assert len(events) == 3
|
|
||||||
|
|
||||||
ef3a = events[0]
|
|
||||||
assert ef3a.adapter == "swpc_alerts"
|
|
||||||
assert ef3a.category == "space.alert"
|
|
||||||
assert ef3a.id == "EF3A|2026-05-19 05:14:59.780"
|
|
||||||
assert ef3a.time == datetime(2026, 5, 19, 5, 14, 59, 780000, tzinfo=timezone.utc)
|
|
||||||
assert ef3a.severity == 0
|
|
||||||
assert ef3a.data["product_id"] == "EF3A"
|
|
||||||
assert ef3a.geo.centroid is None
|
|
||||||
assert ef3a.geo.regions == []
|
|
||||||
assert ef3a.geo.primary_region is None
|
|
||||||
|
|
||||||
k05a = events[1]
|
|
||||||
assert k05a.severity == 1
|
|
||||||
k07a = events[2]
|
|
||||||
assert k07a.severity == 3
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_alerts_dedup(self, tmp_path: Path):
|
|
||||||
from central.adapters.swpc_alerts import SWPCAlertsAdapter
|
|
||||||
|
|
||||||
adapter = SWPCAlertsAdapter(
|
|
||||||
_config("swpc_alerts", 300), MagicMock(), tmp_path / "cursors.db"
|
|
||||||
)
|
|
||||||
adapter._fetch = AsyncMock(return_value=SAMPLE_ALERTS)
|
|
||||||
|
|
||||||
await adapter.startup()
|
|
||||||
first_pass = [e async for e in adapter.poll()]
|
|
||||||
second_pass = [e async for e in adapter.poll()]
|
|
||||||
await adapter.shutdown()
|
|
||||||
|
|
||||||
assert len(first_pass) == 3
|
|
||||||
assert len(second_pass) == 0
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_alerts_subject_for(self, tmp_path: Path):
|
|
||||||
from central.adapters.swpc_alerts import SWPCAlertsAdapter
|
|
||||||
from central.models import Geo
|
|
||||||
|
|
||||||
adapter = SWPCAlertsAdapter(
|
|
||||||
_config("swpc_alerts", 300), MagicMock(), tmp_path / "cursors.db"
|
|
||||||
)
|
|
||||||
event = Event(
|
|
||||||
id="EF3A|2026-05-19 05:14:59.780",
|
|
||||||
adapter="swpc_alerts",
|
|
||||||
category="space.alert",
|
|
||||||
time=datetime(2026, 5, 19, 5, 14, 59, tzinfo=timezone.utc),
|
|
||||||
severity=0,
|
|
||||||
geo=Geo(),
|
|
||||||
data={"product_id": "EF3A"},
|
|
||||||
)
|
|
||||||
assert adapter.subject_for(event) == "central.space.alert.ef3a"
|
|
||||||
|
|
||||||
event_k = Event(
|
|
||||||
id="K05A|...",
|
|
||||||
adapter="swpc_alerts",
|
|
||||||
category="space.alert",
|
|
||||||
time=datetime(2026, 5, 15, tzinfo=timezone.utc),
|
|
||||||
severity=1,
|
|
||||||
geo=Geo(),
|
|
||||||
data={"product_id": "K05A"},
|
|
||||||
)
|
|
||||||
assert adapter.subject_for(event_k) == "central.space.alert.k05a"
|
|
||||||
|
|
||||||
|
|
||||||
class TestSWPCKindexAdapter:
|
|
||||||
"""Tests for SWPCKindexAdapter."""
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_kindex_normalization(self, tmp_path: Path):
|
|
||||||
from central.adapters.swpc_kindex import SWPCKindexAdapter
|
|
||||||
|
|
||||||
adapter = SWPCKindexAdapter(
|
|
||||||
_config("swpc_kindex", 600), MagicMock(), tmp_path / "cursors.db"
|
|
||||||
)
|
|
||||||
adapter._fetch = AsyncMock(return_value=SAMPLE_KINDEX)
|
|
||||||
|
|
||||||
await adapter.startup()
|
|
||||||
events = [e async for e in adapter.poll()]
|
|
||||||
await adapter.shutdown()
|
|
||||||
|
|
||||||
assert len(events) == 3
|
|
||||||
quiet, g1, g4 = events
|
|
||||||
assert quiet.category == "space.kindex"
|
|
||||||
assert quiet.id == "2026-05-12T00:00:00"
|
|
||||||
assert quiet.severity == 0
|
|
||||||
assert quiet.data["Kp"] == 0.67
|
|
||||||
assert g1.severity == 1
|
|
||||||
assert g4.severity == 4
|
|
||||||
assert g4.time == datetime(2026, 5, 12, 6, 0, 0, tzinfo=timezone.utc)
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_kindex_dedup(self, tmp_path: Path):
|
|
||||||
from central.adapters.swpc_kindex import SWPCKindexAdapter
|
|
||||||
|
|
||||||
adapter = SWPCKindexAdapter(
|
|
||||||
_config("swpc_kindex", 600), MagicMock(), tmp_path / "cursors.db"
|
|
||||||
)
|
|
||||||
adapter._fetch = AsyncMock(return_value=SAMPLE_KINDEX)
|
|
||||||
|
|
||||||
await adapter.startup()
|
|
||||||
first_pass = [e async for e in adapter.poll()]
|
|
||||||
second_pass = [e async for e in adapter.poll()]
|
|
||||||
await adapter.shutdown()
|
|
||||||
|
|
||||||
assert len(first_pass) == 3
|
|
||||||
assert len(second_pass) == 0
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_kindex_subject_for(self, tmp_path: Path):
|
|
||||||
from central.adapters.swpc_kindex import SWPCKindexAdapter
|
|
||||||
from central.models import Geo
|
|
||||||
|
|
||||||
adapter = SWPCKindexAdapter(
|
|
||||||
_config("swpc_kindex", 600), MagicMock(), tmp_path / "cursors.db"
|
|
||||||
)
|
|
||||||
event = Event(
|
|
||||||
id="2026-05-12T03:00:00",
|
|
||||||
adapter="swpc_kindex",
|
|
||||||
category="space.kindex",
|
|
||||||
time=datetime(2026, 5, 12, 3, tzinfo=timezone.utc),
|
|
||||||
severity=1,
|
|
||||||
geo=Geo(),
|
|
||||||
data={"Kp": 5.33},
|
|
||||||
)
|
|
||||||
assert adapter.subject_for(event) == "central.space.kindex"
|
|
||||||
|
|
||||||
|
|
||||||
class TestSWPCProtonsAdapter:
|
|
||||||
"""Tests for SWPCProtonsAdapter."""
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_protons_normalization(self, tmp_path: Path):
|
|
||||||
from central.adapters.swpc_protons import SWPCProtonsAdapter
|
|
||||||
|
|
||||||
adapter = SWPCProtonsAdapter(
|
|
||||||
_config("swpc_protons", 600), MagicMock(), tmp_path / "cursors.db"
|
|
||||||
)
|
|
||||||
adapter._fetch = AsyncMock(return_value=SAMPLE_PROTONS)
|
|
||||||
|
|
||||||
await adapter.startup()
|
|
||||||
events = [e async for e in adapter.poll()]
|
|
||||||
await adapter.shutdown()
|
|
||||||
|
|
||||||
assert len(events) == 3
|
|
||||||
first = events[0]
|
|
||||||
assert first.category == "space.proton_flux"
|
|
||||||
assert first.id == "2026-05-18T05:35:00Z|>=1 MeV"
|
|
||||||
assert first.severity == 0
|
|
||||||
assert first.data["energy"] == ">=1 MeV"
|
|
||||||
assert first.data["flux"] == 7.09
|
|
||||||
assert first.time == datetime(2026, 5, 18, 5, 35, 0, tzinfo=timezone.utc)
|
|
||||||
assert first.geo.centroid is None
|
|
||||||
assert first.geo.regions == []
|
|
||||||
|
|
||||||
# Same time_tag, different energy -> distinct event_id
|
|
||||||
assert events[1].id == "2026-05-18T05:35:00Z|>=10 MeV"
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_protons_dedup(self, tmp_path: Path):
|
|
||||||
from central.adapters.swpc_protons import SWPCProtonsAdapter
|
|
||||||
|
|
||||||
adapter = SWPCProtonsAdapter(
|
|
||||||
_config("swpc_protons", 600), MagicMock(), tmp_path / "cursors.db"
|
|
||||||
)
|
|
||||||
adapter._fetch = AsyncMock(return_value=SAMPLE_PROTONS)
|
|
||||||
|
|
||||||
await adapter.startup()
|
|
||||||
first_pass = [e async for e in adapter.poll()]
|
|
||||||
second_pass = [e async for e in adapter.poll()]
|
|
||||||
await adapter.shutdown()
|
|
||||||
|
|
||||||
assert len(first_pass) == 3
|
|
||||||
assert len(second_pass) == 0
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_protons_subject_for(self, tmp_path: Path):
|
|
||||||
from central.adapters.swpc_protons import SWPCProtonsAdapter
|
|
||||||
from central.models import Geo
|
|
||||||
|
|
||||||
adapter = SWPCProtonsAdapter(
|
|
||||||
_config("swpc_protons", 600), MagicMock(), tmp_path / "cursors.db"
|
|
||||||
)
|
|
||||||
event = Event(
|
|
||||||
id="2026-05-18T05:35:00Z|>=10 MeV",
|
|
||||||
adapter="swpc_protons",
|
|
||||||
category="space.proton_flux",
|
|
||||||
time=datetime(2026, 5, 18, 5, 35, 0, tzinfo=timezone.utc),
|
|
||||||
severity=0,
|
|
||||||
geo=Geo(),
|
|
||||||
data={"energy": ">=10 MeV", "flux": 0.21},
|
|
||||||
)
|
|
||||||
assert adapter.subject_for(event) == "central.space.proton_flux"
|
|
||||||
|
|
@ -1,534 +0,0 @@
|
||||||
"""Tests for WFIGS adapters."""
|
|
||||||
|
|
||||||
import sqlite3
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
from pathlib import Path
|
|
||||||
from unittest.mock import AsyncMock, MagicMock, patch
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from central.config_models import AdapterConfig, RegionConfig
|
|
||||||
from central.models import Event, Geo
|
|
||||||
|
|
||||||
|
|
||||||
# Sample GeoJSON response with incidents using real WFIGS format
|
|
||||||
# Note: POOState comes as ISO 3166-2 ("US-MT"), IncidentTypeCategory as codes ("WF")
|
|
||||||
SAMPLE_INCIDENTS_RESPONSE = {
|
|
||||||
"type": "FeatureCollection",
|
|
||||||
"features": [
|
|
||||||
{
|
|
||||||
"type": "Feature",
|
|
||||||
"geometry": {"type": "Point", "coordinates": [-113.5, 48.5]},
|
|
||||||
"properties": {
|
|
||||||
"IrwinID": "GUID-001-GLACIER",
|
|
||||||
"IncidentName": "Glacier Fire",
|
|
||||||
"IncidentTypeCategory": "WF", # Real format: 2-letter code
|
|
||||||
"DailyAcres": 150,
|
|
||||||
"PercentContained": 25,
|
|
||||||
"FireDiscoveryDateTime": 1716000000000,
|
|
||||||
"ModifiedOnDateTime": 1716100000000,
|
|
||||||
"POOState": "US-MT", # Real format: ISO 3166-2
|
|
||||||
"POOCounty": "Glacier",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "Feature",
|
|
||||||
"geometry": {"type": "Point", "coordinates": [-116.5, 43.5]},
|
|
||||||
"properties": {
|
|
||||||
"IrwinID": "GUID-002-OWYHEE",
|
|
||||||
"IncidentName": "Owyhee Rx",
|
|
||||||
"IncidentTypeCategory": "RX", # Prescribed fire
|
|
||||||
"DailyAcres": 5,
|
|
||||||
"PercentContained": 100,
|
|
||||||
"FireDiscoveryDateTime": 1716200000000,
|
|
||||||
"ModifiedOnDateTime": 1716300000000,
|
|
||||||
"POOState": "US-ID",
|
|
||||||
"POOCounty": "Owyhee",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "Feature",
|
|
||||||
"geometry": {"type": "Point", "coordinates": [-80.0, 26.0]},
|
|
||||||
"properties": {
|
|
||||||
"IrwinID": "GUID-003-FLORIDA",
|
|
||||||
"IncidentName": "Florida Fire",
|
|
||||||
"IncidentTypeCategory": "WF",
|
|
||||||
"DailyAcres": 50,
|
|
||||||
"PercentContained": 0,
|
|
||||||
"FireDiscoveryDateTime": 1716400000000,
|
|
||||||
"ModifiedOnDateTime": 1716500000000,
|
|
||||||
"POOState": "US-FL",
|
|
||||||
"POOCounty": "Miami-Dade",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
}
|
|
||||||
|
|
||||||
# Perimeters API uses prefixed field names (attr_*, poly_*)
|
|
||||||
SAMPLE_PERIMETERS_RESPONSE = {
|
|
||||||
"type": "FeatureCollection",
|
|
||||||
"features": [
|
|
||||||
{
|
|
||||||
"type": "Feature",
|
|
||||||
"geometry": {
|
|
||||||
"type": "Polygon",
|
|
||||||
"coordinates": [[
|
|
||||||
[-113.6, 48.4],
|
|
||||||
[-113.4, 48.4],
|
|
||||||
[-113.4, 48.6],
|
|
||||||
[-113.6, 48.6],
|
|
||||||
[-113.6, 48.4],
|
|
||||||
]],
|
|
||||||
},
|
|
||||||
"properties": {
|
|
||||||
"attr_IrwinID": "GUID-001-GLACIER",
|
|
||||||
"attr_IncidentName": "Glacier Fire",
|
|
||||||
"attr_IncidentTypeCategory": "WF", # Real format: 2-letter code
|
|
||||||
"attr_IncidentSize": 150,
|
|
||||||
"poly_GISAcres": 148.5,
|
|
||||||
"attr_PercentContained": 25,
|
|
||||||
"attr_FireDiscoveryDateTime": 1716000000000,
|
|
||||||
"attr_ModifiedOnDateTime_dt": 1716100000000,
|
|
||||||
"attr_POOState": "US-MT", # Real format: ISO 3166-2
|
|
||||||
"attr_POOCounty": "Glacier",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class TestWFIGSCommon:
|
|
||||||
"""Tests for WFIGS common utilities."""
|
|
||||||
|
|
||||||
def test_severity_from_acres_none(self):
|
|
||||||
from central.adapters.wfigs_common import severity_from_acres
|
|
||||||
assert severity_from_acres(None) == 0
|
|
||||||
assert severity_from_acres(0) == 0
|
|
||||||
|
|
||||||
def test_severity_from_acres_small(self):
|
|
||||||
from central.adapters.wfigs_common import severity_from_acres
|
|
||||||
assert severity_from_acres(5) == 1
|
|
||||||
assert severity_from_acres(9.9) == 1
|
|
||||||
|
|
||||||
def test_severity_from_acres_medium(self):
|
|
||||||
from central.adapters.wfigs_common import severity_from_acres
|
|
||||||
assert severity_from_acres(10) == 2
|
|
||||||
assert severity_from_acres(99) == 2
|
|
||||||
|
|
||||||
def test_severity_from_acres_large(self):
|
|
||||||
from central.adapters.wfigs_common import severity_from_acres
|
|
||||||
assert severity_from_acres(100) == 3
|
|
||||||
assert severity_from_acres(999) == 3
|
|
||||||
|
|
||||||
def test_severity_from_acres_very_large(self):
|
|
||||||
from central.adapters.wfigs_common import severity_from_acres
|
|
||||||
assert severity_from_acres(1000) == 4
|
|
||||||
assert severity_from_acres(100000) == 4
|
|
||||||
|
|
||||||
def test_parse_wfigs_timestamp(self):
|
|
||||||
from central.adapters.wfigs_common import parse_wfigs_timestamp
|
|
||||||
ts = parse_wfigs_timestamp(1716000000000)
|
|
||||||
assert ts is not None
|
|
||||||
assert ts.tzinfo == timezone.utc
|
|
||||||
assert ts.year == 2024
|
|
||||||
|
|
||||||
def test_parse_wfigs_timestamp_none(self):
|
|
||||||
from central.adapters.wfigs_common import parse_wfigs_timestamp
|
|
||||||
assert parse_wfigs_timestamp(None) is None
|
|
||||||
|
|
||||||
def test_build_regions_full(self):
|
|
||||||
from central.adapters.wfigs_common import build_regions
|
|
||||||
# Expects normalized 2-letter state code
|
|
||||||
regions, primary = build_regions("MT", "Glacier")
|
|
||||||
assert regions == ["US-MT-GLACIER"]
|
|
||||||
assert primary == "US-MT-GLACIER"
|
|
||||||
|
|
||||||
def test_build_regions_state_only(self):
|
|
||||||
from central.adapters.wfigs_common import build_regions
|
|
||||||
regions, primary = build_regions("MT", None)
|
|
||||||
assert regions == ["US-MT"]
|
|
||||||
assert primary == "US-MT"
|
|
||||||
|
|
||||||
def test_build_regions_none(self):
|
|
||||||
from central.adapters.wfigs_common import build_regions
|
|
||||||
regions, primary = build_regions(None, None)
|
|
||||||
assert regions == []
|
|
||||||
assert primary is None
|
|
||||||
|
|
||||||
def test_subject_suffix(self):
|
|
||||||
from central.adapters.wfigs_common import subject_suffix
|
|
||||||
# Expects normalized 2-letter state code
|
|
||||||
assert subject_suffix("MT", "Glacier") == "mt.glacier"
|
|
||||||
assert subject_suffix("ID", "Ada County") == "id.ada_county"
|
|
||||||
assert subject_suffix("ID", None) == "id"
|
|
||||||
assert subject_suffix(None, None) == "unknown"
|
|
||||||
|
|
||||||
def test_point_in_bbox(self):
|
|
||||||
from central.adapters.wfigs_common import point_in_bbox
|
|
||||||
assert point_in_bbox(-116.5, 43.5, -124, 31, -102, 49) is True
|
|
||||||
assert point_in_bbox(-80.0, 26.0, -124, 31, -102, 49) is False
|
|
||||||
|
|
||||||
# Normalization tests
|
|
||||||
def test_normalize_state_iso_3166(self):
|
|
||||||
"""normalize_state strips US- prefix from ISO 3166-2 codes."""
|
|
||||||
from central.adapters.wfigs_common import normalize_state
|
|
||||||
assert normalize_state("US-MT") == "MT"
|
|
||||||
assert normalize_state("US-ID") == "ID"
|
|
||||||
assert normalize_state("US-CA") == "CA"
|
|
||||||
|
|
||||||
def test_normalize_state_already_2letter(self):
|
|
||||||
"""normalize_state passes through 2-letter codes."""
|
|
||||||
from central.adapters.wfigs_common import normalize_state
|
|
||||||
assert normalize_state("MT") == "MT"
|
|
||||||
assert normalize_state("ID") == "ID"
|
|
||||||
|
|
||||||
def test_normalize_state_none_empty(self):
|
|
||||||
"""normalize_state handles None and empty strings."""
|
|
||||||
from central.adapters.wfigs_common import normalize_state
|
|
||||||
assert normalize_state(None) is None
|
|
||||||
assert normalize_state("") is None
|
|
||||||
|
|
||||||
def test_normalize_state_unknown_format(self):
|
|
||||||
"""normalize_state passes through unknown formats."""
|
|
||||||
from central.adapters.wfigs_common import normalize_state
|
|
||||||
assert normalize_state("Montana") == "Montana"
|
|
||||||
assert normalize_state("US-MONTANA") == "US-MONTANA"
|
|
||||||
|
|
||||||
def test_normalize_incident_type_wf(self):
|
|
||||||
"""normalize_incident_type maps WF to wildfire."""
|
|
||||||
from central.adapters.wfigs_common import normalize_incident_type
|
|
||||||
assert normalize_incident_type("WF") == "wildfire"
|
|
||||||
assert normalize_incident_type("wf") == "wildfire"
|
|
||||||
|
|
||||||
def test_normalize_incident_type_rx(self):
|
|
||||||
"""normalize_incident_type maps RX to prescribed_fire."""
|
|
||||||
from central.adapters.wfigs_common import normalize_incident_type
|
|
||||||
assert normalize_incident_type("RX") == "prescribed_fire"
|
|
||||||
assert normalize_incident_type("rx") == "prescribed_fire"
|
|
||||||
|
|
||||||
def test_normalize_incident_type_cx(self):
|
|
||||||
"""normalize_incident_type maps CX to complex."""
|
|
||||||
from central.adapters.wfigs_common import normalize_incident_type
|
|
||||||
assert normalize_incident_type("CX") == "complex"
|
|
||||||
|
|
||||||
def test_normalize_incident_type_fa(self):
|
|
||||||
"""normalize_incident_type maps FA to false_alarm."""
|
|
||||||
from central.adapters.wfigs_common import normalize_incident_type
|
|
||||||
assert normalize_incident_type("FA") == "false_alarm"
|
|
||||||
|
|
||||||
def test_normalize_incident_type_unknown_code(self):
|
|
||||||
"""normalize_incident_type lowercases unknown codes."""
|
|
||||||
from central.adapters.wfigs_common import normalize_incident_type
|
|
||||||
assert normalize_incident_type("UNKNOWN_CODE") == "unknown_code"
|
|
||||||
assert normalize_incident_type("Wildfire") == "wildfire"
|
|
||||||
|
|
||||||
def test_normalize_incident_type_none(self):
|
|
||||||
"""normalize_incident_type returns unknown for None."""
|
|
||||||
from central.adapters.wfigs_common import normalize_incident_type
|
|
||||||
assert normalize_incident_type(None) == "unknown"
|
|
||||||
assert normalize_incident_type("") == "unknown"
|
|
||||||
|
|
||||||
|
|
||||||
class TestWFIGSIncidentsAdapter:
|
|
||||||
"""Tests for WFIGS Incidents adapter."""
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def mock_config(self) -> AdapterConfig:
|
|
||||||
return AdapterConfig(
|
|
||||||
name="wfigs_incidents",
|
|
||||||
enabled=True,
|
|
||||||
cadence_s=300,
|
|
||||||
settings={
|
|
||||||
"region": {"north": 49.0, "south": 31.0, "east": -102.0, "west": -124.0}
|
|
||||||
},
|
|
||||||
updated_at=datetime.now(timezone.utc),
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def mock_config_store(self) -> MagicMock:
|
|
||||||
return MagicMock()
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def cursor_db_path(self, tmp_path: Path) -> Path:
|
|
||||||
return tmp_path / "cursors.db"
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_normalization_incidents(
|
|
||||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
|
||||||
):
|
|
||||||
"""Incidents are correctly normalized to Events."""
|
|
||||||
from central.adapters.wfigs_incidents import WFIGSIncidentsAdapter
|
|
||||||
|
|
||||||
adapter = WFIGSIncidentsAdapter(mock_config, mock_config_store, cursor_db_path)
|
|
||||||
await adapter.startup()
|
|
||||||
|
|
||||||
mock_response = AsyncMock()
|
|
||||||
mock_response.raise_for_status = MagicMock()
|
|
||||||
mock_response.json = AsyncMock(return_value=SAMPLE_INCIDENTS_RESPONSE)
|
|
||||||
|
|
||||||
with patch.object(adapter._session, "get", return_value=AsyncMock(__aenter__=AsyncMock(return_value=mock_response), __aexit__=AsyncMock())):
|
|
||||||
events = [e async for e in adapter.poll()]
|
|
||||||
|
|
||||||
await adapter.shutdown()
|
|
||||||
|
|
||||||
# Should have 2 events (Florida filtered out by bbox)
|
|
||||||
assert len(events) == 2
|
|
||||||
|
|
||||||
# First event: Glacier Fire
|
|
||||||
event = events[0]
|
|
||||||
assert event.id == "GUID-001-GLACIER"
|
|
||||||
assert event.adapter == "wfigs_incidents"
|
|
||||||
# Category uses normalized incident type
|
|
||||||
assert event.category == "fire.incident.wildfire" # NOT fire.incident.wf
|
|
||||||
assert event.severity == 3 # 150 acres = severity 3 (100-999 range)
|
|
||||||
# Region uses normalized state (no double US-)
|
|
||||||
assert event.geo.primary_region == "US-MT-GLACIER" # NOT US-US-MT-GLACIER
|
|
||||||
# Data contains both normalized and raw values
|
|
||||||
assert event.data["POOState"] == "MT" # normalized
|
|
||||||
assert event.data["POOState_raw"] == "US-MT" # raw
|
|
||||||
assert event.data["IncidentTypeCategory"] == "wildfire" # normalized
|
|
||||||
assert event.data["IncidentTypeCategory_raw"] == "WF" # raw
|
|
||||||
|
|
||||||
# Second event: Owyhee Rx
|
|
||||||
event2 = events[1]
|
|
||||||
assert event2.category == "fire.incident.prescribed_fire" # NOT fire.incident.rx
|
|
||||||
assert event2.data["POOState"] == "ID"
|
|
||||||
assert event2.data["POOState_raw"] == "US-ID"
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_is_published_dedup(
|
|
||||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
|
||||||
):
|
|
||||||
"""is_published/mark_published provides dedup functionality."""
|
|
||||||
from central.adapters.wfigs_incidents import WFIGSIncidentsAdapter
|
|
||||||
|
|
||||||
adapter = WFIGSIncidentsAdapter(mock_config, mock_config_store, cursor_db_path)
|
|
||||||
await adapter.startup()
|
|
||||||
|
|
||||||
# Initially not published
|
|
||||||
assert adapter.is_published("test-id") is False
|
|
||||||
|
|
||||||
# Mark as published
|
|
||||||
adapter.mark_published("test-id")
|
|
||||||
|
|
||||||
# Now it should be published
|
|
||||||
assert adapter.is_published("test-id") is True
|
|
||||||
|
|
||||||
await adapter.shutdown()
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_fall_off_emits_removal(
|
|
||||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
|
||||||
):
|
|
||||||
"""Fall-off detection emits removal events."""
|
|
||||||
from central.adapters.wfigs_incidents import WFIGSIncidentsAdapter
|
|
||||||
|
|
||||||
adapter = WFIGSIncidentsAdapter(mock_config, mock_config_store, cursor_db_path)
|
|
||||||
await adapter.startup()
|
|
||||||
|
|
||||||
# First poll with 2 incidents
|
|
||||||
mock_response1 = AsyncMock()
|
|
||||||
mock_response1.raise_for_status = MagicMock()
|
|
||||||
mock_response1.json = AsyncMock(return_value=SAMPLE_INCIDENTS_RESPONSE)
|
|
||||||
|
|
||||||
# Second poll with only 1 incident (GUID-002 fell off)
|
|
||||||
reduced_response = {
|
|
||||||
"type": "FeatureCollection",
|
|
||||||
"features": [SAMPLE_INCIDENTS_RESPONSE["features"][0]],
|
|
||||||
}
|
|
||||||
mock_response2 = AsyncMock()
|
|
||||||
mock_response2.raise_for_status = MagicMock()
|
|
||||||
mock_response2.json = AsyncMock(return_value=reduced_response)
|
|
||||||
|
|
||||||
with patch.object(adapter._session, "get", return_value=AsyncMock(__aenter__=AsyncMock(return_value=mock_response1), __aexit__=AsyncMock())):
|
|
||||||
events1 = [e async for e in adapter.poll()]
|
|
||||||
|
|
||||||
with patch.object(adapter._session, "get", return_value=AsyncMock(__aenter__=AsyncMock(return_value=mock_response2), __aexit__=AsyncMock())):
|
|
||||||
events2 = [e async for e in adapter.poll()]
|
|
||||||
|
|
||||||
await adapter.shutdown()
|
|
||||||
|
|
||||||
# First poll: 2 incident events
|
|
||||||
assert len(events1) == 2
|
|
||||||
|
|
||||||
# Second poll: 1 incident (seen again) + 1 removal for GUID-002
|
|
||||||
# The incident event is yielded (supervisor does dedup via is_published)
|
|
||||||
# The removal is yielded for GUID-002
|
|
||||||
removal_events = [e for e in events2 if e.category == "fire.incident.removed"]
|
|
||||||
assert len(removal_events) == 1
|
|
||||||
assert removal_events[0].data["irwin_id"] == "GUID-002-OWYHEE"
|
|
||||||
|
|
||||||
def test_subject_for_incidents_normalized(
|
|
||||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
|
||||||
):
|
|
||||||
"""subject_for uses normalized state codes."""
|
|
||||||
from central.adapters.wfigs_incidents import WFIGSIncidentsAdapter
|
|
||||||
|
|
||||||
adapter = WFIGSIncidentsAdapter(mock_config, mock_config_store, cursor_db_path)
|
|
||||||
|
|
||||||
# Event data contains normalized state (MT not US-MT)
|
|
||||||
event = Event(
|
|
||||||
id="test-id",
|
|
||||||
adapter="wfigs_incidents",
|
|
||||||
category="fire.incident.wildfire",
|
|
||||||
time=datetime.now(timezone.utc),
|
|
||||||
severity=2,
|
|
||||||
geo=Geo(primary_region="US-MT-GLACIER"),
|
|
||||||
data={"POOState": "MT", "POOCounty": "Glacier"},
|
|
||||||
)
|
|
||||||
|
|
||||||
subject = adapter.subject_for(event)
|
|
||||||
# Subject uses normalized state: mt.glacier not us-mt.glacier
|
|
||||||
assert subject == "central.fire.incident.mt.glacier"
|
|
||||||
|
|
||||||
def test_subject_for_removal(
|
|
||||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
|
||||||
):
|
|
||||||
from central.adapters.wfigs_incidents import WFIGSIncidentsAdapter
|
|
||||||
|
|
||||||
adapter = WFIGSIncidentsAdapter(mock_config, mock_config_store, cursor_db_path)
|
|
||||||
|
|
||||||
event = Event(
|
|
||||||
id="test-id:removed:2024-01-01",
|
|
||||||
adapter="wfigs_incidents",
|
|
||||||
category="fire.incident.removed",
|
|
||||||
time=datetime.now(timezone.utc),
|
|
||||||
severity=0,
|
|
||||||
geo=Geo(),
|
|
||||||
data={"irwin_id": "test-id", "state": "MT"},
|
|
||||||
)
|
|
||||||
|
|
||||||
subject = adapter.subject_for(event)
|
|
||||||
assert subject == "central.fire.incident.removed.mt"
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_bbox_post_filter(
|
|
||||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
|
||||||
):
|
|
||||||
"""Features outside bbox are filtered out."""
|
|
||||||
from central.adapters.wfigs_incidents import WFIGSIncidentsAdapter
|
|
||||||
|
|
||||||
adapter = WFIGSIncidentsAdapter(mock_config, mock_config_store, cursor_db_path)
|
|
||||||
await adapter.startup()
|
|
||||||
|
|
||||||
mock_response = AsyncMock()
|
|
||||||
mock_response.raise_for_status = MagicMock()
|
|
||||||
mock_response.json = AsyncMock(return_value=SAMPLE_INCIDENTS_RESPONSE)
|
|
||||||
|
|
||||||
with patch.object(adapter._session, "get", return_value=AsyncMock(__aenter__=AsyncMock(return_value=mock_response), __aexit__=AsyncMock())):
|
|
||||||
events = [e async for e in adapter.poll()]
|
|
||||||
|
|
||||||
await adapter.shutdown()
|
|
||||||
|
|
||||||
# Florida incident should be filtered out
|
|
||||||
assert len(events) == 2
|
|
||||||
irwin_ids = {e.id for e in events}
|
|
||||||
assert "GUID-003-FLORIDA" not in irwin_ids
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_apply_config_region_change(
|
|
||||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
|
||||||
):
|
|
||||||
from central.adapters.wfigs_incidents import WFIGSIncidentsAdapter
|
|
||||||
|
|
||||||
adapter = WFIGSIncidentsAdapter(mock_config, mock_config_store, cursor_db_path)
|
|
||||||
|
|
||||||
assert adapter.region.north == 49.0
|
|
||||||
|
|
||||||
new_config = AdapterConfig(
|
|
||||||
name="wfigs_incidents",
|
|
||||||
enabled=True,
|
|
||||||
cadence_s=300,
|
|
||||||
settings={
|
|
||||||
"region": {"north": 50.0, "south": 35.0, "east": -100.0, "west": -120.0}
|
|
||||||
},
|
|
||||||
updated_at=datetime.now(timezone.utc),
|
|
||||||
)
|
|
||||||
await adapter.apply_config(new_config)
|
|
||||||
|
|
||||||
assert adapter.region.north == 50.0
|
|
||||||
assert adapter.region.south == 35.0
|
|
||||||
|
|
||||||
|
|
||||||
class TestWFIGSPerimetersAdapter:
|
|
||||||
"""Tests for WFIGS Perimeters adapter."""
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def mock_config(self) -> AdapterConfig:
|
|
||||||
return AdapterConfig(
|
|
||||||
name="wfigs_perimeters",
|
|
||||||
enabled=True,
|
|
||||||
cadence_s=300,
|
|
||||||
settings={
|
|
||||||
"region": {"north": 49.0, "south": 31.0, "east": -102.0, "west": -124.0}
|
|
||||||
},
|
|
||||||
updated_at=datetime.now(timezone.utc),
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def mock_config_store(self) -> MagicMock:
|
|
||||||
return MagicMock()
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def cursor_db_path(self, tmp_path: Path) -> Path:
|
|
||||||
return tmp_path / "cursors.db"
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_normalization_perimeters(
|
|
||||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
|
||||||
):
|
|
||||||
"""Perimeters are correctly normalized to Events with geometry."""
|
|
||||||
from central.adapters.wfigs_perimeters import WFIGSPerimetersAdapter
|
|
||||||
|
|
||||||
adapter = WFIGSPerimetersAdapter(mock_config, mock_config_store, cursor_db_path)
|
|
||||||
await adapter.startup()
|
|
||||||
|
|
||||||
mock_response = AsyncMock()
|
|
||||||
mock_response.raise_for_status = MagicMock()
|
|
||||||
mock_response.json = AsyncMock(return_value=SAMPLE_PERIMETERS_RESPONSE)
|
|
||||||
|
|
||||||
with patch.object(adapter._session, "get", return_value=AsyncMock(__aenter__=AsyncMock(return_value=mock_response), __aexit__=AsyncMock())):
|
|
||||||
events = [e async for e in adapter.poll()]
|
|
||||||
|
|
||||||
await adapter.shutdown()
|
|
||||||
|
|
||||||
assert len(events) == 1
|
|
||||||
|
|
||||||
event = events[0]
|
|
||||||
assert event.id == "GUID-001-GLACIER"
|
|
||||||
assert event.adapter == "wfigs_perimeters"
|
|
||||||
# Category uses normalized incident type
|
|
||||||
assert event.category == "fire.perimeter.wildfire" # NOT fire.perimeter.wf
|
|
||||||
# Region uses normalized state (no double US-)
|
|
||||||
assert event.geo.primary_region == "US-MT-GLACIER" # NOT US-US-MT-GLACIER
|
|
||||||
# Data contains both normalized and raw values
|
|
||||||
assert event.data["POOState"] == "MT" # normalized
|
|
||||||
assert event.data["POOState_raw"] == "US-MT" # raw
|
|
||||||
assert event.data["IncidentTypeCategory"] == "wildfire" # normalized
|
|
||||||
assert event.data["IncidentTypeCategory_raw"] == "WF" # raw
|
|
||||||
# Geometry is included
|
|
||||||
assert "geometry" in event.data
|
|
||||||
assert event.data["geometry"]["type"] == "Polygon"
|
|
||||||
|
|
||||||
def test_subject_for_perimeters_normalized(
|
|
||||||
self, mock_config: AdapterConfig, mock_config_store: MagicMock, cursor_db_path: Path
|
|
||||||
):
|
|
||||||
"""subject_for uses normalized state codes."""
|
|
||||||
from central.adapters.wfigs_perimeters import WFIGSPerimetersAdapter
|
|
||||||
|
|
||||||
adapter = WFIGSPerimetersAdapter(mock_config, mock_config_store, cursor_db_path)
|
|
||||||
|
|
||||||
# Event data contains normalized state (MT not US-MT)
|
|
||||||
event = Event(
|
|
||||||
id="test-id",
|
|
||||||
adapter="wfigs_perimeters",
|
|
||||||
category="fire.perimeter.wildfire",
|
|
||||||
time=datetime.now(timezone.utc),
|
|
||||||
severity=2,
|
|
||||||
geo=Geo(primary_region="US-MT-GLACIER"),
|
|
||||||
data={"POOState": "MT", "POOCounty": "Glacier", "geometry": {}},
|
|
||||||
)
|
|
||||||
|
|
||||||
subject = adapter.subject_for(event)
|
|
||||||
# Subject uses normalized state: mt.glacier not us-mt.glacier
|
|
||||||
assert subject == "central.fire.perimeter.mt.glacier"
|
|
||||||
|
|
@ -199,357 +199,3 @@ class TestSetupGateMiddlewareWizard:
|
||||||
response = client.get("/setup/operator")
|
response = client.get("/setup/operator")
|
||||||
assert response.status_code == 302
|
assert response.status_code == 302
|
||||||
assert response.headers["location"] == "/"
|
assert response.headers["location"] == "/"
|
||||||
|
|
||||||
class TestSetupAdaptersErrorRerender:
|
|
||||||
"""Test wizard adapters form error re-render path."""
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_invalid_cadence_rerenders_with_error(self):
|
|
||||||
"""POST /setup/adapters with cadence_s=5 re-renders form with error, no DB write."""
|
|
||||||
from central.gui.routes import setup_adapters_submit
|
|
||||||
|
|
||||||
mock_request = MagicMock()
|
|
||||||
mock_request.cookies = {}
|
|
||||||
mock_request.state = MagicMock()
|
|
||||||
|
|
||||||
# Mock form data with invalid cadence
|
|
||||||
mock_form = MagicMock()
|
|
||||||
mock_form.get.side_effect = lambda k, d="": {
|
|
||||||
"csrf_token": "test_csrf_token",
|
|
||||||
"nws_enabled": "on",
|
|
||||||
"nws_cadence_s": "5", # Invalid: below ge=10
|
|
||||||
"nws_contact_email": "test@example.com",
|
|
||||||
"nws_region_north": "49.0",
|
|
||||||
"nws_region_south": "31.0",
|
|
||||||
"nws_region_east": "-102.0",
|
|
||||||
"nws_region_west": "-124.0",
|
|
||||||
"firms_cadence_s": "300",
|
|
||||||
"firms_region_north": "49.0",
|
|
||||||
"firms_region_south": "31.0",
|
|
||||||
"firms_region_east": "-102.0",
|
|
||||||
"firms_region_west": "-124.0",
|
|
||||||
"usgs_quake_cadence_s": "300",
|
|
||||||
"usgs_quake_feed": "all_hour",
|
|
||||||
"usgs_quake_region_north": "49.0",
|
|
||||||
"usgs_quake_region_south": "31.0",
|
|
||||||
"usgs_quake_region_east": "-102.0",
|
|
||||||
"usgs_quake_region_west": "-124.0",
|
|
||||||
}.get(k, d)
|
|
||||||
mock_form.getlist.side_effect = lambda k: {
|
|
||||||
"firms_satellites": ["VIIRS_SNPP_NRT"],
|
|
||||||
}.get(k, [])
|
|
||||||
mock_form.__contains__ = lambda self, k: k in ["nws_enabled"]
|
|
||||||
|
|
||||||
mock_request.form = AsyncMock(return_value=mock_form)
|
|
||||||
|
|
||||||
# Mock wizard state
|
|
||||||
mock_state = MagicMock()
|
|
||||||
mock_state.operator = {"username": "test", "password_hash": "hash"}
|
|
||||||
mock_state.api_keys = []
|
|
||||||
mock_state.adapters = None
|
|
||||||
mock_state.system = None
|
|
||||||
|
|
||||||
# Mock pool with no actual DB access (should not be called for writes)
|
|
||||||
mock_pool = MagicMock()
|
|
||||||
mock_conn = MagicMock()
|
|
||||||
mock_conn.fetch = AsyncMock(return_value=[
|
|
||||||
{"name": "nws", "enabled": False, "cadence_s": 300, "settings": {}},
|
|
||||||
{"name": "firms", "enabled": False, "cadence_s": 300, "settings": {}},
|
|
||||||
{"name": "usgs_quake", "enabled": False, "cadence_s": 300, "settings": {}},
|
|
||||||
])
|
|
||||||
mock_conn.__aenter__ = AsyncMock(return_value=mock_conn)
|
|
||||||
mock_conn.__aexit__ = AsyncMock()
|
|
||||||
mock_pool.acquire = MagicMock(return_value=mock_conn)
|
|
||||||
|
|
||||||
mock_templates = MagicMock()
|
|
||||||
mock_response = MagicMock()
|
|
||||||
mock_response.status_code = 200
|
|
||||||
mock_templates.TemplateResponse.return_value = mock_response
|
|
||||||
|
|
||||||
with patch("central.gui.routes._get_templates", return_value=mock_templates):
|
|
||||||
with patch("central.gui.routes.get_pool", return_value=mock_pool):
|
|
||||||
with patch("central.gui.routes.get_settings") as mock_settings:
|
|
||||||
mock_settings.return_value.csrf_secret = "testsecret12345678901234567890ab"
|
|
||||||
with patch("central.gui.routes.validate_pre_auth_csrf", return_value=True):
|
|
||||||
with patch("central.gui.wizard.get_wizard_state", return_value=mock_state):
|
|
||||||
with patch("central.gui.routes.reuse_or_generate_pre_auth_csrf", return_value=("csrf", None)):
|
|
||||||
result = await setup_adapters_submit(mock_request)
|
|
||||||
|
|
||||||
# Should return 200 (re-render), not 302 (redirect)
|
|
||||||
assert result.status_code == 200
|
|
||||||
|
|
||||||
# Check that template was called with errors
|
|
||||||
call_args = mock_templates.TemplateResponse.call_args
|
|
||||||
context = call_args.kwargs.get("context", call_args[1].get("context"))
|
|
||||||
|
|
||||||
assert context["error"] == "Please fix the errors below."
|
|
||||||
assert "errors" in context
|
|
||||||
assert context["errors"] is not None
|
|
||||||
assert "nws_cadence_s" in context["errors"]
|
|
||||||
assert "10" in context["errors"]["nws_cadence_s"] # Should mention min value
|
|
||||||
|
|
||||||
# Verify adapters have correct shape (with fields)
|
|
||||||
assert "adapters" in context
|
|
||||||
for adapter in context["adapters"]:
|
|
||||||
assert "name" in adapter
|
|
||||||
assert "display_name" in adapter
|
|
||||||
assert "enabled" in adapter
|
|
||||||
assert "cadence_s" in adapter
|
|
||||||
assert "settings" in adapter
|
|
||||||
assert "fields" in adapter
|
|
||||||
|
|
||||||
# Verify no DB execute was called (no writes)
|
|
||||||
mock_conn.execute.assert_not_called()
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_invalid_region_bounds_shows_pydantic_error(self):
|
|
||||||
"""POST /setup/adapters with inverted region bounds shows RegionConfig error."""
|
|
||||||
from central.gui.routes import setup_adapters_submit
|
|
||||||
|
|
||||||
mock_request = MagicMock()
|
|
||||||
mock_request.cookies = {}
|
|
||||||
mock_request.state = MagicMock()
|
|
||||||
|
|
||||||
# Mock form data with inverted region (south > north)
|
|
||||||
mock_form = MagicMock()
|
|
||||||
mock_form.get.side_effect = lambda k, d="": {
|
|
||||||
"csrf_token": "test_csrf_token",
|
|
||||||
"nws_cadence_s": "300",
|
|
||||||
"nws_contact_email": "test@example.com",
|
|
||||||
"nws_region_north": "10.0", # Invalid: north < south
|
|
||||||
"nws_region_south": "20.0",
|
|
||||||
"nws_region_east": "-102.0",
|
|
||||||
"nws_region_west": "-124.0",
|
|
||||||
"firms_cadence_s": "300",
|
|
||||||
"firms_region_north": "49.0",
|
|
||||||
"firms_region_south": "31.0",
|
|
||||||
"firms_region_east": "-102.0",
|
|
||||||
"firms_region_west": "-124.0",
|
|
||||||
"usgs_quake_cadence_s": "300",
|
|
||||||
"usgs_quake_feed": "all_hour",
|
|
||||||
"usgs_quake_region_north": "49.0",
|
|
||||||
"usgs_quake_region_south": "31.0",
|
|
||||||
"usgs_quake_region_east": "-102.0",
|
|
||||||
"usgs_quake_region_west": "-124.0",
|
|
||||||
}.get(k, d)
|
|
||||||
mock_form.getlist.side_effect = lambda k: {
|
|
||||||
"firms_satellites": ["VIIRS_SNPP_NRT"],
|
|
||||||
}.get(k, [])
|
|
||||||
mock_form.__contains__ = lambda self, k: False
|
|
||||||
|
|
||||||
mock_request.form = AsyncMock(return_value=mock_form)
|
|
||||||
|
|
||||||
mock_state = MagicMock()
|
|
||||||
mock_state.operator = {"username": "test", "password_hash": "hash"}
|
|
||||||
mock_state.api_keys = []
|
|
||||||
mock_state.adapters = None
|
|
||||||
mock_state.system = None
|
|
||||||
|
|
||||||
mock_pool = MagicMock()
|
|
||||||
mock_conn = MagicMock()
|
|
||||||
mock_conn.fetch = AsyncMock(return_value=[
|
|
||||||
{"name": "nws", "enabled": False, "cadence_s": 300, "settings": {}},
|
|
||||||
{"name": "firms", "enabled": False, "cadence_s": 300, "settings": {}},
|
|
||||||
{"name": "usgs_quake", "enabled": False, "cadence_s": 300, "settings": {}},
|
|
||||||
])
|
|
||||||
mock_conn.__aenter__ = AsyncMock(return_value=mock_conn)
|
|
||||||
mock_conn.__aexit__ = AsyncMock()
|
|
||||||
mock_pool.acquire = MagicMock(return_value=mock_conn)
|
|
||||||
|
|
||||||
mock_templates = MagicMock()
|
|
||||||
mock_response = MagicMock()
|
|
||||||
mock_response.status_code = 200
|
|
||||||
mock_templates.TemplateResponse.return_value = mock_response
|
|
||||||
|
|
||||||
with patch("central.gui.routes._get_templates", return_value=mock_templates):
|
|
||||||
with patch("central.gui.routes.get_pool", return_value=mock_pool):
|
|
||||||
with patch("central.gui.routes.get_settings") as mock_settings:
|
|
||||||
mock_settings.return_value.csrf_secret = "testsecret12345678901234567890ab"
|
|
||||||
with patch("central.gui.routes.validate_pre_auth_csrf", return_value=True):
|
|
||||||
with patch("central.gui.wizard.get_wizard_state", return_value=mock_state):
|
|
||||||
with patch("central.gui.routes.reuse_or_generate_pre_auth_csrf", return_value=("csrf", None)):
|
|
||||||
result = await setup_adapters_submit(mock_request)
|
|
||||||
|
|
||||||
assert result.status_code == 200
|
|
||||||
|
|
||||||
call_args = mock_templates.TemplateResponse.call_args
|
|
||||||
context = call_args.kwargs.get("context", call_args[1].get("context"))
|
|
||||||
|
|
||||||
assert context["errors"] is not None
|
|
||||||
assert "nws_region" in context["errors"]
|
|
||||||
# Error should come from RegionConfig validator, mentioning bounds
|
|
||||||
assert "north" in context["errors"]["nws_region"].lower() or "south" in context["errors"]["nws_region"].lower()
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_invalid_contact_email_via_pydantic_pattern(self):
|
|
||||||
"""POST /setup/adapters with NWS contact_email='not-an-email' shows Pydantic pattern error."""
|
|
||||||
from central.gui.routes import setup_adapters_submit
|
|
||||||
|
|
||||||
mock_request = MagicMock()
|
|
||||||
mock_request.cookies = {}
|
|
||||||
mock_request.state = MagicMock()
|
|
||||||
|
|
||||||
mock_form = MagicMock()
|
|
||||||
mock_form.get.side_effect = lambda k, d="": {
|
|
||||||
"csrf_token": "test_csrf_token",
|
|
||||||
"nws_enabled": "on",
|
|
||||||
"nws_cadence_s": "300",
|
|
||||||
"nws_contact_email": "not-an-email", # Invalid email format
|
|
||||||
"nws_region_north": "49.0",
|
|
||||||
"nws_region_south": "31.0",
|
|
||||||
"nws_region_east": "-102.0",
|
|
||||||
"nws_region_west": "-124.0",
|
|
||||||
"firms_cadence_s": "300",
|
|
||||||
"firms_region_north": "49.0",
|
|
||||||
"firms_region_south": "31.0",
|
|
||||||
"firms_region_east": "-102.0",
|
|
||||||
"firms_region_west": "-124.0",
|
|
||||||
"usgs_quake_cadence_s": "300",
|
|
||||||
"usgs_quake_feed": "all_hour",
|
|
||||||
"usgs_quake_region_north": "49.0",
|
|
||||||
"usgs_quake_region_south": "31.0",
|
|
||||||
"usgs_quake_region_east": "-102.0",
|
|
||||||
"usgs_quake_region_west": "-124.0",
|
|
||||||
}.get(k, d)
|
|
||||||
mock_form.getlist.side_effect = lambda k: {
|
|
||||||
"firms_satellites": ["VIIRS_SNPP_NRT"],
|
|
||||||
}.get(k, [])
|
|
||||||
mock_form.__contains__ = lambda self, k: k in ["nws_enabled"]
|
|
||||||
|
|
||||||
mock_request.form = AsyncMock(return_value=mock_form)
|
|
||||||
|
|
||||||
mock_state = MagicMock()
|
|
||||||
mock_state.operator = {"username": "test", "password_hash": "hash"}
|
|
||||||
mock_state.api_keys = []
|
|
||||||
mock_state.adapters = None
|
|
||||||
mock_state.system = None
|
|
||||||
|
|
||||||
mock_pool = MagicMock()
|
|
||||||
mock_conn = MagicMock()
|
|
||||||
mock_conn.fetch = AsyncMock(return_value=[
|
|
||||||
{"name": "nws", "enabled": False, "cadence_s": 300, "settings": {}},
|
|
||||||
{"name": "firms", "enabled": False, "cadence_s": 300, "settings": {}},
|
|
||||||
{"name": "usgs_quake", "enabled": False, "cadence_s": 300, "settings": {}},
|
|
||||||
])
|
|
||||||
mock_conn.__aenter__ = AsyncMock(return_value=mock_conn)
|
|
||||||
mock_conn.__aexit__ = AsyncMock()
|
|
||||||
mock_pool.acquire = MagicMock(return_value=mock_conn)
|
|
||||||
|
|
||||||
mock_templates = MagicMock()
|
|
||||||
mock_response = MagicMock()
|
|
||||||
mock_response.status_code = 200
|
|
||||||
mock_templates.TemplateResponse.return_value = mock_response
|
|
||||||
|
|
||||||
with patch("central.gui.routes._get_templates", return_value=mock_templates):
|
|
||||||
with patch("central.gui.routes.get_pool", return_value=mock_pool):
|
|
||||||
with patch("central.gui.routes.get_settings") as mock_settings:
|
|
||||||
mock_settings.return_value.csrf_secret = "testsecret12345678901234567890ab"
|
|
||||||
with patch("central.gui.routes.validate_pre_auth_csrf", return_value=True):
|
|
||||||
with patch("central.gui.wizard.get_wizard_state", return_value=mock_state):
|
|
||||||
with patch("central.gui.routes.reuse_or_generate_pre_auth_csrf", return_value=("csrf", None)):
|
|
||||||
result = await setup_adapters_submit(mock_request)
|
|
||||||
|
|
||||||
assert result.status_code == 200
|
|
||||||
|
|
||||||
call_args = mock_templates.TemplateResponse.call_args
|
|
||||||
context = call_args.kwargs.get("context", call_args[1].get("context"))
|
|
||||||
|
|
||||||
assert context["errors"] is not None
|
|
||||||
assert "nws_contact_email" in context["errors"]
|
|
||||||
# Error should be from Pydantic pattern validation
|
|
||||||
error_msg = context["errors"]["nws_contact_email"].lower()
|
|
||||||
assert "pattern" in error_msg or "string" in error_msg or "match" in error_msg
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_invalid_api_key_alias_generic(self):
|
|
||||||
"""POST /setup/adapters with FIRMS api_key_alias='bogus' shows generic error."""
|
|
||||||
from central.gui.routes import setup_adapters_submit
|
|
||||||
|
|
||||||
mock_request = MagicMock()
|
|
||||||
mock_request.cookies = {}
|
|
||||||
mock_request.state = MagicMock()
|
|
||||||
|
|
||||||
mock_form = MagicMock()
|
|
||||||
mock_form.get.side_effect = lambda k, d="": {
|
|
||||||
"csrf_token": "test_csrf_token",
|
|
||||||
"nws_cadence_s": "300",
|
|
||||||
"nws_contact_email": "test@example.com",
|
|
||||||
"nws_region_north": "49.0",
|
|
||||||
"nws_region_south": "31.0",
|
|
||||||
"nws_region_east": "-102.0",
|
|
||||||
"nws_region_west": "-124.0",
|
|
||||||
"firms_cadence_s": "300",
|
|
||||||
"firms_api_key_alias": "bogus-alias-not-in-state", # Invalid alias
|
|
||||||
"firms_region_north": "49.0",
|
|
||||||
"firms_region_south": "31.0",
|
|
||||||
"firms_region_east": "-102.0",
|
|
||||||
"firms_region_west": "-124.0",
|
|
||||||
"usgs_quake_cadence_s": "300",
|
|
||||||
"usgs_quake_feed": "all_hour",
|
|
||||||
"usgs_quake_region_north": "49.0",
|
|
||||||
"usgs_quake_region_south": "31.0",
|
|
||||||
"usgs_quake_region_east": "-102.0",
|
|
||||||
"usgs_quake_region_west": "-124.0",
|
|
||||||
}.get(k, d)
|
|
||||||
mock_form.getlist.side_effect = lambda k: {
|
|
||||||
"firms_satellites": ["VIIRS_SNPP_NRT"],
|
|
||||||
}.get(k, [])
|
|
||||||
mock_form.__contains__ = lambda self, k: False
|
|
||||||
|
|
||||||
mock_request.form = AsyncMock(return_value=mock_form)
|
|
||||||
|
|
||||||
mock_state = MagicMock()
|
|
||||||
mock_state.operator = {"username": "test", "password_hash": "hash"}
|
|
||||||
mock_state.api_keys = [{"alias": "valid_key"}] # Only valid_key exists
|
|
||||||
mock_state.adapters = None
|
|
||||||
mock_state.system = None
|
|
||||||
|
|
||||||
mock_pool = MagicMock()
|
|
||||||
mock_conn = MagicMock()
|
|
||||||
mock_conn.fetch = AsyncMock(return_value=[
|
|
||||||
{"name": "nws", "enabled": False, "cadence_s": 300, "settings": {}},
|
|
||||||
{"name": "firms", "enabled": False, "cadence_s": 300, "settings": {}},
|
|
||||||
{"name": "usgs_quake", "enabled": False, "cadence_s": 300, "settings": {}},
|
|
||||||
])
|
|
||||||
mock_conn.__aenter__ = AsyncMock(return_value=mock_conn)
|
|
||||||
mock_conn.__aexit__ = AsyncMock()
|
|
||||||
mock_pool.acquire = MagicMock(return_value=mock_conn)
|
|
||||||
|
|
||||||
mock_templates = MagicMock()
|
|
||||||
mock_response = MagicMock()
|
|
||||||
mock_response.status_code = 200
|
|
||||||
mock_templates.TemplateResponse.return_value = mock_response
|
|
||||||
|
|
||||||
with patch("central.gui.routes._get_templates", return_value=mock_templates):
|
|
||||||
with patch("central.gui.routes.get_pool", return_value=mock_pool):
|
|
||||||
with patch("central.gui.routes.get_settings") as mock_settings:
|
|
||||||
mock_settings.return_value.csrf_secret = "testsecret12345678901234567890ab"
|
|
||||||
with patch("central.gui.routes.validate_pre_auth_csrf", return_value=True):
|
|
||||||
with patch("central.gui.wizard.get_wizard_state", return_value=mock_state):
|
|
||||||
with patch("central.gui.routes.reuse_or_generate_pre_auth_csrf", return_value=("csrf", None)):
|
|
||||||
result = await setup_adapters_submit(mock_request)
|
|
||||||
|
|
||||||
assert result.status_code == 200
|
|
||||||
|
|
||||||
call_args = mock_templates.TemplateResponse.call_args
|
|
||||||
context = call_args.kwargs.get("context", call_args[1].get("context"))
|
|
||||||
|
|
||||||
assert context["errors"] is not None
|
|
||||||
assert "firms_api_key_alias" in context["errors"]
|
|
||||||
assert "API key alias does not exist" in context["errors"]["firms_api_key_alias"]
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_api_key_field_none_no_check(self):
|
|
||||||
"""Adapters with api_key_field=None do not trigger the api_key check."""
|
|
||||||
# Verify that NWSAdapter has api_key_field=None
|
|
||||||
from central.adapters.nws import NWSAdapter
|
|
||||||
from central.adapters.firms import FIRMSAdapter
|
|
||||||
from central.adapters.usgs_quake import USGSQuakeAdapter
|
|
||||||
|
|
||||||
# NWS and USGS should have api_key_field=None
|
|
||||||
assert NWSAdapter.api_key_field is None
|
|
||||||
assert USGSQuakeAdapter.api_key_field is None
|
|
||||||
|
|
||||||
# FIRMS should have api_key_field set
|
|
||||||
assert FIRMSAdapter.api_key_field == "api_key_alias"
|
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue