mirror of
https://github.com/zvx-echo6/central.git
synced 2026-05-22 02:24:38 +02:00
refactor(nws): migrate from states to bbox region filtering
- Add RegionConfig pydantic model with validators - NWSAdapter now uses bbox for client-side alert filtering - Implement apply_config for hot-reload of region changes - Remove states-based filtering logic Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Matt Johnson
parent
1ea56b67fd
commit
dfcc0c3a5c
2 changed files with 572 additions and 488 deletions
|
|
@ -1,449 +1,502 @@
|
||||||
"""NWS (National Weather Service) alert adapter."""
|
"""NWS (National Weather Service) alert adapter."""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
import sqlite3
|
import sqlite3
|
||||||
from collections.abc import AsyncIterator
|
from collections.abc import AsyncIterator
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
from tenacity import (
|
from tenacity import (
|
||||||
retry,
|
retry,
|
||||||
stop_after_attempt,
|
stop_after_attempt,
|
||||||
wait_exponential_jitter,
|
wait_exponential_jitter,
|
||||||
retry_if_exception_type,
|
retry_if_exception_type,
|
||||||
)
|
)
|
||||||
|
|
||||||
from central import __version__
|
from central import __version__
|
||||||
from central.adapter import SourceAdapter
|
from central.adapter import SourceAdapter
|
||||||
from central.config import NWSAdapterConfig
|
from central.config_models import AdapterConfig, RegionConfig
|
||||||
from central.models import Event, Geo
|
from central.models import Event, Geo
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# FIPS state codes to postal abbreviations
|
# FIPS state codes to postal abbreviations
|
||||||
FIPS_TO_STATE: dict[str, str] = {
|
FIPS_TO_STATE: dict[str, str] = {
|
||||||
"01": "AL", "02": "AK", "04": "AZ", "05": "AR", "06": "CA",
|
"01": "AL", "02": "AK", "04": "AZ", "05": "AR", "06": "CA",
|
||||||
"08": "CO", "09": "CT", "10": "DE", "11": "DC", "12": "FL",
|
"08": "CO", "09": "CT", "10": "DE", "11": "DC", "12": "FL",
|
||||||
"13": "GA", "15": "HI", "16": "ID", "17": "IL", "18": "IN",
|
"13": "GA", "15": "HI", "16": "ID", "17": "IL", "18": "IN",
|
||||||
"19": "IA", "20": "KS", "21": "KY", "22": "LA", "23": "ME",
|
"19": "IA", "20": "KS", "21": "KY", "22": "LA", "23": "ME",
|
||||||
"24": "MD", "25": "MA", "26": "MI", "27": "MN", "28": "MS",
|
"24": "MD", "25": "MA", "26": "MI", "27": "MN", "28": "MS",
|
||||||
"29": "MO", "30": "MT", "31": "NE", "32": "NV", "33": "NH",
|
"29": "MO", "30": "MT", "31": "NE", "32": "NV", "33": "NH",
|
||||||
"34": "NJ", "35": "NM", "36": "NY", "37": "NC", "38": "ND",
|
"34": "NJ", "35": "NM", "36": "NY", "37": "NC", "38": "ND",
|
||||||
"39": "OH", "40": "OK", "41": "OR", "42": "PA", "44": "RI",
|
"39": "OH", "40": "OK", "41": "OR", "42": "PA", "44": "RI",
|
||||||
"45": "SC", "46": "SD", "47": "TN", "48": "TX", "49": "UT",
|
"45": "SC", "46": "SD", "47": "TN", "48": "TX", "49": "UT",
|
||||||
"50": "VT", "51": "VA", "53": "WA", "54": "WV", "55": "WI",
|
"50": "VT", "51": "VA", "53": "WA", "54": "WV", "55": "WI",
|
||||||
"56": "WY", "60": "AS", "66": "GU", "69": "MP", "72": "PR",
|
"56": "WY", "60": "AS", "66": "GU", "69": "MP", "72": "PR",
|
||||||
"78": "VI",
|
"78": "VI",
|
||||||
}
|
}
|
||||||
|
|
||||||
SEVERITY_MAP: dict[str, int | None] = {
|
SEVERITY_MAP: dict[str, int | None] = {
|
||||||
"Extreme": 4,
|
"Extreme": 4,
|
||||||
"Severe": 3,
|
"Severe": 3,
|
||||||
"Moderate": 2,
|
"Moderate": 2,
|
||||||
"Minor": 1,
|
"Minor": 1,
|
||||||
"Unknown": None,
|
"Unknown": None,
|
||||||
}
|
}
|
||||||
|
|
||||||
NWS_API_URL = "https://api.weather.gov/alerts/active"
|
NWS_API_URL = "https://api.weather.gov/alerts/active"
|
||||||
|
|
||||||
|
|
||||||
def _snake_case(s: str) -> str:
|
def _snake_case(s: str) -> str:
|
||||||
"""Convert a string to snake_case."""
|
"""Convert a string to snake_case."""
|
||||||
s = re.sub(r"[^a-zA-Z0-9\s]", "", s)
|
s = re.sub(r"[^a-zA-Z0-9\s]", "", s)
|
||||||
s = re.sub(r"\s+", "_", s.strip())
|
s = re.sub(r"\s+", "_", s.strip())
|
||||||
return s.lower()
|
return s.lower()
|
||||||
|
|
||||||
|
|
||||||
def _parse_datetime(s: str | None) -> datetime | None:
|
def _parse_datetime(s: str | None) -> datetime | None:
|
||||||
"""Parse an ISO datetime string to UTC datetime."""
|
"""Parse an ISO datetime string to UTC datetime."""
|
||||||
if not s:
|
if not s:
|
||||||
return None
|
return None
|
||||||
try:
|
try:
|
||||||
dt = datetime.fromisoformat(s.replace("Z", "+00:00"))
|
dt = datetime.fromisoformat(s.replace("Z", "+00:00"))
|
||||||
return dt.astimezone(timezone.utc)
|
return dt.astimezone(timezone.utc)
|
||||||
except (ValueError, TypeError):
|
except (ValueError, TypeError):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _compute_centroid(geometry: dict[str, Any] | None) -> tuple[float, float] | None:
|
def _compute_centroid(geometry: dict[str, Any] | None) -> tuple[float, float] | None:
|
||||||
"""Compute centroid from GeoJSON geometry using arithmetic mean of vertices."""
|
"""Compute centroid from GeoJSON geometry using arithmetic mean of vertices."""
|
||||||
if not geometry:
|
if not geometry:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
geom_type = geometry.get("type")
|
geom_type = geometry.get("type")
|
||||||
coords = geometry.get("coordinates")
|
coords = geometry.get("coordinates")
|
||||||
|
|
||||||
if not coords:
|
if not coords:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
all_points: list[tuple[float, float]] = []
|
all_points: list[tuple[float, float]] = []
|
||||||
|
|
||||||
if geom_type == "Point":
|
if geom_type == "Point":
|
||||||
return (coords[0], coords[1])
|
return (coords[0], coords[1])
|
||||||
elif geom_type == "Polygon":
|
elif geom_type == "Polygon":
|
||||||
for ring in coords:
|
for ring in coords:
|
||||||
for point in ring:
|
for point in ring:
|
||||||
all_points.append((point[0], point[1]))
|
all_points.append((point[0], point[1]))
|
||||||
elif geom_type == "MultiPolygon":
|
elif geom_type == "MultiPolygon":
|
||||||
for polygon in coords:
|
for polygon in coords:
|
||||||
for ring in polygon:
|
for ring in polygon:
|
||||||
for point in ring:
|
for point in ring:
|
||||||
all_points.append((point[0], point[1]))
|
all_points.append((point[0], point[1]))
|
||||||
else:
|
else:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
if not all_points:
|
if not all_points:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
avg_lon = sum(p[0] for p in all_points) / len(all_points)
|
avg_lon = sum(p[0] for p in all_points) / len(all_points)
|
||||||
avg_lat = sum(p[1] for p in all_points) / len(all_points)
|
avg_lat = sum(p[1] for p in all_points) / len(all_points)
|
||||||
return (avg_lon, avg_lat)
|
return (avg_lon, avg_lat)
|
||||||
|
|
||||||
|
|
||||||
def _compute_bbox(
|
def _compute_bbox(
|
||||||
geometry: dict[str, Any] | None
|
geometry: dict[str, Any] | None
|
||||||
) -> tuple[float, float, float, float] | None:
|
) -> tuple[float, float, float, float] | None:
|
||||||
"""Compute bounding box from GeoJSON geometry."""
|
"""Compute bounding box from GeoJSON geometry."""
|
||||||
if not geometry:
|
if not geometry:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
geom_type = geometry.get("type")
|
geom_type = geometry.get("type")
|
||||||
coords = geometry.get("coordinates")
|
coords = geometry.get("coordinates")
|
||||||
|
|
||||||
if not coords:
|
if not coords:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
all_points: list[tuple[float, float]] = []
|
all_points: list[tuple[float, float]] = []
|
||||||
|
|
||||||
if geom_type == "Point":
|
if geom_type == "Point":
|
||||||
return (coords[0], coords[1], coords[0], coords[1])
|
return (coords[0], coords[1], coords[0], coords[1])
|
||||||
elif geom_type == "Polygon":
|
elif geom_type == "Polygon":
|
||||||
for ring in coords:
|
for ring in coords:
|
||||||
for point in ring:
|
for point in ring:
|
||||||
all_points.append((point[0], point[1]))
|
all_points.append((point[0], point[1]))
|
||||||
elif geom_type == "MultiPolygon":
|
elif geom_type == "MultiPolygon":
|
||||||
for polygon in coords:
|
for polygon in coords:
|
||||||
for ring in polygon:
|
for ring in polygon:
|
||||||
for point in ring:
|
for point in ring:
|
||||||
all_points.append((point[0], point[1]))
|
all_points.append((point[0], point[1]))
|
||||||
else:
|
else:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
if not all_points:
|
if not all_points:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
min_lon = min(p[0] for p in all_points)
|
min_lon = min(p[0] for p in all_points)
|
||||||
max_lon = max(p[0] for p in all_points)
|
max_lon = max(p[0] for p in all_points)
|
||||||
min_lat = min(p[1] for p in all_points)
|
min_lat = min(p[1] for p in all_points)
|
||||||
max_lat = max(p[1] for p in all_points)
|
max_lat = max(p[1] for p in all_points)
|
||||||
return (min_lon, min_lat, max_lon, max_lat)
|
return (min_lon, min_lat, max_lon, max_lat)
|
||||||
|
|
||||||
|
|
||||||
def _extract_states_from_codes(
|
def _extract_states_from_codes(
|
||||||
same_codes: list[str], ugc_codes: list[str]
|
same_codes: list[str], ugc_codes: list[str]
|
||||||
) -> set[str]:
|
) -> set[str]:
|
||||||
"""Extract state abbreviations from SAME and UGC codes."""
|
"""Extract state abbreviations from SAME and UGC codes."""
|
||||||
states: set[str] = set()
|
states: set[str] = set()
|
||||||
|
|
||||||
for code in same_codes:
|
for code in same_codes:
|
||||||
if len(code) >= 2:
|
if len(code) >= 2:
|
||||||
fips_state = code[:2]
|
fips_state = code[:2]
|
||||||
if fips_state in FIPS_TO_STATE:
|
if fips_state in FIPS_TO_STATE:
|
||||||
states.add(FIPS_TO_STATE[fips_state])
|
states.add(FIPS_TO_STATE[fips_state])
|
||||||
|
|
||||||
for code in ugc_codes:
|
for code in ugc_codes:
|
||||||
if len(code) >= 2 and code[:2].isalpha():
|
if len(code) >= 2 and code[:2].isalpha():
|
||||||
states.add(code[:2].upper())
|
states.add(code[:2].upper())
|
||||||
|
|
||||||
return states
|
return states
|
||||||
|
|
||||||
|
|
||||||
def _build_regions(same_codes: list[str], ugc_codes: list[str]) -> list[str]:
|
def _build_regions(same_codes: list[str], ugc_codes: list[str]) -> list[str]:
|
||||||
"""Build sorted list of region strings from geocodes."""
|
"""Build sorted list of region strings from geocodes."""
|
||||||
regions: set[str] = set()
|
regions: set[str] = set()
|
||||||
|
|
||||||
for code in same_codes:
|
for code in same_codes:
|
||||||
if len(code) >= 2:
|
if len(code) >= 2:
|
||||||
fips_state = code[:2]
|
fips_state = code[:2]
|
||||||
if fips_state in FIPS_TO_STATE:
|
if fips_state in FIPS_TO_STATE:
|
||||||
state = FIPS_TO_STATE[fips_state]
|
state = FIPS_TO_STATE[fips_state]
|
||||||
regions.add(f"US-{state}-FIPS{code}")
|
regions.add(f"US-{state}-FIPS{code}")
|
||||||
|
|
||||||
for code in ugc_codes:
|
for code in ugc_codes:
|
||||||
if len(code) >= 3 and code[:2].isalpha():
|
if len(code) >= 3 and code[:2].isalpha():
|
||||||
state = code[:2].upper()
|
state = code[:2].upper()
|
||||||
rest = code[2:]
|
rest = code[2:]
|
||||||
if rest.startswith("C"):
|
if rest.startswith("C"):
|
||||||
regions.add(f"US-{state}-C{rest[1:]}")
|
regions.add(f"US-{state}-C{rest[1:]}")
|
||||||
elif rest.startswith("Z"):
|
elif rest.startswith("Z"):
|
||||||
regions.add(f"US-{state}-Z{rest[1:]}")
|
regions.add(f"US-{state}-Z{rest[1:]}")
|
||||||
else:
|
else:
|
||||||
regions.add(f"US-{state}-{rest}")
|
regions.add(f"US-{state}-{rest}")
|
||||||
|
|
||||||
return sorted(regions)
|
return sorted(regions)
|
||||||
|
|
||||||
|
|
||||||
class NWSAdapter(SourceAdapter):
|
class NWSAdapter(SourceAdapter):
|
||||||
"""National Weather Service alerts adapter."""
|
"""National Weather Service alerts adapter."""
|
||||||
|
|
||||||
name = "nws"
|
name = "nws"
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
config: NWSAdapterConfig,
|
config: AdapterConfig,
|
||||||
cursor_db_path: Path,
|
cursor_db_path: Path,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.config = config
|
self.cursor_db_path = cursor_db_path
|
||||||
self.states = set(s.upper() for s in config.states)
|
self._session: aiohttp.ClientSession | None = None
|
||||||
self.cursor_db_path = cursor_db_path
|
self._db: sqlite3.Connection | None = None
|
||||||
self._session: aiohttp.ClientSession | None = None
|
|
||||||
self._db: sqlite3.Connection | None = None
|
# Extract settings from unified config
|
||||||
|
self.contact_email: str = config.settings.get("contact_email", "")
|
||||||
async def startup(self) -> None:
|
|
||||||
"""Initialize HTTP session and cursor database."""
|
# Parse region from settings
|
||||||
user_agent = f"Central/{__version__} ({self.config.contact_email})"
|
region_dict = config.settings.get("region")
|
||||||
self._session = aiohttp.ClientSession(
|
if region_dict:
|
||||||
headers={"User-Agent": user_agent},
|
self.region: RegionConfig | None = RegionConfig(**region_dict)
|
||||||
timeout=aiohttp.ClientTimeout(total=30),
|
else:
|
||||||
)
|
self.region = None
|
||||||
|
|
||||||
self._db = sqlite3.connect(str(self.cursor_db_path))
|
async def apply_config(self, new_config: AdapterConfig) -> None:
|
||||||
self._db.execute("""
|
"""Apply new configuration from hot-reload."""
|
||||||
CREATE TABLE IF NOT EXISTS adapter_cursors (
|
# Update contact email
|
||||||
adapter TEXT PRIMARY KEY,
|
self.contact_email = new_config.settings.get("contact_email", "")
|
||||||
cursor_data TEXT NOT NULL,
|
|
||||||
updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
# Update region
|
||||||
)
|
region_dict = new_config.settings.get("region")
|
||||||
""")
|
if region_dict:
|
||||||
self._db.execute("""
|
self.region = RegionConfig(**region_dict)
|
||||||
CREATE TABLE IF NOT EXISTS published_ids (
|
else:
|
||||||
adapter TEXT NOT NULL,
|
self.region = None
|
||||||
event_id TEXT NOT NULL,
|
|
||||||
first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
logger.info(
|
||||||
last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
"NWS config applied",
|
||||||
PRIMARY KEY (adapter, event_id)
|
extra={
|
||||||
)
|
"region": region_dict,
|
||||||
""")
|
"contact_email": self.contact_email,
|
||||||
self._db.execute("""
|
},
|
||||||
CREATE INDEX IF NOT EXISTS published_ids_last_seen
|
)
|
||||||
ON published_ids (last_seen)
|
|
||||||
""")
|
def _point_in_region(self, centroid: tuple[float, float] | None) -> bool:
|
||||||
self._db.commit()
|
"""Check if centroid is within configured region bbox."""
|
||||||
|
if self.region is None:
|
||||||
logger.info("NWS adapter started", extra={"states": list(self.states)})
|
# No region configured = accept all
|
||||||
|
return True
|
||||||
async def shutdown(self) -> None:
|
if centroid is None:
|
||||||
"""Close HTTP session and database."""
|
return False
|
||||||
if self._session:
|
lon, lat = centroid
|
||||||
await self._session.close()
|
return (
|
||||||
self._session = None
|
self.region.west <= lon <= self.region.east
|
||||||
if self._db:
|
and self.region.south <= lat <= self.region.north
|
||||||
self._db.close()
|
)
|
||||||
self._db = None
|
|
||||||
logger.info("NWS adapter shut down")
|
async def startup(self) -> None:
|
||||||
|
"""Initialize HTTP session and cursor database."""
|
||||||
def _get_cursor(self) -> str | None:
|
user_agent = f"Central/{__version__} ({self.contact_email})"
|
||||||
"""Get the stored If-Modified-Since cursor."""
|
self._session = aiohttp.ClientSession(
|
||||||
if not self._db:
|
headers={"User-Agent": user_agent},
|
||||||
return None
|
timeout=aiohttp.ClientTimeout(total=30),
|
||||||
cur = self._db.execute(
|
)
|
||||||
"SELECT cursor_data FROM adapter_cursors WHERE adapter = ?",
|
|
||||||
(self.name,)
|
self._db = sqlite3.connect(str(self.cursor_db_path))
|
||||||
)
|
self._db.execute("""
|
||||||
row = cur.fetchone()
|
CREATE TABLE IF NOT EXISTS adapter_cursors (
|
||||||
return row[0] if row else None
|
adapter TEXT PRIMARY KEY,
|
||||||
|
cursor_data TEXT NOT NULL,
|
||||||
def _set_cursor(self, last_modified: str) -> None:
|
updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||||
"""Store the Last-Modified header for next request."""
|
)
|
||||||
if not self._db:
|
""")
|
||||||
return
|
self._db.execute("""
|
||||||
self._db.execute(
|
CREATE TABLE IF NOT EXISTS published_ids (
|
||||||
"""
|
adapter TEXT NOT NULL,
|
||||||
INSERT INTO adapter_cursors (adapter, cursor_data, updated)
|
event_id TEXT NOT NULL,
|
||||||
VALUES (?, ?, CURRENT_TIMESTAMP)
|
first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
ON CONFLICT (adapter) DO UPDATE SET
|
last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
cursor_data = excluded.cursor_data,
|
PRIMARY KEY (adapter, event_id)
|
||||||
updated = CURRENT_TIMESTAMP
|
)
|
||||||
""",
|
""")
|
||||||
(self.name, last_modified)
|
self._db.execute("""
|
||||||
)
|
CREATE INDEX IF NOT EXISTS published_ids_last_seen
|
||||||
self._db.commit()
|
ON published_ids (last_seen)
|
||||||
|
""")
|
||||||
def is_published(self, event_id: str) -> bool:
|
self._db.commit()
|
||||||
"""Check if an event has already been published."""
|
|
||||||
if not self._db:
|
logger.info(
|
||||||
return False
|
"NWS adapter started",
|
||||||
cur = self._db.execute(
|
extra={
|
||||||
"SELECT 1 FROM published_ids WHERE adapter = ? AND event_id = ?",
|
"region": {
|
||||||
(self.name, event_id)
|
"north": self.region.north,
|
||||||
)
|
"south": self.region.south,
|
||||||
return cur.fetchone() is not None
|
"east": self.region.east,
|
||||||
|
"west": self.region.west,
|
||||||
def mark_published(self, event_id: str) -> None:
|
} if self.region else None,
|
||||||
"""Mark an event as published."""
|
},
|
||||||
if not self._db:
|
)
|
||||||
return
|
|
||||||
self._db.execute(
|
async def shutdown(self) -> None:
|
||||||
"""
|
"""Close HTTP session and database."""
|
||||||
INSERT INTO published_ids (adapter, event_id, first_seen, last_seen)
|
if self._session:
|
||||||
VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
await self._session.close()
|
||||||
ON CONFLICT (adapter, event_id) DO UPDATE SET
|
self._session = None
|
||||||
last_seen = CURRENT_TIMESTAMP
|
if self._db:
|
||||||
""",
|
self._db.close()
|
||||||
(self.name, event_id)
|
self._db = None
|
||||||
)
|
logger.info("NWS adapter shut down")
|
||||||
self._db.commit()
|
|
||||||
|
def _get_cursor(self) -> str | None:
|
||||||
def bump_last_seen(self, event_id: str) -> None:
|
"""Get the stored If-Modified-Since cursor."""
|
||||||
"""Bump the last_seen timestamp for an event."""
|
if not self._db:
|
||||||
if not self._db:
|
return None
|
||||||
return
|
cur = self._db.execute(
|
||||||
self._db.execute(
|
"SELECT cursor_data FROM adapter_cursors WHERE adapter = ?",
|
||||||
"UPDATE published_ids SET last_seen = CURRENT_TIMESTAMP WHERE adapter = ? AND event_id = ?",
|
(self.name,)
|
||||||
(self.name, event_id)
|
)
|
||||||
)
|
row = cur.fetchone()
|
||||||
self._db.commit()
|
return row[0] if row else None
|
||||||
|
|
||||||
def sweep_old_ids(self) -> int:
|
def _set_cursor(self, last_modified: str) -> None:
|
||||||
"""Remove published_ids older than 8 days. Returns count deleted."""
|
"""Store the Last-Modified header for next request."""
|
||||||
if not self._db:
|
if not self._db:
|
||||||
return 0
|
return
|
||||||
cur = self._db.execute(
|
self._db.execute(
|
||||||
"DELETE FROM published_ids WHERE last_seen < datetime('now', '-8 days')"
|
"""
|
||||||
)
|
INSERT INTO adapter_cursors (adapter, cursor_data, updated)
|
||||||
self._db.commit()
|
VALUES (?, ?, CURRENT_TIMESTAMP)
|
||||||
return cur.rowcount
|
ON CONFLICT (adapter) DO UPDATE SET
|
||||||
|
cursor_data = excluded.cursor_data,
|
||||||
@retry(
|
updated = CURRENT_TIMESTAMP
|
||||||
stop=stop_after_attempt(5),
|
""",
|
||||||
wait=wait_exponential_jitter(initial=1, max=60),
|
(self.name, last_modified)
|
||||||
retry=retry_if_exception_type((aiohttp.ClientError, asyncio.TimeoutError)),
|
)
|
||||||
reraise=True,
|
self._db.commit()
|
||||||
)
|
|
||||||
async def _fetch_alerts(self) -> tuple[int, dict[str, Any] | None, str | None]:
|
def is_published(self, event_id: str) -> bool:
|
||||||
"""Fetch alerts from NWS API with conditional request."""
|
"""Check if an event has already been published."""
|
||||||
if not self._session:
|
if not self._db:
|
||||||
raise RuntimeError("Session not initialized")
|
return False
|
||||||
|
cur = self._db.execute(
|
||||||
headers: dict[str, str] = {}
|
"SELECT 1 FROM published_ids WHERE adapter = ? AND event_id = ?",
|
||||||
cursor = self._get_cursor()
|
(self.name, event_id)
|
||||||
if cursor:
|
)
|
||||||
headers["If-Modified-Since"] = cursor
|
return cur.fetchone() is not None
|
||||||
|
|
||||||
async with self._session.get(NWS_API_URL, headers=headers) as resp:
|
def mark_published(self, event_id: str) -> None:
|
||||||
if resp.status in (429, 403):
|
"""Mark an event as published."""
|
||||||
retry_after = resp.headers.get("Retry-After", "60")
|
if not self._db:
|
||||||
try:
|
return
|
||||||
wait_time = int(retry_after)
|
self._db.execute(
|
||||||
except ValueError:
|
"""
|
||||||
wait_time = 60
|
INSERT INTO published_ids (adapter, event_id, first_seen, last_seen)
|
||||||
logger.warning(
|
VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||||
"Rate limited by NWS",
|
ON CONFLICT (adapter, event_id) DO UPDATE SET
|
||||||
extra={"status": resp.status, "retry_after": wait_time}
|
last_seen = CURRENT_TIMESTAMP
|
||||||
)
|
""",
|
||||||
await asyncio.sleep(wait_time)
|
(self.name, event_id)
|
||||||
raise aiohttp.ClientError(f"Rate limited: {resp.status}")
|
)
|
||||||
|
self._db.commit()
|
||||||
if resp.status == 304:
|
|
||||||
return (304, None, None)
|
def bump_last_seen(self, event_id: str) -> None:
|
||||||
|
"""Bump the last_seen timestamp for an event."""
|
||||||
resp.raise_for_status()
|
if not self._db:
|
||||||
|
return
|
||||||
data = await resp.json()
|
self._db.execute(
|
||||||
last_modified = resp.headers.get("Last-Modified")
|
"UPDATE published_ids SET last_seen = CURRENT_TIMESTAMP WHERE adapter = ? AND event_id = ?",
|
||||||
|
(self.name, event_id)
|
||||||
return (resp.status, data, last_modified)
|
)
|
||||||
|
self._db.commit()
|
||||||
def _normalize_feature(self, feature: dict[str, Any]) -> Event | None:
|
|
||||||
"""Normalize a GeoJSON feature to an Event."""
|
def sweep_old_ids(self) -> int:
|
||||||
props = feature.get("properties", {})
|
"""Remove published_ids older than 8 days. Returns count deleted."""
|
||||||
geocode = props.get("geocode", {})
|
if not self._db:
|
||||||
|
return 0
|
||||||
same_codes = geocode.get("SAME", [])
|
cur = self._db.execute(
|
||||||
ugc_codes = geocode.get("UGC", [])
|
"DELETE FROM published_ids WHERE last_seen < datetime('now', '-8 days')"
|
||||||
|
)
|
||||||
feature_states = _extract_states_from_codes(same_codes, ugc_codes)
|
self._db.commit()
|
||||||
if not feature_states.intersection(self.states):
|
return cur.rowcount
|
||||||
return None
|
|
||||||
|
@retry(
|
||||||
event_id = feature.get("id")
|
stop=stop_after_attempt(5),
|
||||||
if not event_id:
|
wait=wait_exponential_jitter(initial=1, max=60),
|
||||||
logger.warning("Feature missing id", extra={"properties": props})
|
retry=retry_if_exception_type((aiohttp.ClientError, asyncio.TimeoutError)),
|
||||||
return None
|
reraise=True,
|
||||||
|
)
|
||||||
event_type = props.get("event", "Unknown")
|
async def _fetch_alerts(self) -> tuple[int, dict[str, Any] | None, str | None]:
|
||||||
category = f"wx.alert.{_snake_case(event_type)}"
|
"""Fetch alerts from NWS API with conditional request."""
|
||||||
|
if not self._session:
|
||||||
time = _parse_datetime(props.get("sent"))
|
raise RuntimeError("Session not initialized")
|
||||||
if not time:
|
|
||||||
logger.warning("Feature missing sent time", extra={"id": event_id})
|
headers: dict[str, str] = {}
|
||||||
return None
|
cursor = self._get_cursor()
|
||||||
|
if cursor:
|
||||||
expires = _parse_datetime(props.get("expires"))
|
headers["If-Modified-Since"] = cursor
|
||||||
|
|
||||||
severity_str = props.get("severity", "Unknown")
|
async with self._session.get(NWS_API_URL, headers=headers) as resp:
|
||||||
severity = SEVERITY_MAP.get(severity_str)
|
if resp.status in (429, 403):
|
||||||
|
retry_after = resp.headers.get("Retry-After", "60")
|
||||||
geometry = feature.get("geometry")
|
try:
|
||||||
centroid = _compute_centroid(geometry)
|
wait_time = int(retry_after)
|
||||||
bbox = _compute_bbox(geometry)
|
except ValueError:
|
||||||
regions = _build_regions(same_codes, ugc_codes)
|
wait_time = 60
|
||||||
primary_region = regions[0] if regions else None
|
logger.warning(
|
||||||
|
"Rate limited by NWS",
|
||||||
geo = Geo(
|
extra={"status": resp.status, "retry_after": wait_time}
|
||||||
centroid=centroid,
|
)
|
||||||
bbox=bbox,
|
await asyncio.sleep(wait_time)
|
||||||
regions=regions,
|
raise aiohttp.ClientError(f"Rate limited: {resp.status}")
|
||||||
primary_region=primary_region,
|
|
||||||
)
|
if resp.status == 304:
|
||||||
|
return (304, None, None)
|
||||||
return Event(
|
|
||||||
id=event_id,
|
resp.raise_for_status()
|
||||||
source="central/adapters/nws",
|
|
||||||
category=category,
|
data = await resp.json()
|
||||||
time=time,
|
last_modified = resp.headers.get("Last-Modified")
|
||||||
expires=expires,
|
|
||||||
severity=severity,
|
return (resp.status, data, last_modified)
|
||||||
geo=geo,
|
|
||||||
data=props,
|
def _normalize_feature(self, feature: dict[str, Any]) -> Event | None:
|
||||||
)
|
"""Normalize a GeoJSON feature to an Event."""
|
||||||
|
props = feature.get("properties", {})
|
||||||
async def poll(self) -> AsyncIterator[Event]:
|
geocode = props.get("geocode", {})
|
||||||
"""Poll NWS API for active alerts."""
|
|
||||||
try:
|
same_codes = geocode.get("SAME", [])
|
||||||
status, data, last_modified = await self._fetch_alerts()
|
ugc_codes = geocode.get("UGC", [])
|
||||||
except Exception as e:
|
|
||||||
logger.error("Failed to fetch NWS alerts", extra={"error": str(e)})
|
# Compute geometry data first
|
||||||
raise
|
geometry = feature.get("geometry")
|
||||||
|
centroid = _compute_centroid(geometry)
|
||||||
if status == 304:
|
bbox = _compute_bbox(geometry)
|
||||||
logger.info("NWS returned 304 Not Modified")
|
|
||||||
return
|
# Filter by region bbox (client-side filtering)
|
||||||
|
if not self._point_in_region(centroid):
|
||||||
if last_modified:
|
return None
|
||||||
self._set_cursor(last_modified)
|
|
||||||
|
event_id = feature.get("id")
|
||||||
features = data.get("features", []) if data else []
|
if not event_id:
|
||||||
logger.info(
|
logger.warning("Feature missing id", extra={"properties": props})
|
||||||
"NWS poll completed",
|
return None
|
||||||
extra={"status": status, "feature_count": len(features)}
|
|
||||||
)
|
event_type = props.get("event", "Unknown")
|
||||||
|
category = f"wx.alert.{_snake_case(event_type)}"
|
||||||
yielded = 0
|
|
||||||
for feature in features:
|
time = _parse_datetime(props.get("sent"))
|
||||||
try:
|
if not time:
|
||||||
event = self._normalize_feature(feature)
|
logger.warning("Feature missing sent time", extra={"id": event_id})
|
||||||
if event:
|
return None
|
||||||
yield event
|
|
||||||
yielded += 1
|
expires = _parse_datetime(props.get("expires"))
|
||||||
except Exception as e:
|
|
||||||
logger.warning(
|
severity_str = props.get("severity", "Unknown")
|
||||||
"Failed to normalize feature",
|
severity = SEVERITY_MAP.get(severity_str)
|
||||||
extra={"error": str(e), "feature_id": feature.get("id")}
|
|
||||||
)
|
regions = _build_regions(same_codes, ugc_codes)
|
||||||
|
primary_region = regions[0] if regions else None
|
||||||
logger.info("NWS yielded events", extra={"count": yielded})
|
|
||||||
|
geo = Geo(
|
||||||
|
centroid=centroid,
|
||||||
|
bbox=bbox,
|
||||||
|
regions=regions,
|
||||||
|
primary_region=primary_region,
|
||||||
|
)
|
||||||
|
|
||||||
|
return Event(
|
||||||
|
id=event_id,
|
||||||
|
source="central/adapters/nws",
|
||||||
|
category=category,
|
||||||
|
time=time,
|
||||||
|
expires=expires,
|
||||||
|
severity=severity,
|
||||||
|
geo=geo,
|
||||||
|
data=props,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def poll(self) -> AsyncIterator[Event]:
|
||||||
|
"""Poll NWS API for active alerts."""
|
||||||
|
try:
|
||||||
|
status, data, last_modified = await self._fetch_alerts()
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("Failed to fetch NWS alerts", extra={"error": str(e)})
|
||||||
|
raise
|
||||||
|
|
||||||
|
if status == 304:
|
||||||
|
logger.info("NWS returned 304 Not Modified")
|
||||||
|
return
|
||||||
|
|
||||||
|
if last_modified:
|
||||||
|
self._set_cursor(last_modified)
|
||||||
|
|
||||||
|
features = data.get("features", []) if data else []
|
||||||
|
logger.info(
|
||||||
|
"NWS poll completed",
|
||||||
|
extra={"status": status, "feature_count": len(features)}
|
||||||
|
)
|
||||||
|
|
||||||
|
yielded = 0
|
||||||
|
for feature in features:
|
||||||
|
try:
|
||||||
|
event = self._normalize_feature(feature)
|
||||||
|
if event:
|
||||||
|
yield event
|
||||||
|
yielded += 1
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(
|
||||||
|
"Failed to normalize feature",
|
||||||
|
extra={"error": str(e), "feature_id": feature.get("id")}
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("NWS yielded events", extra={"count": yielded})
|
||||||
|
|
|
||||||
|
|
@ -1,39 +1,70 @@
|
||||||
"""Pydantic models for database-backed configuration."""
|
"""Pydantic models for database-backed configuration."""
|
||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field, model_validator
|
||||||
|
|
||||||
|
|
||||||
class AdapterConfig(BaseModel):
|
class RegionConfig(BaseModel):
|
||||||
"""Configuration for a single adapter."""
|
"""Geographic bounding box for adapter region filtering."""
|
||||||
|
|
||||||
name: str = Field(description="Unique adapter identifier")
|
north: float = Field(ge=-90, le=90, description="Northern latitude bound")
|
||||||
enabled: bool = Field(default=True, description="Whether adapter is active")
|
south: float = Field(ge=-90, le=90, description="Southern latitude bound")
|
||||||
cadence_s: int = Field(description="Poll interval in seconds")
|
east: float = Field(ge=-180, le=180, description="Eastern longitude bound")
|
||||||
settings: dict[str, Any] = Field(
|
west: float = Field(ge=-180, le=180, description="Western longitude bound")
|
||||||
default_factory=dict, description="Adapter-specific settings"
|
|
||||||
)
|
@model_validator(mode="after")
|
||||||
paused_at: datetime | None = Field(
|
def validate_bounds(self) -> "RegionConfig":
|
||||||
default=None, description="When adapter was paused, if paused"
|
if self.north <= self.south:
|
||||||
)
|
raise ValueError(
|
||||||
updated_at: datetime = Field(description="Last configuration update time")
|
f"north ({self.north}) must be greater than south ({self.south})"
|
||||||
|
)
|
||||||
@property
|
if self.east == self.west:
|
||||||
def is_paused(self) -> bool:
|
raise ValueError("east and west cannot be equal (zero-width bbox)")
|
||||||
"""Check if adapter is currently paused."""
|
return self
|
||||||
return self.paused_at is not None
|
|
||||||
|
|
||||||
|
class AdapterConfig(BaseModel):
|
||||||
class ApiKeyInfo(BaseModel):
|
"""Configuration for a single adapter."""
|
||||||
"""Metadata about an API key (without the decrypted value)."""
|
|
||||||
|
name: str = Field(description="Unique adapter identifier")
|
||||||
alias: str = Field(description="Key identifier/alias")
|
enabled: bool = Field(default=True, description="Whether adapter is active")
|
||||||
created_at: datetime = Field(description="When key was created")
|
cadence_s: int = Field(description="Poll interval in seconds")
|
||||||
rotated_at: datetime | None = Field(
|
settings: dict[str, Any] = Field(
|
||||||
default=None, description="Last rotation time"
|
default_factory=dict, description="Adapter-specific settings"
|
||||||
)
|
)
|
||||||
last_used_at: datetime | None = Field(
|
paused_at: datetime | None = Field(
|
||||||
default=None, description="Last usage time"
|
default=None, description="When adapter was paused, if paused"
|
||||||
)
|
)
|
||||||
|
updated_at: datetime = Field(description="Last configuration update time")
|
||||||
|
|
||||||
|
@property
|
||||||
|
def is_paused(self) -> bool:
|
||||||
|
"""Check if adapter is currently paused."""
|
||||||
|
return self.paused_at is not None
|
||||||
|
|
||||||
|
|
||||||
|
class StreamConfig(BaseModel):
|
||||||
|
"""Configuration for a JetStream stream."""
|
||||||
|
|
||||||
|
name: str = Field(description="Stream name")
|
||||||
|
max_age_s: int = Field(description="Maximum message age in seconds")
|
||||||
|
max_bytes: int = Field(description="Maximum stream size in bytes")
|
||||||
|
managed_max_bytes: bool = Field(
|
||||||
|
default=True, description="Whether max_bytes is auto-managed by supervisor"
|
||||||
|
)
|
||||||
|
updated_at: datetime = Field(description="Last configuration update time")
|
||||||
|
|
||||||
|
|
||||||
|
class ApiKeyInfo(BaseModel):
|
||||||
|
"""Metadata about an API key (without the decrypted value)."""
|
||||||
|
|
||||||
|
alias: str = Field(description="Key identifier/alias")
|
||||||
|
created_at: datetime = Field(description="When key was created")
|
||||||
|
rotated_at: datetime | None = Field(
|
||||||
|
default=None, description="Last rotation time"
|
||||||
|
)
|
||||||
|
last_used_at: datetime | None = Field(
|
||||||
|
default=None, description="Last usage time"
|
||||||
|
)
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue