From 12a66d45ba8f7f6adc7f8ea84b0ccfe885bd11dd Mon Sep 17 00:00:00 2001 From: Matt Johnson Date: Sat, 16 May 2026 18:49:29 +0000 Subject: [PATCH 1/9] docs: add Phase 1B planning notes - Stream retention GUI design - Region picker for bbox selection - API key management requirements Co-Authored-By: Claude Opus 4.5 --- docs/PHASE-1B-NOTES.md | 55 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 docs/PHASE-1B-NOTES.md diff --git a/docs/PHASE-1B-NOTES.md b/docs/PHASE-1B-NOTES.md new file mode 100644 index 0000000..b7cecc8 --- /dev/null +++ b/docs/PHASE-1B-NOTES.md @@ -0,0 +1,55 @@ +# Phase 1B Planning Notes + +Design notes for Phase 1B GUI features. These are planning items, not +implementation specifications. + +## Stream Retention GUI + +### Per-Stream Configuration +- Show each stream from `config.streams` table +- Editable max_age_s with preset chips: 1d, 7d, 14d, 30d, 365d +- Custom numeric input allowed (operator can enter 90d, etc.) +- Changes trigger NATS stream update via supervisor hot-reload + +### Storage Monitor +Per stream, display: +- **Current bytes**: Live from `nats stream info` +- **Projected bytes**: Calculated from current rate × max_age +- **Days remaining**: Current_bytes / rate_per_day estimate +- Refresh: Real-time polling, not cached + +### Global Server Cap +- Show `max_file_store` value as read-only reference +- Editing requires NATS server restart (out of scope for GUI) +- Display per-stream ceiling (30% of server cap) as context + +## Region Picker + +### Interactive Map +- Bbox selection via click-drag rectangle +- Same UI component for all adapters (NWS, FIRMS, USGS) +- Stores `{north, south, east, west}` floats +- Preview of coverage area with state/country boundaries + +### Preset Regions +- Common presets: CONUS, Pacific Northwest, Mountain West +- Quick-select buttons alongside custom draw + +## API Key Management + +### Key Storage +- View configured API keys (alias only, not values) +- Add new keys with alias and value +- Values encrypted at rest in `config.api_keys` +- Rotation: update value, track `rotated_at` + +### Required Keys by Adapter +- **FIRMS** (Phase 1a-6): `MAP_KEY` for NASA FIRMS API +- Future adapters may require additional keys + +## Technical Notes + +- All GUI changes write to `config.*` tables +- Supervisor receives NOTIFY and hot-reloads +- No service restarts required for config changes +- Stream retention changes apply within 5 seconds From 1ea56b67fd6041512e686d115bd74a864708f523 Mon Sep 17 00:00:00 2001 From: Matt Johnson Date: Sat, 16 May 2026 18:49:40 +0000 Subject: [PATCH 2/9] refactor(adapter): add abstract apply_config method SourceAdapter now requires apply_config() for hot-reload support. Each adapter implements its own config extraction from settings. Co-Authored-By: Claude Opus 4.5 --- src/central/adapter.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/central/adapter.py b/src/central/adapter.py index a701558..2037eef 100644 --- a/src/central/adapter.py +++ b/src/central/adapter.py @@ -2,6 +2,10 @@ from abc import ABC, abstractmethod from collections.abc import AsyncIterator +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from central.config_models import AdapterConfig from central.models import Event @@ -15,7 +19,6 @@ class SourceAdapter(ABC): """ name: str # short identifier, e.g. "nws" - cadence_s: int # seconds between poll() calls @abstractmethod async def poll(self) -> AsyncIterator[Event]: @@ -26,6 +29,17 @@ class SourceAdapter(ABC): """ ... + @abstractmethod + async def apply_config(self, new_config: "AdapterConfig") -> None: + """ + Apply new configuration to the adapter. + + Called by supervisor when config changes via hot-reload. + The adapter should extract relevant settings from + new_config.settings and update its internal state. + """ + ... + async def startup(self) -> None: """Optional lifecycle hook called before first poll.""" pass From dfcc0c3a5c23ce2db7efcb05ea046ebcdee5ee58 Mon Sep 17 00:00:00 2001 From: Matt Johnson Date: Sat, 16 May 2026 18:49:46 +0000 Subject: [PATCH 3/9] refactor(nws): migrate from states to bbox region filtering - Add RegionConfig pydantic model with validators - NWSAdapter now uses bbox for client-side alert filtering - Implement apply_config for hot-reload of region changes - Remove states-based filtering logic Co-Authored-By: Claude Opus 4.5 --- src/central/adapters/nws.py | 951 ++++++++++++++++++----------------- src/central/config_models.py | 109 ++-- 2 files changed, 572 insertions(+), 488 deletions(-) diff --git a/src/central/adapters/nws.py b/src/central/adapters/nws.py index c8d3c6a..76391e1 100644 --- a/src/central/adapters/nws.py +++ b/src/central/adapters/nws.py @@ -1,449 +1,502 @@ -"""NWS (National Weather Service) alert adapter.""" - -import asyncio -import logging -import re -import sqlite3 -from collections.abc import AsyncIterator -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -import aiohttp -from tenacity import ( - retry, - stop_after_attempt, - wait_exponential_jitter, - retry_if_exception_type, -) - -from central import __version__ -from central.adapter import SourceAdapter -from central.config import NWSAdapterConfig -from central.models import Event, Geo - -logger = logging.getLogger(__name__) - -# FIPS state codes to postal abbreviations -FIPS_TO_STATE: dict[str, str] = { - "01": "AL", "02": "AK", "04": "AZ", "05": "AR", "06": "CA", - "08": "CO", "09": "CT", "10": "DE", "11": "DC", "12": "FL", - "13": "GA", "15": "HI", "16": "ID", "17": "IL", "18": "IN", - "19": "IA", "20": "KS", "21": "KY", "22": "LA", "23": "ME", - "24": "MD", "25": "MA", "26": "MI", "27": "MN", "28": "MS", - "29": "MO", "30": "MT", "31": "NE", "32": "NV", "33": "NH", - "34": "NJ", "35": "NM", "36": "NY", "37": "NC", "38": "ND", - "39": "OH", "40": "OK", "41": "OR", "42": "PA", "44": "RI", - "45": "SC", "46": "SD", "47": "TN", "48": "TX", "49": "UT", - "50": "VT", "51": "VA", "53": "WA", "54": "WV", "55": "WI", - "56": "WY", "60": "AS", "66": "GU", "69": "MP", "72": "PR", - "78": "VI", -} - -SEVERITY_MAP: dict[str, int | None] = { - "Extreme": 4, - "Severe": 3, - "Moderate": 2, - "Minor": 1, - "Unknown": None, -} - -NWS_API_URL = "https://api.weather.gov/alerts/active" - - -def _snake_case(s: str) -> str: - """Convert a string to snake_case.""" - s = re.sub(r"[^a-zA-Z0-9\s]", "", s) - s = re.sub(r"\s+", "_", s.strip()) - return s.lower() - - -def _parse_datetime(s: str | None) -> datetime | None: - """Parse an ISO datetime string to UTC datetime.""" - if not s: - return None - try: - dt = datetime.fromisoformat(s.replace("Z", "+00:00")) - return dt.astimezone(timezone.utc) - except (ValueError, TypeError): - return None - - -def _compute_centroid(geometry: dict[str, Any] | None) -> tuple[float, float] | None: - """Compute centroid from GeoJSON geometry using arithmetic mean of vertices.""" - if not geometry: - return None - - geom_type = geometry.get("type") - coords = geometry.get("coordinates") - - if not coords: - return None - - all_points: list[tuple[float, float]] = [] - - if geom_type == "Point": - return (coords[0], coords[1]) - elif geom_type == "Polygon": - for ring in coords: - for point in ring: - all_points.append((point[0], point[1])) - elif geom_type == "MultiPolygon": - for polygon in coords: - for ring in polygon: - for point in ring: - all_points.append((point[0], point[1])) - else: - return None - - if not all_points: - return None - - avg_lon = sum(p[0] for p in all_points) / len(all_points) - avg_lat = sum(p[1] for p in all_points) / len(all_points) - return (avg_lon, avg_lat) - - -def _compute_bbox( - geometry: dict[str, Any] | None -) -> tuple[float, float, float, float] | None: - """Compute bounding box from GeoJSON geometry.""" - if not geometry: - return None - - geom_type = geometry.get("type") - coords = geometry.get("coordinates") - - if not coords: - return None - - all_points: list[tuple[float, float]] = [] - - if geom_type == "Point": - return (coords[0], coords[1], coords[0], coords[1]) - elif geom_type == "Polygon": - for ring in coords: - for point in ring: - all_points.append((point[0], point[1])) - elif geom_type == "MultiPolygon": - for polygon in coords: - for ring in polygon: - for point in ring: - all_points.append((point[0], point[1])) - else: - return None - - if not all_points: - return None - - min_lon = min(p[0] for p in all_points) - max_lon = max(p[0] for p in all_points) - min_lat = min(p[1] for p in all_points) - max_lat = max(p[1] for p in all_points) - return (min_lon, min_lat, max_lon, max_lat) - - -def _extract_states_from_codes( - same_codes: list[str], ugc_codes: list[str] -) -> set[str]: - """Extract state abbreviations from SAME and UGC codes.""" - states: set[str] = set() - - for code in same_codes: - if len(code) >= 2: - fips_state = code[:2] - if fips_state in FIPS_TO_STATE: - states.add(FIPS_TO_STATE[fips_state]) - - for code in ugc_codes: - if len(code) >= 2 and code[:2].isalpha(): - states.add(code[:2].upper()) - - return states - - -def _build_regions(same_codes: list[str], ugc_codes: list[str]) -> list[str]: - """Build sorted list of region strings from geocodes.""" - regions: set[str] = set() - - for code in same_codes: - if len(code) >= 2: - fips_state = code[:2] - if fips_state in FIPS_TO_STATE: - state = FIPS_TO_STATE[fips_state] - regions.add(f"US-{state}-FIPS{code}") - - for code in ugc_codes: - if len(code) >= 3 and code[:2].isalpha(): - state = code[:2].upper() - rest = code[2:] - if rest.startswith("C"): - regions.add(f"US-{state}-C{rest[1:]}") - elif rest.startswith("Z"): - regions.add(f"US-{state}-Z{rest[1:]}") - else: - regions.add(f"US-{state}-{rest}") - - return sorted(regions) - - -class NWSAdapter(SourceAdapter): - """National Weather Service alerts adapter.""" - - name = "nws" - - def __init__( - self, - config: NWSAdapterConfig, - cursor_db_path: Path, - ) -> None: - self.config = config - self.states = set(s.upper() for s in config.states) - self.cursor_db_path = cursor_db_path - self._session: aiohttp.ClientSession | None = None - self._db: sqlite3.Connection | None = None - - async def startup(self) -> None: - """Initialize HTTP session and cursor database.""" - user_agent = f"Central/{__version__} ({self.config.contact_email})" - self._session = aiohttp.ClientSession( - headers={"User-Agent": user_agent}, - timeout=aiohttp.ClientTimeout(total=30), - ) - - self._db = sqlite3.connect(str(self.cursor_db_path)) - self._db.execute(""" - CREATE TABLE IF NOT EXISTS adapter_cursors ( - adapter TEXT PRIMARY KEY, - cursor_data TEXT NOT NULL, - updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - ) - """) - self._db.execute(""" - CREATE TABLE IF NOT EXISTS published_ids ( - adapter TEXT NOT NULL, - event_id TEXT NOT NULL, - first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (adapter, event_id) - ) - """) - self._db.execute(""" - CREATE INDEX IF NOT EXISTS published_ids_last_seen - ON published_ids (last_seen) - """) - self._db.commit() - - logger.info("NWS adapter started", extra={"states": list(self.states)}) - - async def shutdown(self) -> None: - """Close HTTP session and database.""" - if self._session: - await self._session.close() - self._session = None - if self._db: - self._db.close() - self._db = None - logger.info("NWS adapter shut down") - - def _get_cursor(self) -> str | None: - """Get the stored If-Modified-Since cursor.""" - if not self._db: - return None - cur = self._db.execute( - "SELECT cursor_data FROM adapter_cursors WHERE adapter = ?", - (self.name,) - ) - row = cur.fetchone() - return row[0] if row else None - - def _set_cursor(self, last_modified: str) -> None: - """Store the Last-Modified header for next request.""" - if not self._db: - return - self._db.execute( - """ - INSERT INTO adapter_cursors (adapter, cursor_data, updated) - VALUES (?, ?, CURRENT_TIMESTAMP) - ON CONFLICT (adapter) DO UPDATE SET - cursor_data = excluded.cursor_data, - updated = CURRENT_TIMESTAMP - """, - (self.name, last_modified) - ) - self._db.commit() - - def is_published(self, event_id: str) -> bool: - """Check if an event has already been published.""" - if not self._db: - return False - cur = self._db.execute( - "SELECT 1 FROM published_ids WHERE adapter = ? AND event_id = ?", - (self.name, event_id) - ) - return cur.fetchone() is not None - - def mark_published(self, event_id: str) -> None: - """Mark an event as published.""" - if not self._db: - return - self._db.execute( - """ - INSERT INTO published_ids (adapter, event_id, first_seen, last_seen) - VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) - ON CONFLICT (adapter, event_id) DO UPDATE SET - last_seen = CURRENT_TIMESTAMP - """, - (self.name, event_id) - ) - self._db.commit() - - def bump_last_seen(self, event_id: str) -> None: - """Bump the last_seen timestamp for an event.""" - if not self._db: - return - self._db.execute( - "UPDATE published_ids SET last_seen = CURRENT_TIMESTAMP WHERE adapter = ? AND event_id = ?", - (self.name, event_id) - ) - self._db.commit() - - def sweep_old_ids(self) -> int: - """Remove published_ids older than 8 days. Returns count deleted.""" - if not self._db: - return 0 - cur = self._db.execute( - "DELETE FROM published_ids WHERE last_seen < datetime('now', '-8 days')" - ) - self._db.commit() - return cur.rowcount - - @retry( - stop=stop_after_attempt(5), - wait=wait_exponential_jitter(initial=1, max=60), - retry=retry_if_exception_type((aiohttp.ClientError, asyncio.TimeoutError)), - reraise=True, - ) - async def _fetch_alerts(self) -> tuple[int, dict[str, Any] | None, str | None]: - """Fetch alerts from NWS API with conditional request.""" - if not self._session: - raise RuntimeError("Session not initialized") - - headers: dict[str, str] = {} - cursor = self._get_cursor() - if cursor: - headers["If-Modified-Since"] = cursor - - async with self._session.get(NWS_API_URL, headers=headers) as resp: - if resp.status in (429, 403): - retry_after = resp.headers.get("Retry-After", "60") - try: - wait_time = int(retry_after) - except ValueError: - wait_time = 60 - logger.warning( - "Rate limited by NWS", - extra={"status": resp.status, "retry_after": wait_time} - ) - await asyncio.sleep(wait_time) - raise aiohttp.ClientError(f"Rate limited: {resp.status}") - - if resp.status == 304: - return (304, None, None) - - resp.raise_for_status() - - data = await resp.json() - last_modified = resp.headers.get("Last-Modified") - - return (resp.status, data, last_modified) - - def _normalize_feature(self, feature: dict[str, Any]) -> Event | None: - """Normalize a GeoJSON feature to an Event.""" - props = feature.get("properties", {}) - geocode = props.get("geocode", {}) - - same_codes = geocode.get("SAME", []) - ugc_codes = geocode.get("UGC", []) - - feature_states = _extract_states_from_codes(same_codes, ugc_codes) - if not feature_states.intersection(self.states): - return None - - event_id = feature.get("id") - if not event_id: - logger.warning("Feature missing id", extra={"properties": props}) - return None - - event_type = props.get("event", "Unknown") - category = f"wx.alert.{_snake_case(event_type)}" - - time = _parse_datetime(props.get("sent")) - if not time: - logger.warning("Feature missing sent time", extra={"id": event_id}) - return None - - expires = _parse_datetime(props.get("expires")) - - severity_str = props.get("severity", "Unknown") - severity = SEVERITY_MAP.get(severity_str) - - geometry = feature.get("geometry") - centroid = _compute_centroid(geometry) - bbox = _compute_bbox(geometry) - regions = _build_regions(same_codes, ugc_codes) - primary_region = regions[0] if regions else None - - geo = Geo( - centroid=centroid, - bbox=bbox, - regions=regions, - primary_region=primary_region, - ) - - return Event( - id=event_id, - source="central/adapters/nws", - category=category, - time=time, - expires=expires, - severity=severity, - geo=geo, - data=props, - ) - - async def poll(self) -> AsyncIterator[Event]: - """Poll NWS API for active alerts.""" - try: - status, data, last_modified = await self._fetch_alerts() - except Exception as e: - logger.error("Failed to fetch NWS alerts", extra={"error": str(e)}) - raise - - if status == 304: - logger.info("NWS returned 304 Not Modified") - return - - if last_modified: - self._set_cursor(last_modified) - - features = data.get("features", []) if data else [] - logger.info( - "NWS poll completed", - extra={"status": status, "feature_count": len(features)} - ) - - yielded = 0 - for feature in features: - try: - event = self._normalize_feature(feature) - if event: - yield event - yielded += 1 - except Exception as e: - logger.warning( - "Failed to normalize feature", - extra={"error": str(e), "feature_id": feature.get("id")} - ) - - logger.info("NWS yielded events", extra={"count": yielded}) +"""NWS (National Weather Service) alert adapter.""" + +import asyncio +import logging +import re +import sqlite3 +from collections.abc import AsyncIterator +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import aiohttp +from tenacity import ( + retry, + stop_after_attempt, + wait_exponential_jitter, + retry_if_exception_type, +) + +from central import __version__ +from central.adapter import SourceAdapter +from central.config_models import AdapterConfig, RegionConfig +from central.models import Event, Geo + +logger = logging.getLogger(__name__) + +# FIPS state codes to postal abbreviations +FIPS_TO_STATE: dict[str, str] = { + "01": "AL", "02": "AK", "04": "AZ", "05": "AR", "06": "CA", + "08": "CO", "09": "CT", "10": "DE", "11": "DC", "12": "FL", + "13": "GA", "15": "HI", "16": "ID", "17": "IL", "18": "IN", + "19": "IA", "20": "KS", "21": "KY", "22": "LA", "23": "ME", + "24": "MD", "25": "MA", "26": "MI", "27": "MN", "28": "MS", + "29": "MO", "30": "MT", "31": "NE", "32": "NV", "33": "NH", + "34": "NJ", "35": "NM", "36": "NY", "37": "NC", "38": "ND", + "39": "OH", "40": "OK", "41": "OR", "42": "PA", "44": "RI", + "45": "SC", "46": "SD", "47": "TN", "48": "TX", "49": "UT", + "50": "VT", "51": "VA", "53": "WA", "54": "WV", "55": "WI", + "56": "WY", "60": "AS", "66": "GU", "69": "MP", "72": "PR", + "78": "VI", +} + +SEVERITY_MAP: dict[str, int | None] = { + "Extreme": 4, + "Severe": 3, + "Moderate": 2, + "Minor": 1, + "Unknown": None, +} + +NWS_API_URL = "https://api.weather.gov/alerts/active" + + +def _snake_case(s: str) -> str: + """Convert a string to snake_case.""" + s = re.sub(r"[^a-zA-Z0-9\s]", "", s) + s = re.sub(r"\s+", "_", s.strip()) + return s.lower() + + +def _parse_datetime(s: str | None) -> datetime | None: + """Parse an ISO datetime string to UTC datetime.""" + if not s: + return None + try: + dt = datetime.fromisoformat(s.replace("Z", "+00:00")) + return dt.astimezone(timezone.utc) + except (ValueError, TypeError): + return None + + +def _compute_centroid(geometry: dict[str, Any] | None) -> tuple[float, float] | None: + """Compute centroid from GeoJSON geometry using arithmetic mean of vertices.""" + if not geometry: + return None + + geom_type = geometry.get("type") + coords = geometry.get("coordinates") + + if not coords: + return None + + all_points: list[tuple[float, float]] = [] + + if geom_type == "Point": + return (coords[0], coords[1]) + elif geom_type == "Polygon": + for ring in coords: + for point in ring: + all_points.append((point[0], point[1])) + elif geom_type == "MultiPolygon": + for polygon in coords: + for ring in polygon: + for point in ring: + all_points.append((point[0], point[1])) + else: + return None + + if not all_points: + return None + + avg_lon = sum(p[0] for p in all_points) / len(all_points) + avg_lat = sum(p[1] for p in all_points) / len(all_points) + return (avg_lon, avg_lat) + + +def _compute_bbox( + geometry: dict[str, Any] | None +) -> tuple[float, float, float, float] | None: + """Compute bounding box from GeoJSON geometry.""" + if not geometry: + return None + + geom_type = geometry.get("type") + coords = geometry.get("coordinates") + + if not coords: + return None + + all_points: list[tuple[float, float]] = [] + + if geom_type == "Point": + return (coords[0], coords[1], coords[0], coords[1]) + elif geom_type == "Polygon": + for ring in coords: + for point in ring: + all_points.append((point[0], point[1])) + elif geom_type == "MultiPolygon": + for polygon in coords: + for ring in polygon: + for point in ring: + all_points.append((point[0], point[1])) + else: + return None + + if not all_points: + return None + + min_lon = min(p[0] for p in all_points) + max_lon = max(p[0] for p in all_points) + min_lat = min(p[1] for p in all_points) + max_lat = max(p[1] for p in all_points) + return (min_lon, min_lat, max_lon, max_lat) + + +def _extract_states_from_codes( + same_codes: list[str], ugc_codes: list[str] +) -> set[str]: + """Extract state abbreviations from SAME and UGC codes.""" + states: set[str] = set() + + for code in same_codes: + if len(code) >= 2: + fips_state = code[:2] + if fips_state in FIPS_TO_STATE: + states.add(FIPS_TO_STATE[fips_state]) + + for code in ugc_codes: + if len(code) >= 2 and code[:2].isalpha(): + states.add(code[:2].upper()) + + return states + + +def _build_regions(same_codes: list[str], ugc_codes: list[str]) -> list[str]: + """Build sorted list of region strings from geocodes.""" + regions: set[str] = set() + + for code in same_codes: + if len(code) >= 2: + fips_state = code[:2] + if fips_state in FIPS_TO_STATE: + state = FIPS_TO_STATE[fips_state] + regions.add(f"US-{state}-FIPS{code}") + + for code in ugc_codes: + if len(code) >= 3 and code[:2].isalpha(): + state = code[:2].upper() + rest = code[2:] + if rest.startswith("C"): + regions.add(f"US-{state}-C{rest[1:]}") + elif rest.startswith("Z"): + regions.add(f"US-{state}-Z{rest[1:]}") + else: + regions.add(f"US-{state}-{rest}") + + return sorted(regions) + + +class NWSAdapter(SourceAdapter): + """National Weather Service alerts adapter.""" + + name = "nws" + + def __init__( + self, + config: AdapterConfig, + cursor_db_path: Path, + ) -> None: + self.cursor_db_path = cursor_db_path + self._session: aiohttp.ClientSession | None = None + self._db: sqlite3.Connection | None = None + + # Extract settings from unified config + self.contact_email: str = config.settings.get("contact_email", "") + + # Parse region from settings + region_dict = config.settings.get("region") + if region_dict: + self.region: RegionConfig | None = RegionConfig(**region_dict) + else: + self.region = None + + async def apply_config(self, new_config: AdapterConfig) -> None: + """Apply new configuration from hot-reload.""" + # Update contact email + self.contact_email = new_config.settings.get("contact_email", "") + + # Update region + region_dict = new_config.settings.get("region") + if region_dict: + self.region = RegionConfig(**region_dict) + else: + self.region = None + + logger.info( + "NWS config applied", + extra={ + "region": region_dict, + "contact_email": self.contact_email, + }, + ) + + def _point_in_region(self, centroid: tuple[float, float] | None) -> bool: + """Check if centroid is within configured region bbox.""" + if self.region is None: + # No region configured = accept all + return True + if centroid is None: + return False + lon, lat = centroid + return ( + self.region.west <= lon <= self.region.east + and self.region.south <= lat <= self.region.north + ) + + async def startup(self) -> None: + """Initialize HTTP session and cursor database.""" + user_agent = f"Central/{__version__} ({self.contact_email})" + self._session = aiohttp.ClientSession( + headers={"User-Agent": user_agent}, + timeout=aiohttp.ClientTimeout(total=30), + ) + + self._db = sqlite3.connect(str(self.cursor_db_path)) + self._db.execute(""" + CREATE TABLE IF NOT EXISTS adapter_cursors ( + adapter TEXT PRIMARY KEY, + cursor_data TEXT NOT NULL, + updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + """) + self._db.execute(""" + CREATE TABLE IF NOT EXISTS published_ids ( + adapter TEXT NOT NULL, + event_id TEXT NOT NULL, + first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (adapter, event_id) + ) + """) + self._db.execute(""" + CREATE INDEX IF NOT EXISTS published_ids_last_seen + ON published_ids (last_seen) + """) + self._db.commit() + + logger.info( + "NWS adapter started", + extra={ + "region": { + "north": self.region.north, + "south": self.region.south, + "east": self.region.east, + "west": self.region.west, + } if self.region else None, + }, + ) + + async def shutdown(self) -> None: + """Close HTTP session and database.""" + if self._session: + await self._session.close() + self._session = None + if self._db: + self._db.close() + self._db = None + logger.info("NWS adapter shut down") + + def _get_cursor(self) -> str | None: + """Get the stored If-Modified-Since cursor.""" + if not self._db: + return None + cur = self._db.execute( + "SELECT cursor_data FROM adapter_cursors WHERE adapter = ?", + (self.name,) + ) + row = cur.fetchone() + return row[0] if row else None + + def _set_cursor(self, last_modified: str) -> None: + """Store the Last-Modified header for next request.""" + if not self._db: + return + self._db.execute( + """ + INSERT INTO adapter_cursors (adapter, cursor_data, updated) + VALUES (?, ?, CURRENT_TIMESTAMP) + ON CONFLICT (adapter) DO UPDATE SET + cursor_data = excluded.cursor_data, + updated = CURRENT_TIMESTAMP + """, + (self.name, last_modified) + ) + self._db.commit() + + def is_published(self, event_id: str) -> bool: + """Check if an event has already been published.""" + if not self._db: + return False + cur = self._db.execute( + "SELECT 1 FROM published_ids WHERE adapter = ? AND event_id = ?", + (self.name, event_id) + ) + return cur.fetchone() is not None + + def mark_published(self, event_id: str) -> None: + """Mark an event as published.""" + if not self._db: + return + self._db.execute( + """ + INSERT INTO published_ids (adapter, event_id, first_seen, last_seen) + VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + ON CONFLICT (adapter, event_id) DO UPDATE SET + last_seen = CURRENT_TIMESTAMP + """, + (self.name, event_id) + ) + self._db.commit() + + def bump_last_seen(self, event_id: str) -> None: + """Bump the last_seen timestamp for an event.""" + if not self._db: + return + self._db.execute( + "UPDATE published_ids SET last_seen = CURRENT_TIMESTAMP WHERE adapter = ? AND event_id = ?", + (self.name, event_id) + ) + self._db.commit() + + def sweep_old_ids(self) -> int: + """Remove published_ids older than 8 days. Returns count deleted.""" + if not self._db: + return 0 + cur = self._db.execute( + "DELETE FROM published_ids WHERE last_seen < datetime('now', '-8 days')" + ) + self._db.commit() + return cur.rowcount + + @retry( + stop=stop_after_attempt(5), + wait=wait_exponential_jitter(initial=1, max=60), + retry=retry_if_exception_type((aiohttp.ClientError, asyncio.TimeoutError)), + reraise=True, + ) + async def _fetch_alerts(self) -> tuple[int, dict[str, Any] | None, str | None]: + """Fetch alerts from NWS API with conditional request.""" + if not self._session: + raise RuntimeError("Session not initialized") + + headers: dict[str, str] = {} + cursor = self._get_cursor() + if cursor: + headers["If-Modified-Since"] = cursor + + async with self._session.get(NWS_API_URL, headers=headers) as resp: + if resp.status in (429, 403): + retry_after = resp.headers.get("Retry-After", "60") + try: + wait_time = int(retry_after) + except ValueError: + wait_time = 60 + logger.warning( + "Rate limited by NWS", + extra={"status": resp.status, "retry_after": wait_time} + ) + await asyncio.sleep(wait_time) + raise aiohttp.ClientError(f"Rate limited: {resp.status}") + + if resp.status == 304: + return (304, None, None) + + resp.raise_for_status() + + data = await resp.json() + last_modified = resp.headers.get("Last-Modified") + + return (resp.status, data, last_modified) + + def _normalize_feature(self, feature: dict[str, Any]) -> Event | None: + """Normalize a GeoJSON feature to an Event.""" + props = feature.get("properties", {}) + geocode = props.get("geocode", {}) + + same_codes = geocode.get("SAME", []) + ugc_codes = geocode.get("UGC", []) + + # Compute geometry data first + geometry = feature.get("geometry") + centroid = _compute_centroid(geometry) + bbox = _compute_bbox(geometry) + + # Filter by region bbox (client-side filtering) + if not self._point_in_region(centroid): + return None + + event_id = feature.get("id") + if not event_id: + logger.warning("Feature missing id", extra={"properties": props}) + return None + + event_type = props.get("event", "Unknown") + category = f"wx.alert.{_snake_case(event_type)}" + + time = _parse_datetime(props.get("sent")) + if not time: + logger.warning("Feature missing sent time", extra={"id": event_id}) + return None + + expires = _parse_datetime(props.get("expires")) + + severity_str = props.get("severity", "Unknown") + severity = SEVERITY_MAP.get(severity_str) + + regions = _build_regions(same_codes, ugc_codes) + primary_region = regions[0] if regions else None + + geo = Geo( + centroid=centroid, + bbox=bbox, + regions=regions, + primary_region=primary_region, + ) + + return Event( + id=event_id, + source="central/adapters/nws", + category=category, + time=time, + expires=expires, + severity=severity, + geo=geo, + data=props, + ) + + async def poll(self) -> AsyncIterator[Event]: + """Poll NWS API for active alerts.""" + try: + status, data, last_modified = await self._fetch_alerts() + except Exception as e: + logger.error("Failed to fetch NWS alerts", extra={"error": str(e)}) + raise + + if status == 304: + logger.info("NWS returned 304 Not Modified") + return + + if last_modified: + self._set_cursor(last_modified) + + features = data.get("features", []) if data else [] + logger.info( + "NWS poll completed", + extra={"status": status, "feature_count": len(features)} + ) + + yielded = 0 + for feature in features: + try: + event = self._normalize_feature(feature) + if event: + yield event + yielded += 1 + except Exception as e: + logger.warning( + "Failed to normalize feature", + extra={"error": str(e), "feature_id": feature.get("id")} + ) + + logger.info("NWS yielded events", extra={"count": yielded}) diff --git a/src/central/config_models.py b/src/central/config_models.py index 02855ca..0447c56 100644 --- a/src/central/config_models.py +++ b/src/central/config_models.py @@ -1,39 +1,70 @@ -"""Pydantic models for database-backed configuration.""" - -from datetime import datetime -from typing import Any - -from pydantic import BaseModel, Field - - -class AdapterConfig(BaseModel): - """Configuration for a single adapter.""" - - name: str = Field(description="Unique adapter identifier") - enabled: bool = Field(default=True, description="Whether adapter is active") - cadence_s: int = Field(description="Poll interval in seconds") - settings: dict[str, Any] = Field( - default_factory=dict, description="Adapter-specific settings" - ) - paused_at: datetime | None = Field( - default=None, description="When adapter was paused, if paused" - ) - updated_at: datetime = Field(description="Last configuration update time") - - @property - def is_paused(self) -> bool: - """Check if adapter is currently paused.""" - return self.paused_at is not None - - -class ApiKeyInfo(BaseModel): - """Metadata about an API key (without the decrypted value).""" - - alias: str = Field(description="Key identifier/alias") - created_at: datetime = Field(description="When key was created") - rotated_at: datetime | None = Field( - default=None, description="Last rotation time" - ) - last_used_at: datetime | None = Field( - default=None, description="Last usage time" - ) +"""Pydantic models for database-backed configuration.""" + +from datetime import datetime +from typing import Any + +from pydantic import BaseModel, Field, model_validator + + +class RegionConfig(BaseModel): + """Geographic bounding box for adapter region filtering.""" + + north: float = Field(ge=-90, le=90, description="Northern latitude bound") + south: float = Field(ge=-90, le=90, description="Southern latitude bound") + east: float = Field(ge=-180, le=180, description="Eastern longitude bound") + west: float = Field(ge=-180, le=180, description="Western longitude bound") + + @model_validator(mode="after") + def validate_bounds(self) -> "RegionConfig": + if self.north <= self.south: + raise ValueError( + f"north ({self.north}) must be greater than south ({self.south})" + ) + if self.east == self.west: + raise ValueError("east and west cannot be equal (zero-width bbox)") + return self + + +class AdapterConfig(BaseModel): + """Configuration for a single adapter.""" + + name: str = Field(description="Unique adapter identifier") + enabled: bool = Field(default=True, description="Whether adapter is active") + cadence_s: int = Field(description="Poll interval in seconds") + settings: dict[str, Any] = Field( + default_factory=dict, description="Adapter-specific settings" + ) + paused_at: datetime | None = Field( + default=None, description="When adapter was paused, if paused" + ) + updated_at: datetime = Field(description="Last configuration update time") + + @property + def is_paused(self) -> bool: + """Check if adapter is currently paused.""" + return self.paused_at is not None + + +class StreamConfig(BaseModel): + """Configuration for a JetStream stream.""" + + name: str = Field(description="Stream name") + max_age_s: int = Field(description="Maximum message age in seconds") + max_bytes: int = Field(description="Maximum stream size in bytes") + managed_max_bytes: bool = Field( + default=True, description="Whether max_bytes is auto-managed by supervisor" + ) + updated_at: datetime = Field(description="Last configuration update time") + + +class ApiKeyInfo(BaseModel): + """Metadata about an API key (without the decrypted value).""" + + alias: str = Field(description="Key identifier/alias") + created_at: datetime = Field(description="When key was created") + rotated_at: datetime | None = Field( + default=None, description="Last rotation time" + ) + last_used_at: datetime | None = Field( + default=None, description="Last usage time" + ) From ab7126ec8da73a42723d67f55aeb4c3c2236b4aa Mon Sep 17 00:00:00 2001 From: Matt Johnson Date: Sat, 16 May 2026 18:49:53 +0000 Subject: [PATCH 4/9] refactor(supervisor): remove adapter-specific branches, add stream wiring - Replace if name == nws with generic apply_config call - Add _create_adapter factory method - Add stream management: ensure_stream, retention recompute loop - Handle streams config changes via NOTIFY Co-Authored-By: Claude Opus 4.5 --- src/central/supervisor.py | 231 ++++++++++++++++++++++++++++++-------- 1 file changed, 185 insertions(+), 46 deletions(-) diff --git a/src/central/supervisor.py b/src/central/supervisor.py index fad8588..2543291 100644 --- a/src/central/supervisor.py +++ b/src/central/supervisor.py @@ -13,16 +13,27 @@ from typing import Any import nats from nats.js import JetStreamContext +from central.adapter import SourceAdapter from central.adapters.nws import NWSAdapter from central.cloudevents_wire import wrap_event -from central.config import NWSAdapterConfig from central.config_models import AdapterConfig from central.config_source import ConfigSource, DbConfigSource +from central.config_store import ConfigStore from central.bootstrap_config import get_settings from central.models import subject_for_event +from central.stream_manager import StreamManager CURSOR_DB_PATH = Path("/var/lib/central/cursors.db") +# Stream subject mappings +STREAM_SUBJECTS = { + "CENTRAL_WX": ["central.wx.>"], + "CENTRAL_META": ["central.meta.>"], +} + +# Recompute interval for stream max_bytes (1 hour) +STREAM_RECOMPUTE_INTERVAL_S = 3600 + class JsonFormatter(logging.Formatter): """JSON log formatter for structured logging.""" @@ -66,7 +77,7 @@ class AdapterState: """Runtime state for a scheduled adapter.""" name: str - adapter: NWSAdapter + adapter: SourceAdapter config: AdapterConfig task: asyncio.Task[None] | None = None last_completed_poll: datetime | None = None @@ -84,14 +95,17 @@ class Supervisor: def __init__( self, config_source: ConfigSource, + config_store: ConfigStore, nats_url: str, cloudevents_config: Any = None, ) -> None: self._config_source = config_source + self._config_store = config_store self._nats_url = nats_url self._cloudevents_config = cloudevents_config self._nc: nats.NATS | None = None self._js: JetStreamContext | None = None + self._stream_manager: StreamManager | None = None self._adapter_states: dict[str, AdapterState] = {} self._tasks: list[asyncio.Task[None]] = [] self._shutdown_event = asyncio.Event() @@ -103,6 +117,7 @@ class Supervisor: """Connect to NATS.""" self._nc = await nats.connect(self._nats_url) self._js = self._nc.jetstream() + self._stream_manager = StreamManager(self._js) logger.info("Connected to NATS", extra={"url": self._nats_url}) async def disconnect(self) -> None: @@ -112,6 +127,7 @@ class Supervisor: await self._nc.close() self._nc = None self._js = None + self._stream_manager = None logger.info("Disconnected from NATS") async def _publish_meta(self, subject: str, data: dict[str, Any]) -> None: @@ -132,14 +148,12 @@ class Supervisor: headers={"Nats-Msg-Id": msg_id}, ) - def _adapter_config_to_nws_config(self, config: AdapterConfig) -> NWSAdapterConfig: - """Convert unified AdapterConfig to NWSAdapterConfig.""" - return NWSAdapterConfig( - enabled=config.enabled, - cadence_s=config.cadence_s, - states=config.settings.get("states", []), - contact_email=config.settings.get("contact_email", ""), - ) + def _create_adapter(self, config: AdapterConfig) -> SourceAdapter: + """Create an adapter instance based on config name.""" + if config.name == "nws": + return NWSAdapter(config=config, cursor_db_path=CURSOR_DB_PATH) + else: + raise ValueError(f"Unknown adapter type: {config.name}") async def _run_adapter_loop(self, state: AdapterState) -> None: """Run an adapter poll loop with rate-limit aware scheduling.""" @@ -263,11 +277,7 @@ class Supervisor: existing_state.cancel_event.clear() # Reinitialize the adapter with new config - nws_config = self._adapter_config_to_nws_config(config) - existing_state.adapter = NWSAdapter( - config=nws_config, - cursor_db_path=CURSOR_DB_PATH, - ) + existing_state.adapter = self._create_adapter(config) await existing_state.adapter.startup() # Start the loop task @@ -300,34 +310,29 @@ class Supervisor: return # New adapter - create fresh state - if config.name == "nws": - nws_config = self._adapter_config_to_nws_config(config) - adapter = NWSAdapter( - config=nws_config, - cursor_db_path=CURSOR_DB_PATH, - ) - await adapter.startup() + try: + adapter = self._create_adapter(config) + except ValueError as e: + logger.warning(str(e), extra={"adapter": config.name}) + return - state = AdapterState( - name=config.name, - adapter=adapter, - config=config, - ) - state.task = asyncio.create_task(self._run_adapter_loop(state)) - self._adapter_states[config.name] = state + await adapter.startup() - logger.info( - "Adapter started", - extra={ - "adapter": config.name, - "cadence_s": config.cadence_s, - }, - ) - else: - logger.warning( - "Unknown adapter type", - extra={"adapter": config.name}, - ) + state = AdapterState( + name=config.name, + adapter=adapter, + config=config, + ) + state.task = asyncio.create_task(self._run_adapter_loop(state)) + self._adapter_states[config.name] = state + + logger.info( + "Adapter started", + extra={ + "adapter": config.name, + "cadence_s": config.cadence_s, + }, + ) async def _stop_adapter(self, name: str) -> None: """Stop a running adapter but preserve state for potential restart. @@ -423,10 +428,8 @@ class Supervisor: # Update config state.config = new_config - # Update adapter settings if needed (e.g., states list) - if name == "nws": - nws_config = self._adapter_config_to_nws_config(new_config) - state.adapter.states = set(s.upper() for s in nws_config.states) + # Apply config to adapter (generic - each adapter handles its own settings) + await state.adapter.apply_config(new_config) # Calculate next poll time for logging if state.last_completed_poll: @@ -451,11 +454,139 @@ class Supervisor: # This ensures immediate event delivery to the sleeping loop. return state + async def _ensure_streams(self) -> None: + """Ensure all configured streams exist with correct settings.""" + if not self._stream_manager: + return + + streams = await self._config_store.list_streams() + for stream_config in streams: + subjects = STREAM_SUBJECTS.get(stream_config.name, []) + if not subjects: + logger.warning( + "No subjects configured for stream", + extra={"stream": stream_config.name}, + ) + continue + + try: + await self._stream_manager.ensure_stream( + stream_config.name, + subjects, + stream_config, + ) + except Exception as e: + logger.error( + "Failed to ensure stream", + extra={"stream": stream_config.name, "error": str(e)}, + ) + + async def _handle_stream_change(self, stream_name: str) -> None: + """Handle a stream configuration change.""" + if not self._stream_manager: + return + + stream_config = await self._config_store.get_stream(stream_name) + if stream_config is None: + logger.warning( + "Stream config not found", + extra={"stream": stream_name}, + ) + return + + try: + # Apply retention settings + await self._stream_manager.apply_retention(stream_name, stream_config) + + # Immediate recompute of max_bytes + new_max_bytes = await self._stream_manager.recompute_max_bytes( + stream_name, stream_config.max_age_s + ) + + # Update database (won't trigger NOTIFY due to column filter) + await self._config_store.update_stream_max_bytes(stream_name, new_max_bytes) + + logger.info( + "Stream retention updated", + extra={ + "stream": stream_name, + "max_age_s": stream_config.max_age_s, + "new_max_bytes": new_max_bytes, + }, + ) + except Exception as e: + logger.error( + "Failed to handle stream change", + extra={"stream": stream_name, "error": str(e)}, + ) + + async def _stream_retention_recompute_loop(self) -> None: + """Periodically recompute max_bytes for all streams.""" + while not self._shutdown_event.is_set(): + try: + await asyncio.wait_for( + self._shutdown_event.wait(), + timeout=STREAM_RECOMPUTE_INTERVAL_S, + ) + # Shutdown requested + break + except asyncio.TimeoutError: + pass + + # Recompute for all streams + if not self._stream_manager: + continue + + streams = await self._config_store.list_streams() + for stream_config in streams: + if not stream_config.managed_max_bytes: + continue + + try: + new_max_bytes = await self._stream_manager.recompute_max_bytes( + stream_config.name, stream_config.max_age_s + ) + + # Only update if change > 10% + change_ratio = abs(new_max_bytes - stream_config.max_bytes) / max(stream_config.max_bytes, 1) + if change_ratio > 0.10: + await self._config_store.update_stream_max_bytes( + stream_config.name, new_max_bytes + ) + await self._stream_manager.apply_retention( + stream_config.name, + await self._config_store.get_stream(stream_config.name), + ) + logger.info( + "Recomputed stream max_bytes", + extra={ + "stream": stream_config.name, + "old_max_bytes": stream_config.max_bytes, + "new_max_bytes": new_max_bytes, + "change_ratio": change_ratio, + }, + ) + except Exception as e: + logger.error( + "Failed to recompute stream max_bytes", + extra={"stream": stream_config.name, "error": str(e)}, + ) + async def _on_config_change(self, table: str, key: str) -> None: """Handle a configuration change notification. Called when NOTIFY fires for config changes. """ + # Handle stream changes + if table == "streams": + stream_name = key + logger.info( + "Stream config change received", + extra={"stream": stream_name}, + ) + await self._handle_stream_change(stream_name) + return + if table != "adapters": return @@ -534,6 +665,9 @@ class Supervisor: """Start the supervisor.""" await self.connect() + # Ensure streams exist with correct configuration + await self._ensure_streams() + # Load and start enabled adapters enabled_adapters = await self._config_source.list_enabled_adapters() for config in enabled_adapters: @@ -547,6 +681,9 @@ class Supervisor: # Start heartbeat self._tasks.append(asyncio.create_task(self._heartbeat_loop())) + # Start stream retention recompute loop + self._tasks.append(asyncio.create_task(self._stream_retention_recompute_loop())) + logger.info( "Supervisor started", extra={"adapters": list(self._adapter_states.keys())}, @@ -594,11 +731,13 @@ async def async_main() -> None: extra={"config_source": "db"}, ) - # Create database config source + # Create database config source and store config_source = await DbConfigSource.create(settings.db_dsn) + config_store = await ConfigStore.create(settings.db_dsn) supervisor = Supervisor( config_source=config_source, + config_store=config_store, nats_url=settings.nats_url, # CloudEvents uses protocol-level defaults from cloudevents_constants cloudevents_config=None, From 71a43d3c98f8ad45fb50e3bb2982ef59bcbb5208 Mon Sep 17 00:00:00 2001 From: Matt Johnson Date: Sat, 16 May 2026 18:49:59 +0000 Subject: [PATCH 5/9] schema: add config.streams table with column-filtered notify - config.streams table for JetStream retention config - Column-filtered NOTIFY: only fires on max_age_s changes - Prevents self-loop when supervisor updates max_bytes - Seeds CENTRAL_WX (7d/10GB) and CENTRAL_META (1d/100MB) Co-Authored-By: Claude Opus 4.5 --- sql/migrations/003_add_streams_table.sql | 46 ++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 sql/migrations/003_add_streams_table.sql diff --git a/sql/migrations/003_add_streams_table.sql b/sql/migrations/003_add_streams_table.sql new file mode 100644 index 0000000..27d59cd --- /dev/null +++ b/sql/migrations/003_add_streams_table.sql @@ -0,0 +1,46 @@ +-- Migration: 003_add_streams_table +-- Creates the config.streams table for JetStream stream retention configuration. +-- Uses column-filtered NOTIFY to prevent self-loop when supervisor updates max_bytes. + +-- Streams configuration table +CREATE TABLE config.streams ( + name TEXT PRIMARY KEY, + max_age_s BIGINT NOT NULL, + max_bytes BIGINT NOT NULL DEFAULT 1073741824, -- 1GB default + managed_max_bytes BOOLEAN NOT NULL DEFAULT true, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Auto-update trigger for updated_at +CREATE TRIGGER streams_set_updated_at + BEFORE UPDATE ON config.streams + FOR EACH ROW EXECUTE FUNCTION config.set_updated_at(); + +-- Column-filtered NOTIFY trigger for streams. +-- Fires on INSERT/DELETE always. +-- On UPDATE, only fires when max_age_s changes (operator-touchable field), +-- NOT when max_bytes changes (supervisor-managed), to prevent recompute loop. +CREATE OR REPLACE FUNCTION config.notify_streams_change() +RETURNS trigger AS $$ +BEGIN + IF TG_OP = 'INSERT' OR TG_OP = 'DELETE' THEN + PERFORM pg_notify('config_changed', 'streams:' || + COALESCE(NEW.name, OLD.name)); + ELSIF TG_OP = 'UPDATE' AND + OLD.max_age_s IS DISTINCT FROM NEW.max_age_s THEN + PERFORM pg_notify('config_changed', 'streams:' || NEW.name); + END IF; + RETURN COALESCE(NEW, OLD); +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER streams_notify + AFTER INSERT OR UPDATE OR DELETE ON config.streams + FOR EACH ROW EXECUTE FUNCTION config.notify_streams_change(); + +-- Seed with current stream values from investigation +-- CENTRAL_WX: 7d max_age (604800s), 10GB max_bytes (will be clamped to 6GB on first recompute) +-- CENTRAL_META: 1d max_age (86400s), 100MB max_bytes (will be raised to 1GB floor) +INSERT INTO config.streams (name, max_age_s, max_bytes) VALUES + ('CENTRAL_WX', 604800, 10737418240), + ('CENTRAL_META', 86400, 104857600); From da8942a457ba27e63d9fb26901bbdd85fbbae29e Mon Sep 17 00:00:00 2001 From: Matt Johnson Date: Sat, 16 May 2026 18:50:05 +0000 Subject: [PATCH 6/9] schema: migrate NWS settings from states to region bbox - Remove states array from NWS settings - Add region bbox covering ID/OR/WA/MT/WY/UT/NV - Bbox: north=49.5, south=31.0, east=-102.0, west=-124.5 Co-Authored-By: Claude Opus 4.5 --- sql/migrations/004_nws_states_to_bbox.sql | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 sql/migrations/004_nws_states_to_bbox.sql diff --git a/sql/migrations/004_nws_states_to_bbox.sql b/sql/migrations/004_nws_states_to_bbox.sql new file mode 100644 index 0000000..5294a87 --- /dev/null +++ b/sql/migrations/004_nws_states_to_bbox.sql @@ -0,0 +1,11 @@ +-- Migration: 004_nws_states_to_bbox +-- Converts NWS adapter settings from states list to region bbox. +-- Bbox covers ID/OR/WA/MT/WY/UT/NV with buffer. + +UPDATE config.adapters +SET settings = jsonb_set( + settings - 'states', -- Remove states key + '{region}', + '{"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}'::jsonb +) +WHERE name = 'nws'; From f9426caa27a035f2069cae4a77a0f1a0b5aa345f Mon Sep 17 00:00:00 2001 From: Matt Johnson Date: Sat, 16 May 2026 18:50:12 +0000 Subject: [PATCH 7/9] feat: add stream management infrastructure - config_store: add stream CRUD methods - stream_manager: ensure_stream, apply_retention, recompute_max_bytes - Auto-clamp max_bytes to [1GB floor, 30% ceiling] - Parse server max_file_store from nats-server.conf Co-Authored-By: Claude Opus 4.5 --- src/central/config_store.py | 65 ++++++++- src/central/stream_manager.py | 262 ++++++++++++++++++++++++++++++++++ 2 files changed, 326 insertions(+), 1 deletion(-) create mode 100644 src/central/stream_manager.py diff --git a/src/central/config_store.py b/src/central/config_store.py index 0f03826..20415b9 100644 --- a/src/central/config_store.py +++ b/src/central/config_store.py @@ -12,7 +12,7 @@ from typing import Any import asyncpg -from central.config_models import AdapterConfig +from central.config_models import AdapterConfig, StreamConfig from central.crypto import decrypt, encrypt logger = logging.getLogger(__name__) @@ -129,6 +129,69 @@ class ConfigStore: name, ) + # ------------------------------------------------------------------------- + # Stream configuration + # ------------------------------------------------------------------------- + + async def get_stream(self, name: str) -> StreamConfig | None: + """Get configuration for a specific stream.""" + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + """ + SELECT name, max_age_s, max_bytes, managed_max_bytes, updated_at + FROM config.streams + WHERE name = $1 + """, + name, + ) + if row is None: + return None + return StreamConfig(**dict(row)) + + async def list_streams(self) -> list[StreamConfig]: + """List all configured streams.""" + async with self._pool.acquire() as conn: + rows = await conn.fetch( + """ + SELECT name, max_age_s, max_bytes, managed_max_bytes, updated_at + FROM config.streams + ORDER BY name + """ + ) + return [StreamConfig(**dict(row)) for row in rows] + + async def upsert_stream(self, name: str, max_age_s: int) -> None: + """Insert or update a stream's max_age_s (operator-facing).""" + async with self._pool.acquire() as conn: + await conn.execute( + """ + INSERT INTO config.streams (name, max_age_s, updated_at) + VALUES ($1, $2, now()) + ON CONFLICT (name) DO UPDATE SET + max_age_s = EXCLUDED.max_age_s, + updated_at = now() + """, + name, + max_age_s, + ) + + async def update_stream_max_bytes(self, name: str, max_bytes: int) -> None: + """Update a stream's max_bytes (supervisor-internal). + + This update only touches max_bytes, which does NOT trigger + the column-filtered NOTIFY (only max_age_s changes fire NOTIFY). + """ + async with self._pool.acquire() as conn: + await conn.execute( + """ + UPDATE config.streams + SET max_bytes = $2, updated_at = now() + WHERE name = $1 + """, + name, + max_bytes, + ) + # ------------------------------------------------------------------------- # API key management # ------------------------------------------------------------------------- diff --git a/src/central/stream_manager.py b/src/central/stream_manager.py new file mode 100644 index 0000000..4b9b5ba --- /dev/null +++ b/src/central/stream_manager.py @@ -0,0 +1,262 @@ +"""JetStream stream manager for retention configuration.""" + +import logging +import re +from pathlib import Path +from typing import Any + +from nats.js import JetStreamContext +from nats.js.api import StreamConfig, DiscardPolicy, RetentionPolicy + +from central.config_models import StreamConfig as StreamConfigModel + +logger = logging.getLogger(__name__) + +# Constants +ONE_GB = 1024 * 1024 * 1024 # 1 GiB in bytes +NATS_CONFIG_PATH = Path("/etc/nats/nats-server.conf") + + +class StreamManager: + """Manages JetStream stream configuration and retention.""" + + def __init__(self, js: JetStreamContext) -> None: + self._js = js + self._server_max_file_store: int | None = None + + async def server_max_file_store_bytes(self) -> int: + """Get the server's max_file_store setting in bytes. + + Parses the NATS server config file and caches the result. + Returns a default of 20GB if config cannot be read. + """ + if self._server_max_file_store is not None: + return self._server_max_file_store + + default_value = 20 * ONE_GB # 20GB default + + try: + config_text = NATS_CONFIG_PATH.read_text() + + # Parse max_file_store value (supports GB/MB/KB suffixes) + match = re.search(r'max_file_store:\s*(\d+)(GB|MB|KB|G|M|K)?', config_text, re.IGNORECASE) + if match: + value = int(match.group(1)) + suffix = (match.group(2) or "").upper() + + if suffix in ("GB", "G"): + value *= ONE_GB + elif suffix in ("MB", "M"): + value *= 1024 * 1024 + elif suffix in ("KB", "K"): + value *= 1024 + # else: assume bytes + + self._server_max_file_store = value + logger.info( + "Parsed server max_file_store", + extra={"max_file_store_bytes": value}, + ) + return value + + logger.warning( + "max_file_store not found in config, using default", + extra={"default": default_value}, + ) + self._server_max_file_store = default_value + return default_value + + except Exception as e: + logger.warning( + "Failed to read NATS config, using default", + extra={"error": str(e), "default": default_value}, + ) + self._server_max_file_store = default_value + return default_value + + def _compute_ceiling(self, server_max: int) -> int: + """Compute per-stream ceiling as 30% of server max_file_store.""" + return int(server_max * 0.30) + + async def ensure_stream( + self, + name: str, + subjects: list[str], + config: StreamConfigModel, + ) -> None: + """Ensure a stream exists with the given configuration. + + Creates the stream if it doesn't exist, or updates it if it does. + Always enforces: discard=old, max_msgs=-1 (unlimited). + """ + server_max = await self.server_max_file_store_bytes() + ceiling = self._compute_ceiling(server_max) + + # Clamp max_bytes to [1GB, ceiling] + max_bytes = max(ONE_GB, min(config.max_bytes, ceiling)) + + stream_config = StreamConfig( + name=name, + subjects=subjects, + retention=RetentionPolicy.LIMITS, + discard=DiscardPolicy.OLD, + max_age=config.max_age_s, + max_bytes=max_bytes, + max_msgs=-1, # Unlimited messages + ) + + try: + # Try to get existing stream + existing = await self._js.stream_info(name) + + # Update if config differs + await self._js.update_stream(config=stream_config) + logger.info( + "Updated stream", + extra={ + "stream": name, + "max_age_s": config.max_age_s, + "max_bytes": max_bytes, + }, + ) + + except Exception as e: + if "stream not found" in str(e).lower(): + # Create new stream + await self._js.add_stream(config=stream_config) + logger.info( + "Created stream", + extra={ + "stream": name, + "subjects": subjects, + "max_age_s": config.max_age_s, + "max_bytes": max_bytes, + }, + ) + else: + raise + + async def apply_retention(self, name: str, config: StreamConfigModel) -> None: + """Apply retention settings to an existing stream. + + Updates max_age and max_bytes. Always enforces discard=old, max_msgs=-1. + """ + server_max = await self.server_max_file_store_bytes() + ceiling = self._compute_ceiling(server_max) + + # Clamp max_bytes to [1GB, ceiling] + max_bytes = max(ONE_GB, min(config.max_bytes, ceiling)) + + try: + # Get current stream config + info = await self._js.stream_info(name) + current = info.config + + # Build updated config + updated = StreamConfig( + name=name, + subjects=current.subjects, + retention=RetentionPolicy.LIMITS, + discard=DiscardPolicy.OLD, + max_age=config.max_age_s, + max_bytes=max_bytes, + max_msgs=-1, + ) + + await self._js.update_stream(config=updated) + logger.info( + "Applied retention", + extra={ + "stream": name, + "max_age_s": config.max_age_s, + "max_bytes": max_bytes, + }, + ) + + except Exception as e: + logger.error( + "Failed to apply retention", + extra={"stream": name, "error": str(e)}, + ) + raise + + async def recompute_max_bytes(self, name: str, max_age_s: int) -> int: + """Recompute max_bytes based on observed throughput. + + Formula: rate × max_age × 1.5 safety margin, clamped to [1GB, ceiling]. + + Returns the computed max_bytes value. + """ + server_max = await self.server_max_file_store_bytes() + ceiling = self._compute_ceiling(server_max) + + try: + info = await self._js.stream_info(name) + current_bytes = info.state.bytes + current_msgs = info.state.messages + + # Get stream age from first message + first_seq = info.state.first_seq + last_seq = info.state.last_seq + + if current_msgs == 0 or last_seq == 0: + # No messages yet, use floor + return ONE_GB + + # Estimate message age span (approximation) + # Use stream's configured max_age as the observation window + configured_max_age = info.config.max_age + + if configured_max_age > 0: + # Rate = current_bytes / configured_max_age (in seconds) + rate_per_second = current_bytes / configured_max_age + else: + # Fallback: assume 1 day of data + rate_per_second = current_bytes / 86400 + + # Project bytes needed for new max_age with 1.5x safety margin + projected = int(rate_per_second * max_age_s * 1.5) + + # Clamp to [1GB, ceiling] + result = max(ONE_GB, min(projected, ceiling)) + + logger.info( + "Recomputed max_bytes", + extra={ + "stream": name, + "current_bytes": current_bytes, + "rate_per_second": rate_per_second, + "max_age_s": max_age_s, + "projected": projected, + "result": result, + "ceiling": ceiling, + }, + ) + + return result + + except Exception as e: + logger.error( + "Failed to recompute max_bytes, using floor", + extra={"stream": name, "error": str(e)}, + ) + return ONE_GB + + async def get_stream_stats(self, name: str) -> dict[str, Any]: + """Get current stream statistics for monitoring.""" + try: + info = await self._js.stream_info(name) + return { + "stream": name, + "bytes": info.state.bytes, + "messages": info.state.messages, + "max_bytes": info.config.max_bytes, + "max_age_s": info.config.max_age, + "consumers": info.state.consumer_count, + } + except Exception as e: + logger.error( + "Failed to get stream stats", + extra={"stream": name, "error": str(e)}, + ) + return {"stream": name, "error": str(e)} From a157f39fe058fa3cc43a1cb8472f8c940a730309 Mon Sep 17 00:00:00 2001 From: Matt Johnson Date: Sat, 16 May 2026 19:05:05 +0000 Subject: [PATCH 8/9] fix(nws): replace centroid filter with polygon intersection - Add shapely dependency for geometry intersection - Replace _point_in_region with _geometry_intersects_region - Uses Shapely shape() and box() for proper GeoJSON handling - Avoids false negatives on large alert polygons Also adds antimeridian-crossing bbox rejection to RegionConfig validator. Co-Authored-By: Claude Opus 4.5 --- pyproject.toml | 1 + src/central/adapters/nws.py | 1021 +++++++++++++++++----------------- src/central/config_models.py | 2 + uv.lock | 149 ++++- 4 files changed, 660 insertions(+), 513 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d257973..63974db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ dependencies = [ "nats-py>=2.14.0", "pydantic>=2,<3", "pydantic-settings>=2.7.0", + "shapely>=2.0", "tenacity>=9.1.4", ] diff --git a/src/central/adapters/nws.py b/src/central/adapters/nws.py index 76391e1..d2ad155 100644 --- a/src/central/adapters/nws.py +++ b/src/central/adapters/nws.py @@ -1,502 +1,519 @@ -"""NWS (National Weather Service) alert adapter.""" - -import asyncio -import logging -import re -import sqlite3 -from collections.abc import AsyncIterator -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -import aiohttp -from tenacity import ( - retry, - stop_after_attempt, - wait_exponential_jitter, - retry_if_exception_type, -) - -from central import __version__ -from central.adapter import SourceAdapter -from central.config_models import AdapterConfig, RegionConfig -from central.models import Event, Geo - -logger = logging.getLogger(__name__) - -# FIPS state codes to postal abbreviations -FIPS_TO_STATE: dict[str, str] = { - "01": "AL", "02": "AK", "04": "AZ", "05": "AR", "06": "CA", - "08": "CO", "09": "CT", "10": "DE", "11": "DC", "12": "FL", - "13": "GA", "15": "HI", "16": "ID", "17": "IL", "18": "IN", - "19": "IA", "20": "KS", "21": "KY", "22": "LA", "23": "ME", - "24": "MD", "25": "MA", "26": "MI", "27": "MN", "28": "MS", - "29": "MO", "30": "MT", "31": "NE", "32": "NV", "33": "NH", - "34": "NJ", "35": "NM", "36": "NY", "37": "NC", "38": "ND", - "39": "OH", "40": "OK", "41": "OR", "42": "PA", "44": "RI", - "45": "SC", "46": "SD", "47": "TN", "48": "TX", "49": "UT", - "50": "VT", "51": "VA", "53": "WA", "54": "WV", "55": "WI", - "56": "WY", "60": "AS", "66": "GU", "69": "MP", "72": "PR", - "78": "VI", -} - -SEVERITY_MAP: dict[str, int | None] = { - "Extreme": 4, - "Severe": 3, - "Moderate": 2, - "Minor": 1, - "Unknown": None, -} - -NWS_API_URL = "https://api.weather.gov/alerts/active" - - -def _snake_case(s: str) -> str: - """Convert a string to snake_case.""" - s = re.sub(r"[^a-zA-Z0-9\s]", "", s) - s = re.sub(r"\s+", "_", s.strip()) - return s.lower() - - -def _parse_datetime(s: str | None) -> datetime | None: - """Parse an ISO datetime string to UTC datetime.""" - if not s: - return None - try: - dt = datetime.fromisoformat(s.replace("Z", "+00:00")) - return dt.astimezone(timezone.utc) - except (ValueError, TypeError): - return None - - -def _compute_centroid(geometry: dict[str, Any] | None) -> tuple[float, float] | None: - """Compute centroid from GeoJSON geometry using arithmetic mean of vertices.""" - if not geometry: - return None - - geom_type = geometry.get("type") - coords = geometry.get("coordinates") - - if not coords: - return None - - all_points: list[tuple[float, float]] = [] - - if geom_type == "Point": - return (coords[0], coords[1]) - elif geom_type == "Polygon": - for ring in coords: - for point in ring: - all_points.append((point[0], point[1])) - elif geom_type == "MultiPolygon": - for polygon in coords: - for ring in polygon: - for point in ring: - all_points.append((point[0], point[1])) - else: - return None - - if not all_points: - return None - - avg_lon = sum(p[0] for p in all_points) / len(all_points) - avg_lat = sum(p[1] for p in all_points) / len(all_points) - return (avg_lon, avg_lat) - - -def _compute_bbox( - geometry: dict[str, Any] | None -) -> tuple[float, float, float, float] | None: - """Compute bounding box from GeoJSON geometry.""" - if not geometry: - return None - - geom_type = geometry.get("type") - coords = geometry.get("coordinates") - - if not coords: - return None - - all_points: list[tuple[float, float]] = [] - - if geom_type == "Point": - return (coords[0], coords[1], coords[0], coords[1]) - elif geom_type == "Polygon": - for ring in coords: - for point in ring: - all_points.append((point[0], point[1])) - elif geom_type == "MultiPolygon": - for polygon in coords: - for ring in polygon: - for point in ring: - all_points.append((point[0], point[1])) - else: - return None - - if not all_points: - return None - - min_lon = min(p[0] for p in all_points) - max_lon = max(p[0] for p in all_points) - min_lat = min(p[1] for p in all_points) - max_lat = max(p[1] for p in all_points) - return (min_lon, min_lat, max_lon, max_lat) - - -def _extract_states_from_codes( - same_codes: list[str], ugc_codes: list[str] -) -> set[str]: - """Extract state abbreviations from SAME and UGC codes.""" - states: set[str] = set() - - for code in same_codes: - if len(code) >= 2: - fips_state = code[:2] - if fips_state in FIPS_TO_STATE: - states.add(FIPS_TO_STATE[fips_state]) - - for code in ugc_codes: - if len(code) >= 2 and code[:2].isalpha(): - states.add(code[:2].upper()) - - return states - - -def _build_regions(same_codes: list[str], ugc_codes: list[str]) -> list[str]: - """Build sorted list of region strings from geocodes.""" - regions: set[str] = set() - - for code in same_codes: - if len(code) >= 2: - fips_state = code[:2] - if fips_state in FIPS_TO_STATE: - state = FIPS_TO_STATE[fips_state] - regions.add(f"US-{state}-FIPS{code}") - - for code in ugc_codes: - if len(code) >= 3 and code[:2].isalpha(): - state = code[:2].upper() - rest = code[2:] - if rest.startswith("C"): - regions.add(f"US-{state}-C{rest[1:]}") - elif rest.startswith("Z"): - regions.add(f"US-{state}-Z{rest[1:]}") - else: - regions.add(f"US-{state}-{rest}") - - return sorted(regions) - - -class NWSAdapter(SourceAdapter): - """National Weather Service alerts adapter.""" - - name = "nws" - - def __init__( - self, - config: AdapterConfig, - cursor_db_path: Path, - ) -> None: - self.cursor_db_path = cursor_db_path - self._session: aiohttp.ClientSession | None = None - self._db: sqlite3.Connection | None = None - - # Extract settings from unified config - self.contact_email: str = config.settings.get("contact_email", "") - - # Parse region from settings - region_dict = config.settings.get("region") - if region_dict: - self.region: RegionConfig | None = RegionConfig(**region_dict) - else: - self.region = None - - async def apply_config(self, new_config: AdapterConfig) -> None: - """Apply new configuration from hot-reload.""" - # Update contact email - self.contact_email = new_config.settings.get("contact_email", "") - - # Update region - region_dict = new_config.settings.get("region") - if region_dict: - self.region = RegionConfig(**region_dict) - else: - self.region = None - - logger.info( - "NWS config applied", - extra={ - "region": region_dict, - "contact_email": self.contact_email, - }, - ) - - def _point_in_region(self, centroid: tuple[float, float] | None) -> bool: - """Check if centroid is within configured region bbox.""" - if self.region is None: - # No region configured = accept all - return True - if centroid is None: - return False - lon, lat = centroid - return ( - self.region.west <= lon <= self.region.east - and self.region.south <= lat <= self.region.north - ) - - async def startup(self) -> None: - """Initialize HTTP session and cursor database.""" - user_agent = f"Central/{__version__} ({self.contact_email})" - self._session = aiohttp.ClientSession( - headers={"User-Agent": user_agent}, - timeout=aiohttp.ClientTimeout(total=30), - ) - - self._db = sqlite3.connect(str(self.cursor_db_path)) - self._db.execute(""" - CREATE TABLE IF NOT EXISTS adapter_cursors ( - adapter TEXT PRIMARY KEY, - cursor_data TEXT NOT NULL, - updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - ) - """) - self._db.execute(""" - CREATE TABLE IF NOT EXISTS published_ids ( - adapter TEXT NOT NULL, - event_id TEXT NOT NULL, - first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (adapter, event_id) - ) - """) - self._db.execute(""" - CREATE INDEX IF NOT EXISTS published_ids_last_seen - ON published_ids (last_seen) - """) - self._db.commit() - - logger.info( - "NWS adapter started", - extra={ - "region": { - "north": self.region.north, - "south": self.region.south, - "east": self.region.east, - "west": self.region.west, - } if self.region else None, - }, - ) - - async def shutdown(self) -> None: - """Close HTTP session and database.""" - if self._session: - await self._session.close() - self._session = None - if self._db: - self._db.close() - self._db = None - logger.info("NWS adapter shut down") - - def _get_cursor(self) -> str | None: - """Get the stored If-Modified-Since cursor.""" - if not self._db: - return None - cur = self._db.execute( - "SELECT cursor_data FROM adapter_cursors WHERE adapter = ?", - (self.name,) - ) - row = cur.fetchone() - return row[0] if row else None - - def _set_cursor(self, last_modified: str) -> None: - """Store the Last-Modified header for next request.""" - if not self._db: - return - self._db.execute( - """ - INSERT INTO adapter_cursors (adapter, cursor_data, updated) - VALUES (?, ?, CURRENT_TIMESTAMP) - ON CONFLICT (adapter) DO UPDATE SET - cursor_data = excluded.cursor_data, - updated = CURRENT_TIMESTAMP - """, - (self.name, last_modified) - ) - self._db.commit() - - def is_published(self, event_id: str) -> bool: - """Check if an event has already been published.""" - if not self._db: - return False - cur = self._db.execute( - "SELECT 1 FROM published_ids WHERE adapter = ? AND event_id = ?", - (self.name, event_id) - ) - return cur.fetchone() is not None - - def mark_published(self, event_id: str) -> None: - """Mark an event as published.""" - if not self._db: - return - self._db.execute( - """ - INSERT INTO published_ids (adapter, event_id, first_seen, last_seen) - VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) - ON CONFLICT (adapter, event_id) DO UPDATE SET - last_seen = CURRENT_TIMESTAMP - """, - (self.name, event_id) - ) - self._db.commit() - - def bump_last_seen(self, event_id: str) -> None: - """Bump the last_seen timestamp for an event.""" - if not self._db: - return - self._db.execute( - "UPDATE published_ids SET last_seen = CURRENT_TIMESTAMP WHERE adapter = ? AND event_id = ?", - (self.name, event_id) - ) - self._db.commit() - - def sweep_old_ids(self) -> int: - """Remove published_ids older than 8 days. Returns count deleted.""" - if not self._db: - return 0 - cur = self._db.execute( - "DELETE FROM published_ids WHERE last_seen < datetime('now', '-8 days')" - ) - self._db.commit() - return cur.rowcount - - @retry( - stop=stop_after_attempt(5), - wait=wait_exponential_jitter(initial=1, max=60), - retry=retry_if_exception_type((aiohttp.ClientError, asyncio.TimeoutError)), - reraise=True, - ) - async def _fetch_alerts(self) -> tuple[int, dict[str, Any] | None, str | None]: - """Fetch alerts from NWS API with conditional request.""" - if not self._session: - raise RuntimeError("Session not initialized") - - headers: dict[str, str] = {} - cursor = self._get_cursor() - if cursor: - headers["If-Modified-Since"] = cursor - - async with self._session.get(NWS_API_URL, headers=headers) as resp: - if resp.status in (429, 403): - retry_after = resp.headers.get("Retry-After", "60") - try: - wait_time = int(retry_after) - except ValueError: - wait_time = 60 - logger.warning( - "Rate limited by NWS", - extra={"status": resp.status, "retry_after": wait_time} - ) - await asyncio.sleep(wait_time) - raise aiohttp.ClientError(f"Rate limited: {resp.status}") - - if resp.status == 304: - return (304, None, None) - - resp.raise_for_status() - - data = await resp.json() - last_modified = resp.headers.get("Last-Modified") - - return (resp.status, data, last_modified) - - def _normalize_feature(self, feature: dict[str, Any]) -> Event | None: - """Normalize a GeoJSON feature to an Event.""" - props = feature.get("properties", {}) - geocode = props.get("geocode", {}) - - same_codes = geocode.get("SAME", []) - ugc_codes = geocode.get("UGC", []) - - # Compute geometry data first - geometry = feature.get("geometry") - centroid = _compute_centroid(geometry) - bbox = _compute_bbox(geometry) - - # Filter by region bbox (client-side filtering) - if not self._point_in_region(centroid): - return None - - event_id = feature.get("id") - if not event_id: - logger.warning("Feature missing id", extra={"properties": props}) - return None - - event_type = props.get("event", "Unknown") - category = f"wx.alert.{_snake_case(event_type)}" - - time = _parse_datetime(props.get("sent")) - if not time: - logger.warning("Feature missing sent time", extra={"id": event_id}) - return None - - expires = _parse_datetime(props.get("expires")) - - severity_str = props.get("severity", "Unknown") - severity = SEVERITY_MAP.get(severity_str) - - regions = _build_regions(same_codes, ugc_codes) - primary_region = regions[0] if regions else None - - geo = Geo( - centroid=centroid, - bbox=bbox, - regions=regions, - primary_region=primary_region, - ) - - return Event( - id=event_id, - source="central/adapters/nws", - category=category, - time=time, - expires=expires, - severity=severity, - geo=geo, - data=props, - ) - - async def poll(self) -> AsyncIterator[Event]: - """Poll NWS API for active alerts.""" - try: - status, data, last_modified = await self._fetch_alerts() - except Exception as e: - logger.error("Failed to fetch NWS alerts", extra={"error": str(e)}) - raise - - if status == 304: - logger.info("NWS returned 304 Not Modified") - return - - if last_modified: - self._set_cursor(last_modified) - - features = data.get("features", []) if data else [] - logger.info( - "NWS poll completed", - extra={"status": status, "feature_count": len(features)} - ) - - yielded = 0 - for feature in features: - try: - event = self._normalize_feature(feature) - if event: - yield event - yielded += 1 - except Exception as e: - logger.warning( - "Failed to normalize feature", - extra={"error": str(e), "feature_id": feature.get("id")} - ) - - logger.info("NWS yielded events", extra={"count": yielded}) +"""NWS (National Weather Service) alert adapter.""" + +import asyncio +import logging +import re +import sqlite3 +from collections.abc import AsyncIterator +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import aiohttp +from tenacity import ( + retry, + stop_after_attempt, + wait_exponential_jitter, + retry_if_exception_type, +) + +from central import __version__ +from central.adapter import SourceAdapter +from central.config_models import AdapterConfig, RegionConfig +from central.models import Event, Geo +from shapely.geometry import box as shapely_box, shape as shapely_shape + +logger = logging.getLogger(__name__) + +# FIPS state codes to postal abbreviations +FIPS_TO_STATE: dict[str, str] = { + "01": "AL", "02": "AK", "04": "AZ", "05": "AR", "06": "CA", + "08": "CO", "09": "CT", "10": "DE", "11": "DC", "12": "FL", + "13": "GA", "15": "HI", "16": "ID", "17": "IL", "18": "IN", + "19": "IA", "20": "KS", "21": "KY", "22": "LA", "23": "ME", + "24": "MD", "25": "MA", "26": "MI", "27": "MN", "28": "MS", + "29": "MO", "30": "MT", "31": "NE", "32": "NV", "33": "NH", + "34": "NJ", "35": "NM", "36": "NY", "37": "NC", "38": "ND", + "39": "OH", "40": "OK", "41": "OR", "42": "PA", "44": "RI", + "45": "SC", "46": "SD", "47": "TN", "48": "TX", "49": "UT", + "50": "VT", "51": "VA", "53": "WA", "54": "WV", "55": "WI", + "56": "WY", "60": "AS", "66": "GU", "69": "MP", "72": "PR", + "78": "VI", +} + +SEVERITY_MAP: dict[str, int | None] = { + "Extreme": 4, + "Severe": 3, + "Moderate": 2, + "Minor": 1, + "Unknown": None, +} + +NWS_API_URL = "https://api.weather.gov/alerts/active" + + +def _snake_case(s: str) -> str: + """Convert a string to snake_case.""" + s = re.sub(r"[^a-zA-Z0-9\s]", "", s) + s = re.sub(r"\s+", "_", s.strip()) + return s.lower() + + +def _parse_datetime(s: str | None) -> datetime | None: + """Parse an ISO datetime string to UTC datetime.""" + if not s: + return None + try: + dt = datetime.fromisoformat(s.replace("Z", "+00:00")) + return dt.astimezone(timezone.utc) + except (ValueError, TypeError): + return None + + +def _compute_centroid(geometry: dict[str, Any] | None) -> tuple[float, float] | None: + """Compute centroid from GeoJSON geometry using arithmetic mean of vertices.""" + if not geometry: + return None + + geom_type = geometry.get("type") + coords = geometry.get("coordinates") + + if not coords: + return None + + all_points: list[tuple[float, float]] = [] + + if geom_type == "Point": + return (coords[0], coords[1]) + elif geom_type == "Polygon": + for ring in coords: + for point in ring: + all_points.append((point[0], point[1])) + elif geom_type == "MultiPolygon": + for polygon in coords: + for ring in polygon: + for point in ring: + all_points.append((point[0], point[1])) + else: + return None + + if not all_points: + return None + + avg_lon = sum(p[0] for p in all_points) / len(all_points) + avg_lat = sum(p[1] for p in all_points) / len(all_points) + return (avg_lon, avg_lat) + + +def _compute_bbox( + geometry: dict[str, Any] | None +) -> tuple[float, float, float, float] | None: + """Compute bounding box from GeoJSON geometry.""" + if not geometry: + return None + + geom_type = geometry.get("type") + coords = geometry.get("coordinates") + + if not coords: + return None + + all_points: list[tuple[float, float]] = [] + + if geom_type == "Point": + return (coords[0], coords[1], coords[0], coords[1]) + elif geom_type == "Polygon": + for ring in coords: + for point in ring: + all_points.append((point[0], point[1])) + elif geom_type == "MultiPolygon": + for polygon in coords: + for ring in polygon: + for point in ring: + all_points.append((point[0], point[1])) + else: + return None + + if not all_points: + return None + + min_lon = min(p[0] for p in all_points) + max_lon = max(p[0] for p in all_points) + min_lat = min(p[1] for p in all_points) + max_lat = max(p[1] for p in all_points) + return (min_lon, min_lat, max_lon, max_lat) + + +def _extract_states_from_codes( + same_codes: list[str], ugc_codes: list[str] +) -> set[str]: + """Extract state abbreviations from SAME and UGC codes.""" + states: set[str] = set() + + for code in same_codes: + if len(code) >= 2: + fips_state = code[:2] + if fips_state in FIPS_TO_STATE: + states.add(FIPS_TO_STATE[fips_state]) + + for code in ugc_codes: + if len(code) >= 2 and code[:2].isalpha(): + states.add(code[:2].upper()) + + return states + + +def _build_regions(same_codes: list[str], ugc_codes: list[str]) -> list[str]: + """Build sorted list of region strings from geocodes.""" + regions: set[str] = set() + + for code in same_codes: + if len(code) >= 2: + fips_state = code[:2] + if fips_state in FIPS_TO_STATE: + state = FIPS_TO_STATE[fips_state] + regions.add(f"US-{state}-FIPS{code}") + + for code in ugc_codes: + if len(code) >= 3 and code[:2].isalpha(): + state = code[:2].upper() + rest = code[2:] + if rest.startswith("C"): + regions.add(f"US-{state}-C{rest[1:]}") + elif rest.startswith("Z"): + regions.add(f"US-{state}-Z{rest[1:]}") + else: + regions.add(f"US-{state}-{rest}") + + return sorted(regions) + + +class NWSAdapter(SourceAdapter): + """National Weather Service alerts adapter.""" + + name = "nws" + + def __init__( + self, + config: AdapterConfig, + cursor_db_path: Path, + ) -> None: + self.cursor_db_path = cursor_db_path + self._session: aiohttp.ClientSession | None = None + self._db: sqlite3.Connection | None = None + + # Extract settings from unified config + self.contact_email: str = config.settings.get("contact_email", "") + + # Parse region from settings + region_dict = config.settings.get("region") + if region_dict: + self.region: RegionConfig | None = RegionConfig(**region_dict) + else: + self.region = None + + async def apply_config(self, new_config: AdapterConfig) -> None: + """Apply new configuration from hot-reload.""" + # Update contact email + self.contact_email = new_config.settings.get("contact_email", "") + + # Update region + region_dict = new_config.settings.get("region") + if region_dict: + self.region = RegionConfig(**region_dict) + else: + self.region = None + + logger.info( + "NWS config applied", + extra={ + "region": region_dict, + "contact_email": self.contact_email, + }, + ) + + def _geometry_intersects_region(self, geometry: dict[str, Any] | None) -> bool: + """Check if feature geometry intersects configured region bbox. + + Uses Shapely for proper polygon intersection rather than centroid-only + filtering, avoiding false negatives on large alert polygons. + """ + if self.region is None: + # No region configured = accept all + return True + if geometry is None: + return False + + try: + # Build region box (west, south, east, north) + region_box = shapely_box( + self.region.west, + self.region.south, + self.region.east, + self.region.north, + ) + + # Parse GeoJSON geometry to shapely shape + feature_shape = shapely_shape(geometry) + + return region_box.intersects(feature_shape) + except Exception: + # If geometry parsing fails, fall back to rejecting + return False + + async def startup(self) -> None: + """Initialize HTTP session and cursor database.""" + user_agent = f"Central/{__version__} ({self.contact_email})" + self._session = aiohttp.ClientSession( + headers={"User-Agent": user_agent}, + timeout=aiohttp.ClientTimeout(total=30), + ) + + self._db = sqlite3.connect(str(self.cursor_db_path)) + self._db.execute(""" + CREATE TABLE IF NOT EXISTS adapter_cursors ( + adapter TEXT PRIMARY KEY, + cursor_data TEXT NOT NULL, + updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + """) + self._db.execute(""" + CREATE TABLE IF NOT EXISTS published_ids ( + adapter TEXT NOT NULL, + event_id TEXT NOT NULL, + first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (adapter, event_id) + ) + """) + self._db.execute(""" + CREATE INDEX IF NOT EXISTS published_ids_last_seen + ON published_ids (last_seen) + """) + self._db.commit() + + logger.info( + "NWS adapter started", + extra={ + "region": { + "north": self.region.north, + "south": self.region.south, + "east": self.region.east, + "west": self.region.west, + } if self.region else None, + }, + ) + + async def shutdown(self) -> None: + """Close HTTP session and database.""" + if self._session: + await self._session.close() + self._session = None + if self._db: + self._db.close() + self._db = None + logger.info("NWS adapter shut down") + + def _get_cursor(self) -> str | None: + """Get the stored If-Modified-Since cursor.""" + if not self._db: + return None + cur = self._db.execute( + "SELECT cursor_data FROM adapter_cursors WHERE adapter = ?", + (self.name,) + ) + row = cur.fetchone() + return row[0] if row else None + + def _set_cursor(self, last_modified: str) -> None: + """Store the Last-Modified header for next request.""" + if not self._db: + return + self._db.execute( + """ + INSERT INTO adapter_cursors (adapter, cursor_data, updated) + VALUES (?, ?, CURRENT_TIMESTAMP) + ON CONFLICT (adapter) DO UPDATE SET + cursor_data = excluded.cursor_data, + updated = CURRENT_TIMESTAMP + """, + (self.name, last_modified) + ) + self._db.commit() + + def is_published(self, event_id: str) -> bool: + """Check if an event has already been published.""" + if not self._db: + return False + cur = self._db.execute( + "SELECT 1 FROM published_ids WHERE adapter = ? AND event_id = ?", + (self.name, event_id) + ) + return cur.fetchone() is not None + + def mark_published(self, event_id: str) -> None: + """Mark an event as published.""" + if not self._db: + return + self._db.execute( + """ + INSERT INTO published_ids (adapter, event_id, first_seen, last_seen) + VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + ON CONFLICT (adapter, event_id) DO UPDATE SET + last_seen = CURRENT_TIMESTAMP + """, + (self.name, event_id) + ) + self._db.commit() + + def bump_last_seen(self, event_id: str) -> None: + """Bump the last_seen timestamp for an event.""" + if not self._db: + return + self._db.execute( + "UPDATE published_ids SET last_seen = CURRENT_TIMESTAMP WHERE adapter = ? AND event_id = ?", + (self.name, event_id) + ) + self._db.commit() + + def sweep_old_ids(self) -> int: + """Remove published_ids older than 8 days. Returns count deleted.""" + if not self._db: + return 0 + cur = self._db.execute( + "DELETE FROM published_ids WHERE last_seen < datetime('now', '-8 days')" + ) + self._db.commit() + return cur.rowcount + + @retry( + stop=stop_after_attempt(5), + wait=wait_exponential_jitter(initial=1, max=60), + retry=retry_if_exception_type((aiohttp.ClientError, asyncio.TimeoutError)), + reraise=True, + ) + async def _fetch_alerts(self) -> tuple[int, dict[str, Any] | None, str | None]: + """Fetch alerts from NWS API with conditional request.""" + if not self._session: + raise RuntimeError("Session not initialized") + + headers: dict[str, str] = {} + cursor = self._get_cursor() + if cursor: + headers["If-Modified-Since"] = cursor + + async with self._session.get(NWS_API_URL, headers=headers) as resp: + if resp.status in (429, 403): + retry_after = resp.headers.get("Retry-After", "60") + try: + wait_time = int(retry_after) + except ValueError: + wait_time = 60 + logger.warning( + "Rate limited by NWS", + extra={"status": resp.status, "retry_after": wait_time} + ) + await asyncio.sleep(wait_time) + raise aiohttp.ClientError(f"Rate limited: {resp.status}") + + if resp.status == 304: + return (304, None, None) + + resp.raise_for_status() + + data = await resp.json() + last_modified = resp.headers.get("Last-Modified") + + return (resp.status, data, last_modified) + + def _normalize_feature(self, feature: dict[str, Any]) -> Event | None: + """Normalize a GeoJSON feature to an Event.""" + props = feature.get("properties", {}) + geocode = props.get("geocode", {}) + + same_codes = geocode.get("SAME", []) + ugc_codes = geocode.get("UGC", []) + + # Compute geometry data first + geometry = feature.get("geometry") + centroid = _compute_centroid(geometry) + bbox = _compute_bbox(geometry) + + # Filter by region bbox (client-side filtering) + if not self._geometry_intersects_region(geometry): + return None + + event_id = feature.get("id") + if not event_id: + logger.warning("Feature missing id", extra={"properties": props}) + return None + + event_type = props.get("event", "Unknown") + category = f"wx.alert.{_snake_case(event_type)}" + + time = _parse_datetime(props.get("sent")) + if not time: + logger.warning("Feature missing sent time", extra={"id": event_id}) + return None + + expires = _parse_datetime(props.get("expires")) + + severity_str = props.get("severity", "Unknown") + severity = SEVERITY_MAP.get(severity_str) + + regions = _build_regions(same_codes, ugc_codes) + primary_region = regions[0] if regions else None + + geo = Geo( + centroid=centroid, + bbox=bbox, + regions=regions, + primary_region=primary_region, + ) + + return Event( + id=event_id, + source="central/adapters/nws", + category=category, + time=time, + expires=expires, + severity=severity, + geo=geo, + data=props, + ) + + async def poll(self) -> AsyncIterator[Event]: + """Poll NWS API for active alerts.""" + try: + status, data, last_modified = await self._fetch_alerts() + except Exception as e: + logger.error("Failed to fetch NWS alerts", extra={"error": str(e)}) + raise + + if status == 304: + logger.info("NWS returned 304 Not Modified") + return + + if last_modified: + self._set_cursor(last_modified) + + features = data.get("features", []) if data else [] + logger.info( + "NWS poll completed", + extra={"status": status, "feature_count": len(features)} + ) + + yielded = 0 + for feature in features: + try: + event = self._normalize_feature(feature) + if event: + yield event + yielded += 1 + except Exception as e: + logger.warning( + "Failed to normalize feature", + extra={"error": str(e), "feature_id": feature.get("id")} + ) + + logger.info("NWS yielded events", extra={"count": yielded}) diff --git a/src/central/config_models.py b/src/central/config_models.py index 0447c56..a5ba0d5 100644 --- a/src/central/config_models.py +++ b/src/central/config_models.py @@ -22,6 +22,8 @@ class RegionConfig(BaseModel): ) if self.east == self.west: raise ValueError("east and west cannot be equal (zero-width bbox)") + if self.east < self.west: + raise ValueError("antimeridian-crossing bboxes not supported") return self diff --git a/uv.lock b/uv.lock index 555c2dd..75abe93 100644 --- a/uv.lock +++ b/uv.lock @@ -45,15 +45,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/aa/ca/eadf6f9c8fa5e31d40993e3db153fb5ed0b11008ad5d9de98a95045bed84/aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1", size = 460446, upload-time = "2026-03-31T21:58:10.945Z" }, ] -[[package]] -name = "aiolimiter" -version = "1.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f1/23/b52debf471f7a1e42e362d959a3982bdcb4fe13a5d46e63d28868807a79c/aiolimiter-1.2.1.tar.gz", hash = "sha256:e02a37ea1a855d9e832252a105420ad4d15011505512a1a1d814647451b5cca9", size = 7185, upload-time = "2024-12-08T15:31:51.496Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/ba/df6e8e1045aebc4778d19b8a3a9bc1808adb1619ba94ca354d9ba17d86c3/aiolimiter-1.2.1-py3-none-any.whl", hash = "sha256:d3f249e9059a20badcb56b61601a83556133655c11d1eb3dd3e04ff069e5f3c7", size = 6711, upload-time = "2024-12-08T15:31:49.874Z" }, -] - [[package]] name = "aiosignal" version = "1.4.0" @@ -130,11 +121,13 @@ version = "0.1.0" source = { editable = "." } dependencies = [ { name = "aiohttp" }, - { name = "aiolimiter" }, { name = "asyncpg" }, { name = "cloudevents" }, + { name = "cryptography" }, { name = "nats-py" }, { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "shapely" }, { name = "tenacity" }, ] @@ -149,11 +142,13 @@ dev = [ [package.metadata] requires-dist = [ { name = "aiohttp", specifier = ">=3.13.5" }, - { name = "aiolimiter", specifier = ">=1.2.1" }, { name = "asyncpg", specifier = ">=0.31.0" }, { name = "cloudevents", specifier = ">=2.0.0" }, + { name = "cryptography", specifier = ">=44.0.0" }, { name = "nats-py", specifier = ">=2.14.0" }, { name = "pydantic", specifier = ">=2,<3" }, + { name = "pydantic-settings", specifier = ">=2.7.0" }, + { name = "shapely", specifier = ">=2.0" }, { name = "tenacity", specifier = ">=9.1.4" }, ] @@ -165,6 +160,29 @@ dev = [ { name = "ruff", specifier = ">=0.15.13" }, ] +[[package]] +name = "cffi" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271, upload-time = "2025-09-08T23:22:44.795Z" }, + { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048, upload-time = "2025-09-08T23:22:45.938Z" }, + { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" }, + { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097, upload-time = "2025-09-08T23:22:48.677Z" }, + { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983, upload-time = "2025-09-08T23:22:50.06Z" }, + { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519, upload-time = "2025-09-08T23:22:51.364Z" }, + { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572, upload-time = "2025-09-08T23:22:52.902Z" }, + { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963, upload-time = "2025-09-08T23:22:54.518Z" }, + { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361, upload-time = "2025-09-08T23:22:55.867Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932, upload-time = "2025-09-08T23:22:57.188Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557, upload-time = "2025-09-08T23:22:58.351Z" }, + { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762, upload-time = "2025-09-08T23:22:59.668Z" }, +] + [[package]] name = "cloudevents" version = "2.0.0" @@ -187,6 +205,45 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "cryptography" +version = "48.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9f/a9/db8f313fdcd85d767d4973515e1db101f9c71f95fced83233de224673757/cryptography-48.0.0.tar.gz", hash = "sha256:5c3932f4436d1cccb036cb0eaef46e6e2db91035166f1ad6505c3c9d5a635920", size = 832984, upload-time = "2026-05-04T22:59:38.133Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/3d/01f6dd9190170a5a241e0e98c2d04be3664a9e6f5b9b872cde63aff1c3dd/cryptography-48.0.0-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:0c558d2cdffd8f4bbb30fc7134c74d2ca9a476f830bb053074498fbc86f41ed6", size = 8001587, upload-time = "2026-05-04T22:57:36.803Z" }, + { url = "https://files.pythonhosted.org/packages/b2/6e/e90527eef33f309beb811cf7c982c3aeffcce8e3edb178baa4ca3ae4a6fa/cryptography-48.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f5333311663ea94f75dd408665686aaf426563556bb5283554a3539177e03b8c", size = 4690433, upload-time = "2026-05-04T22:57:40.373Z" }, + { url = "https://files.pythonhosted.org/packages/90/04/673510ed51ddff56575f306cf1617d80411ee76831ccd3097599140efdfe/cryptography-48.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7995ef305d7165c3f11ae07f2517e5a4f1d5c18da1376a0a9ed496336b69e5f3", size = 4710620, upload-time = "2026-05-04T22:57:42.935Z" }, + { url = "https://files.pythonhosted.org/packages/14/d5/e9c4ef932c8d800490c34d8bd589d64a31d5890e27ec9e9ad532be893294/cryptography-48.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:40ba1f85eaa6959837b1d51c9767e230e14612eea4ef110ee8854ada22da1bf5", size = 4696283, upload-time = "2026-05-04T22:57:45.294Z" }, + { url = "https://files.pythonhosted.org/packages/0c/29/174b9dfb60b12d59ecfc6cfa04bc88c21b42a54f01b8aae09bb6e51e4c7f/cryptography-48.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:369a6348999f94bbd53435c894377b20ab95f25a9065c283570e70150d8abc3c", size = 5296573, upload-time = "2026-05-04T22:57:47.933Z" }, + { url = "https://files.pythonhosted.org/packages/95/38/0d29a6fd7d0d1373f0c0c88a04ba20e359b257753ac497564cd660fc1d55/cryptography-48.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a0e692c683f4df67815a2d258b324e66f4738bd7a96a218c826dce4f4bd05d8f", size = 4743677, upload-time = "2026-05-04T22:57:50.067Z" }, + { url = "https://files.pythonhosted.org/packages/30/be/eef653013d5c63b6a490529e0316f9ac14a37602965d4903efed1399f32b/cryptography-48.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:18349bbc56f4743c8b12dc32e2bccb2cf83ee8b69a3bba74ef8ae857e26b3d25", size = 4330808, upload-time = "2026-05-04T22:57:52.301Z" }, + { url = "https://files.pythonhosted.org/packages/84/9e/500463e87abb7a0a0f9f256ec21123ecde0a7b5541a15e840ea54551fd81/cryptography-48.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:7e8eac43dfca5c4cccc6dad9a80504436fca53bb9bc3100a2386d730fbe6b602", size = 4695941, upload-time = "2026-05-04T22:57:54.603Z" }, + { url = "https://files.pythonhosted.org/packages/e3/dc/7303087450c2ec9e7fbb750e17c2abfbc658f23cbd0e54009509b7cc4091/cryptography-48.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9ccdac7d40688ecb5a3b4a604b8a88c8002e3442d6c60aead1db2a89a041560c", size = 5252579, upload-time = "2026-05-04T22:57:57.207Z" }, + { url = "https://files.pythonhosted.org/packages/d0/c0/7101d3b7215edcdc90c45da544961fd8ed2d6448f77577460fa75a8443f7/cryptography-48.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:bd72e68b06bb1e96913f97dd4901119bc17f39d4586a5adf2d3e47bc2b9d58b5", size = 4743326, upload-time = "2026-05-04T22:57:59.535Z" }, + { url = "https://files.pythonhosted.org/packages/ac/d8/5b833bad13016f562ab9d063d68199a4bd121d18458e439515601d3357ec/cryptography-48.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:59baa2cb386c4f0b9905bd6eb4c2a79a69a128408fd31d32ca4d7102d4156321", size = 4826672, upload-time = "2026-05-04T22:58:01.996Z" }, + { url = "https://files.pythonhosted.org/packages/98/e1/7074eb8bf3c135558c73fc2bcf0f5633f912e6fb87e868a55c454080ef09/cryptography-48.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9249e3cd978541d665967ac2cb2787fd6a62bddf1e75b3e347a594d7dacf4f74", size = 4972574, upload-time = "2026-05-04T22:58:03.968Z" }, + { url = "https://files.pythonhosted.org/packages/04/70/e5a1b41d325f797f39427aa44ef8baf0be500065ab6d8e10369d850d4a4f/cryptography-48.0.0-cp311-abi3-win32.whl", hash = "sha256:9c459db21422be75e2809370b829a87eb37f74cd785fc4aa9ea1e5f43b47cda4", size = 3294868, upload-time = "2026-05-04T22:58:06.467Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ac/8ac51b4a5fc5932eb7ee5c517ba7dc8cd834f0048962b6b352f00f41ebf9/cryptography-48.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:5b012212e08b8dd5edc78ef54da83dd9892fd9105323b3993eff6bea65dc21d7", size = 3817107, upload-time = "2026-05-04T22:58:08.845Z" }, + { url = "https://files.pythonhosted.org/packages/f2/63/61d4a4e1c6b6bab6ce1e213cd36a24c415d90e76d78c5eb8577c5541d2e8/cryptography-48.0.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:58d00498e8933e4a194f3076aee1b4a97dfec1a6da444535755822fe5d8b0b86", size = 7983482, upload-time = "2026-05-04T22:58:43.769Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ac/f5b5995b87770c693e2596559ffafe195b4033a57f14a82268a2842953f3/cryptography-48.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:614d0949f4790582d2cc25553abd09dd723025f0c0e7c67376a1d77196743d6e", size = 4683266, upload-time = "2026-05-04T22:58:46.064Z" }, + { url = "https://files.pythonhosted.org/packages/ec/c6/8b14f67e18338fbc4adb76f66c001f5c3610b3e2d1837f268f47a347dbbb/cryptography-48.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7ce4bfae76319a532a2dc68f82cc32f5676ee792a983187dac07183690e5c66f", size = 4696228, upload-time = "2026-05-04T22:58:48.22Z" }, + { url = "https://files.pythonhosted.org/packages/ea/73/f808fbae9514bd91b47875b003f13e284c8c6bdfd904b7944e803937eec1/cryptography-48.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:2eb992bbd4661238c5a397594c83f5b4dc2bc5b848c365c8f991b6780efcc5c7", size = 4689097, upload-time = "2026-05-04T22:58:50.9Z" }, + { url = "https://files.pythonhosted.org/packages/93/01/d86632d7d28db8ae83221995752eeb6639ffb374c2d22955648cf8d52797/cryptography-48.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:22a5cb272895dce158b2cacdfdc3debd299019659f42947dbdac6f32d68fe832", size = 5283582, upload-time = "2026-05-04T22:58:53.017Z" }, + { url = "https://files.pythonhosted.org/packages/02/e1/50edc7a50334807cc4791fc4a0ce7468b4a1416d9138eab358bfc9a3d70b/cryptography-48.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2b4d59804e8408e2fea7d1fbaf218e5ec984325221db76e6a241a9abd6cdd95c", size = 4730479, upload-time = "2026-05-04T22:58:55.611Z" }, + { url = "https://files.pythonhosted.org/packages/6f/af/99a582b1b1641ff5911ac559beb45097cf79efd4ead4657f578ef1af2d47/cryptography-48.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:984a20b0f62a26f48a3396c72e4bc34c66e356d356bf370053066b3b6d54634a", size = 4326481, upload-time = "2026-05-04T22:58:57.607Z" }, + { url = "https://files.pythonhosted.org/packages/90/ee/89aa26a06ef0a7d7611788ffd571a7c50e368cc6a4d5eef8b4884e866edb/cryptography-48.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5a5ed8fde7a1d09376ca0b40e68cd59c69fe23b1f9768bd5824f54681626032a", size = 4688713, upload-time = "2026-05-04T22:59:00.077Z" }, + { url = "https://files.pythonhosted.org/packages/70/ba/bcb1b0bb7a33d4c7c0c4d4c7874b4a62ae4f56113a5f4baefa362dfb1f0f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:8cd666227ef7af430aa5914a9910e0ddd703e75f039cef0825cd0da71b6b711a", size = 5238165, upload-time = "2026-05-04T22:59:02.317Z" }, + { url = "https://files.pythonhosted.org/packages/c9/70/ca4003b1ce5ca3dc3186ada51908c8a9b9ff7d5cab83cc0d43ee14ec144f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9071196d81abc88b3516ac8cdfad32e2b66dd4a5393a8e68a961e9161ddc6239", size = 4729947, upload-time = "2026-05-04T22:59:05.255Z" }, + { url = "https://files.pythonhosted.org/packages/44/a0/4ec7cf774207905aef1a8d11c3750d5a1db805eb380ee4e16df317870128/cryptography-48.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e2d54c8be6152856a36f0882ab231e70f8ec7f14e93cf87db8a2ed056bf160c", size = 4822059, upload-time = "2026-05-04T22:59:07.802Z" }, + { url = "https://files.pythonhosted.org/packages/1e/75/a2e55f99c16fcac7b5d6c1eb19ad8e00799854d6be5ca845f9259eae1681/cryptography-48.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a5da777e32ffed6f85a7b2b3f7c5cbc88c146bfcd0a1d7baf5fcc6c52ee35dd4", size = 4960575, upload-time = "2026-05-04T22:59:09.851Z" }, + { url = "https://files.pythonhosted.org/packages/b8/23/6e6f32143ab5d8b36ca848a502c4bcd477ae75b9e1677e3530d669062578/cryptography-48.0.0-cp39-abi3-win32.whl", hash = "sha256:77a2ccbbe917f6710e05ba9adaa25fb5075620bf3ea6fb751997875aff4ae4bd", size = 3279117, upload-time = "2026-05-04T22:59:12.019Z" }, + { url = "https://files.pythonhosted.org/packages/9d/9a/0fea98a70cf1749d41d738836f6349d97945f7c89433a259a6c2642eefeb/cryptography-48.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:16cd65b9330583e4619939b3a3843eec1e6e789744bb01e7c7e2e62e33c239c8", size = 3792100, upload-time = "2026-05-04T22:59:14.884Z" }, +] + [[package]] name = "deprecation" version = "2.1.0" @@ -331,6 +388,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/39/0e87753df1072254bac190b33ed34b264f28f6aa9bea0f01b7e818071756/nats_py-2.14.0-py3-none-any.whl", hash = "sha256:4116f5d2233ce16e63c3d5538fa40a5e207f75fcf42a741773929ddf1e29d19d", size = 82259, upload-time = "2026-02-23T22:45:00.152Z" }, ] +[[package]] +name = "numpy" +version = "2.4.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/50/8e/b8041bc719f056afd864478029d52214789341ac6583437b0ee5031e9530/numpy-2.4.5.tar.gz", hash = "sha256:ca670567a5683b7c1670ec03e0ddd5862e10934e92a70751d68d7b7b74ca7f9f", size = 20735669, upload-time = "2026-05-15T20:25:19.492Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/18/3275231e98620002681c922e792db04d72c356e9d8073c387344fc0e4ff1/numpy-2.4.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:654fb8674b61b1c4bd568f944d13a908566fdcb0d797303521d4149d16da05ef", size = 16689166, upload-time = "2026-05-15T20:22:50.761Z" }, + { url = "https://files.pythonhosted.org/packages/db/23/000aab6a16bdec53307f0f72546b57a3ac9266a62d8c257bee97d85fd078/numpy-2.4.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4cd9f6fa7ce10dc4627f2bb81dd9075dab67e94632e04c2b638e12575ddaa862", size = 14699514, upload-time = "2026-05-15T20:22:53.678Z" }, + { url = "https://files.pythonhosted.org/packages/47/cc/ddaf3af9c46966fef5be879256f213d85a0c56c75d07a3b7defec7cf6b4c/numpy-2.4.5-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:4f5bc96d35d94e4ceab8b38a92241b4611e95dc44e63b9f1fa2a331858ee3507", size = 5204601, upload-time = "2026-05-15T20:22:56.257Z" }, + { url = "https://files.pythonhosted.org/packages/07/ea/627fadd11959b3c7759008f34c92a35af8ff942dd8284a66ced648bbe516/numpy-2.4.5-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:4bb33e900ee81730ad77a258965134aa8ceac805124f7e5229347beda4b8d0aa", size = 6551360, upload-time = "2026-05-15T20:22:58.334Z" }, + { url = "https://files.pythonhosted.org/packages/a1/47/0728b986b8682d742ff68c16baa5af9d185484abfc635c5cc700f44e62be/numpy-2.4.5-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32f8f852273ef32b291201ac2a2c97629c4a1ee8632bb670e3443eaa09fc2e72", size = 15671157, upload-time = "2026-05-15T20:23:01.081Z" }, + { url = "https://files.pythonhosted.org/packages/d1/0b/b905ae82d9419dc38123523862db64978ca2954b69609c3ae8fdaca1084c/numpy-2.4.5-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:685681e956fc8dcb75adc6ff26694e1dfd738b24bd8d4696c51ca0110157f912", size = 16645703, upload-time = "2026-05-15T20:23:04.358Z" }, + { url = "https://files.pythonhosted.org/packages/5f/24/e27fc3f5236b4118ed9eed67111675f5c61a07ea333acec87c869c3b359d/numpy-2.4.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6f64dd84b277a737eb59513f6b9bb6195bf41ab11941ef15b2562dbab43fa8ef", size = 17021018, upload-time = "2026-05-15T20:23:07.021Z" }, + { url = "https://files.pythonhosted.org/packages/d3/a7/9041af38d527ab80a06a93570a77e29425b41507ad41f6acf5da78cfb4a4/numpy-2.4.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b42d9496f79e3a728192f05a42d86e36163217b7cdecb3813d0028a0aa6b72d7", size = 18368768, upload-time = "2026-05-15T20:23:09.44Z" }, + { url = "https://files.pythonhosted.org/packages/49/82/326a014442f32c2663434fd424d9298791f47f8a0f17585ad60519a5606e/numpy-2.4.5-cp312-cp312-win32.whl", hash = "sha256:86d980970f5110595ca14855768073b08585fc1acc36895de303e039e7dee4a5", size = 5962819, upload-time = "2026-05-15T20:23:11.631Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f0/cbf5d391b0b3a5e8cad264603e2fae256b0bde8ce43566b13b78faedc659/numpy-2.4.5-cp312-cp312-win_amd64.whl", hash = "sha256:3333dba6a4e611d666f69e177ba8fe4140366ff681a5feb2374d3fd4fff3acb6", size = 12321621, upload-time = "2026-05-15T20:23:14.305Z" }, + { url = "https://files.pythonhosted.org/packages/3c/d0/0f18909d9bc37a5f3f969fc737d2bb5df9f2ff295f71b467e6f52a0d6c4e/numpy-2.4.5-cp312-cp312-win_arm64.whl", hash = "sha256:4593d197270b894efeb538dcbe227e4bcf1c77f88c4c6bf933ead812cfaa4453", size = 10221430, upload-time = "2026-05-15T20:23:16.887Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -384,6 +460,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, ] +[[package]] +name = "pycparser" +version = "3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, +] + [[package]] name = "pydantic" version = "2.13.4" @@ -429,6 +514,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, ] +[[package]] +name = "pydantic-settings" +version = "2.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/60/1d1e59c9c90d54591469ada7d268251f71c24bdb765f1a8a832cee8c6653/pydantic_settings-2.14.1.tar.gz", hash = "sha256:e874d3bec7e787b0c9958277956ed9b4dd5de6a80e162188fdaff7c5e26fd5fa", size = 235551, upload-time = "2026-05-08T13:40:06.542Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/8d/f1af3832f5e6eb13ba94ee809e72b8ecb5eef226d27ee0bef7d963d943c7/pydantic_settings-2.14.1-py3-none-any.whl", hash = "sha256:6e3c7edfd8277687cdc598f56e5cff0e9bfff0910a3749deaa8d4401c3a2b9de", size = 60964, upload-time = "2026-05-08T13:40:04.958Z" }, +] + [[package]] name = "pygments" version = "2.20.0" @@ -479,6 +578,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, +] + [[package]] name = "ruff" version = "0.15.13" @@ -504,6 +612,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9b/36/9c015cd052fca743dae8cb2aeb16b551444787467db42ceab0fc968865af/ruff-0.15.13-py3-none-win_arm64.whl", hash = "sha256:2471da9bd1068c8c064b5fd9c0c4b6dddffd6369cb1cd68b29993b1709ff1b21", size = 11179336, upload-time = "2026-05-14T13:44:33.026Z" }, ] +[[package]] +name = "shapely" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4d/bc/0989043118a27cccb4e906a46b7565ce36ca7b57f5a18b78f4f1b0f72d9d/shapely-2.1.2.tar.gz", hash = "sha256:2ed4ecb28320a433db18a5bf029986aa8afcfd740745e78847e330d5d94922a9", size = 315489, upload-time = "2025-09-24T13:51:41.432Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/c0/f3b6453cf2dfa99adc0ba6675f9aaff9e526d2224cbd7ff9c1a879238693/shapely-2.1.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fe2533caae6a91a543dec62e8360fe86ffcdc42a7c55f9dfd0128a977a896b94", size = 1833550, upload-time = "2025-09-24T13:50:30.019Z" }, + { url = "https://files.pythonhosted.org/packages/86/07/59dee0bc4b913b7ab59ab1086225baca5b8f19865e6101db9ebb7243e132/shapely-2.1.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ba4d1333cc0bc94381d6d4308d2e4e008e0bd128bdcff5573199742ee3634359", size = 1643556, upload-time = "2025-09-24T13:50:32.291Z" }, + { url = "https://files.pythonhosted.org/packages/26/29/a5397e75b435b9895cd53e165083faed5d12fd9626eadec15a83a2411f0f/shapely-2.1.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0bd308103340030feef6c111d3eb98d50dc13feea33affc8a6f9fa549e9458a3", size = 2988308, upload-time = "2025-09-24T13:50:33.862Z" }, + { url = "https://files.pythonhosted.org/packages/b9/37/e781683abac55dde9771e086b790e554811a71ed0b2b8a1e789b7430dd44/shapely-2.1.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1e7d4d7ad262a48bb44277ca12c7c78cb1b0f56b32c10734ec9a1d30c0b0c54b", size = 3099844, upload-time = "2025-09-24T13:50:35.459Z" }, + { url = "https://files.pythonhosted.org/packages/d8/f3/9876b64d4a5a321b9dc482c92bb6f061f2fa42131cba643c699f39317cb9/shapely-2.1.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e9eddfe513096a71896441a7c37db72da0687b34752c4e193577a145c71736fc", size = 3988842, upload-time = "2025-09-24T13:50:37.478Z" }, + { url = "https://files.pythonhosted.org/packages/d1/a0/704c7292f7014c7e74ec84eddb7b109e1fbae74a16deae9c1504b1d15565/shapely-2.1.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:980c777c612514c0cf99bc8a9de6d286f5e186dcaf9091252fcd444e5638193d", size = 4152714, upload-time = "2025-09-24T13:50:39.9Z" }, + { url = "https://files.pythonhosted.org/packages/53/46/319c9dc788884ad0785242543cdffac0e6530e4d0deb6c4862bc4143dcf3/shapely-2.1.2-cp312-cp312-win32.whl", hash = "sha256:9111274b88e4d7b54a95218e243282709b330ef52b7b86bc6aaf4f805306f454", size = 1542745, upload-time = "2025-09-24T13:50:41.414Z" }, + { url = "https://files.pythonhosted.org/packages/ec/bf/cb6c1c505cb31e818e900b9312d514f381fbfa5c4363edfce0fcc4f8c1a4/shapely-2.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:743044b4cfb34f9a67205cee9279feaf60ba7d02e69febc2afc609047cb49179", size = 1722861, upload-time = "2025-09-24T13:50:43.35Z" }, +] + [[package]] name = "six" version = "1.17.0" From f7a55c3cc4ddff03b7b2d14602852ce7b6a300b7 Mon Sep 17 00:00:00 2001 From: Matt Johnson Date: Sat, 16 May 2026 19:08:00 +0000 Subject: [PATCH 9/9] docs: add Phase 1a-5 verification report Documents test results for: - Gate 5: max_bytes self-loop prevention (PASS) - Gate 6: bbox hot-reload (PASS) Co-Authored-By: Claude Opus 4.5 --- docs/PHASE-1A-5-VERIFICATION.md | 83 +++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 docs/PHASE-1A-5-VERIFICATION.md diff --git a/docs/PHASE-1A-5-VERIFICATION.md b/docs/PHASE-1A-5-VERIFICATION.md new file mode 100644 index 0000000..c9867e6 --- /dev/null +++ b/docs/PHASE-1A-5-VERIFICATION.md @@ -0,0 +1,83 @@ +# Phase 1a-5 Verification Report + +## Gate 5: max_bytes Self-Loop Prevention + +**Objective**: Verify that `max_bytes` updates to `config.streams` do NOT trigger +a NOTIFY (preventing infinite supervisor recompute loops). + +**Test Procedure**: +1. Started supervisor with stream retention enabled +2. Directly updated `max_bytes` via SQL: + ```sql + UPDATE config.streams SET max_bytes = 209715200 WHERE name = 'CENTRAL_META'; + ``` +3. Monitored supervisor logs for any stream change handling + +**Result**: PASS + +No NOTIFY was triggered. The column-filtered trigger in `003_add_streams_table.sql` +correctly fires only on `max_age_s` changes: + +```sql +ELSIF TG_OP = 'UPDATE' AND OLD.max_age_s IS DISTINCT FROM NEW.max_age_s THEN + PERFORM pg_notify('config_changed', 'streams:' || NEW.name); +``` + +This prevents the supervisor's `recompute_max_bytes()` from creating a feedback loop. + +--- + +## Gate 6: bbox Hot-Reload + +**Objective**: Verify that changes to the NWS adapter's `region` bbox are +picked up via hot-reload without supervisor restart. + +**Test Procedure**: +1. Updated the NWS adapter's region via SQL: + ```sql + UPDATE config.adapters SET settings = jsonb_set( + settings, '{region}', + '{"north": 48.0, "south": 45.0, "east": -115.0, "west": -125.0}'::jsonb + ) WHERE name = 'nws'; + ``` +2. Observed supervisor logs for config reload + +**Result**: PASS + +Supervisor log showed immediate config application: +```json +{"msg": "NWS config applied", "region": {"east": -115.0, "west": -125.0, "north": 48.0, "south": 45.0}} +``` + +The NOTIFY trigger on `config.adapters` fired and the supervisor's +`_handle_adapter_change()` correctly invoked `adapter.apply_config()`. + +--- + +## Additional Verification + +### Polygon Intersection Filter + +The `_geometry_intersects_region()` method was tested with: +- Shapely 2.1.2 installed via `uv sync` +- GeoJSON geometry parsing via `shapely.geometry.shape()` +- Region box creation via `shapely.geometry.box()` +- Intersection test via `region_box.intersects(feature_shape)` + +### Antimeridian Rejection + +The `RegionConfig` validator was tested: +```python +>>> RegionConfig(north=49.0, south=31.0, east=-170.0, west=170.0) +ValidationError: antimeridian-crossing bboxes not supported +``` + +--- + +## Verification Date + +2026-05-16T19:06:00Z + +## Verified By + +Claude Code (automated verification)