Merge pull request #7 from zvx-echo6/feature/1a-6-firms-adapter

feat: FIRMS fire hotspot adapter (Phase 1a-6)
This commit is contained in:
malice 2026-05-16 14:25:55 -06:00 committed by GitHub
commit 2fd5bc01c0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 1771 additions and 841 deletions

View file

@ -53,3 +53,41 @@ Per stream, display:
- Supervisor receives NOTIFY and hot-reloads
- No service restarts required for config changes
- Stream retention changes apply within 5 seconds
## FIRMS Adapter Configuration
### MAP_KEY Management
- Display key alias () and timestamp
- Allow operator to rotate key value (re-encrypt new key)
- Show warning if key not present (polling disabled)
- No key value display (security)
### Satellite Selection
- Toggle individual satellites: VIIRS_SNPP, VIIRS_NOAA20, VIIRS_NOAA21
- Stored in array
- Changes hot-reload to adapter without restart
### SNPP End-of-Life Notice
- NASA timeline: SNPP mission ends ~October 2026
- GUI should display warning banner when SNPP is enabled and date approaches
- Recommend adding NOAA-21 to satellites list before SNPP EOL
- After EOL, adapter will fail to fetch SNPP data (404); GUI should surface this
## FIRMS Adapter Configuration
### MAP_KEY Management
- Display key alias (firms) and last_used_at timestamp
- Allow operator to rotate key value (re-encrypt new key)
- Show warning if key not present (polling disabled)
- No key value display (security)
### Satellite Selection
- Toggle individual satellites: VIIRS_SNPP, VIIRS_NOAA20, VIIRS_NOAA21
- Stored in config.adapters.settings.satellites array
- Changes hot-reload to adapter without restart
### SNPP End-of-Life Notice
- NASA timeline: SNPP mission ends ~October 2026
- GUI should display warning banner when SNPP is enabled and date approaches
- Recommend adding NOAA-21 to satellites list before SNPP EOL
- After EOL, adapter will fail to fetch SNPP data (404); GUI should surface this

View file

@ -0,0 +1,27 @@
-- Migration: 005_add_firms_adapter
-- Seeds FIRMS adapter configuration and CENTRAL_FIRE stream.
-- Seed FIRMS adapter row
INSERT INTO config.adapters (name, enabled, cadence_s, settings)
VALUES (
'firms',
true,
300,
jsonb_build_object(
'region', jsonb_build_object(
'north', 49.5,
'south', 31.0,
'east', -102.0,
'west', -124.5
),
'api_key_alias', 'firms',
'satellites', jsonb_build_array(
'VIIRS_SNPP_NRT',
'VIIRS_NOAA20_NRT'
)
)
);
-- Seed CENTRAL_FIRE stream row
INSERT INTO config.streams (name, max_age_s, max_bytes)
VALUES ('CENTRAL_FIRE', 604800, 1073741824);

View file

@ -0,0 +1,430 @@
"""FIRMS (Fire Information for Resource Management System) adapter."""
import csv
import logging
import sqlite3
from collections.abc import AsyncIterator
from datetime import datetime, timezone
from io import StringIO
from pathlib import Path
from typing import Any
import aiohttp
from tenacity import (
retry,
stop_after_attempt,
wait_exponential_jitter,
retry_if_exception_type,
)
from central.adapter import SourceAdapter
from central.config_models import AdapterConfig, RegionConfig
from central.config_store import ConfigStore
from central.models import Event, Geo
logger = logging.getLogger(__name__)
# FIRMS API base URL
FIRMS_API_BASE = "https://firms.modaps.eosdis.nasa.gov/api/area/csv"
# Satellite name mapping
SATELLITE_SHORT = {
"VIIRS_SNPP_NRT": "viirs_snpp",
"VIIRS_NOAA20_NRT": "viirs_noaa20",
"VIIRS_NOAA21_NRT": "viirs_noaa21",
}
# Confidence mapping
CONFIDENCE_MAP = {
"l": "low",
"n": "nominal",
"h": "high",
}
# Severity mapping (confidence -> severity level)
SEVERITY_MAP = {
"high": 3,
"nominal": 2,
"low": 1,
}
class FIRMSAdapter(SourceAdapter):
"""NASA FIRMS fire hotspot adapter."""
name = "firms"
def __init__(
self,
config: AdapterConfig,
config_store: ConfigStore,
cursor_db_path: Path,
) -> None:
self._config_store = config_store
self._cursor_db_path = cursor_db_path
self._session: aiohttp.ClientSession | None = None
self._db: sqlite3.Connection | None = None
self._api_key: str | None = None
# Extract settings from config
self._api_key_alias: str = config.settings.get("api_key_alias", "firms")
self._satellites: list[str] = config.settings.get(
"satellites", ["VIIRS_SNPP_NRT", "VIIRS_NOAA20_NRT"]
)
# Parse region from settings
region_dict = config.settings.get("region")
if region_dict:
self.region: RegionConfig | None = RegionConfig(**region_dict)
else:
self.region = None
async def apply_config(self, new_config: AdapterConfig) -> None:
"""Apply new configuration from hot-reload."""
old_alias = self._api_key_alias
# Update settings
self._api_key_alias = new_config.settings.get("api_key_alias", "firms")
self._satellites = new_config.settings.get(
"satellites", ["VIIRS_SNPP_NRT", "VIIRS_NOAA20_NRT"]
)
# Update region
region_dict = new_config.settings.get("region")
if region_dict:
self.region = RegionConfig(**region_dict)
else:
self.region = None
# If API key alias changed, re-fetch the key
if self._api_key_alias != old_alias:
self._api_key = await self._config_store.get_api_key(self._api_key_alias)
if self._api_key:
logger.info("FIRMS API key reloaded", extra={"alias": self._api_key_alias})
else:
logger.warning(
"FIRMS API key not found after alias change",
extra={"alias": self._api_key_alias},
)
logger.info(
"FIRMS config applied",
extra={
"region": region_dict,
"satellites": self._satellites,
"api_key_alias": self._api_key_alias,
},
)
async def startup(self) -> None:
"""Initialize HTTP session, dedup tracker, and fetch API key."""
# Fetch API key
self._api_key = await self._config_store.get_api_key(self._api_key_alias)
if not self._api_key:
logger.error(
"FIRMS API key not found - polling will be skipped until key is set",
extra={"alias": self._api_key_alias},
)
# Initialize HTTP session
self._session = aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=60),
)
# Initialize dedup tracker (shared sqlite DB with NWS)
self._db = sqlite3.connect(str(self._cursor_db_path))
self._db.execute("""
CREATE TABLE IF NOT EXISTS published_ids (
adapter TEXT NOT NULL,
event_id TEXT NOT NULL,
first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (adapter, event_id)
)
""")
self._db.execute("""
CREATE INDEX IF NOT EXISTS published_ids_last_seen
ON published_ids (last_seen)
""")
self._db.commit()
# Sweep old entries on startup (48h for FIRMS)
self.sweep_old_ids()
logger.info(
"FIRMS adapter started",
extra={
"region": {
"north": self.region.north,
"south": self.region.south,
"east": self.region.east,
"west": self.region.west,
} if self.region else None,
"satellites": self._satellites,
"api_key_present": self._api_key is not None,
},
)
async def shutdown(self) -> None:
"""Close HTTP session and database."""
if self._session:
await self._session.close()
self._session = None
if self._db:
self._db.close()
self._db = None
logger.info("FIRMS adapter shut down")
def is_published(self, stable_id: str) -> bool:
"""Check if an event has already been published."""
if not self._db:
return False
cur = self._db.execute(
"SELECT 1 FROM published_ids WHERE adapter = ? AND event_id = ?",
(self.name, stable_id),
)
return cur.fetchone() is not None
def mark_published(self, stable_id: str) -> None:
"""Mark an event as published."""
if not self._db:
return
self._db.execute(
"""
INSERT INTO published_ids (adapter, event_id, first_seen, last_seen)
VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (adapter, event_id) DO UPDATE SET
last_seen = CURRENT_TIMESTAMP
""",
(self.name, stable_id),
)
self._db.commit()
def sweep_old_ids(self) -> int:
"""Remove published_ids older than 48 hours. Returns count deleted."""
if not self._db:
return 0
cur = self._db.execute(
"DELETE FROM published_ids WHERE adapter = ? AND last_seen < datetime('now', '-48 hours')",
(self.name,),
)
self._db.commit()
count = cur.rowcount
if count > 0:
logger.info("FIRMS swept old dedup entries", extra={"count": count})
return count
def _build_stable_id(
self, satellite: str, acq_date: str, acq_time: str, lat: float, lon: float
) -> str:
"""Build stable ID for deduplication."""
# Round lat/lon to 0.001 degrees to handle floating-point comparison
lat_rounded = round(lat, 3)
lon_rounded = round(lon, 3)
return f"{satellite}:{acq_date}:{acq_time}:{lat_rounded}:{lon_rounded}"
def _build_url(self, satellite: str) -> str | None:
"""Build FIRMS API URL for a satellite."""
if not self._api_key or not self.region:
return None
# Area format: west,south,east,north
area = f"{self.region.west},{self.region.south},{self.region.east},{self.region.north}"
return f"{FIRMS_API_BASE}/{self._api_key}/{satellite}/{area}/1"
@retry(
stop=stop_after_attempt(3),
wait=wait_exponential_jitter(initial=2, max=30),
retry=retry_if_exception_type((aiohttp.ClientError,)),
reraise=True,
)
async def _fetch_csv(self, url: str) -> str:
"""Fetch CSV data from FIRMS API."""
if not self._session:
raise RuntimeError("Session not initialized")
async with self._session.get(url) as resp:
# Check for error responses
content_type = resp.headers.get("Content-Type", "")
if "text/html" in content_type:
text = await resp.text()
logger.error(
"FIRMS returned HTML (likely auth error)",
extra={"status": resp.status, "preview": text[:200]},
)
raise ValueError("FIRMS returned HTML instead of CSV")
resp.raise_for_status()
return await resp.text()
def _parse_csv(self, csv_text: str, satellite: str) -> list[dict[str, Any]]:
"""Parse FIRMS CSV response into list of dicts."""
rows = []
reader = csv.DictReader(StringIO(csv_text))
for row in reader:
try:
# Parse required fields
lat = float(row["latitude"])
lon = float(row["longitude"])
acq_date = row["acq_date"]
acq_time = row["acq_time"]
confidence_raw = row.get("confidence", "n").lower()
confidence = CONFIDENCE_MAP.get(confidence_raw, "nominal")
rows.append({
"latitude": lat,
"longitude": lon,
"bright_ti4": float(row.get("bright_ti4", 0)) if row.get("bright_ti4") else None,
"bright_ti5": float(row.get("bright_ti5", 0)) if row.get("bright_ti5") else None,
"scan": float(row.get("scan", 0)) if row.get("scan") else None,
"track": float(row.get("track", 0)) if row.get("track") else None,
"acq_date": acq_date,
"acq_time": acq_time,
"satellite": row.get("satellite", satellite),
"instrument": row.get("instrument", "VIIRS"),
"confidence": confidence,
"confidence_raw": confidence_raw,
"version": row.get("version", ""),
"frp": float(row.get("frp", 0)) if row.get("frp") else None,
"daynight": row.get("daynight", ""),
})
except (KeyError, ValueError) as e:
logger.warning(
"Failed to parse FIRMS row",
extra={"error": str(e), "row": dict(row)},
)
continue
return rows
def _row_to_event(self, row: dict[str, Any], satellite: str) -> Event:
"""Convert a parsed CSV row to an Event."""
satellite_short = SATELLITE_SHORT.get(satellite, satellite.lower().replace("_nrt", ""))
confidence = row["confidence"]
severity = SEVERITY_MAP.get(confidence, 1)
# Parse acquisition time
acq_date = row["acq_date"]
acq_time = row["acq_time"]
# acq_time is HHMM format
try:
time = datetime.strptime(
f"{acq_date} {acq_time}", "%Y-%m-%d %H%M"
).replace(tzinfo=timezone.utc)
except ValueError:
time = datetime.now(timezone.utc)
lat = row["latitude"]
lon = row["longitude"]
# Build stable ID
stable_id = self._build_stable_id(satellite, acq_date, acq_time, lat, lon)
geo = Geo(
centroid=(lon, lat), # GeoJSON order: lon, lat
bbox=(lon, lat, lon, lat), # Point bbox
regions=[],
primary_region=None,
)
return Event(
id=stable_id,
source="central/adapters/firms",
category=f"fire.hotspot.{satellite_short}.{confidence}",
time=time,
expires=None,
severity=severity,
geo=geo,
data=row,
)
async def poll(self) -> AsyncIterator[Event]:
"""Poll FIRMS API for fire hotspots."""
# Check API key
if not self._api_key:
# Try to fetch again in case it was added
self._api_key = await self._config_store.get_api_key(self._api_key_alias)
if not self._api_key:
logger.warning(
"FIRMS API key still not available, skipping poll",
extra={"alias": self._api_key_alias},
)
return
if not self.region:
logger.warning("FIRMS region not configured, skipping poll")
return
# Sweep old dedup entries periodically
self.sweep_old_ids()
total_features = 0
total_new = 0
for satellite in self._satellites:
url = self._build_url(satellite)
if not url:
continue
try:
csv_text = await self._fetch_csv(url)
rows = self._parse_csv(csv_text, satellite)
feature_count = len(rows)
total_features += feature_count
new_count = 0
for row in rows:
stable_id = self._build_stable_id(
satellite,
row["acq_date"],
row["acq_time"],
row["latitude"],
row["longitude"],
)
if self.is_published(stable_id):
continue
event = self._row_to_event(row, satellite)
yield event
self.mark_published(stable_id)
new_count += 1
total_new += new_count
logger.info(
"FIRMS satellite poll completed",
extra={
"satellite": satellite,
"feature_count": feature_count,
"new_count": new_count,
},
)
except Exception as e:
logger.error(
"FIRMS poll failed for satellite",
extra={"satellite": satellite, "error": str(e)},
)
continue
logger.info(
"FIRMS poll completed",
extra={
"total_features": total_features,
"total_new": total_new,
"satellites": self._satellites,
},
)
def subject_for_fire_hotspot(ev: Event) -> str:
"""Compute the NATS subject for a fire hotspot event.
Subject format: central.fire.hotspot.<satellite>.<confidence>
The category already contains the satellite and confidence info,
so we just prefix with 'central.'.
"""
# category is "fire.hotspot.<satellite>.<confidence>"
return f"central.{ev.category}"

View file

@ -20,6 +20,7 @@ from tenacity import (
from central import __version__
from central.adapter import SourceAdapter
from central.config_models import AdapterConfig, RegionConfig
from central.config_store import ConfigStore
from central.models import Event, Geo
from shapely.geometry import box as shapely_box, shape as shapely_shape
@ -196,6 +197,7 @@ class NWSAdapter(SourceAdapter):
def __init__(
self,
config: AdapterConfig,
config_store: ConfigStore,
cursor_db_path: Path,
) -> None:
self.cursor_db_path = cursor_db_path

View file

@ -24,7 +24,7 @@ class Event(BaseModel):
id: str # unique, stable across republish
source: str # adapter identity, e.g. "central/adapters/nws"
category: str # e.g. "wx.alert.severe_thunderstorm_warning"
category: str # e.g. "wx.alert.severe_thunderstorm_warning" or "fire.hotspot.viirs_snpp.high"
time: datetime # event-time UTC, not processing-time
expires: datetime | None = None
severity: int | None = None # 0..4 or None for "Unknown"
@ -32,19 +32,30 @@ class Event(BaseModel):
data: dict[str, Any] # adapter-specific payload
def subject_for_event(ev: Event, prefix: str = "central.wx") -> str:
def subject_for_event(ev: Event) -> str:
"""
Compute the NATS subject for an alert-style event.
Compute the NATS subject for an event based on its category.
For weather alerts the subject is:
Dispatch by category prefix:
- fire.*: returns central.<category> directly
- wx.*: uses weather alert subject logic
Weather alert subjects:
central.wx.alert.us.<state_lower>.county.<county_lower>
or
central.wx.alert.us.<state_lower>.zone.<zone_lower>
based on whether the primary_region encodes a county or a zone.
If primary_region is None or unparseable, returns:
central.wx.alert.us.unknown
Fire hotspot subjects:
central.fire.hotspot.<satellite>.<confidence>
"""
# Fire events: subject is just central.<category>
if ev.category.startswith("fire."):
return f"central.{ev.category}"
# Weather events: use geo-based subject logic
prefix = "central.wx"
if ev.geo.primary_region is None:
return f"{prefix}.alert.us.unknown"

View file

@ -15,6 +15,7 @@ from nats.js import JetStreamContext
from central.adapter import SourceAdapter
from central.adapters.nws import NWSAdapter
from central.adapters.firms import FIRMSAdapter
from central.cloudevents_wire import wrap_event
from central.config_models import AdapterConfig
from central.config_source import ConfigSource, DbConfigSource
@ -23,12 +24,19 @@ from central.bootstrap_config import get_settings
from central.models import subject_for_event
from central.stream_manager import StreamManager
# Adapter registry - add new adapters here
_ADAPTER_REGISTRY: dict[str, type[SourceAdapter]] = {
"nws": NWSAdapter,
"firms": FIRMSAdapter,
}
CURSOR_DB_PATH = Path("/var/lib/central/cursors.db")
# Stream subject mappings
STREAM_SUBJECTS = {
"CENTRAL_WX": ["central.wx.>"],
"CENTRAL_META": ["central.meta.>"],
"CENTRAL_FIRE": ["central.fire.>"],
}
# Recompute interval for stream max_bytes (1 hour)
@ -150,10 +158,14 @@ class Supervisor:
def _create_adapter(self, config: AdapterConfig) -> SourceAdapter:
"""Create an adapter instance based on config name."""
if config.name == "nws":
return NWSAdapter(config=config, cursor_db_path=CURSOR_DB_PATH)
else:
cls = _ADAPTER_REGISTRY.get(config.name)
if cls is None:
raise ValueError(f"Unknown adapter type: {config.name}")
return cls(
config=config,
config_store=self._config_store,
cursor_db_path=CURSOR_DB_PATH,
)
async def _run_adapter_loop(self, state: AdapterState) -> None:
"""Run an adapter poll loop with rate-limit aware scheduling."""

410
tests/test_firms.py Normal file
View file

@ -0,0 +1,410 @@
"""Tests for FIRMS adapter."""
import pytest
from datetime import datetime, timezone
from unittest.mock import AsyncMock, MagicMock, patch
from pathlib import Path
import tempfile
from central.adapters.firms import (
FIRMSAdapter,
CONFIDENCE_MAP,
SATELLITE_SHORT,
subject_for_fire_hotspot,
)
from central.config_models import AdapterConfig, RegionConfig
from central.models import Event, Geo
# Sample FIRMS CSV response
SAMPLE_CSV = """latitude,longitude,bright_ti4,scan,track,acq_date,acq_time,satellite,instrument,confidence,version,bright_ti5,frp,daynight
45.123,-116.456,320.5,0.39,0.36,2026-05-16,1430,N,VIIRS,h,2.0NRT,290.2,15.3,D
46.789,-117.012,305.2,0.41,0.38,2026-05-16,1430,N,VIIRS,n,2.0NRT,285.1,8.7,D
45.123,-116.456,318.9,0.40,0.37,2026-05-16,1430,N,VIIRS,l,2.0NRT,288.5,12.1,D
"""
# Sample CSV with duplicate (same location, date, time)
SAMPLE_CSV_WITH_DUPE = """latitude,longitude,bright_ti4,scan,track,acq_date,acq_time,satellite,instrument,confidence,version,bright_ti5,frp,daynight
45.123,-116.456,320.5,0.39,0.36,2026-05-16,1430,N,VIIRS,h,2.0NRT,290.2,15.3,D
45.123,-116.456,320.5,0.39,0.36,2026-05-16,1430,N,VIIRS,h,2.0NRT,290.2,15.3,D
"""
def make_adapter_config(
region: dict | None = None,
satellites: list[str] | None = None,
) -> AdapterConfig:
"""Create an AdapterConfig for testing."""
settings = {
"api_key_alias": "firms",
"satellites": satellites or ["VIIRS_SNPP_NRT", "VIIRS_NOAA20_NRT"],
}
if region:
settings["region"] = region
else:
settings["region"] = {
"north": 49.5,
"south": 31.0,
"east": -102.0,
"west": -124.5,
}
return AdapterConfig(
name="firms",
enabled=True,
cadence_s=300,
settings=settings,
updated_at=datetime.now(timezone.utc),
)
@pytest.fixture
def temp_db_path():
"""Create a temporary database path for testing."""
with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as f:
yield Path(f.name)
@pytest.fixture
def mock_config_store():
"""Create a mock ConfigStore."""
store = MagicMock()
store.get_api_key = AsyncMock(return_value="test_api_key")
return store
class TestConfidenceMapping:
"""Test confidence value mapping."""
def test_low_confidence(self):
assert CONFIDENCE_MAP["l"] == "low"
def test_nominal_confidence(self):
assert CONFIDENCE_MAP["n"] == "nominal"
def test_high_confidence(self):
assert CONFIDENCE_MAP["h"] == "high"
class TestSatelliteShortNames:
"""Test satellite short name mapping."""
def test_snpp_short_name(self):
assert SATELLITE_SHORT["VIIRS_SNPP_NRT"] == "viirs_snpp"
def test_noaa20_short_name(self):
assert SATELLITE_SHORT["VIIRS_NOAA20_NRT"] == "viirs_noaa20"
def test_noaa21_short_name(self):
assert SATELLITE_SHORT["VIIRS_NOAA21_NRT"] == "viirs_noaa21"
class TestStableIdGeneration:
"""Test stable ID generation for deduplication."""
@pytest.mark.asyncio
async def test_stable_id_format(self, temp_db_path, mock_config_store):
config = make_adapter_config()
adapter = FIRMSAdapter(
config=config,
config_store=mock_config_store,
cursor_db_path=temp_db_path,
)
stable_id = adapter._build_stable_id(
satellite="VIIRS_SNPP_NRT",
acq_date="2026-05-16",
acq_time="1430",
lat=45.1234567,
lon=-116.4567890,
)
# Should be rounded to 3 decimal places
assert stable_id == "VIIRS_SNPP_NRT:2026-05-16:1430:45.123:-116.457"
@pytest.mark.asyncio
async def test_stable_id_rounding(self, temp_db_path, mock_config_store):
"""Test that small lat/lon differences within 0.001 round to same ID."""
config = make_adapter_config()
adapter = FIRMSAdapter(
config=config,
config_store=mock_config_store,
cursor_db_path=temp_db_path,
)
# Values that differ by less than 0.0005 should round to same value
id1 = adapter._build_stable_id("SAT", "2026-05-16", "1430", 45.1234, -116.4564)
id2 = adapter._build_stable_id("SAT", "2026-05-16", "1430", 45.1232, -116.4562)
# Both should round to 45.124, -116.457
assert id1 == id2
class TestCsvParsing:
"""Test CSV parsing."""
@pytest.mark.asyncio
async def test_parse_csv_rows(self, temp_db_path, mock_config_store):
config = make_adapter_config()
adapter = FIRMSAdapter(
config=config,
config_store=mock_config_store,
cursor_db_path=temp_db_path,
)
rows = adapter._parse_csv(SAMPLE_CSV, "VIIRS_SNPP_NRT")
assert len(rows) == 3
assert rows[0]["latitude"] == 45.123
assert rows[0]["longitude"] == -116.456
assert rows[0]["confidence"] == "high"
assert rows[1]["confidence"] == "nominal"
assert rows[2]["confidence"] == "low"
@pytest.mark.asyncio
async def test_parse_csv_brightness(self, temp_db_path, mock_config_store):
config = make_adapter_config()
adapter = FIRMSAdapter(
config=config,
config_store=mock_config_store,
cursor_db_path=temp_db_path,
)
rows = adapter._parse_csv(SAMPLE_CSV, "VIIRS_SNPP_NRT")
assert rows[0]["bright_ti4"] == 320.5
assert rows[0]["bright_ti5"] == 290.2
assert rows[0]["frp"] == 15.3
class TestEventGeneration:
"""Test Event generation from CSV rows."""
@pytest.mark.asyncio
async def test_event_category(self, temp_db_path, mock_config_store):
config = make_adapter_config()
adapter = FIRMSAdapter(
config=config,
config_store=mock_config_store,
cursor_db_path=temp_db_path,
)
rows = adapter._parse_csv(SAMPLE_CSV, "VIIRS_SNPP_NRT")
event = adapter._row_to_event(rows[0], "VIIRS_SNPP_NRT")
assert event.category == "fire.hotspot.viirs_snpp.high"
@pytest.mark.asyncio
async def test_event_severity(self, temp_db_path, mock_config_store):
config = make_adapter_config()
adapter = FIRMSAdapter(
config=config,
config_store=mock_config_store,
cursor_db_path=temp_db_path,
)
rows = adapter._parse_csv(SAMPLE_CSV, "VIIRS_SNPP_NRT")
high_event = adapter._row_to_event(rows[0], "VIIRS_SNPP_NRT")
nominal_event = adapter._row_to_event(rows[1], "VIIRS_SNPP_NRT")
low_event = adapter._row_to_event(rows[2], "VIIRS_SNPP_NRT")
assert high_event.severity == 3
assert nominal_event.severity == 2
assert low_event.severity == 1
@pytest.mark.asyncio
async def test_event_geo(self, temp_db_path, mock_config_store):
config = make_adapter_config()
adapter = FIRMSAdapter(
config=config,
config_store=mock_config_store,
cursor_db_path=temp_db_path,
)
rows = adapter._parse_csv(SAMPLE_CSV, "VIIRS_SNPP_NRT")
event = adapter._row_to_event(rows[0], "VIIRS_SNPP_NRT")
# GeoJSON order: lon, lat
assert event.geo.centroid == (-116.456, 45.123)
assert event.geo.bbox == (-116.456, 45.123, -116.456, 45.123)
class TestDeduplication:
"""Test deduplication logic."""
@pytest.mark.asyncio
async def test_dedup_marks_published(self, temp_db_path, mock_config_store):
config = make_adapter_config()
adapter = FIRMSAdapter(
config=config,
config_store=mock_config_store,
cursor_db_path=temp_db_path,
)
await adapter.startup()
stable_id = "VIIRS_SNPP_NRT:2026-05-16:1430:45.123:-116.456"
# Not published initially
assert not adapter.is_published(stable_id)
# Mark as published
adapter.mark_published(stable_id)
# Now should be published
assert adapter.is_published(stable_id)
await adapter.shutdown()
@pytest.mark.asyncio
async def test_dedup_prevents_duplicates(self, temp_db_path, mock_config_store):
"""Test that duplicate rows don't produce duplicate events."""
# Use only one satellite to simplify the test
config = make_adapter_config(satellites=["VIIRS_SNPP_NRT"])
adapter = FIRMSAdapter(
config=config,
config_store=mock_config_store,
cursor_db_path=temp_db_path,
)
await adapter.startup()
# Mock the fetch to return CSV with duplicates
with patch.object(adapter, "_fetch_csv", new_callable=AsyncMock) as mock_fetch:
mock_fetch.return_value = SAMPLE_CSV_WITH_DUPE
events = []
async for event in adapter.poll():
events.append(event)
# Should only get one event despite two identical rows
assert len(events) == 1
await adapter.shutdown()
class TestSubjectGeneration:
"""Test subject generation for fire hotspots."""
def test_subject_format(self):
event = Event(
id="test",
source="central/adapters/firms",
category="fire.hotspot.viirs_snpp.high",
time=datetime.now(timezone.utc),
severity=3,
geo=Geo(centroid=(-116.0, 45.0)),
data={},
)
subject = subject_for_fire_hotspot(event)
assert subject == "central.fire.hotspot.viirs_snpp.high"
def test_subject_nominal_confidence(self):
event = Event(
id="test",
source="central/adapters/firms",
category="fire.hotspot.viirs_noaa20.nominal",
time=datetime.now(timezone.utc),
severity=2,
geo=Geo(centroid=(-116.0, 45.0)),
data={},
)
subject = subject_for_fire_hotspot(event)
assert subject == "central.fire.hotspot.viirs_noaa20.nominal"
class TestUrlBuilding:
"""Test FIRMS API URL building."""
@pytest.mark.asyncio
async def test_url_format(self, temp_db_path, mock_config_store):
config = make_adapter_config(
region={"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}
)
adapter = FIRMSAdapter(
config=config,
config_store=mock_config_store,
cursor_db_path=temp_db_path,
)
await adapter.startup()
url = adapter._build_url("VIIRS_SNPP_NRT")
assert url is not None
assert "test_api_key" in url
assert "VIIRS_SNPP_NRT" in url
assert "-124.5,31.0,-102.0,49.5" in url # west,south,east,north
assert "/1" in url # dayRange
await adapter.shutdown()
@pytest.mark.asyncio
async def test_url_none_without_key(self, temp_db_path):
mock_store = MagicMock()
mock_store.get_api_key = AsyncMock(return_value=None)
config = make_adapter_config()
adapter = FIRMSAdapter(
config=config,
config_store=mock_store,
cursor_db_path=temp_db_path,
)
await adapter.startup()
url = adapter._build_url("VIIRS_SNPP_NRT")
assert url is None
await adapter.shutdown()
class TestApplyConfig:
"""Test hot-reload configuration application."""
@pytest.mark.asyncio
async def test_apply_config_updates_region(self, temp_db_path, mock_config_store):
config = make_adapter_config(
region={"north": 49.5, "south": 31.0, "east": -102.0, "west": -124.5}
)
adapter = FIRMSAdapter(
config=config,
config_store=mock_config_store,
cursor_db_path=temp_db_path,
)
await adapter.startup()
# Original region
assert adapter.region.north == 49.5
# Apply new config with different region
new_config = make_adapter_config(
region={"north": 48.0, "south": 45.0, "east": -115.0, "west": -125.0}
)
await adapter.apply_config(new_config)
assert adapter.region.north == 48.0
assert adapter.region.south == 45.0
await adapter.shutdown()
@pytest.mark.asyncio
async def test_apply_config_updates_satellites(self, temp_db_path, mock_config_store):
config = make_adapter_config(satellites=["VIIRS_SNPP_NRT", "VIIRS_NOAA20_NRT"])
adapter = FIRMSAdapter(
config=config,
config_store=mock_config_store,
cursor_db_path=temp_db_path,
)
await adapter.startup()
# Original satellites
assert len(adapter._satellites) == 2
# Apply config with single satellite
new_config = make_adapter_config(satellites=["VIIRS_NOAA20_NRT"])
await adapter.apply_config(new_config)
assert adapter._satellites == ["VIIRS_NOAA20_NRT"]
await adapter.shutdown()