From 0dd83a340e5c45d1d41d24afeca41185aa99129b Mon Sep 17 00:00:00 2001 From: malice Date: Sat, 6 Jun 2026 14:44:00 -0600 Subject: [PATCH] v0.10.3: rip out state_511_atis adapter (superseded by itd_511 v0.10.0; Castle Rock legacy shape EOL per sister-site discovery) (#88) Closes #88 Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/CONSUMER-INTEGRATION.md | 87 +---- .../032_remove_state_511_atis_adapters.sql | 13 + src/central/adapters/itd_511_cameras.py | 5 +- src/central/adapters/state_511_atis.py | 310 ------------------ .../adapters/state_511_atis_cameras.py | 249 -------------- src/central/gui/routes.py | 2 +- .../templates/_event_rows/state_511_atis.html | 10 - .../_event_rows/state_511_atis_cameras.html | 8 - .../_event_summaries/state_511_atis.html | 7 - .../state_511_atis_cameras.html | 4 - .../state_511_atis_cameras_sample.json | 1 - tests/fixtures/state_511_atis_closures.json | 1 - .../fixtures/state_511_atis_construction.json | 1 - tests/fixtures/state_511_atis_incidents.json | 1 - .../state_511_atis_markers_closures.json | 1 - .../state_511_atis_markers_construction.json | 1 - .../state_511_atis_markers_incidents.json | 1 - tests/test_consumer_doc.py | 24 ++ tests/test_events_feed_frontend.py | 3 - tests/test_gui_adapter_edit.py | 9 - tests/test_state_511_atis.py | 207 ------------ tests/test_state_511_atis_cameras.py | 134 -------- tests/test_telemetry_separation.py | 2 +- 23 files changed, 45 insertions(+), 1036 deletions(-) create mode 100644 sql/migrations/032_remove_state_511_atis_adapters.sql delete mode 100644 src/central/adapters/state_511_atis.py delete mode 100644 src/central/adapters/state_511_atis_cameras.py delete mode 100644 src/central/gui/templates/_event_rows/state_511_atis.html delete mode 100644 src/central/gui/templates/_event_rows/state_511_atis_cameras.html delete mode 100644 src/central/gui/templates/_event_summaries/state_511_atis.html delete mode 100644 src/central/gui/templates/_event_summaries/state_511_atis_cameras.html delete mode 100644 tests/fixtures/state_511_atis_cameras_sample.json delete mode 100644 tests/fixtures/state_511_atis_closures.json delete mode 100644 tests/fixtures/state_511_atis_construction.json delete mode 100644 tests/fixtures/state_511_atis_incidents.json delete mode 100644 tests/fixtures/state_511_atis_markers_closures.json delete mode 100644 tests/fixtures/state_511_atis_markers_construction.json delete mode 100644 tests/fixtures/state_511_atis_markers_incidents.json delete mode 100644 tests/test_state_511_atis.py delete mode 100644 tests/test_state_511_atis_cameras.py diff --git a/docs/CONSUMER-INTEGRATION.md b/docs/CONSUMER-INTEGRATION.md index 7b2502b..bb32b6a 100644 --- a/docs/CONSUMER-INTEGRATION.md +++ b/docs/CONSUMER-INTEGRATION.md @@ -1478,91 +1478,12 @@ already running can disable those overlap categories via `EONETSettings.category } ``` -### state_511_atis — State 511 incidents / closures / road work (Castle Rock ATIS) - -State-DOT 511 traffic events from the Castle Rock ATIS platform. Each layer is a -two-endpoint join: `GET /map/mapIcons/` (markers: `itemId` + `location` -`[lat,lon]`) joined on id with `POST /List/GetData/` (DataTables detail: -road name, description, county, severity). Verified for Idaho only. - -- **Stream:** `CENTRAL_TRAFFIC` -- **Layers / event_types:** Incidents -> `incident`, Closures -> `closure`, - Construction (`type":"Roadwork"`) -> `work_zone`. (Cameras are telemetry and - ship as a separate adapter later.) -- **Subject pattern:** `central.traffic..` (e.g. - `central.traffic.incident.id`); `` is the lowercased config `code`. -- **GUI event_type:** from `category = ".state_511_atis"` (first - dotted segment). `incident` and `closure` are new event_types (query-derived; - no hardcoded enum); `work_zone` is shared with wzdx. -- **Cadence default:** 300s (5 min). -- **Dedup key shape:** composite `::` - (e.g. `ID:Incidents:33579`); reused as the inner `Event.id`. -- **Event.data fields:** - - | key | type | nullable | description | - |---|---|---|---| - | `roadway_name` | str | yes | Road name, e.g. `US-95` | - | `description` | str | yes | Operator-readable narrative | - | `event_sub_type` | str | yes | e.g. `roadwayBlocked`, `longTermRoadConstruction` | - | `direction` | str | yes | `Both` / `North` / `Unknown` … | - | `location_description` | str | yes | Cross-street / landmark, e.g. `Five Mile Creek \| US-20` | - | `county` / `state` | str | yes | Upstream-supplied; populate the Location column | - | `start_date` / `last_updated` | str | yes | US-format local strings (no TZ; parsed naive->UTC, approximate) | - | `is_full_closure` | bool | yes | Closures only; drives severity | - | `layer` / `state_code` | str | no | Source layer + 2-letter state code (subject routing) | - | `latitude` / `longitude` | float | yes | From the marker join (enrichment input) | - -- **Severity:** `is_full_closure == true` -> 3, else 1 (the upstream `severity` - string is "None" on most records; not mapped in v1). -- **Decipherable as-is:** mostly. Road + location + description + county/state are - user-ready; the geocoder fills `city` from the joined coordinates. -- **Removal semantics:** none in v1. Events age out of the upstream feed; the - 14-day dedup sweep expires stale ids. - -### state_511_atis_cameras — State 511 traffic cameras (Castle Rock ATIS, telemetry) - -State-DOT 511 traffic cameras (Idaho). One telemetry event per camera per UTC day; -the `/telemetry` detail drawer renders the live image inline (`` fetched -direct from the source -- Central stores the URL, never the image bytes). Pairs -with state_511_atis incidents: see an incident, click a nearby camera, see road -conditions. - -- **Stream:** `CENTRAL_TRAFFIC_CAMERAS` (telemetry; `/telemetry` tab). -- **Subject pattern:** `central.traffic_cameras..` -- subscribe to - one camera or `central.traffic_cameras.id.>` for all Idaho. -- **GUI event_type:** `camera` (from `category = "camera.state_511_atis_cameras"`). -- **Source:** full state list via `POST /List/GetData/Cameras` (DataTables, - **paginated** at 100/page; Idaho ~455). Public-unauth. **Cadence 600s.** -- **Dedup key shape:** `:cam::` -- one event per - camera per UTC day. The table always shows today's cameras; no per-poll flooding - and no dedup-window/retention coordination needed. -- **Event.data fields:** - - | key | type | nullable | description | - |---|---|---|---| - | `camera_id` | int | no | Stable upstream id | - | `roadway_name` | str | yes | e.g. `I-84` | - | `location_description` | str | yes | e.g. `I-84 Mountain Home` (embeds the roadway) | - | `direction` | str | yes | `North` / `Unknown` / ... | - | `source` | str | yes | Owning agency: `Idaho511`, `ITDNET`, `RWIS`, `UDOT`, `ODOT`, ... (border cameras Idaho 511 surfaces) | - | `image_url` | str | yes | Full live image URL (`/map/Cctv/`); browser fetches direct | - | `image_count` | int | no | Number of camera angles (1-4) | - | `record_updated` | str | yes | Camera config edit time (NOT image-capture time) | - | `state_code` | str | no | Routing | - | `latitude` / `longitude` | float | yes | From the WKT `POINT (lon lat)` | - -- **Severity:** always `1` (cameras have no severity signal; telemetry styling). -- **Decipherable as-is:** yes -- location + source are user-ready; the live image - is one click away. **No image-capture timestamp** is available upstream (the - `lastUpdated` field is camera-config time), so the drawer shows no "captured at". -- **Removal semantics:** none; offline cameras serve an empty image but stay listed. - ### itd_511 — Idaho 511 official DOT API (events + advisories) Idaho Transportation Department's official 511 REST API. Polls roadwork, closures, incidents, special events, and advisories statewide. First -official-state-DOT-API adapter (v0.10.0); runs in parallel with state_511_atis -(Castle Rock) post-deploy for A/B comparison. Idaho-only; subject suffix is +official-state-DOT-API adapter (v0.10.0; superseded the Castle Rock-backed +state_511_atis adapter, removed in v0.10.3). Idaho-only; subject suffix is uniformly `us.id`. - **Stream:** `CENTRAL_TRAFFIC` (event class). **event_type:** one of @@ -1660,11 +1581,11 @@ never the image bytes). Sibling adapter of itd_511 (shared API key alias Real-time incidents (closures, jams, hazards, road work, accidents) from TomTom Orbis `incidentDetails`, polled per metro bbox. Complements wzdx (federal work -zones) and state_511_atis (state-DOT reports) with commercial vehicle-telematics +zones) and itd_511 (state-DOT reports) with commercial vehicle-telematics coverage. One event per incident. - **Stream:** `CENTRAL_TRAFFIC` (event class). **event_type:** `incident` (from - `category = "incident.tomtom_incidents"`); shares the type with state_511_atis. + `category = "incident.tomtom_incidents"`); shares the type with itd_511. - **Subject pattern:** `central.traffic.incident.` (e.g. `central.traffic.incident.id`); `` is the per-bbox `state_code`. - **Coverage:** configured metro bboxes, **each <= 10,000 km^2** (TomTom rejects diff --git a/sql/migrations/032_remove_state_511_atis_adapters.sql b/sql/migrations/032_remove_state_511_atis_adapters.sql new file mode 100644 index 0000000..2a0fbb6 --- /dev/null +++ b/sql/migrations/032_remove_state_511_atis_adapters.sql @@ -0,0 +1,13 @@ +-- v0.10.3: rip out state_511_atis + state_511_atis_cameras (Castle Rock legacy +-- shape EOL; superseded by itd_511 + itd_511_cameras from migration 031 / v0.10.0). +-- +-- Idempotent: the DELETE succeeds whether the rows are present or not. Historical +-- events in public.events stay (preserved as historical record per Matt's call); +-- only the config.adapters rows that would otherwise be hot-reloaded into the +-- supervisor are removed. +-- +-- Note: cursors.db cleanup (published_ids for both adapters) is a SQLite-side +-- step handled at deploy time, NOT in this Postgres migration. + +DELETE FROM config.adapters +WHERE name IN ('state_511_atis', 'state_511_atis_cameras'); diff --git a/src/central/adapters/itd_511_cameras.py b/src/central/adapters/itd_511_cameras.py index 3238e26..4fbf3b1 100644 --- a/src/central/adapters/itd_511_cameras.py +++ b/src/central/adapters/itd_511_cameras.py @@ -15,9 +15,8 @@ Subject convention (v0.9.20 forward, locked v0.10.0 architectural call A): central.traffic_cameras.us.id. Dedup id: ``idaho_511:cam::`` — one telemetry event per -camera per UTC day, matching the state_511_atis_cameras precedent. Image URLs -ship straight through; the partial renders from Views[0].Url -(jpeg/gif/png all confirmed publicly reachable). +camera per UTC day. Image URLs ship straight through; the partial renders + from Views[0].Url (jpeg/gif/png all confirmed publicly reachable). Retry predicate matches itd_511's (architectural call B): no retry on 4xx except 429 with Retry-After. Shares the ``_Transient`` sentinel and diff --git a/src/central/adapters/state_511_atis.py b/src/central/adapters/state_511_atis.py deleted file mode 100644 index 4150a7e..0000000 --- a/src/central/adapters/state_511_atis.py +++ /dev/null @@ -1,310 +0,0 @@ -"""State 511 (Castle Rock ATIS) adapter — Idaho first. - -Castle Rock's ATIS platform exposes two endpoints per layer that must be joined: - - GET /map/mapIcons/ -> thin markers: {itemId, location:[lat,lon], ...} - - POST /List/GetData/ -> rich DataTables rows keyed by id==itemId -The marker feed has coordinates but no text; the List feed has road name / -description / county / severity but no coordinates. We join on id. - -Layers map to traffic event_types (wzdx precedent — category drives the GUI -event_type via split_part, subject is central.traffic..): - Incidents -> incident, Closures -> closure, Construction -> work_zone. -Cameras are telemetry (data_class) and ship as a separate adapter later. - -Templatized per state via settings {"states":[{"code","base_url"}]}; only Idaho -is verified (Oregon/Wyoming are not Castle Rock). Add states as settings rows -once each host's URL shape is confirmed. Dedup is inherited from SourceAdapter. -""" - -import asyncio -import logging -import sqlite3 -from collections.abc import AsyncIterator -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -import aiohttp -from pydantic import BaseModel -from tenacity import ( - retry, - retry_if_exception_type, - stop_after_attempt, - wait_exponential_jitter, -) - -from central.adapter import SourceAdapter -from central.config_models import AdapterConfig -from central.config_store import ConfigStore -from central.models import Event, Geo - -logger = logging.getLogger(__name__) - -# Castle Rock layer -> Central event_type (category = ".state_511_atis"). -LAYER_EVENT_TYPE: dict[str, str] = { - "Incidents": "incident", - "Closures": "closure", - "Construction": "work_zone", -} - -# DataTables server-side body. POST is required (GET returns an empty data array). -# Castle Rock caps each page at 100 rows regardless of `length`, so we paginate. -_LIST_PAGE_LENGTH = 100 -_MAX_PAGES = 50 # defensive ceiling (~5,000 rows/layer) - - -def _list_body(start: int) -> dict[str, str]: - return { - "draw": "1", "start": str(start), "length": str(_LIST_PAGE_LENGTH), - "columns[0][data]": "0", "order[0][column]": "0", - "order[0][dir]": "asc", "search[value]": "", - } - - -_XHR = {"X-Requested-With": "XMLHttpRequest"} - -_FETCH_CONCURRENCY = 4 -_FETCH_TIMEOUT_S = 30 - -_DEDUP_DDL = ( - "CREATE TABLE IF NOT EXISTS published_ids (" - "adapter TEXT NOT NULL, event_id TEXT NOT NULL, " - "first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, " - "last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, " - "PRIMARY KEY (adapter, event_id))" -) - - -def _parse_us_dt(value: str | None) -> datetime | None: - """Parse Castle Rock's US-format local timestamp (e.g. "5/25/26, 2:32 PM"). - - No timezone is supplied; treated as naive -> UTC (approximate — the freshness - signal is last_updated ordering, not absolute TZ). Returns None on failure. - """ - if not value: - return None - try: - return datetime.strptime(value.strip(), "%m/%d/%y, %I:%M %p").replace(tzinfo=timezone.utc) - except (ValueError, TypeError): - return None - - -class StateConfig(BaseModel): - """One Castle Rock 511 deployment to poll.""" - - code: str # 2-letter state code, e.g. "ID" - base_url: str # e.g. "https://511.idaho.gov" - - -class State511ATISSettings(BaseModel): - """states: verified Castle Rock deployments. Empty = nothing to poll.""" - - states: list[StateConfig] = [] - - -class State511ATISAdapter(SourceAdapter): - """Castle Rock ATIS 511 adapter (incidents / closures / construction).""" - - name = "state_511_atis" - display_name = "State 511 (Castle Rock ATIS)" - description = ( - "State DOT 511 incidents, closures, and road work from the Castle Rock " - "ATIS platform. Joins the map-marker and detail-list endpoints per layer. " - "Verified for Idaho; add states as settings rows once each is confirmed." - ) - settings_schema = State511ATISSettings - requires_api_key = None - api_key_field = None - wizard_order = None # Ships disabled - default_cadence_s = 300 - data_class = "event" - # Coords come from the marker join; geocoder fills city (county/state are upstream). - enrichment_locations = [("latitude", "longitude")] - - def __init__( - self, - config: AdapterConfig, - config_store: ConfigStore, - cursor_db_path: Path, - ) -> None: - self._config_store = config_store - self._cursor_db_path = cursor_db_path - self._session: aiohttp.ClientSession | None = None - self._db: sqlite3.Connection | None = None - self._states: list[StateConfig] = self._read_states(config) - - @staticmethod - def _read_states(config: AdapterConfig) -> list[StateConfig]: - raw = config.settings.get("states") or [] - return [StateConfig(**s) for s in raw] - - async def startup(self) -> None: - self._session = aiohttp.ClientSession( - timeout=aiohttp.ClientTimeout(total=_FETCH_TIMEOUT_S), - headers={"User-Agent": "Central/0.9 (+state_511_atis)"}, - ) - self._db = sqlite3.connect(self._cursor_db_path) - self._db.execute(_DEDUP_DDL) - self._db.execute("CREATE INDEX IF NOT EXISTS published_ids_last_seen ON published_ids (last_seen)") - self._db.commit() - logger.info("state_511_atis adapter started", - extra={"states": [s.code for s in self._states]}) - - async def shutdown(self) -> None: - if self._session: - await self._session.close() - self._session = None - if self._db: - self._db.close() - self._db = None - - async def apply_config(self, new_config: AdapterConfig) -> None: - self._states = self._read_states(new_config) - logger.info("state_511_atis config updated", - extra={"states": [s.code for s in self._states]}) - - @retry( - stop=stop_after_attempt(3), - wait=wait_exponential_jitter(initial=1, max=30), - retry=retry_if_exception_type((aiohttp.ClientError, TimeoutError)), - ) - async def _fetch_markers(self, base_url: str, layer: str) -> dict[str, tuple[float, float]]: - """GET /map/mapIcons/ -> {itemId: (lat, lon)}.""" - assert self._session is not None - async with self._session.get(f"{base_url}/map/mapIcons/{layer}") as resp: - resp.raise_for_status() - doc = await resp.json(content_type=None) - out: dict[str, tuple[float, float]] = {} - for m in (doc.get("item2") or []): - loc = m.get("location") - if isinstance(loc, list) and len(loc) == 2 and m.get("itemId") is not None: - out[str(m["itemId"])] = (float(loc[0]), float(loc[1])) - return out - - @retry( - stop=stop_after_attempt(3), - wait=wait_exponential_jitter(initial=1, max=30), - retry=retry_if_exception_type((aiohttp.ClientError, TimeoutError)), - ) - async def _fetch_page(self, base_url: str, layer: str, start: int) -> dict[str, Any]: - assert self._session is not None - async with self._session.post( - f"{base_url}/List/GetData/{layer}", data=_list_body(start), headers=_XHR - ) as resp: - resp.raise_for_status() - return await resp.json(content_type=None) - - async def _fetch_details(self, base_url: str, layer: str) -> list[dict[str, Any]]: - """POST /List/GetData/ (DataTables), paginated -> all rows. [] on failure. - - Castle Rock caps each page at 100 rows regardless of `length`, so we loop - until recordsFiltered is collected or a page returns empty, with a - defensive _MAX_PAGES ceiling. A mid-pagination failure returns the rows - gathered so far (retried next poll). - """ - rows: list[dict[str, Any]] = [] - start = 0 - total: int | None = None - for _ in range(_MAX_PAGES): - try: - doc = await self._fetch_page(base_url, layer, start) - except (aiohttp.ClientError, TimeoutError, asyncio.TimeoutError, ValueError) as exc: - logger.warning("state_511_atis detail fetch failed", - extra={"layer": layer, "base_url": base_url, "start": start, "error": str(exc)}) - break - if total is None: - total = doc.get("recordsFiltered") or doc.get("recordsTotal") or 0 - page = doc.get("data") or [] - if not page: - break - rows.extend(page) - start += _LIST_PAGE_LENGTH - if len(rows) >= total: - break - else: - logger.warning("state_511_atis pagination hit max_pages cap", - extra={"layer": layer, "max_pages": _MAX_PAGES, - "collected": len(rows), "recordsFiltered": total}) - return rows - - def _build_event( - self, detail: dict[str, Any], coords: tuple[float, float] | None, - state_code: str, layer: str, - ) -> Event | None: - record_id = detail.get("id") - if record_id is None: - return None - event_type = LAYER_EVENT_TYPE[layer] - lat, lon = (coords if coords else (None, None)) - return Event( - id=f"{state_code}:{layer}:{record_id}", - adapter=self.name, - category=f"{event_type}.state_511_atis", - time=(_parse_us_dt(detail.get("lastUpdated")) - or _parse_us_dt(detail.get("startDate")) - or datetime.now(timezone.utc)), - expires=_parse_us_dt(detail.get("endDate")), - severity=(3 if detail.get("isFullClosure") else 1), - geo=Geo( - centroid=(lon, lat) if lat is not None and lon is not None else None, - regions=[f"US-{state_code}"], - primary_region=f"US-{state_code}", - ), - data={ - "roadway_name": detail.get("roadwayName"), - "description": (detail.get("description") or "").strip() or None, - "event_sub_type": detail.get("eventSubType"), - "direction": detail.get("direction"), - "location_description": detail.get("locationDescription"), - "county": detail.get("county"), - "state": detail.get("state"), - "start_date": detail.get("startDate"), - "last_updated": detail.get("lastUpdated"), - "is_full_closure": detail.get("isFullClosure"), - "layer": layer, - "state_code": state_code, - "latitude": lat, # enrichment_locations pair (canonical) - "longitude": lon, - }, - ) - - async def poll(self) -> AsyncIterator[Event]: - if not self._session: - raise RuntimeError("Session not initialized") - sem = asyncio.Semaphore(_FETCH_CONCURRENCY) - - async def _layer(state: StateConfig, layer: str): - async with sem: - try: - markers = await self._fetch_markers(state.base_url, layer) - except (aiohttp.ClientError, TimeoutError) as exc: - logger.warning("state_511_atis marker fetch failed", - extra={"layer": layer, "state": state.code, "error": str(exc)}) - markers = {} - details = await self._fetch_details(state.base_url, layer) - return state.code, layer, markers, details - - tasks = [_layer(s, layer) for s in self._states for layer in LAYER_EVENT_TYPE] - yielded = 0 - for state_code, layer, markers, details in await asyncio.gather(*tasks): - for detail in details: - try: - coords = markers.get(str(detail.get("id"))) - event = self._build_event(detail, coords, state_code, layer) - except Exception: # one bad record never sinks the poll - logger.exception("state_511_atis record parse failed", - extra={"layer": layer, "state": state_code}) - continue - if event is None: - continue - yield event - yielded += 1 - - self.sweep_old_ids() - logger.info("state_511_atis poll completed", extra={"events_yielded": yielded}) - - def subject_for(self, event: Event) -> str: - d = event.data - event_type = LAYER_EVENT_TYPE.get(d.get("layer"), "incident") - code = (d.get("state_code") or "").lower() or "unknown" - return f"central.traffic.{event_type}.{code}" diff --git a/src/central/adapters/state_511_atis_cameras.py b/src/central/adapters/state_511_atis_cameras.py deleted file mode 100644 index 6ff7e91..0000000 --- a/src/central/adapters/state_511_atis_cameras.py +++ /dev/null @@ -1,249 +0,0 @@ -"""State 511 (Castle Rock ATIS) cameras adapter — telemetry. - -Polls the full state camera directory via POST /List/GetData/Cameras (paginated, -100/page), emitting one telemetry Event per camera to CENTRAL_TRAFFIC_CAMERAS -(subject central.traffic_cameras.{state}.{camera_id}). data_class="telemetry" -> -the /telemetry tab. The detail drawer renders straight from the upstream -image URL -- no blob storage or proxy in Central. Idaho-only; templatized per -state via settings, same shape as state_511_atis. - -Dedup is per-UTC-day ({state}:cam:{id}:{YYYY-MM-DD}): one event per camera per day --- the table always shows today's cameras, with no dedup-window/retention -coordination and no per-poll flooding. Inherits the v0.9.1 dedup mixin. -""" - -import asyncio -import logging -import re -import sqlite3 -from collections.abc import AsyncIterator -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -import aiohttp -from pydantic import BaseModel -from tenacity import ( - retry, - retry_if_exception_type, - stop_after_attempt, - wait_exponential_jitter, -) - -from central.adapter import SourceAdapter -from central.config_models import AdapterConfig -from central.config_store import ConfigStore -from central.models import Event, Geo - -logger = logging.getLogger(__name__) - -_PAGE_LENGTH = 100 # Castle Rock caps the DataTables page at 100 rows -_MAX_PAGES = 20 # defensive ceiling (~2,000 cameras) -_XHR = {"X-Requested-With": "XMLHttpRequest"} -_FETCH_CONCURRENCY = 4 -_FETCH_TIMEOUT_S = 30 -_WKT_POINT = re.compile(r"POINT \(([-0-9.]+) ([-0-9.]+)\)") - -_DEDUP_DDL = ( - "CREATE TABLE IF NOT EXISTS published_ids (" - "adapter TEXT NOT NULL, event_id TEXT NOT NULL, " - "first_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, " - "last_seen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, " - "PRIMARY KEY (adapter, event_id))" -) - - -def _parse_wkt(wkt: str | None) -> tuple[float | None, float | None]: - """'POINT (lon lat)' -> (lat, lon); (None, None) on failure.""" - if not wkt: - return (None, None) - m = _WKT_POINT.match(wkt) - if not m: - return (None, None) - try: - return (float(m.group(2)), float(m.group(1))) - except ValueError: - return (None, None) - - -class StateConfig(BaseModel): - code: str - base_url: str - - -class State511CamerasSettings(BaseModel): - """states: Castle Rock 511 deployments to poll for cameras.""" - - states: list[StateConfig] = [] - - -class State511ATISCamerasAdapter(SourceAdapter): - """Castle Rock ATIS 511 camera directory adapter (telemetry).""" - - name = "state_511_atis_cameras" - display_name = "State 511 Cameras (Castle Rock ATIS)" - description = ( - "State DOT 511 traffic cameras from the Castle Rock ATIS platform. One " - "telemetry event per camera (per UTC day); the detail drawer shows the live " - "image direct from the source. Verified for Idaho." - ) - settings_schema = State511CamerasSettings - requires_api_key = None - api_key_field = None - wizard_order = None # Ships disabled - default_cadence_s = 600 - data_class = "telemetry" - enrichment_locations = [("latitude", "longitude")] - - def __init__( - self, - config: AdapterConfig, - config_store: ConfigStore, - cursor_db_path: Path, - ) -> None: - self._config_store = config_store - self._cursor_db_path = cursor_db_path - self._session: aiohttp.ClientSession | None = None - self._db: sqlite3.Connection | None = None - self._states: list[StateConfig] = self._read_states(config) - - @staticmethod - def _read_states(config: AdapterConfig) -> list[StateConfig]: - return [StateConfig(**s) for s in (config.settings.get("states") or [])] - - async def startup(self) -> None: - self._session = aiohttp.ClientSession( - timeout=aiohttp.ClientTimeout(total=_FETCH_TIMEOUT_S), - headers={"User-Agent": "Central/0.9 (+state_511_atis_cameras)"}, - ) - self._db = sqlite3.connect(self._cursor_db_path) - self._db.execute(_DEDUP_DDL) - self._db.execute("CREATE INDEX IF NOT EXISTS published_ids_last_seen ON published_ids (last_seen)") - self._db.commit() - logger.info("state_511_atis_cameras adapter started", - extra={"states": [s.code for s in self._states]}) - - async def shutdown(self) -> None: - if self._session: - await self._session.close() - self._session = None - if self._db: - self._db.close() - self._db = None - - async def apply_config(self, new_config: AdapterConfig) -> None: - self._states = self._read_states(new_config) - logger.info("state_511_atis_cameras config updated", - extra={"states": [s.code for s in self._states]}) - - @retry( - stop=stop_after_attempt(3), - wait=wait_exponential_jitter(initial=1, max=30), - retry=retry_if_exception_type((aiohttp.ClientError, TimeoutError)), - ) - async def _fetch_page(self, base_url: str, start: int) -> dict[str, Any]: - assert self._session is not None - body = { - "draw": "1", "start": str(start), "length": str(_PAGE_LENGTH), - "columns[0][data]": "0", "order[0][column]": "0", - "order[0][dir]": "asc", "search[value]": "", - } - async with self._session.post( - f"{base_url}/List/GetData/Cameras", data=body, headers=_XHR - ) as resp: - resp.raise_for_status() - return await resp.json(content_type=None) - - async def _fetch_all(self, base_url: str) -> list[dict[str, Any]]: - """Paginate the camera directory (100/page) -> all rows. [] on failure.""" - try: - first = await self._fetch_page(base_url, 0) - except (aiohttp.ClientError, TimeoutError, asyncio.TimeoutError, ValueError) as exc: - logger.warning("state_511_atis_cameras page fetch failed", - extra={"base_url": base_url, "start": 0, "error": str(exc)}) - return [] - total = first.get("recordsTotal") or 0 - rows = list(first.get("data") or []) - start = _PAGE_LENGTH - pages = 1 - while start < total and pages < _MAX_PAGES: - try: - page = await self._fetch_page(base_url, start) - except (aiohttp.ClientError, TimeoutError, asyncio.TimeoutError, ValueError) as exc: - logger.warning("state_511_atis_cameras page fetch failed", - extra={"base_url": base_url, "start": start, "error": str(exc)}) - break - rows.extend(page.get("data") or []) - start += _PAGE_LENGTH - pages += 1 - return rows - - def _build_event(self, cam: dict[str, Any], state: StateConfig) -> Event | None: - cam_id = cam.get("id") - if cam_id is None: - return None - lat, lon = _parse_wkt((cam.get("latLng") or {}).get("geography", {}).get("wellKnownText")) - images = cam.get("images") or [] - image_url = None - if images and images[0].get("imageUrl"): - image_url = state.base_url + images[0]["imageUrl"] - day = datetime.now(timezone.utc).strftime("%Y-%m-%d") - return Event( - id=f"{state.code}:cam:{cam_id}:{day}", - adapter=self.name, - category="camera.state_511_atis_cameras", - time=datetime.now(timezone.utc), - severity=1, # telemetry; cameras have no severity signal - geo=Geo( - centroid=(lon, lat) if lat is not None and lon is not None else None, - regions=[f"US-{state.code}"], - primary_region=f"US-{state.code}", - ), - data={ - "camera_id": cam_id, - "roadway_name": cam.get("roadway"), - "location_description": cam.get("location"), - "direction": cam.get("direction"), - "source": cam.get("source"), - "image_url": image_url, - "image_count": len(images), - "record_updated": cam.get("lastUpdated"), # camera config edit time (not capture time) - "state_code": state.code, - "latitude": lat, - "longitude": lon, - }, - ) - - async def poll(self) -> AsyncIterator[Event]: - if not self._session: - raise RuntimeError("Session not initialized") - sem = asyncio.Semaphore(_FETCH_CONCURRENCY) - - async def _one(state: StateConfig) -> list[Event]: - async with sem: - cams = await self._fetch_all(state.base_url) - out: list[Event] = [] - for cam in cams: - try: - ev = self._build_event(cam, state) - except Exception: - logger.exception("state_511_atis_cameras parse failed", extra={"state": state.code}) - continue - if ev is not None: - out.append(ev) - return out - - results = await asyncio.gather(*[_one(s) for s in self._states]) - yielded = 0 - for evs in results: - for ev in evs: - yield ev - yielded += 1 - - self.sweep_old_ids() - logger.info("state_511_atis_cameras poll completed", extra={"events_yielded": yielded}) - - def subject_for(self, event: Event) -> str: - d = event.data - code = (d.get("state_code") or "").lower() or "unknown" - return f"central.traffic_cameras.{code}.{d.get('camera_id')}" diff --git a/src/central/gui/routes.py b/src/central/gui/routes.py index 64635d8..eaa92f0 100644 --- a/src/central/gui/routes.py +++ b/src/central/gui/routes.py @@ -2851,7 +2851,7 @@ ADAPTER_GROUPS = { "Space": ["swpc_alerts", "swpc_kindex", "swpc_protons"], "Geophysical": ["usgs_quake", "nwis"], "Earth Observation": ["eonet"], - "Transportation": ["wzdx", "state_511_atis", "tomtom_flow", "tomtom_incidents", "state_511_atis_cameras", "itd_511", "itd_511_cameras"], + "Transportation": ["wzdx", "tomtom_flow", "tomtom_incidents", "itd_511", "itd_511_cameras"], } # Same palette the map legend uses, indexed by sorted-adapter position. EVENTS_PALETTE = [ diff --git a/src/central/gui/templates/_event_rows/state_511_atis.html b/src/central/gui/templates/_event_rows/state_511_atis.html deleted file mode 100644 index c8cd844..0000000 --- a/src/central/gui/templates/_event_rows/state_511_atis.html +++ /dev/null @@ -1,10 +0,0 @@ -{# State 511 (Castle Rock ATIS) detail rows. Fields from payload->data->data; - every block guarded. direction "Unknown"/"None" suppressed (wzdx lesson). #} -{% set d = (event.data.get('data') or {}).get('data') or {} %} -{% if d.get('roadway_name') %}
Road
{{ d.roadway_name }}{% if d.get('direction') and d.direction not in ['Unknown', 'None'] %} ({{ d.direction }}){% endif %}
{% endif %} -{% if d.get('location_description') %}
Location
{{ d.location_description }}
{% endif %} -{% if d.get('event_sub_type') %}
Type
{{ d.event_sub_type }}
{% endif %} -{% if d.get('is_full_closure') %}
Full closure
Yes
{% endif %} -{% if d.get('start_date') %}
Started
{{ d.start_date }}
{% endif %} -{% if d.get('last_updated') %}
Updated
{{ d.last_updated }}
{% endif %} -{% if d.get('description') %}
Description
{{ d.description | truncate(220) }}
{% endif %} diff --git a/src/central/gui/templates/_event_rows/state_511_atis_cameras.html b/src/central/gui/templates/_event_rows/state_511_atis_cameras.html deleted file mode 100644 index 37cff1e..0000000 --- a/src/central/gui/templates/_event_rows/state_511_atis_cameras.html +++ /dev/null @@ -1,8 +0,0 @@ -{# State 511 camera detail. Live fetched direct from the source (no proxy / - blob storage in Central). Fields from payload->data->data. #} -{% set d = (event.data.get('data') or {}).get('data') or {} %} -{% if d.get('image_url') %}
View
Camera view
{% endif %} -{% if d.get('roadway_name') %}
Road
{{ d.roadway_name }}{% if d.get('direction') and d.direction not in ['Unknown', 'None'] %} ({{ d.direction }}){% endif %}
{% endif %} -{% if d.get('location_description') %}
Location
{{ d.location_description }}
{% endif %} -{% if d.get('source') %}
Source
{{ d.source }}
{% endif %} -{% if d.get('image_count') and d.image_count > 1 %}
Views
{{ d.image_count }} camera angles
{% endif %} diff --git a/src/central/gui/templates/_event_summaries/state_511_atis.html b/src/central/gui/templates/_event_summaries/state_511_atis.html deleted file mode 100644 index 76a793a..0000000 --- a/src/central/gui/templates/_event_summaries/state_511_atis.html +++ /dev/null @@ -1,7 +0,0 @@ -{# State 511 (Castle Rock ATIS) one-line subject. Lead label by layer + road + - cross-street location; county/state render in the generic Location column - (it falls back to d.county/d.state). Fields from payload->data->data. #} -{% set d = (event.data.get('data') or {}).get('data') or {} %} -{%- set labels = {'Incidents': 'Incident', 'Closures': 'Closure', 'Construction': 'Road work'} -%} -{%- set lead = labels.get(d.get('layer'), 'Traffic event') -%} -{{ lead }}{% if d.get('roadway_name') %} on {{ d.roadway_name }}{% endif %}{% if d.get('location_description') %} — {{ d.location_description }}{% endif %} diff --git a/src/central/gui/templates/_event_summaries/state_511_atis_cameras.html b/src/central/gui/templates/_event_summaries/state_511_atis_cameras.html deleted file mode 100644 index 6974dc7..0000000 --- a/src/central/gui/templates/_event_summaries/state_511_atis_cameras.html +++ /dev/null @@ -1,4 +0,0 @@ -{# State 511 camera one-line subject. The `location` field already embeds the - roadway (e.g. "I-84 Mountain Home"), so it stands alone. Fields from payload->data->data. #} -{% set d = (event.data.get('data') or {}).get('data') or {} %} -Camera: {{ d.get('location_description') or d.get('roadway_name') or ('#' ~ d.get('camera_id')) }} diff --git a/tests/fixtures/state_511_atis_cameras_sample.json b/tests/fixtures/state_511_atis_cameras_sample.json deleted file mode 100644 index 5990691..0000000 --- a/tests/fixtures/state_511_atis_cameras_sample.json +++ /dev/null @@ -1 +0,0 @@ -{"recordsTotal":2,"recordsFiltered":2,"data":[{"DT_RowId":"1","tooltipUrl":"/tooltip/Cameras/1?lang=%7Blang%7D&noCss=true","agencyLogoEnabled":false,"visible":true,"isDefault":false,"images":[{"id":1,"cameraSiteId":1,"sortOrder":1,"description":"N/A","imageUrl":"/map/Cctv/1","imageType":0,"isVideoAuthRequired":false,"videoDisabled":false,"disabled":false,"blocked":false,"language":"en"}],"id":1,"sourceId":"10.C1","source":"UDOT","type":"UDOT","areaId":null,"area":null,"sortOrder":1,"roadway":"I-15","direction":"Unknown","location":"I-15 UT/ID State Line UT","latLng":{"geography":{"coordinateSystemId":4326,"wellKnownText":"POINT (-112.198 42.0011)"}},"linkId1":"112984999F","linkId2":"114769197T","created":"2023-01-23T13:51:47.3533333+00:00","lastUpdated":"2023-01-23T13:51:47.3533333+00:00","lastEditedBy":"zeeshawn.ahmad@gmail.com","defaultCameraSite":false,"nickname":null,"language":"en","jsonData":{},"jsonDataSerialized":null,"region":null,"state":null,"county":null,"city":null,"dotDistrict":null},{"DT_RowId":"2","tooltipUrl":"/tooltip/Cameras/2?lang=%7Blang%7D&noCss=true","agencyLogoEnabled":false,"visible":true,"isDefault":false,"images":[{"id":2,"cameraSiteId":2,"sortOrder":325,"imageUrl":"/map/Cctv/2","imageType":0,"refreshRateMs":300000,"isVideoAuthRequired":false,"videoDisabled":false,"disabled":false,"blocked":false,"language":"en"},{"id":3,"cameraSiteId":2,"sortOrder":325,"imageUrl":"/map/Cctv/3","imageType":0,"refreshRateMs":300000,"isVideoAuthRequired":false,"videoDisabled":false,"disabled":false,"blocked":false,"language":"en"},{"id":4,"cameraSiteId":2,"sortOrder":325,"imageUrl":"/map/Cctv/4","imageType":0,"refreshRateMs":300000,"isVideoAuthRequired":false,"videoDisabled":false,"disabled":false,"blocked":false,"language":"en"}],"id":2,"sourceId":"100.C1","source":"RWIS","type":"RWIS","areaId":null,"area":null,"sortOrder":325,"roadway":"SH-75","direction":"Unknown","location":"SH-75 Wood River","latLng":{"geography":{"coordinateSystemId":4326,"wellKnownText":"POINT (-114.345 43.5946)"}},"linkId1":"41105278T","linkId2":"851613715T","created":"2024-08-12T07:57:34.1833333+00:00","lastUpdated":"2024-08-12T07:57:34.1833333+00:00","lastEditedBy":"barton.phelps@itd.idaho.gov","defaultCameraSite":false,"nickname":null,"language":"en","jsonData":{"rwisType":"Normal","status":"Normal"},"jsonDataSerialized":"{\"RwisType\":\"Normal\",\"Status\":\"Normal\"}","region":null,"state":null,"county":null,"city":null,"dotDistrict":null}]} \ No newline at end of file diff --git a/tests/fixtures/state_511_atis_closures.json b/tests/fixtures/state_511_atis_closures.json deleted file mode 100644 index f9a2397..0000000 --- a/tests/fixtures/state_511_atis_closures.json +++ /dev/null @@ -1 +0,0 @@ -{"data":[{"DT_RowId":"17","tooltipUrl":"/tooltip/Closures/17?lang=%7Blang%7D&noCss=true","id":17,"type":"Closures","layerName":"Closures","roadwayName":"N McDermott Rd","description":" Long term road construction on N McDermott Rd Both Directions from Five Mile Creek to US-20. All lanes closed. 1/30/2023 2:24 PM Mon, Tue, Wed, Thu, Fri, Sat, Sun: Active all day
Comments: Open to local traffic only.
","sourceId":"469","source":"ERS","comment":"Open to local traffic only.","eventSubType":"longTermRoadConstruction","startDate":"1/30/23, 2:24 PM","endDate":null,"lastUpdated":"6/18/24, 6:48 PM","isFullClosure":true,"severity":"None","direction":"Both","locationDescription":"Five Mile Creek | US-20","detourDescription":null,"laneDescription":"All lanes closed","recurrenceDescription":"Mon, Tue, Wed, Thu, Fri, Sat, Sun:
Active all day

","widthRestriction":null,"heightRestriction":null,"heightUnderRestriction":null,"lengthRestriction":null,"weightRestriction":null,"majorEvent":null,"county":"Ada","region":"Idaho Falls/Pocatello","state":"Idaho","country":"United States","showOnMap":true,"restrictions":"Width Restriction:
Height Restriction:
Length Restriction:
Weight Restriction:
Speed Restriction:
"}]} \ No newline at end of file diff --git a/tests/fixtures/state_511_atis_construction.json b/tests/fixtures/state_511_atis_construction.json deleted file mode 100644 index 5ace0c4..0000000 --- a/tests/fixtures/state_511_atis_construction.json +++ /dev/null @@ -1 +0,0 @@ -{"data":[{"DT_RowId":"23","tooltipUrl":"/tooltip/Construction/23?lang=%7Blang%7D&noCss=true","id":23,"type":"Roadwork","layerName":"Construction","roadwayName":"SH-81","description":" Work on the shoulder on SH-81 near Poverty Gulch. 7/8/2024 9:29 PM Mon: 12:00 PM - 5:00 PM, Tue, Wed, Thu, Fri, Sat, Sun: Active all day Activities: use caution, warning.","sourceId":"4277","source":"ERS","comment":null,"eventSubType":"workOnTheShoulder","startDate":"7/8/24, 9:29 PM","endDate":null,"lastUpdated":"6/11/25, 3:02 PM","isFullClosure":false,"severity":"None","direction":"Unknown","locationDescription":"Poverty Gulch","detourDescription":null,"laneDescription":"","recurrenceDescription":"Mon:
12:00 PM - 5:00 PM

Tue, Wed, Thu, Fri, Sat, Sun:
Active all day

","widthRestriction":null,"heightRestriction":null,"heightUnderRestriction":null,"lengthRestriction":null,"weightRestriction":null,"majorEvent":null,"county":"Cassia","region":"Idaho Falls/Pocatello","state":"Idaho","country":"United States","showOnMap":true,"restrictions":"Width Restriction:
Height Restriction:
Length Restriction:
Weight Restriction:
Speed Restriction:
"}]} \ No newline at end of file diff --git a/tests/fixtures/state_511_atis_incidents.json b/tests/fixtures/state_511_atis_incidents.json deleted file mode 100644 index 7dbe120..0000000 --- a/tests/fixtures/state_511_atis_incidents.json +++ /dev/null @@ -1 +0,0 @@ -{"data":[{"DT_RowId":"33579","tooltipUrl":"/tooltip/Incidents/33579?lang=%7Blang%7D&noCss=true","id":33579,"type":"Incidents","layerName":"Incidents","roadwayName":"US-95","description":" Roadway Blocked on US-95 Both Directions near MM (469). All lanes blocked. Activities: Expect Delays, Reduced to Single Lane, Alternating Direction of Travel, Use Caution.
Comments: Milepost 469, roadway blocked. Expect delays. Use caution.
","sourceId":"10991","source":"ERS","comment":"Milepost 469, roadway blocked. Expect delays. Use caution.","eventSubType":"roadwayBlocked","startDate":"5/25/26, 2:32 PM","endDate":null,"lastUpdated":"5/25/26, 3:40 PM","isFullClosure":false,"severity":"None","direction":"Both","locationDescription":"Ponderosa Mobile Home Park","detourDescription":null,"laneDescription":"All lanes blocked","recurrenceDescription":null,"widthRestriction":null,"heightRestriction":null,"heightUnderRestriction":null,"lengthRestriction":null,"weightRestriction":null,"majorEvent":null,"county":"Bonner","region":"Idaho Falls/Pocatello","state":"Idaho","country":"United States","showOnMap":true,"restrictions":"Width Restriction:
Height Restriction:
Length Restriction:
Weight Restriction:
Speed Restriction:
"}]} \ No newline at end of file diff --git a/tests/fixtures/state_511_atis_markers_closures.json b/tests/fixtures/state_511_atis_markers_closures.json deleted file mode 100644 index a1842bd..0000000 --- a/tests/fixtures/state_511_atis_markers_closures.json +++ /dev/null @@ -1 +0,0 @@ -{"item2":[{"polyline":{"path":"qbliGhv{eUsk@MeTEsBAsAGiAQw@Qi@MoAYmA]aA]}@a@[?[JSNKNENCTEzCGnAIX","color":"#CC0004"},"itemId":"17","location":[43.6485700000001,-116.47349],"icon":{"url":"/Generated/Content/Images/511/map_closure.svg"},"title":""}]} \ No newline at end of file diff --git a/tests/fixtures/state_511_atis_markers_construction.json b/tests/fixtures/state_511_atis_markers_construction.json deleted file mode 100644 index 05909c1..0000000 --- a/tests/fixtures/state_511_atis_markers_construction.json +++ /dev/null @@ -1 +0,0 @@ -{"item2":[{"itemId":"23","location":[42.5168038430856,-113.711287649613],"icon":{},"title":""}]} \ No newline at end of file diff --git a/tests/fixtures/state_511_atis_markers_incidents.json b/tests/fixtures/state_511_atis_markers_incidents.json deleted file mode 100644 index 9439e70..0000000 --- a/tests/fixtures/state_511_atis_markers_incidents.json +++ /dev/null @@ -1 +0,0 @@ -{"item2":[{"itemId":"33579","location":[48.2055675659533,-116.563364000498],"icon":{},"title":""}]} \ No newline at end of file diff --git a/tests/test_consumer_doc.py b/tests/test_consumer_doc.py index 3193b0f..0607de0 100644 --- a/tests/test_consumer_doc.py +++ b/tests/test_consumer_doc.py @@ -119,3 +119,27 @@ def test_subsections_appear_in_doc_order_matches_registry_size(): f"duplicate per-adapter sections: {[a for a in doc_adapters if doc_adapters.count(a) > 1]}" ) assert len(doc_adapters) == len(discover_adapters()) + + +def test_castle_rock_legacy_adapters_remain_removed(): + """v0.10.3 regression guard: ``state_511_atis`` and ``state_511_atis_cameras`` + were ripped out because the Castle Rock legacy ``/map/mapIcons/`` + + ``/List/GetData/`` shape is end-of-life on the only Idaho source we cared + about (Idaho 511) -- the official ITD adapters (``itd_511`` + ``itd_511_cameras``, + v0.10.0) supersede them. The sister-site discovery confirmed no other + Castle Rock customer still exposes the legacy shape that this adapter pair + consumed. Re-adding either module would resurrect a dying-upstream dependency.""" + registry = discover_adapters() + assert "state_511_atis" not in registry, ( + "state_511_atis was removed in v0.10.3; use itd_511 (v0.10.0) instead" + ) + assert "state_511_atis_cameras" not in registry, ( + "state_511_atis_cameras was removed in v0.10.3; use itd_511_cameras (v0.10.0) instead" + ) + adapters_dir = Path(__file__).resolve().parents[1] / "src" / "central" / "adapters" + assert not (adapters_dir / "state_511_atis.py").exists(), ( + "state_511_atis.py was removed in v0.10.3; do not re-add" + ) + assert not (adapters_dir / "state_511_atis_cameras.py").exists(), ( + "state_511_atis_cameras.py was removed in v0.10.3; do not re-add" + ) diff --git a/tests/test_events_feed_frontend.py b/tests/test_events_feed_frontend.py index 6639287..22dedd5 100644 --- a/tests/test_events_feed_frontend.py +++ b/tests/test_events_feed_frontend.py @@ -1143,10 +1143,8 @@ _SAMPLE_INNER = { "wfigs_incidents": {"county": "Montezuma", "state": "CO"}, "wfigs_perimeters": {"county": "Carbon", "state": "MT"}, "wzdx": {"road_names": ["I-80"], "direction": "eastbound"}, - "state_511_atis": {"layer": "Incidents", "roadway_name": "US-95", "location_description": "Ponderosa Mobile Home Park"}, "tomtom_flow": {"road_category": "primary", "relative_speed": 0.11}, "tomtom_incidents": {"description": "Roadworks", "from": "Early Road", "to": "Slade Road"}, - "state_511_atis_cameras": {"location_description": "I-84 Mountain Home", "camera_id": 42}, "itd_511": {"event_type_short": "work_zone", "roadway_name": "I-84"}, "itd_511_cameras": {"location": "I-84 Mountain Home", "camera_id": 42}, } @@ -1170,7 +1168,6 @@ _EXPECTED_SUBJECT = { "wzdx": "Work zone on I-80 eastbound", "tomtom_flow": "Traffic flow (primary) — 11% of free-flow", "tomtom_incidents": "Roadworks on Early Road → Slade Road", - "state_511_atis_cameras": "Camera: I-84 Mountain Home", "itd_511": "Road work on I-84", "itd_511_cameras": "Camera: I-84 Mountain Home", } diff --git a/tests/test_gui_adapter_edit.py b/tests/test_gui_adapter_edit.py index 19727b0..d0ee06c 100644 --- a/tests/test_gui_adapter_edit.py +++ b/tests/test_gui_adapter_edit.py @@ -226,12 +226,3 @@ class TestBboxMapPreview: assert "bbox-map" in out # map container present assert "L.map(" in out and "L.rectangle(" in out # Leaflet init present - def test_state_511_atis_no_bbox_map(self): - """Non-bbox model_list (StateConfig) → generic editor, no map (no regression).""" - from central.adapters.state_511_atis import State511ATISAdapter - s = {"states": [{"code": "ID", "base_url": "https://511.idaho.gov"}]} - out = _render("adapters_edit.html", - _ctx(s, describe_fields(State511ATISAdapter.settings_schema, s), None, - name="state_511_atis", display="511 ATIS")) - assert "model-list" in out # generic editor still renders - assert "bbox-map" not in out # but no map div diff --git a/tests/test_state_511_atis.py b/tests/test_state_511_atis.py deleted file mode 100644 index f87ee4b..0000000 --- a/tests/test_state_511_atis.py +++ /dev/null @@ -1,207 +0,0 @@ -"""Tests for the state_511_atis adapter (Castle Rock ATIS, Idaho). - -Fixtures are real captures (one record + its matching marker per layer): - state_511_atis_.json -- POST /List/GetData/ .data[0:1] - state_511_atis_markers_.json -- GET /map/mapIcons/ matching item2 - -No tests/conftest isolation entry is added: dedup uses the supervisor-injected -cursors.db (inherited mixin) and discovery is stateless -- no adapter-owned cache. -""" - -import json -from datetime import datetime, timezone -from pathlib import Path -from unittest.mock import MagicMock - -import pytest - -from central.adapter import SourceAdapter -from central.adapters.state_511_atis import ( - LAYER_EVENT_TYPE, - State511ATISAdapter, - _parse_us_dt, -) -from central.config_models import AdapterConfig - -FIX = Path(__file__).parent / "fixtures" -DETAIL = {lyr: json.loads((FIX / f"state_511_atis_{lyr.lower()}.json").read_text())["data"][0] - for lyr in ("Incidents", "Closures", "Construction")} -MARK = {lyr: json.loads((FIX / f"state_511_atis_markers_{lyr.lower()}.json").read_text())["item2"][0] - for lyr in ("Incidents", "Closures", "Construction")} - - -def _coords(layer): - loc = MARK[layer]["location"] - return (loc[0], loc[1]) - - -def _cfg(): - return AdapterConfig( - name="state_511_atis", enabled=True, cadence_s=300, - settings={"states": [{"code": "ID", "base_url": "https://511.idaho.gov"}]}, - updated_at=datetime.now(timezone.utc), - ) - - -@pytest.fixture -def adapter(tmp_path): - return State511ATISAdapter(_cfg(), MagicMock(), tmp_path / "cursors.db") - - -def test_layer_event_type_map(): - assert LAYER_EVENT_TYPE == {"Incidents": "incident", "Closures": "closure", "Construction": "work_zone"} - - -def test_parse_us_dt(): - assert _parse_us_dt("5/25/26, 2:32 PM") == datetime(2026, 5, 25, 14, 32, tzinfo=timezone.utc) - assert _parse_us_dt("") is None - assert _parse_us_dt("not a date") is None - - -def test_dedup_key(adapter): - e = adapter._build_event(DETAIL["Incidents"], _coords("Incidents"), "ID", "Incidents") - assert e.id == "ID:Incidents:33579" - - -def test_build_incident(adapter): - e = adapter._build_event(DETAIL["Incidents"], _coords("Incidents"), "ID", "Incidents") - assert e.category == "incident.state_511_atis" - assert e.severity == 1 - assert e.data["roadway_name"] == "US-95" - assert e.data["county"] == "Bonner" - assert e.data["latitude"] is not None and e.data["longitude"] is not None - - -def test_build_closure_full_closure_severity(adapter): - e = adapter._build_event(DETAIL["Closures"], _coords("Closures"), "ID", "Closures") - assert e.category == "closure.state_511_atis" - assert e.data["is_full_closure"] is True - assert e.severity == 3 # isFullClosure -> 3 - - -def test_build_construction_maps_to_work_zone(adapter): - e = adapter._build_event(DETAIL["Construction"], _coords("Construction"), "ID", "Construction") - assert e.category == "work_zone.state_511_atis" # layer Construction, type "Roadwork" - assert e.severity == 1 - assert e.data["roadway_name"] == "SH-81" - - -def test_join_missing_coords(adapter): - e = adapter._build_event(DETAIL["Incidents"], None, "ID", "Incidents") - assert e.data["latitude"] is None and e.data["longitude"] is None - assert e.geo.centroid is None # still built, just no map point - - -@pytest.mark.parametrize("layer,et", [("Incidents", "incident"), ("Closures", "closure"), ("Construction", "work_zone")]) -def test_subject_for(adapter, layer, et): - e = adapter._build_event(DETAIL[layer], _coords(layer), "ID", layer) - assert adapter.subject_for(e) == f"central.traffic.{et}.id" - - -def test_summary_partial_renders(): - from central.gui.routes import _derive_subject - inner = {"layer": "Incidents", "roadway_name": "US-95", "location_description": "Ponderosa Mobile Home Park"} - row = {"adapter": "state_511_atis", "data": {"data": {"data": inner}}} - assert _derive_subject(row) == "Incident on US-95 — Ponderosa Mobile Home Park" - - -@pytest.mark.asyncio -async def test_poll_joins_and_yields(adapter): - await adapter.startup() - - async def fake_markers(base_url, layer): - m = MARK[layer] - return {str(m["itemId"]): (m["location"][0], m["location"][1])} - - async def fake_details(base_url, layer): - return [DETAIL[layer]] - - adapter._fetch_markers = fake_markers - adapter._fetch_details = fake_details - events = [e async for e in adapter.poll()] - await adapter.shutdown() - assert len(events) == 3 # one ID state x three layers - assert {e.category for e in events} == { - "incident.state_511_atis", "closure.state_511_atis", "work_zone.state_511_atis", - } - assert all(e.adapter == "state_511_atis" for e in events) - - -def test_inherits_dedup_mixin(): - for m in ("is_published", "mark_published", "sweep_old_ids"): - assert m not in State511ATISAdapter.__dict__, f"redefines {m}" - assert getattr(State511ATISAdapter, m) is getattr(SourceAdapter, m) - - -# --- v0.9.7 pagination ------------------------------------------------------ - -def _rec(i): - return {"id": i, "type": "Roadwork", "roadwayName": "SH-1", "location": f"loc {i}"} - - -def _page(records, records_filtered): - return {"draw": 1, "recordsTotal": records_filtered, - "recordsFiltered": records_filtered, "data": records} - - -@pytest.mark.asyncio -async def test_pagination_collects_all_pages(adapter): - await adapter.startup() - pages = {0: _page([_rec(i) for i in range(100)], 114), - 100: _page([_rec(i) for i in range(100, 114)], 114)} - - async def fake_page(base_url, layer, start): - return pages[start] - - adapter._fetch_page = fake_page - rows = await adapter._fetch_details("https://511.idaho.gov", "Construction") - await adapter.shutdown() - assert len(rows) == 114 # 100 + 14, not truncated at the 100-row cap - assert {r["id"] for r in rows} == set(range(114)) - - -@pytest.mark.asyncio -async def test_pagination_handles_short_final_page(adapter): - await adapter.startup() - pages = {0: _page([_rec(i) for i in range(100)], 130), - 100: _page([_rec(i) for i in range(100, 130)], 130)} - - async def fake_page(base_url, layer, start): - return pages[start] - - adapter._fetch_page = fake_page - rows = await adapter._fetch_details("https://511.idaho.gov", "Construction") - await adapter.shutdown() - assert len(rows) == 130 # short 30-row final page collected - - -@pytest.mark.asyncio -async def test_pagination_empty_page_breaks(adapter): - # recordsFiltered overstates the set; an empty page must stop the loop (no hang). - await adapter.startup() - pages = {0: _page([_rec(i) for i in range(100)], 250), 100: _page([], 250)} - - async def fake_page(base_url, layer, start): - return pages.get(start, _page([], 250)) - - adapter._fetch_page = fake_page - rows = await adapter._fetch_details("https://511.idaho.gov", "Construction") - await adapter.shutdown() - assert len(rows) == 100 # empty page 2 terminates cleanly - - -@pytest.mark.asyncio -async def test_pagination_loop_cap(adapter, caplog): - # recordsFiltered never satisfied -> loop must stop at _MAX_PAGES and warn. - await adapter.startup() - - async def fake_page(base_url, layer, start): - return _page([_rec(start + i) for i in range(100)], 999_999) - - adapter._fetch_page = fake_page - with caplog.at_level("WARNING"): - rows = await adapter._fetch_details("https://511.idaho.gov", "Construction") - await adapter.shutdown() - from central.adapters.state_511_atis import _MAX_PAGES - assert len(rows) == _MAX_PAGES * 100 # capped, not infinite - assert "max_pages" in caplog.text diff --git a/tests/test_state_511_atis_cameras.py b/tests/test_state_511_atis_cameras.py deleted file mode 100644 index 52b3a09..0000000 --- a/tests/test_state_511_atis_cameras.py +++ /dev/null @@ -1,134 +0,0 @@ -"""Tests for the state_511_atis_cameras adapter (v0.9.6). - -Fixture is a real /List/GetData/Cameras capture (2 cameras: one single-image -UDOT border camera, one multi-image RWIS camera): - tests/fixtures/state_511_atis_cameras_sample.json - -No conftest entry: dedup uses the supervisor-injected cursors.db (inherited -mixin); polling is stateless. -""" - -import json -from datetime import datetime, timezone -from pathlib import Path -from unittest.mock import MagicMock - -import pytest - -from central.adapter import SourceAdapter -from central.adapters.state_511_atis_cameras import ( - State511ATISCamerasAdapter, - StateConfig, - _parse_wkt, -) -from central.config_models import AdapterConfig - -FIX = json.loads((Path(__file__).parent / "fixtures" / "state_511_atis_cameras_sample.json").read_text()) -CAMS = FIX["data"] -ID = StateConfig(code="ID", base_url="https://511.idaho.gov") -TODAY = datetime.now(timezone.utc).strftime("%Y-%m-%d") - - -def _cfg(): - return AdapterConfig( - name="state_511_atis_cameras", enabled=True, cadence_s=600, - settings={"states": [ID.model_dump()]}, updated_at=datetime.now(timezone.utc), - ) - - -@pytest.fixture -def adapter(tmp_path): - return State511ATISCamerasAdapter(_cfg(), MagicMock(), tmp_path / "cursors.db") - - -def test_wkt_parse(): - assert _parse_wkt("POINT (-112.198 42.0011)") == (42.0011, -112.198) # (lat, lon) - assert _parse_wkt(None) == (None, None) - assert _parse_wkt("nonsense") == (None, None) - - -def test_dedup_key_shape(adapter): - e = adapter._build_event(CAMS[0], ID) - assert e.id == f"ID:cam:{CAMS[0]['id']}:{TODAY}" # per-UTC-day bucketing - - -def test_build_event_with_image_url(adapter): - e = adapter._build_event(CAMS[0], ID) - assert e.category == "camera.state_511_atis_cameras" - assert e.severity == 1 - assert e.data["image_url"] == "https://511.idaho.gov" + CAMS[0]["images"][0]["imageUrl"] - assert e.data["source"] == CAMS[0]["source"] # provenance surfaced - assert e.data["roadway_name"] == CAMS[0]["roadway"] - assert e.data["latitude"] is not None and e.data["longitude"] is not None - - -def test_build_event_multi_image(adapter): - e = adapter._build_event(CAMS[1], ID) - assert e.data["image_count"] == len(CAMS[1]["images"]) - assert e.data["image_count"] >= 2 - - -def test_no_image_url_handled_gracefully(adapter): - cam = {"id": 999, "roadway": "US-95", "location": "US-95 Somewhere", "source": "ITDNET", - "direction": "Unknown", "images": [], - "latLng": {"geography": {"wellKnownText": "POINT (-116.5 46.4)"}}} - e = adapter._build_event(cam, ID) - assert e is not None - assert e.data["image_url"] is None and e.data["image_count"] == 0 - assert e.data["location_description"] == "US-95 Somewhere" - - -def test_subject_for_state_id(adapter): - e = adapter._build_event(CAMS[0], ID) - assert adapter.subject_for(e) == f"central.traffic_cameras.id.{CAMS[0]['id']}" - - -def test_subject_for_unknown_state(adapter): - e = adapter._build_event(CAMS[0], StateConfig(code="", base_url="https://x")) - assert adapter.subject_for(e) == f"central.traffic_cameras.unknown.{CAMS[0]['id']}" - - -def test_summary_partial_renders(): - from central.gui.routes import _derive_subject - inner = {"location_description": "I-84 Mountain Home", "camera_id": 42} - row = {"adapter": "state_511_atis_cameras", "data": {"data": {"data": inner}}} - assert _derive_subject(row) == "Camera: I-84 Mountain Home" - - -def test_rows_partial_includes_img_tag(): - from central.gui.routes import _get_templates - inner = {"image_url": "https://511.idaho.gov/map/Cctv/1", "roadway_name": "I-84", - "location_description": "I-84 Mountain Home", "source": "ITDNET", "image_count": 1} - row = {"data": {"data": {"data": inner}}} - html = _get_templates().env.get_template("_event_rows/state_511_atis_cameras.html").render(event=row) - assert "