mirror of
https://github.com/zvx-echo6/central.git
synced 2026-06-10 11:54:37 +02:00
v0.10.0: ITD 511 official API adapter (events + advisories + cameras) (#85)
First official-state-DOT-API pattern landing. Two adapters in one PR: - itd_511 (event-class): polls Events (60s) + Advisories (300s) from https://511.idaho.gov/api/v2/get/{event,alerts}. Decodes EncodedPolyline to LineString via the polyline lib (bookend LineString or Point fallback); ITD Severity string mapped None->1 / Minor->2 / Major->3 with IsFullClosure=true forcing 3 regardless; RecurrenceSchedules / Restrictions / DetourPolyline pass through unmodified. Advisories ship as structural pass-through under data.advisory since the upstream /alerts endpoint currently returns []; per-record try/except keeps a surprise shape from sinking the cycle when ITD posts its first one. - itd_511_cameras (telemetry-class): polls Cameras (600s). One event per camera per UTC day; image URL passes straight through to <img src>. Region uniform US-ID with data.source_jurisdiction preserving the raw upstream Source field for the ~1.2% cross-DOT border-region mirrors (UDOT / ODOT / WYDOT / WSDOT / NDot / MTD / DriveBC / Lemhi County). Subject convention (v0.9.20 forward): central.traffic.<event_type>.us.id and central.traffic_cameras.us.id.<camera_id>. Castle Rock state_511_atis keeps its bare-state subject; consumers stay on central.traffic.> wildcards during the A/B comparison window. Retry predicate tightened from the Castle Rock / TomTom precedent: 5xx + connection / timeout retry; 4xx other than 429 skip-with-warn (don't burn quota on permanent errors); 429 honors Retry-After once then retries. API key (alias 'idaho_511') travels in the ?key= query string, so every error log path runs through self._redact() to scrub the URL. Both adapters ship disabled; operator enables via GUI after registering the API key with 'python -m set_api_key idaho_511'. Reuses existing CENTRAL_TRAFFIC and CENTRAL_TRAFFIC_CAMERAS streams -- no archive restart needed. Scope-cap exception: this PR is ~1.5k lines vs. the standard 500-line cap, authorized as a one-time exception for the first official-state-DOT-API pattern landing. Two adapters + their tests + real-API fixtures naturally exceed the v0.9.x adapter-cap budget. Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
GitHub
parent
1f7bccaac6
commit
1d5548c24c
18 changed files with 1783 additions and 2 deletions
1
tests/fixtures/itd_511_alerts_sample.json
vendored
Normal file
1
tests/fixtures/itd_511_alerts_sample.json
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
[]
|
||||
90
tests/fixtures/itd_511_cameras_sample.json
vendored
Normal file
90
tests/fixtures/itd_511_cameras_sample.json
vendored
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
[
|
||||
{
|
||||
"Id": 3,
|
||||
"Source": "ITDNET",
|
||||
"SourceId": "1000",
|
||||
"Roadway": "SH-55 Eagle",
|
||||
"Direction": "Southbound",
|
||||
"Latitude": 43.619167,
|
||||
"Longitude": -116.35478,
|
||||
"Location": "SH-55 Eagle Fairview",
|
||||
"SortOrder": 0,
|
||||
"Views": [
|
||||
{
|
||||
"Id": 1039,
|
||||
"Url": "https://511.idaho.gov/map/Cctv/1039",
|
||||
"Status": "Disabled",
|
||||
"Description": "D3 SH-55 37.9 Eagle Fairview 526"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Id": 436,
|
||||
"Source": "ACHD",
|
||||
"SourceId": "4001",
|
||||
"Roadway": "Local Boise",
|
||||
"Direction": "Unknown",
|
||||
"Latitude": 43.60304,
|
||||
"Longitude": -116.18841,
|
||||
"Location": "Park Parkcenter Front Clearwater",
|
||||
"SortOrder": 0,
|
||||
"Views": [
|
||||
{
|
||||
"Id": 631,
|
||||
"Url": "https://511.idaho.gov/map/Cctv/631",
|
||||
"Status": "Enabled",
|
||||
"Description": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Id": 1,
|
||||
"Source": "UDOT",
|
||||
"SourceId": "10.C1",
|
||||
"Roadway": "I-15",
|
||||
"Direction": "Unknown",
|
||||
"Latitude": 42.0011,
|
||||
"Longitude": -112.198,
|
||||
"Location": "I-15 UT/ID State Line UT",
|
||||
"SortOrder": 1,
|
||||
"Views": [
|
||||
{
|
||||
"Id": 1,
|
||||
"Url": "https://511.idaho.gov/map/Cctv/1",
|
||||
"Status": "Enabled",
|
||||
"Description": "N/A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Id": 2,
|
||||
"Source": "RWIS",
|
||||
"SourceId": "100.C1",
|
||||
"Roadway": "SH-75",
|
||||
"Direction": "Unknown",
|
||||
"Latitude": 43.5946,
|
||||
"Longitude": -114.345,
|
||||
"Location": "SH-75 Wood River",
|
||||
"SortOrder": 325,
|
||||
"Views": [
|
||||
{
|
||||
"Id": 2,
|
||||
"Url": "https://511.idaho.gov/map/Cctv/2",
|
||||
"Status": "Enabled",
|
||||
"Description": ""
|
||||
},
|
||||
{
|
||||
"Id": 3,
|
||||
"Url": "https://511.idaho.gov/map/Cctv/3",
|
||||
"Status": "Enabled",
|
||||
"Description": ""
|
||||
},
|
||||
{
|
||||
"Id": 4,
|
||||
"Url": "https://511.idaho.gov/map/Cctv/4",
|
||||
"Status": "Enabled",
|
||||
"Description": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
214
tests/fixtures/itd_511_event_sample.json
vendored
Normal file
214
tests/fixtures/itd_511_event_sample.json
vendored
Normal file
|
|
@ -0,0 +1,214 @@
|
|||
[
|
||||
{
|
||||
"ID": 23,
|
||||
"SourceId": "4277",
|
||||
"Organization": "ERS",
|
||||
"RoadwayName": "SH-81",
|
||||
"DirectionOfTravel": "Unknown",
|
||||
"Description": " Work on the shoulder on SH-81 near Poverty Gulch. 7/8/2024 9:29 PM Mon: 12:00 PM - 5:00 PM, Tue, Wed, Thu, Fri, Sat, Sun: Active all day Activities: use caution, warning.",
|
||||
"Reported": 1720495740,
|
||||
"LastUpdated": 1749675731,
|
||||
"StartDate": 1720495740,
|
||||
"PlannedEndDate": null,
|
||||
"LanesAffected": "No Data",
|
||||
"Latitude": 42.5168038430856,
|
||||
"Longitude": -113.711287649613,
|
||||
"LatitudeSecondary": null,
|
||||
"LongitudeSecondary": null,
|
||||
"EventType": "roadwork",
|
||||
"EventSubType": "workOnTheShoulder",
|
||||
"IsFullClosure": false,
|
||||
"Severity": "None",
|
||||
"Comment": null,
|
||||
"EncodedPolyline": null,
|
||||
"Restrictions": {
|
||||
"Width": null,
|
||||
"Height": null,
|
||||
"Length": null,
|
||||
"Weight": null,
|
||||
"Speed": null
|
||||
},
|
||||
"DetourPolyline": "",
|
||||
"DetourInstructions": "",
|
||||
"Recurrence": "<b>Mon:</b><br/>12:00 PM - 5:00 PM<br/><br/><b>Tue, Wed, Thu, Fri, Sat, Sun:</b><br/>Active all day<br/><br/>",
|
||||
"RecurrenceSchedules": [
|
||||
{
|
||||
"StartDate": "7/8/2024 9:29:00 PM-06:00:00",
|
||||
"EndDate": null,
|
||||
"Times": [
|
||||
{
|
||||
"StartTime": "12:00:00-06:00:00",
|
||||
"EndTime": "16:59:59-06:00:00"
|
||||
}
|
||||
],
|
||||
"DaysOfWeek": [
|
||||
"Monday"
|
||||
]
|
||||
},
|
||||
{
|
||||
"StartDate": "7/8/2024 9:29:00 PM-06:00:00",
|
||||
"EndDate": null,
|
||||
"Times": [
|
||||
{
|
||||
"StartTime": "00:00:00-06:00:00",
|
||||
"EndTime": "23:59:59-06:00:00"
|
||||
}
|
||||
],
|
||||
"DaysOfWeek": [
|
||||
"Tuesday",
|
||||
"Wednesday",
|
||||
"Thursday",
|
||||
"Friday",
|
||||
"Saturday",
|
||||
"Sunday"
|
||||
]
|
||||
}
|
||||
],
|
||||
"Cause": "roadwork"
|
||||
},
|
||||
{
|
||||
"ID": 17,
|
||||
"SourceId": "469",
|
||||
"Organization": "ERS",
|
||||
"RoadwayName": "N McDermott Rd",
|
||||
"DirectionOfTravel": "Both",
|
||||
"Description": " Long term road construction on N McDermott Rd Both Directions from Five Mile Creek to US-20. All lanes closed. 1/30/2023 2:24 PM Mon, Tue, Wed, Thu, Fri, Sat, Sun: Active all day",
|
||||
"Reported": 1675113840,
|
||||
"LastUpdated": 1718758080,
|
||||
"StartDate": 1675113840,
|
||||
"PlannedEndDate": null,
|
||||
"LanesAffected": "All lanes closed",
|
||||
"Latitude": 43.6485700000001,
|
||||
"Longitude": -116.47349,
|
||||
"LatitudeSecondary": 43.6630000000001,
|
||||
"LongitudeSecondary": -116.47421,
|
||||
"EventType": "closures",
|
||||
"EventSubType": "longTermRoadConstruction",
|
||||
"IsFullClosure": true,
|
||||
"Severity": "None",
|
||||
"Comment": "Open to local traffic only.",
|
||||
"EncodedPolyline": "qbliGhv{eUsk@MeTEsBAsAGiAQw@Qi@MoAYmA]aA]}@a@[?[JSNKNENCTEzCGnAIX",
|
||||
"Restrictions": {
|
||||
"Width": null,
|
||||
"Height": null,
|
||||
"Length": null,
|
||||
"Weight": null,
|
||||
"Speed": null
|
||||
},
|
||||
"DetourPolyline": "",
|
||||
"DetourInstructions": "",
|
||||
"Recurrence": "<b>Mon, Tue, Wed, Thu, Fri, Sat, Sun:</b><br/>Active all day<br/><br/>",
|
||||
"RecurrenceSchedules": [
|
||||
{
|
||||
"StartDate": "1/30/2023 2:24:00 PM-06:00:00",
|
||||
"EndDate": null,
|
||||
"Times": [
|
||||
{
|
||||
"StartTime": "00:00:00-06:00:00",
|
||||
"EndTime": "23:59:59-06:00:00"
|
||||
}
|
||||
],
|
||||
"DaysOfWeek": [
|
||||
"Monday",
|
||||
"Tuesday",
|
||||
"Wednesday",
|
||||
"Thursday",
|
||||
"Friday",
|
||||
"Saturday",
|
||||
"Sunday"
|
||||
]
|
||||
}
|
||||
],
|
||||
"Cause": "roadwork"
|
||||
},
|
||||
{
|
||||
"ID": 34585,
|
||||
"SourceId": "11146",
|
||||
"Organization": "ERS",
|
||||
"RoadwayName": "I-84",
|
||||
"DirectionOfTravel": "East",
|
||||
"Description": " Left Lane Blocked on I-84 Eastbound near N Franklin Blvd. 2 Left Lanes Blocked. Activities: Expect Delays, Look Out for Flagger, Use Caution.",
|
||||
"Reported": 1780543740,
|
||||
"LastUpdated": 1780543820,
|
||||
"StartDate": 1780543740,
|
||||
"PlannedEndDate": null,
|
||||
"LanesAffected": "2 Left Lanes Blocked",
|
||||
"Latitude": 43.5980897425697,
|
||||
"Longitude": -116.542818286917,
|
||||
"LatitudeSecondary": null,
|
||||
"LongitudeSecondary": null,
|
||||
"EventType": "accidentsAndIncidents",
|
||||
"EventSubType": "leftLaneBlocked",
|
||||
"IsFullClosure": false,
|
||||
"Severity": "None",
|
||||
"Comment": "The two left lanes of eastbound I-84 will be blocked near milepost 36.5 tonight for road maintenance. Keep Right.",
|
||||
"EncodedPolyline": null,
|
||||
"Restrictions": {
|
||||
"Width": null,
|
||||
"Height": null,
|
||||
"Length": null,
|
||||
"Weight": null,
|
||||
"Speed": null
|
||||
},
|
||||
"DetourPolyline": "",
|
||||
"DetourInstructions": "",
|
||||
"Recurrence": "",
|
||||
"RecurrenceSchedules": "",
|
||||
"Cause": "Incident"
|
||||
},
|
||||
{
|
||||
"ID": 33663,
|
||||
"SourceId": "11000",
|
||||
"Organization": "ERS",
|
||||
"RoadwayName": "SH-16",
|
||||
"DirectionOfTravel": "Both",
|
||||
"Description": " Special event on SH-16 Both Directions at W Chaparral Rd. 6/8/2026 8:00 AM to 6/15/2026 5:00 PM Mon, Tue, Wed, Thu, Fri, Sat, Sun: Active all day Activities: Use Caution.",
|
||||
"Reported": 1780927200,
|
||||
"LastUpdated": 1779805341,
|
||||
"StartDate": 1780927200,
|
||||
"PlannedEndDate": 1781564400,
|
||||
"LanesAffected": "No Data",
|
||||
"Latitude": 43.780078841046,
|
||||
"Longitude": -116.473209123902,
|
||||
"LatitudeSecondary": null,
|
||||
"LongitudeSecondary": null,
|
||||
"EventType": "specialEvents",
|
||||
"EventSubType": "specialEvent",
|
||||
"IsFullClosure": false,
|
||||
"Severity": "None",
|
||||
"Comment": "Rodeo Traffic entering/leaving roadway ",
|
||||
"EncodedPolyline": null,
|
||||
"Restrictions": {
|
||||
"Width": null,
|
||||
"Height": null,
|
||||
"Length": null,
|
||||
"Weight": null,
|
||||
"Speed": null
|
||||
},
|
||||
"DetourPolyline": "",
|
||||
"DetourInstructions": "",
|
||||
"Recurrence": "<b>Mon, Tue, Wed, Thu, Fri, Sat, Sun:</b><br/>Active all day<br/><br/>",
|
||||
"RecurrenceSchedules": [
|
||||
{
|
||||
"StartDate": "6/8/2026 8:00:00 AM-06:00:00",
|
||||
"EndDate": "6/15/2026 5:00:00 PM-06:00:00",
|
||||
"Times": [
|
||||
{
|
||||
"StartTime": "00:00:00-06:00:00",
|
||||
"EndTime": "23:59:59-06:00:00"
|
||||
}
|
||||
],
|
||||
"DaysOfWeek": [
|
||||
"Monday",
|
||||
"Tuesday",
|
||||
"Wednesday",
|
||||
"Thursday",
|
||||
"Friday",
|
||||
"Saturday",
|
||||
"Sunday"
|
||||
]
|
||||
}
|
||||
],
|
||||
"Cause": "specialEvents"
|
||||
}
|
||||
]
|
||||
|
|
@ -1147,6 +1147,8 @@ _SAMPLE_INNER = {
|
|||
"tomtom_flow": {"road_category": "primary", "relative_speed": 0.11},
|
||||
"tomtom_incidents": {"description": "Roadworks", "from": "Early Road", "to": "Slade Road"},
|
||||
"state_511_atis_cameras": {"location_description": "I-84 Mountain Home", "camera_id": 42},
|
||||
"itd_511": {"event_type_short": "work_zone", "roadway_name": "I-84"},
|
||||
"itd_511_cameras": {"location": "I-84 Mountain Home", "camera_id": 42},
|
||||
}
|
||||
|
||||
# Exact expected subjects for the deterministic adapters. swpc_alerts is omitted
|
||||
|
|
@ -1169,6 +1171,8 @@ _EXPECTED_SUBJECT = {
|
|||
"tomtom_flow": "Traffic flow (primary) — 11% of free-flow",
|
||||
"tomtom_incidents": "Roadworks on Early Road → Slade Road",
|
||||
"state_511_atis_cameras": "Camera: I-84 Mountain Home",
|
||||
"itd_511": "Road work on I-84",
|
||||
"itd_511_cameras": "Camera: I-84 Mountain Home",
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
398
tests/test_itd_511.py
Normal file
398
tests/test_itd_511.py
Normal file
|
|
@ -0,0 +1,398 @@
|
|||
"""Tests for the itd_511 adapter (v0.10.0).
|
||||
|
||||
Fixtures are real captures from https://511.idaho.gov/api/v2/get/event,alerts
|
||||
trimmed to one record per EventType plus an empty advisories baseline:
|
||||
tests/fixtures/itd_511_event_sample.json
|
||||
tests/fixtures/itd_511_alerts_sample.json
|
||||
|
||||
No conftest entry: dedup uses the supervisor-injected cursors.db (inherited
|
||||
mixin); polling is stateless.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import aiohttp
|
||||
import pytest
|
||||
|
||||
from central.adapter import SourceAdapter
|
||||
from central.adapters.itd_511 import (
|
||||
EVENT_TYPE_MAP,
|
||||
Itd511Adapter,
|
||||
_build_geometry,
|
||||
_decode_polyline,
|
||||
_itd_severity,
|
||||
_parse_epoch,
|
||||
_strip_or_none,
|
||||
_Transient,
|
||||
_wait_strategy,
|
||||
)
|
||||
from central.config_models import AdapterConfig
|
||||
from central.models import Event, Geo
|
||||
|
||||
FIX = Path(__file__).parent / "fixtures"
|
||||
EVENT = json.loads((FIX / "itd_511_event_sample.json").read_text())
|
||||
ALERTS = json.loads((FIX / "itd_511_alerts_sample.json").read_text())
|
||||
BY_TYPE = {r["EventType"]: r for r in EVENT}
|
||||
|
||||
|
||||
def _cfg():
|
||||
return AdapterConfig(
|
||||
name="itd_511", enabled=True, cadence_s=60,
|
||||
settings={"api_key_alias": "idaho_511"},
|
||||
updated_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def adapter(tmp_path):
|
||||
cs = MagicMock()
|
||||
cs.get_api_key = AsyncMock(return_value="testkey-32chars-deadbeefdeadbeef")
|
||||
return Itd511Adapter(_cfg(), cs, tmp_path / "cursors.db")
|
||||
|
||||
|
||||
def test_event_type_map_is_complete():
|
||||
assert EVENT_TYPE_MAP == {
|
||||
"roadwork": "work_zone", "closures": "closure",
|
||||
"accidentsAndIncidents": "incident", "specialEvents": "special_event",
|
||||
}
|
||||
|
||||
|
||||
def test_parse_epoch():
|
||||
assert _parse_epoch(1675113840) == datetime(2023, 1, 30, 21, 24, tzinfo=timezone.utc)
|
||||
assert _parse_epoch(None) is None
|
||||
assert _parse_epoch("not-an-int") is None
|
||||
assert _parse_epoch("1675113840") == datetime(2023, 1, 30, 21, 24, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sev,fc,expected", [
|
||||
("None", False, 1), ("Minor", False, 2), ("Major", False, 3),
|
||||
("None", True, 3), ("Minor", True, 3), ("Major", True, 3), # full-closure forces 3
|
||||
(None, False, 1), ("Bogus", False, 1),
|
||||
])
|
||||
def test_severity_mapping(sev, fc, expected):
|
||||
assert _itd_severity(sev, fc) == expected
|
||||
|
||||
|
||||
def test_strip_or_none_handles_eventsubtype_trailing_space():
|
||||
assert _strip_or_none("pavementMarkingOperations ") == "pavementMarkingOperations"
|
||||
assert _strip_or_none("") is None
|
||||
assert _strip_or_none(" ") is None
|
||||
assert _strip_or_none(None) is None
|
||||
assert _strip_or_none(42) == 42 # non-string passthrough
|
||||
|
||||
|
||||
def test_decode_polyline_roundtrip():
|
||||
import polyline as polyline_lib
|
||||
enc = polyline_lib.encode([(43.6, -116.5), (43.7, -116.4)])
|
||||
assert _decode_polyline(enc) == [(43.6, -116.5), (43.7, -116.4)]
|
||||
assert _decode_polyline(None) == []
|
||||
assert _decode_polyline("") == []
|
||||
# malformed string => library raises => caught => []
|
||||
assert _decode_polyline("\x00\x00\x00not-a-polyline") == []
|
||||
|
||||
|
||||
def test_build_geometry_polyline_wins():
|
||||
import polyline as polyline_lib
|
||||
enc = polyline_lib.encode([(43.6, -116.5), (43.7, -116.4)])
|
||||
geom, centroid = _build_geometry(40.0, -100.0, None, None, enc)
|
||||
assert geom["type"] == "LineString"
|
||||
assert len(geom["coordinates"]) == 2
|
||||
assert centroid == geom["coordinates"][0] # first vertex (lon, lat) order
|
||||
|
||||
|
||||
def test_build_geometry_bookend_linestring():
|
||||
geom, centroid = _build_geometry(43.6, -116.5, 43.7, -116.4, None)
|
||||
assert geom == {"type": "LineString",
|
||||
"coordinates": [(-116.5, 43.6), (-116.4, 43.7)]}
|
||||
assert centroid == (-116.5, 43.6)
|
||||
|
||||
|
||||
def test_build_geometry_point_only():
|
||||
geom, centroid = _build_geometry(43.6, -116.5, None, None, None)
|
||||
assert geom == {"type": "Point", "coordinates": (-116.5, 43.6)}
|
||||
assert centroid == (-116.5, 43.6)
|
||||
|
||||
|
||||
def test_build_geometry_missing_all():
|
||||
assert _build_geometry(None, None, None, None, None) == (None, None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("etype,short", [
|
||||
("roadwork", "work_zone"), ("closures", "closure"),
|
||||
("accidentsAndIncidents", "incident"), ("specialEvents", "special_event"),
|
||||
])
|
||||
def test_build_event_category_and_dedup_id(adapter, etype, short):
|
||||
rec = BY_TYPE[etype]
|
||||
e = adapter._build_event_record(rec)
|
||||
assert e.category == f"{short}.itd_511"
|
||||
assert e.id == f"idaho_511:event:{rec['SourceId']}"
|
||||
assert e.adapter == "itd_511"
|
||||
assert e.geo.primary_region == "US-ID"
|
||||
assert e.geo.regions == ["US-ID"]
|
||||
|
||||
|
||||
def test_build_event_closure_has_linestring_geometry(adapter):
|
||||
e = adapter._build_event_record(BY_TYPE["closures"])
|
||||
assert e.geo.geometry is not None
|
||||
assert e.geo.geometry["type"] == "LineString"
|
||||
assert len(e.geo.geometry["coordinates"]) >= 2
|
||||
|
||||
|
||||
def test_build_event_full_closure_forces_severity_3(adapter):
|
||||
e = adapter._build_event_record(BY_TYPE["closures"])
|
||||
assert e.data["is_full_closure"] is True
|
||||
assert e.severity == 3
|
||||
|
||||
|
||||
def test_build_event_unknown_event_type_falls_back_to_incident(adapter):
|
||||
rec = {**BY_TYPE["roadwork"], "EventType": "WhoKnows", "SourceId": "X1"}
|
||||
e = adapter._build_event_record(rec)
|
||||
assert e.category == "incident.itd_511"
|
||||
|
||||
|
||||
def test_build_event_dedup_id_falls_back_to_id_when_sourceid_missing(adapter):
|
||||
rec = {**BY_TYPE["roadwork"], "SourceId": None, "ID": 99999}
|
||||
e = adapter._build_event_record(rec)
|
||||
assert e.id == "idaho_511:event:99999"
|
||||
|
||||
|
||||
def test_build_event_returns_none_without_any_id(adapter):
|
||||
rec = {**BY_TYPE["roadwork"], "SourceId": None, "ID": None}
|
||||
assert adapter._build_event_record(rec) is None
|
||||
|
||||
|
||||
def test_build_event_strips_trailing_space_on_event_sub_type(adapter):
|
||||
rec = {**BY_TYPE["roadwork"], "EventSubType": "pavementMarkingOperations "}
|
||||
e = adapter._build_event_record(rec)
|
||||
assert e.data["event_sub_type"] == "pavementMarkingOperations"
|
||||
|
||||
|
||||
def test_build_event_captures_cause_and_organization(adapter):
|
||||
e = adapter._build_event_record(BY_TYPE["roadwork"])
|
||||
assert e.data["cause"] == BY_TYPE["roadwork"]["Cause"]
|
||||
assert e.data["organization"] == BY_TYPE["roadwork"]["Organization"]
|
||||
|
||||
|
||||
def test_build_event_passes_through_recurrence_and_restrictions(adapter):
|
||||
e = adapter._build_event_record(BY_TYPE["closures"])
|
||||
assert e.data["recurrence_schedules"] == BY_TYPE["closures"]["RecurrenceSchedules"]
|
||||
assert e.data["restrictions"] == BY_TYPE["closures"]["Restrictions"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("short,expected_subject", [
|
||||
("work_zone", "central.traffic.work_zone.us.id"),
|
||||
("closure", "central.traffic.closure.us.id"),
|
||||
("incident", "central.traffic.incident.us.id"),
|
||||
("special_event", "central.traffic.special_event.us.id"),
|
||||
("advisory", "central.traffic.advisory.us.id"),
|
||||
])
|
||||
def test_subject_for(adapter, short, expected_subject):
|
||||
e = Event(id="x", adapter="itd_511", category=f"{short}.itd_511",
|
||||
time=datetime.now(timezone.utc), severity=1, geo=Geo(), data={})
|
||||
assert adapter.subject_for(e) == expected_subject
|
||||
|
||||
|
||||
def test_advisory_structural_passthrough(adapter):
|
||||
# Synthesize an advisory (ITD currently returns []); per-record try/except
|
||||
# in poll() means downstream surprises won't sink the cycle.
|
||||
rec = {"SourceId": "ADV-1", "Description": "Snow event in central Idaho",
|
||||
"Latitude": 44.0, "Longitude": -114.5, "Reported": 1780500000}
|
||||
e = adapter._build_advisory_record(rec)
|
||||
assert e is not None
|
||||
assert e.category == "advisory.itd_511"
|
||||
assert e.id == "idaho_511:advisory:ADV-1"
|
||||
assert e.data["advisory"] == rec # full pass-through, schema-free
|
||||
assert e.data["latitude"] == 44.0
|
||||
assert e.data["event_type_short"] == "advisory"
|
||||
|
||||
|
||||
def test_advisory_returns_none_without_any_id(adapter):
|
||||
assert adapter._build_advisory_record({"Description": "no id"}) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_poll_yields_events_from_both_endpoints(adapter):
|
||||
await adapter.startup()
|
||||
adapter._fetch = AsyncMock(side_effect=lambda ep: {"event": EVENT, "alerts": ALERTS}[ep])
|
||||
events = [e async for e in adapter.poll()]
|
||||
await adapter.shutdown()
|
||||
# alerts fixture is [] so events == EVENT count
|
||||
assert len(events) == len(EVENT)
|
||||
assert all(e.adapter == "itd_511" for e in events)
|
||||
assert {e.category for e in events} == {
|
||||
"work_zone.itd_511", "closure.itd_511",
|
||||
"incident.itd_511", "special_event.itd_511",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_poll_advisory_cadence_throttles_alerts_endpoint(adapter):
|
||||
"""Advisories poll on the 0th, 5th, 10th... event-poll (5x throttle)."""
|
||||
await adapter.startup()
|
||||
calls: list[str] = []
|
||||
|
||||
async def fake_fetch(ep):
|
||||
calls.append(ep)
|
||||
return EVENT if ep == "event" else ALERTS
|
||||
|
||||
adapter._fetch = fake_fetch
|
||||
for _ in range(6):
|
||||
[_ async for _ in adapter.poll()]
|
||||
await adapter.shutdown()
|
||||
# 6 polls: events every time, alerts on poll 0 and poll 5
|
||||
assert calls.count("event") == 6
|
||||
assert calls.count("alerts") == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_poll_skips_cleanly_without_api_key(tmp_path):
|
||||
cs = MagicMock()
|
||||
cs.get_api_key = AsyncMock(return_value=None)
|
||||
adapter = Itd511Adapter(_cfg(), cs, tmp_path / "cursors.db")
|
||||
await adapter.startup()
|
||||
events = [e async for e in adapter.poll()]
|
||||
await adapter.shutdown()
|
||||
assert events == [] # no fetch, clean skip per tomtom_flow precedent
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_key_never_leaks_in_error_path(adapter, caplog):
|
||||
"""The key travels in ?key=, so aiohttp's default error messages include
|
||||
the full URL; every error-log path must run through self._redact().
|
||||
Regression guard via caplog inspection. NOTE: ``caplog.text`` only contains
|
||||
the message field — structured ``extra={}`` kwargs land as attributes on
|
||||
the LogRecord, so we inspect both surfaces."""
|
||||
await adapter.startup()
|
||||
key_value = adapter._api_key # the testkey set up in the adapter fixture
|
||||
assert key_value and len(key_value) > 16
|
||||
|
||||
async def boom(endpoint):
|
||||
raise aiohttp.ClientConnectionError(
|
||||
f"Cannot connect to host 511.idaho.gov ssl:default [key={key_value}]"
|
||||
)
|
||||
|
||||
adapter._fetch = boom
|
||||
with caplog.at_level(logging.WARNING, logger="central.adapters.itd_511"):
|
||||
events = [e async for e in adapter.poll()]
|
||||
await adapter.shutdown()
|
||||
assert events == []
|
||||
surfaces = [r.getMessage() for r in caplog.records]
|
||||
surfaces.extend(str(getattr(r, "error", "")) for r in caplog.records)
|
||||
joined = " ".join(surfaces)
|
||||
assert key_value not in joined, f"key leaked to log: {joined!r}"
|
||||
assert "<KEY>" in joined # redaction marker proves _redact() actually fired
|
||||
|
||||
|
||||
def test_inherits_dedup_mixin_from_source_adapter():
|
||||
for m in ("is_published", "mark_published", "sweep_old_ids"):
|
||||
assert m not in Itd511Adapter.__dict__, f"redefines {m}"
|
||||
assert getattr(Itd511Adapter, m) is getattr(SourceAdapter, m)
|
||||
|
||||
|
||||
def test_summary_partial_renders_per_event_type():
|
||||
from central.gui.routes import _derive_subject
|
||||
cases = [
|
||||
({"event_type_short": "work_zone", "roadway_name": "I-84"},
|
||||
"Road work on I-84"),
|
||||
({"event_type_short": "closure", "roadway_name": "N McDermott Rd",
|
||||
"direction": "Both", "is_full_closure": True},
|
||||
"Closure on N McDermott Rd Both (full closure)"),
|
||||
({"event_type_short": "incident", "roadway_name": "I-84",
|
||||
"direction": "East"},
|
||||
"Incident on I-84 East"),
|
||||
({"event_type_short": "advisory"}, "Advisory"),
|
||||
# Drops "Unknown" direction per wzdx lesson
|
||||
({"event_type_short": "work_zone", "roadway_name": "I-84",
|
||||
"direction": "Unknown"},
|
||||
"Road work on I-84"),
|
||||
]
|
||||
for inner, expected in cases:
|
||||
row = {"adapter": "itd_511", "data": {"data": {"data": inner}}}
|
||||
assert _derive_subject(row) == expected, f"mismatch for {inner!r}"
|
||||
|
||||
|
||||
def test_class_attributes_match_spec():
|
||||
assert Itd511Adapter.name == "itd_511"
|
||||
assert Itd511Adapter.data_class == "event"
|
||||
assert Itd511Adapter.requires_api_key == "idaho_511"
|
||||
assert Itd511Adapter.api_key_field == "api_key_alias"
|
||||
assert Itd511Adapter.default_cadence_s == 60
|
||||
assert Itd511Adapter.wizard_order is None
|
||||
assert Itd511Adapter.enrichment_locations == [("latitude", "longitude")]
|
||||
|
||||
|
||||
# --- BUG C: 429 Retry-After must drive the wait directly; no double-sleep --
|
||||
|
||||
def test_transient_carries_wait_s():
|
||||
t = _Transient("429 retry-after=42", wait_s=42)
|
||||
assert t.wait_s == 42
|
||||
assert str(t) == "429 retry-after=42"
|
||||
assert _Transient("5xx").wait_s is None # default omits
|
||||
|
||||
|
||||
def test_wait_strategy_honors_transient_wait_s():
|
||||
"""BUG C regression: a 429 Retry-After must drive the wait directly via
|
||||
_Transient.wait_s; tenacity must NOT also wait its exponential jitter on
|
||||
top (the previous shape did both, blocking ~120s+ per cycle)."""
|
||||
retry_state = MagicMock()
|
||||
outcome = MagicMock()
|
||||
outcome.exception.return_value = _Transient("429", wait_s=42)
|
||||
retry_state.outcome = outcome
|
||||
assert _wait_strategy(retry_state) == 42.0
|
||||
outcome.exception.return_value = _Transient("429", wait_s=60)
|
||||
assert _wait_strategy(retry_state) == 60.0
|
||||
|
||||
|
||||
def test_wait_strategy_falls_back_for_transient_without_wait_s():
|
||||
"""5xx _Transient (no Retry-After) falls through to exponential jitter."""
|
||||
retry_state = MagicMock()
|
||||
outcome = MagicMock()
|
||||
outcome.exception.return_value = _Transient("503 server error") # wait_s None
|
||||
retry_state.outcome = outcome
|
||||
retry_state.attempt_number = 1
|
||||
retry_state.idle_for = 0
|
||||
wait = _wait_strategy(retry_state)
|
||||
assert isinstance(wait, float) and wait >= 0
|
||||
|
||||
|
||||
def test_wait_strategy_falls_back_for_non_transient():
|
||||
"""Network errors (no wait_s) get exponential jitter."""
|
||||
retry_state = MagicMock()
|
||||
outcome = MagicMock()
|
||||
outcome.exception.return_value = aiohttp.ClientConnectionError("net error")
|
||||
retry_state.outcome = outcome
|
||||
retry_state.attempt_number = 1
|
||||
retry_state.idle_for = 0
|
||||
wait = _wait_strategy(retry_state)
|
||||
assert isinstance(wait, float) and wait >= 0
|
||||
|
||||
|
||||
# --- BUG D3: assert→if-raise (asserts strip under python -O) -----------------
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_session_unset_raises_runtime_not_assert(adapter):
|
||||
"""D3 regression: asserts strip under ``python -O``, so the session-not-
|
||||
started precondition must be enforced with an explicit if-raise."""
|
||||
assert adapter._session is None # precondition: not yet started
|
||||
with pytest.raises(RuntimeError, match="session not started"):
|
||||
await adapter._fetch("event")
|
||||
|
||||
|
||||
# --- BUG D5: tenacity has no default logging hooks (audit guard) -------------
|
||||
|
||||
def test_tenacity_decorator_has_explicit_no_log_hooks():
|
||||
"""D5 audit: tenacity's defaults (before_sleep=None, after=after_nothing)
|
||||
have no logging — so the URL-with-key can't leak via the retry path. We
|
||||
pin them explicitly on @retry; if a future tenacity upgrade changes the
|
||||
defaults, this test fails loudly. Also confirms reraise=True so we get
|
||||
_Transient/ClientError verbatim instead of RetryError."""
|
||||
from tenacity import after_nothing, before_nothing
|
||||
retrying = Itd511Adapter._fetch.retry
|
||||
assert retrying.before_sleep is None
|
||||
assert retrying.after is after_nothing
|
||||
assert retrying.before is before_nothing
|
||||
assert retrying.reraise is True
|
||||
206
tests/test_itd_511_cameras.py
Normal file
206
tests/test_itd_511_cameras.py
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
"""Tests for the itd_511_cameras adapter (v0.10.0).
|
||||
|
||||
Fixture covers 4 cameras: ITDNET, ACHD, UDOT (cross-border per v0.10.0
|
||||
finding 4), and an RWIS multi-view (to exercise the additional_views capture):
|
||||
tests/fixtures/itd_511_cameras_sample.json
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from central.adapter import SourceAdapter
|
||||
from central.adapters.itd_511 import _Transient
|
||||
from central.adapters.itd_511_cameras import NATIVE_SOURCES, Itd511CamerasAdapter
|
||||
from central.config_models import AdapterConfig
|
||||
|
||||
FIX = Path(__file__).parent / "fixtures"
|
||||
CAMS = json.loads((FIX / "itd_511_cameras_sample.json").read_text())
|
||||
|
||||
|
||||
def _cfg():
|
||||
return AdapterConfig(
|
||||
name="itd_511_cameras", enabled=True, cadence_s=600,
|
||||
settings={"api_key_alias": "idaho_511"},
|
||||
updated_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def adapter(tmp_path):
|
||||
cs = MagicMock()
|
||||
cs.get_api_key = AsyncMock(return_value="testkey-32chars-deadbeefdeadbeef")
|
||||
return Itd511CamerasAdapter(_cfg(), cs, tmp_path / "cursors.db")
|
||||
|
||||
|
||||
def test_class_attributes_match_spec():
|
||||
assert Itd511CamerasAdapter.name == "itd_511_cameras"
|
||||
assert Itd511CamerasAdapter.data_class == "telemetry"
|
||||
assert Itd511CamerasAdapter.requires_api_key == "idaho_511"
|
||||
assert Itd511CamerasAdapter.default_cadence_s == 600
|
||||
|
||||
|
||||
def test_build_event_category_and_subject(adapter):
|
||||
e = adapter._build_event(CAMS[0])
|
||||
assert e.category == "camera.itd_511_cameras"
|
||||
assert e.adapter == "itd_511_cameras"
|
||||
assert adapter.subject_for(e) == f"central.traffic_cameras.us.id.{CAMS[0]['Id']}"
|
||||
|
||||
|
||||
def test_dedup_id_per_utc_day(adapter):
|
||||
e = adapter._build_event(CAMS[0])
|
||||
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
assert e.id == f"idaho_511:cam:{CAMS[0]['Id']}:{today}"
|
||||
|
||||
|
||||
def test_image_url_passthrough(adapter):
|
||||
e = adapter._build_event(CAMS[0])
|
||||
assert e.data["image_url"] == CAMS[0]["Views"][0]["Url"]
|
||||
assert e.data["image_url"].startswith("https://511.idaho.gov/map/Cctv/")
|
||||
|
||||
|
||||
def test_source_jurisdiction_preserves_border_cameras(adapter):
|
||||
"""Per v0.10.0 finding 4: ITD aggregates ~1.2% cross-DOT mirrors (UDOT,
|
||||
ODOT, WYDOT, ...). Region stays US-ID; source_jurisdiction preserves the
|
||||
raw upstream Source value for downstream re-bucketing."""
|
||||
udot = next((c for c in CAMS if c["Source"] == "UDOT"), None)
|
||||
assert udot is not None, "fixture must include a UDOT cross-border camera"
|
||||
e = adapter._build_event(udot)
|
||||
assert e.data["source_jurisdiction"] == "UDOT"
|
||||
assert e.data["source"] == "UDOT"
|
||||
assert e.geo.primary_region == "US-ID" # uniform Idaho tagging per locked decision
|
||||
assert e.geo.regions == ["US-ID"]
|
||||
|
||||
|
||||
def test_multiple_views_captured(adapter):
|
||||
multi = next((c for c in CAMS if len(c.get("Views") or []) > 1), None)
|
||||
assert multi is not None, "fixture must include a multi-view camera"
|
||||
e = adapter._build_event(multi)
|
||||
assert e.data["view_count"] == len(multi["Views"])
|
||||
assert e.data["additional_views"] == [v["Url"] for v in multi["Views"][1:]]
|
||||
|
||||
|
||||
def test_single_view_has_empty_additional_views(adapter):
|
||||
single = next((c for c in CAMS if len(c.get("Views") or []) == 1), None)
|
||||
assert single is not None, "fixture must include a single-view camera"
|
||||
e = adapter._build_event(single)
|
||||
assert e.data["additional_views"] == []
|
||||
assert e.data["view_count"] == 1
|
||||
|
||||
|
||||
def test_build_event_returns_none_without_id(adapter):
|
||||
assert adapter._build_event({"Source": "ITDNET"}) is None
|
||||
|
||||
|
||||
def test_severity_is_1_for_telemetry(adapter):
|
||||
e = adapter._build_event(CAMS[0])
|
||||
assert e.severity == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_poll_yields_one_event_per_camera(adapter):
|
||||
await adapter.startup()
|
||||
adapter._fetch_cameras = AsyncMock(return_value=CAMS)
|
||||
events = [e async for e in adapter.poll()]
|
||||
await adapter.shutdown()
|
||||
assert len(events) == len(CAMS)
|
||||
assert all(e.adapter == "itd_511_cameras" for e in events)
|
||||
assert all(e.category == "camera.itd_511_cameras" for e in events)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_poll_skips_cleanly_without_api_key(tmp_path):
|
||||
cs = MagicMock()
|
||||
cs.get_api_key = AsyncMock(return_value=None)
|
||||
adapter = Itd511CamerasAdapter(_cfg(), cs, tmp_path / "cursors.db")
|
||||
await adapter.startup()
|
||||
events = [e async for e in adapter.poll()]
|
||||
await adapter.shutdown()
|
||||
assert events == []
|
||||
|
||||
|
||||
def test_summary_partial_renders():
|
||||
from central.gui.routes import _derive_subject
|
||||
inner = {"camera_id": 1, "location": "I-15 UT/ID State Line UT", "roadway": "I-15"}
|
||||
row = {"adapter": "itd_511_cameras", "data": {"data": {"data": inner}}}
|
||||
assert _derive_subject(row) == "Camera: I-15 UT/ID State Line UT"
|
||||
|
||||
|
||||
def test_summary_partial_falls_back_to_id_when_location_missing():
|
||||
from central.gui.routes import _derive_subject
|
||||
inner = {"camera_id": 42}
|
||||
row = {"adapter": "itd_511_cameras", "data": {"data": {"data": inner}}}
|
||||
assert _derive_subject(row) == "Camera: #42"
|
||||
|
||||
|
||||
def test_inherits_dedup_mixin_from_source_adapter():
|
||||
for m in ("is_published", "mark_published", "sweep_old_ids"):
|
||||
assert m not in Itd511CamerasAdapter.__dict__, f"redefines {m}"
|
||||
assert getattr(Itd511CamerasAdapter, m) is getattr(SourceAdapter, m)
|
||||
|
||||
|
||||
# --- BUG E: poll() must catch _Transient (tenacity reraise after retries) ---
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_poll_catches_transient_after_exhausted_retries(adapter):
|
||||
"""BUG E regression: cameras.poll() except tuple must include _Transient
|
||||
so tenacity's reraise of a persistent 5xx or 429 (after exhausted
|
||||
retries) does NOT crash the whole poll cycle."""
|
||||
await adapter.startup()
|
||||
|
||||
async def boom_transient():
|
||||
raise _Transient("503 persistent")
|
||||
|
||||
adapter._fetch_cameras = boom_transient
|
||||
events = [e async for e in adapter.poll()]
|
||||
await adapter.shutdown()
|
||||
assert events == [] # poll exited cleanly, didn't raise
|
||||
|
||||
|
||||
# --- BUG D2: NATIVE_SOURCES allow-list lives at the adapter, not the partial -
|
||||
|
||||
def test_native_sources_module_constant():
|
||||
assert NATIVE_SOURCES == frozenset({"ITDNET", "Idaho511", "ACHD", "RWIS"})
|
||||
|
||||
|
||||
def test_is_native_source_flag_set_per_camera(adapter):
|
||||
"""Border-region UDOT camera is non-native; ITDNET is native."""
|
||||
udot = next(c for c in CAMS if c["Source"] == "UDOT")
|
||||
itdnet = next(c for c in CAMS if c["Source"] == "ITDNET")
|
||||
assert adapter._build_event(udot).data["is_native_source"] is False
|
||||
assert adapter._build_event(itdnet).data["is_native_source"] is True
|
||||
|
||||
|
||||
def test_row_partial_does_not_hardcode_source_list(adapter):
|
||||
"""D2 regression: the cross-DOT-mirror annotation is driven by
|
||||
data.is_native_source — the partial must NOT carry the source allow-list
|
||||
itself ([[feedback_no_hardcoding]])."""
|
||||
from central.gui.routes import _get_templates
|
||||
tmpl = _get_templates().env.get_template("_event_rows/itd_511_cameras.html")
|
||||
udot_evt = adapter._build_event(next(c for c in CAMS if c["Source"] == "UDOT"))
|
||||
itdnet_evt = adapter._build_event(next(c for c in CAMS if c["Source"] == "ITDNET"))
|
||||
udot_html = tmpl.render(event={"data": {"data": {"data": udot_evt.data}}})
|
||||
itdnet_html = tmpl.render(event={"data": {"data": {"data": itdnet_evt.data}}})
|
||||
assert "(cross-DOT mirror)" in udot_html
|
||||
assert "(cross-DOT mirror)" not in itdnet_html
|
||||
# Audit: the partial source on disk must not contain the allow-list.
|
||||
partial = Path(__file__).resolve().parents[1] / (
|
||||
"src/central/gui/templates/_event_rows/itd_511_cameras.html"
|
||||
)
|
||||
text = partial.read_text()
|
||||
for name in NATIVE_SOURCES:
|
||||
assert name not in text, f"partial hardcodes {name}"
|
||||
|
||||
|
||||
# --- BUG D3: assert→if-raise on the cameras sibling --------------------------
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_session_unset_raises_runtime(adapter):
|
||||
"""D3 regression: asserts strip under python -O; the session-not-started
|
||||
precondition must hold even with optimizations."""
|
||||
assert adapter._session is None
|
||||
with pytest.raises(RuntimeError, match="session not started"):
|
||||
await adapter._fetch_cameras()
|
||||
|
|
@ -11,7 +11,7 @@ from central.adapter_discovery import discover_adapters
|
|||
from central.gui import routes
|
||||
|
||||
# Adapters with data_class="telemetry" (the pinned split; grow as telemetry adapters land).
|
||||
_TELEMETRY = ["nwis", "state_511_atis_cameras", "tomtom_flow"]
|
||||
_TELEMETRY = ["itd_511_cameras", "nwis", "state_511_atis_cameras", "tomtom_flow"]
|
||||
|
||||
|
||||
# --- data_class defaults / registry split -----------------------------------
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue