diff --git a/backend/services/navi_offroute/mvum_parking.py b/backend/services/navi_offroute/mvum_parking.py index 7634eff..8c4ef27 100644 --- a/backend/services/navi_offroute/mvum_parking.py +++ b/backend/services/navi_offroute/mvum_parking.py @@ -7,13 +7,21 @@ can suggest "drive to a parking lot, switch to foot/2w/4w" trips where no MVUM trailhead exists — BLM/state land, urban edges, anywhere OSM has parking but the USFS trailhead layer does not. Read-only, pure spatial lookup; mirrors the MVUMSpatialIndex (Layer 0) / TrailheadIndex (Layer 3a) singleton pattern. + +Coordinates are stored in packed numpy arrays and the per-feature attribute columns +as plain lists; the shapely Point objects exist only long enough to build the +STRtree and are then released. Candidate record dicts are constructed lazily in +query_parking_near_line. This keeps RSS to a few hundred MB for ~1.5M rows instead +of ~1 GB of per-row dicts + Point objects. """ import logging import os import sqlite3 +import sys import time as _time from pathlib import Path +import numpy as np import psutil from shapely.geometry import Point, LineString from shapely.strtree import STRtree @@ -36,9 +44,11 @@ def parking_db_path() -> Path: class OSMParkingIndex: """In-memory STRtree over OSM parking points from osm-parking.db. - Keeps the STRtree plus a parallel ``records`` list of - ``{lat, lon, name, road_class, parking_type, access}`` dicts. Records whose - ``access`` is private/no/permit are dropped at load (useless as candidates). + Storage is columnar: ``_lats``/``_lons`` (float64 numpy arrays) plus + ``_names``/``_parking_types``/``_accesses`` (lists, aligned by index). + ``road_class`` is the constant ``"parking"`` so it is not stored per row. + query_parking_near_line() builds the ``{lat, lon, name, road_class, + parking_type, access}`` record dicts lazily from these columns. """ def __init__(self, db_path=None): @@ -47,8 +57,8 @@ class OSMParkingIndex: rss_before = proc.memory_info().rss self.db_path = Path(db_path) if db_path else parking_db_path() - self.records = [] # aligned with self._points - self._points = [] + lats, lons = [], [] + self._names, self._parking_types, self._accesses = [], [], [] skipped_access = 0 conn = sqlite3.connect(f"file:{self.db_path}?mode=ro", uri=True) @@ -65,23 +75,30 @@ class OSMParkingIndex: if lat is None or lon is None: continue # The ingest already stored representative_point() (an interior point - # of each parking polygon) in the lat/lon columns, so build the STRtree - # straight from them -- parsing the 1.6M WKB shape blobs here would add - # minutes to every worker boot for an identical point. - self.records.append({ - "lat": float(lat), - "lon": float(lon), - "name": row["name"] or "", - "road_class": "parking", - "parking_type": row["parking_type"], - "access": access, - }) - self._points.append(Point(float(lon), float(lat))) + # of each parking polygon) in the lat/lon columns, so the STRtree is + # built straight from them -- parsing the 1.5M WKB shape blobs here + # would add minutes to every worker boot for an identical point. + lats.append(float(lat)) + lons.append(float(lon)) + self._names.append(row["name"] or "") + # intern the small-cardinality attribute strings so duplicate values + # share one object instead of 1.5M separate ones. + pt = row["parking_type"] + self._parking_types.append(sys.intern(pt) if isinstance(pt, str) else pt) + self._accesses.append(sys.intern(access) if isinstance(access, str) else access) finally: conn.close() - self._tree = STRtree(self._points) if self._points else None - self.count = len(self.records) + self._lats = np.asarray(lats, dtype=np.float64) + self._lons = np.asarray(lons, dtype=np.float64) + + # Build the STRtree from transient Point objects, then release them; the tree + # internalizes its own geometry storage and we reconstruct points on demand. + points = [Point(lon, lat) for lon, lat in zip(lons, lats)] + self._tree = STRtree(points) if points else None + del points + + self.count = len(self._lats) self.skipped_access = skipped_access self.build_time_seconds = _time.perf_counter() - t0 self.memory_estimate_mb = max( @@ -90,6 +107,22 @@ class OSMParkingIndex: "OSM parking index loaded: %d parking objects (%d access-blocked skipped) " "in %.2f seconds", self.count, skipped_access, self.build_time_seconds) + def _record(self, i): + """Construct a candidate record dict for column index ``i``.""" + return { + "lat": float(self._lats[i]), + "lon": float(self._lons[i]), + "name": self._names[i], + "road_class": "parking", + "parking_type": self._parking_types[i], + "access": self._accesses[i], + } + + @property + def records(self): + """All records, built lazily (used by tests / introspection — not the hot path).""" + return [self._record(i) for i in range(self.count)] + def query_parking_near_line(self, coords, buffer_m=2000): """Parking records within ~``buffer_m`` of a (lat, lon) polyline. @@ -104,8 +137,8 @@ class OSMParkingIndex: buffer_deg = _buffer_degrees_for_meters(buffer_m, avg_lat) out = [] for i in self._tree.query(geom.buffer(buffer_deg)): - if geom.distance(self._points[i]) <= buffer_deg: - out.append(self.records[i]) + if geom.distance(Point(self._lons[i], self._lats[i])) <= buffer_deg: + out.append(self._record(i)) return out diff --git a/backend/services/navi_offroute/tests/test_mvum_parking.py b/backend/services/navi_offroute/tests/test_mvum_parking.py index 6da6fdd..52b343e 100644 --- a/backend/services/navi_offroute/tests/test_mvum_parking.py +++ b/backend/services/navi_offroute/tests/test_mvum_parking.py @@ -30,7 +30,7 @@ def test_parking_index_loads(tmp_path): ]) idx = OSMParkingIndex(db_path=db) assert idx.count == 3 - assert len(idx.records) == len(idx._points) == 3 + assert len(idx.records) == idx.count == 3 rec = idx.records[0] assert rec["name"] == "Lot A" assert rec["road_class"] == "parking"