navi-offroute: cache MVUM decoded features process-wide (O3a, perf)

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

backend/services/navi_offroute/mvum.py

@@ -11,6 +11,7 @@ import math
 import os
 import re
 import sqlite3
+import threading
 import time as _time
 import warnings
 from datetime import datetime
@@ -550,6 +551,134 @@ class MVUMReader:
             self._conn = None
 
 
+# ── Process-level decoded-feature cache (O3a perf) ────────────────────────────────
+# query_*_bbox ignore the bbox and scan the whole mvum_roads+mvum_trails tables, and the
+# WKB decode (the dominant cost) is bbox- AND mode-independent, so the decoded feature set
+# is cacheable once per worker process. We also pre-compute each motorized mode's access
+# flag at build time (check_date=None, the only value the Auto path ever passes), turning
+# 3 redundant full-table decodes per request into one cold decode + cheap per-request raster.
+_CACHE_MODES = ("mtb", "atv", "vehicle")     # motorized modes baked into the cache
+_FEATURE_CACHE = None                        # list of (geom, (minx,miny,maxx,maxy), {mode: access})
+_FEATURE_CACHE_LOCK = threading.Lock()
+
+_ROADS_SQL = """SELECT ogc_fid, id, name, symbol, operationalmaintlevel, seasonal,
+                       atv, atv_datesopen, motorcycle, motorcycle_datesopen,
+                       highclearancevehicle, highclearancevehicle_datesopen,
+                       passengervehicle, passengervehicle_datesopen,
+                       e_bike_class1, e_bike_class1_dur, shape FROM mvum_roads"""
+_TRAILS_SQL = """SELECT ogc_fid, id, name, symbol, seasonal, trailclass,
+                        atv, atv_datesopen, motorcycle, motorcycle_datesopen,
+                        highclearancevehicle, highclearancevehicle_datesopen,
+                        passengervehicle, passengervehicle_datesopen,
+                        e_bike_class1, e_bike_class1_dur, shape FROM mvum_trails"""
+
+
+def _parse_check_date(check_date):
+    """'MM/DD' -> (month, day) or None (matches the inline parse in the legacy path)."""
+    if check_date:
+        m = re.match(r"(\d{1,2})/(\d{1,2})", check_date)
+        if m:
+            return (int(m.group(1)), int(m.group(2)))
+    return None
+
+
+def _row_access(row, table, mode, check_date):
+    """Per-mode access (True=open / False=closed / None=not applicable) — identical logic to
+    query_roads_bbox / query_trails_bbox."""
+    status_field, dates_field = get_mode_field(mode)
+    keys = row.keys()
+    access = check_access(
+        row[status_field] if status_field in keys else None,
+        row[dates_field] if dates_field in keys else None,
+        row["seasonal"], check_date)
+    if access is None:
+        if table == "mvum_roads":
+            access = symbol_to_access(row["symbol"], mode, row["operationalmaintlevel"])
+        else:
+            access = symbol_to_access(row["symbol"], mode)
+    return access
+
+
+def _rasterize_feature(grid, geom, value, west, north, pixel_lon, pixel_lat):
+    """Bresenham-rasterize a (Multi)LineString into grid — extracted verbatim from the
+    legacy get_mvum_access_grid inner loop so the cache and fallback paths can't diverge."""
+    if geom.geom_type == "MultiLineString":
+        coords_list = [list(line.coords) for line in geom.geoms]
+    elif geom.geom_type == "LineString":
+        coords_list = [list(geom.coords)]
+    else:
+        return
+    for coords in coords_list:
+        for i in range(len(coords) - 1):
+            x1, y1 = coords[i]
+            x2, y2 = coords[i + 1]
+            _draw_line(grid, int((north - y1) / pixel_lat), int((x1 - west) / pixel_lon),
+                       int((north - y2) / pixel_lat), int((x2 - west) / pixel_lon), value)
+
+
+def _load_all_decoded_features(db_path):
+    """Decode all mvum_roads + mvum_trails geometries ONCE; pre-compute each motorized mode's
+    access at check_date=None. Returns [(geom, bounds, {mode: access}), ...]. Raises if the DB
+    is missing (propagated so callers fall back to None — graceful degradation)."""
+    t0 = _time.perf_counter()
+    reader = MVUMReader(db_path)
+    out = []
+    try:
+        conn = reader._get_conn()
+        for table, sql in (("mvum_roads", _ROADS_SQL), ("mvum_trails", _TRAILS_SQL)):
+            if not reader.table_exists(table):
+                continue
+            for row in conn.execute(sql):
+                shape = row["shape"]
+                if shape is None:
+                    continue
+                try:
+                    geom = wkb.loads(shape)
+                except Exception:
+                    continue
+                acc = {m: _row_access(row, table, m, None) for m in _CACHE_MODES}
+                if all(a is None for a in acc.values()):
+                    continue                      # applies to no mode (legacy: excluded)
+                out.append((geom, geom.bounds, acc))
+    finally:
+        reader.close()
+    logger.info("mvum: decoded %d features into process cache in %.2fs",
+                len(out), _time.perf_counter() - t0)
+    return out
+
+
+def _get_feature_cache(db_path=None):
+    """Lazy, threadsafe-init process-level decoded-feature cache. Read-only after init."""
+    global _FEATURE_CACHE
+    if _FEATURE_CACHE is None:
+        with _FEATURE_CACHE_LOCK:
+            if _FEATURE_CACHE is None:
+                _FEATURE_CACHE = _load_all_decoded_features(db_path)
+    return _FEATURE_CACHE
+
+
+def _rasterize_uncached(grid, south, north, west, east, mode, parsed_date,
+                        pixel_lon, pixel_lat, db_path):
+    """Legacy per-request path (query + decode + rasterize) — used for the rare seasonal
+    (check_date set) case so its behaviour is preserved exactly."""
+    reader = MVUMReader(db_path)
+    try:
+        for feats in (reader.query_roads_bbox(south, north, west, east, mode, parsed_date),
+                      reader.query_trails_bbox(south, north, west, east, mode, parsed_date)):
+            for feat in feats:
+                try:
+                    geom = wkb.loads(feat["shape"])
+                except Exception:
+                    continue
+                minx, miny, maxx, maxy = geom.bounds
+                if maxx < west or minx > east or maxy < south or miny > north:
+                    continue
+                _rasterize_feature(grid, geom, 1 if feat["access"] else 255,
+                                   west, north, pixel_lon, pixel_lat)
+    finally:
+        reader.close()
+
+
 def get_mvum_access_grid(
     south: float, north: float, west: float, east: float,
     target_shape: Tuple[int, int],
@@ -557,94 +686,66 @@ def get_mvum_access_grid(
     check_date: Optional[str] = None,
     db_path: Path = None     # None → navi_db_path() (env NAVI_OFFROUTE_NAVI_DB)
 ) -> np.ndarray:
-    """
+    """MVUM access grid (uint8: 0=no data, 1=open, 255=closed) for one mode over a bbox.
-    Get MVUM access grid for pathfinding.
 
-    Args:
+    Cache-backed (O3a) for the common path (check_date=None, motorized mode): rasterizes from
-        south, north, west, east: Bounding box (WGS84)
+    the process-wide decoded-feature cache. Falls back to the legacy per-request query+decode
-        target_shape: (rows, cols) to match elevation grid
+    for seasonal checks or unusual modes. Foot bypasses MVUM (zeros). Signature unchanged."""
-        mode: Travel mode (foot skips MVUM entirely)
-        check_date: Optional "MM/DD" string for seasonal checking
-        db_path: Path to navi.db
-
-    Returns:
-        np.ndarray of uint8:
-            0   = no MVUM data (defer to existing trail/friction logic)
-            1   = road/trail is OPEN to this vehicle mode
-            255 = road/trail EXISTS but is CLOSED to this mode
-    """
-    # Foot mode bypasses MVUM entirely
     if mode == "foot":
         return np.zeros(target_shape, dtype=np.uint8)
-
-    # Parse check_date if provided
-    parsed_date = None
-    if check_date:
-        match = re.match(r"(\d{1,2})/(\d{1,2})", check_date)
-        if match:
-            parsed_date = (int(match.group(1)), int(match.group(2)))
-
-    # Initialize output grid
     grid = np.zeros(target_shape, dtype=np.uint8)
     rows, cols = target_shape
-
-    # Pixel size
     pixel_lat = (north - south) / rows
     pixel_lon = (east - west) / cols
+    parsed_date = _parse_check_date(check_date)
 
-    reader = MVUMReader(db_path)
+    if parsed_date is None and mode in _CACHE_MODES:
-
+        for geom, (minx, miny, maxx, maxy), acc in _get_feature_cache(db_path):
-    try:
+            if maxx < west or minx > east or maxy < south or miny > north:
-        # Query roads and trails
+                continue
-        roads = reader.query_roads_bbox(south, north, west, east, mode, parsed_date)
+            a = acc.get(mode)
-        trails = reader.query_trails_bbox(south, north, west, east, mode, parsed_date)
+            if a is None:
-
+                continue
-        # Rasterize features
+            _rasterize_feature(grid, geom, 1 if a else 255, west, north, pixel_lon, pixel_lat)
-        for features in [roads, trails]:
+    else:
-            for feat in features:
+        _rasterize_uncached(grid, south, north, west, east, mode, parsed_date,
-                try:
+                            pixel_lon, pixel_lat, db_path)
-                    geom = wkb.loads(feat["shape"])
-
-                    # Get geometry bounds
-                    minx, miny, maxx, maxy = geom.bounds
-
-                    # Check if intersects our bbox
-                    if maxx < west or minx > east or maxy < south or miny > north:
-                        continue
-
-                    # Rasterize line
-                    value = 1 if feat["access"] else 255
-
-                    # Simple line rasterization
-                    if geom.geom_type in ("LineString", "MultiLineString"):
-                        if geom.geom_type == "MultiLineString":
-                            coords_list = [list(line.coords) for line in geom.geoms]
-                        else:
-                            coords_list = [list(geom.coords)]
-
-                        for coords in coords_list:
-                            for i in range(len(coords) - 1):
-                                x1, y1 = coords[i]
-                                x2, y2 = coords[i + 1]
-
-                                # Convert to pixel coordinates
-                                col1 = int((x1 - west) / pixel_lon)
-                                row1 = int((north - y1) / pixel_lat)
-                                col2 = int((x2 - west) / pixel_lon)
-                                row2 = int((north - y2) / pixel_lat)
-
-                                # Bresenham's line algorithm
-                                _draw_line(grid, row1, col1, row2, col2, value)
-
-                except Exception as e:
-                    continue
-
-    finally:
-        reader.close()
-
     return grid
 
 
+def get_mvum_access_grids_all_modes(
+    south: float, north: float, west: float, east: float,
+    target_shape: Tuple[int, int],
+    modes: List[str],
+    check_date: Optional[str] = None,
+    db_path: Path = None,
+) -> Dict[str, np.ndarray]:
+    """Build MVUM access grids for several motorized modes in ONE pass over the cached features
+    (vs N independent decodes). Returns {mode: uint8 grid}, same per-grid semantics as
+    get_mvum_access_grid. Seasonal (check_date set) defers per-mode to the legacy path."""
+    rows, cols = target_shape
+    pixel_lat = (north - south) / rows
+    pixel_lon = (east - west) / cols
+    grids = {m: np.zeros(target_shape, dtype=np.uint8) for m in modes}
+    parsed_date = _parse_check_date(check_date)
+
+    if parsed_date is not None:
+        for m in modes:
+            grids[m] = get_mvum_access_grid(south, north, west, east, target_shape,
+                                            m, check_date, db_path)
+        return grids
+
+    for geom, (minx, miny, maxx, maxy), acc in _get_feature_cache(db_path):
+        if maxx < west or minx > east or maxy < south or miny > north:
+            continue
+        for m in modes:
+            a = acc.get(m)
+            if a is None:
+                continue
+            _rasterize_feature(grids[m], geom, 1 if a else 255, west, north, pixel_lon, pixel_lat)
+    return grids
+
+
 def _draw_line(grid: np.ndarray, r1: int, c1: int, r2: int, c2: int, value: int):
     """Draw a line on the grid using Bresenham's algorithm."""
     rows, cols = grid.shape

backend/services/navi_offroute/router.py

@@ -45,7 +45,7 @@ from .transitions import MODE_INDEX
 from .friction import FrictionReader, friction_to_multiplier
 from .barriers import BarrierReader, WildernessReader, wilderness_tif_path
 from .trails import TrailReader
-from .mvum import get_mvum_access_grid
+from .mvum import get_mvum_access_grid, get_mvum_access_grids_all_modes
 from .mvum_annotate import annotate_network_edges
 from .mvum_exclude import build_exclude_polygons
 
@@ -1025,14 +1025,16 @@ class OffrouteRouter:
         mvum_mode = {"2w": "mtb", "4w": "atv", "vehicle": "vehicle"}
         on_date = getattr(self, "mvum_on_date", None)
         check = on_date.strftime("%m/%d") if on_date else None
-        out = {}
+        # O3a: one pass over the process-cached decoded features for all motorized modes,
-        for mode, mv_name in mvum_mode.items():
+        # instead of 3 independent full-table decodes.
-            try:
+        try:
-                out[mode] = get_mvum_access_grid(
+            grids = get_mvum_access_grids_all_modes(
-                    bbox["south"], bbox["north"], bbox["west"], bbox["east"],
+                bbox["south"], bbox["north"], bbox["west"], bbox["east"],
-                    target_shape=shape, mode=mv_name, check_date=check)
+                target_shape=shape, modes=list(mvum_mode.values()), check_date=check)
-            except Exception as e:
+        except Exception as e:
-                logger.info("auto: MVUM grid unavailable for %s: %s", mode, e)
+            logger.info("auto: MVUM grids unavailable: %s", e)
+            return None
+        out = {mode: grids[mv_name] for mode, mv_name in mvum_mode.items() if mv_name in grids}
         return out or None
 
     def _append_unified_segment(self, features, seg_coords, mode_idx):

backend/services/navi_offroute/tests/test_offroute.py

@@ -1478,3 +1478,28 @@ def test_route_auto_e2e_http_in_town_fires_bypass(client, monkeypatch):
     data = resp.get_json()
     assert data["selected_mode"] == "vehicle"
     assert data["summary"]["auto_bypass"] is True
+
+
+def test_mvum_cache_decode_once(monkeypatch):
+    """O3a: the process-level decoded-feature cache decodes WKB once — a second call adds zero
+    wkb.loads (cache hit). Skips without the real navi.db (e.g. matt-desktop)."""
+    import services.navi_offroute.mvum as _mvum
+    from services.navi_offroute.mvum import navi_db_path
+    if not navi_db_path().exists():
+        pytest.skip("navi.db not present locally")
+    monkeypatch.setattr(_mvum, "_FEATURE_CACHE", None)        # force a cold build, auto-reverted
+    n = [0]
+    orig = _mvum.wkb.loads
+    def counting(*a, **k):
+        n[0] += 1
+        return orig(*a, **k)
+    monkeypatch.setattr(_mvum.wkb, "loads", counting)
+    bbox = (43.5, 44.0, -115.0, -114.0)                       # Sawtooth NF area
+    g1 = _mvum.get_mvum_access_grids_all_modes(*bbox, target_shape=(200, 200),
+                                               modes=["mtb", "atv", "vehicle"])
+    first = n[0]
+    g2 = _mvum.get_mvum_access_grids_all_modes(*bbox, target_shape=(200, 200),
+                                               modes=["mtb", "atv", "vehicle"])
+    assert first > 0                                          # cold call decoded features
+    assert n[0] == first                                      # warm call added ZERO decodes
+    assert set(g1) == {"mtb", "atv", "vehicle"} and g1["atv"].shape == (200, 200)