From b741e217f6257cc7f5424af1ff7d1a662e39a197 Mon Sep 17 00:00:00 2001 From: Matt Date: Thu, 7 May 2026 01:32:25 +0000 Subject: [PATCH 01/41] =?UTF-8?q?fix:=20ZIM=20table=20extraction=20?= =?UTF-8?q?=E2=80=94=20pipe-delimited=20cells=20instead=20of=20concatenati?= =?UTF-8?q?on?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-processes HTML tree before lxml .text_content() to prevent element concatenation: - cells joined with ' | ' delimiter, rows with newlines -
tags produce newlines -
  • items get '- ' prefix and newline separation -
    /
    definition list items get newline separation Fixes ~868 mangled Qdrant points where table content was jammed together (e.g. 'Freq51Primary1A==' instead of 'Freq51 | Primary | 1A=='). Co-Authored-By: Claude Opus 4.6 --- lib/processors/zim_processor.py | 66 +++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/lib/processors/zim_processor.py b/lib/processors/zim_processor.py index b258408..6f5c887 100644 --- a/lib/processors/zim_processor.py +++ b/lib/processors/zim_processor.py @@ -77,10 +77,73 @@ def _text_hash(text): return hashlib.md5(text.encode('utf-8')).hexdigest() +def _flatten_table(table_el): + """Convert a
    • element to pipe-delimited text. + + Each becomes a row with cells joined by ' | '. + Returns the formatted table as a string with blank lines around it. + """ + rows = [] + for tr in table_el.iter('tr'): + cells = [] + for cell in tr: + if cell.tag in ('td', 'th'): + cell_text = (cell.text_content() or '').strip() + # Collapse internal whitespace in each cell + cell_text = re.sub(r'\s+', ' ', cell_text) + if cell_text: + cells.append(cell_text) + if cells: + rows.append(' | '.join(cells)) + if not rows: + return '' + return '\n'.join(rows) + + +def _preprocess_tree(doc): + """Pre-process HTML tree to add delimiters before text_content() flattens it. + + Handles:
    ,
    ,
  • ,
    ,
    -- elements that lxml's + text_content() would concatenate without any separators. + """ + from lxml import etree + + # 1. Replace
    • elements with their pipe-delimited text + for table in list(doc.iter('table')): + formatted = _flatten_table(table) + if formatted: + replacement = etree.Element('div') + replacement.text = '\n\n' + formatted + '\n\n' + parent = table.getparent() + if parent is not None: + parent.replace(table, replacement) + else: + table.drop_tree() + + # 2.
    -> inject newline + for br in list(doc.iter('br')): + br.tail = '\n' + (br.tail or '') + + # 3.
  • -> inject newline + "- " prefix + for li in list(doc.iter('li')): + li.text = '- ' + (li.text or '') + li.tail = '\n' + (li.tail or '') + + # 4.
    -> inject newline before + for dt in list(doc.iter('dt')): + dt.tail = '\n' + (dt.tail or '') + + # 5.
    -> inject newline + indent + for dd in list(doc.iter('dd')): + dd.text = ' ' + (dd.text or '') + dd.tail = '\n' + (dd.tail or '') + + def _html_to_text(html_bytes): """Convert HTML bytes to clean text via lxml. Strips nav, footer, script, style elements. Decodes entities. + Pre-processes tables, lists, and line breaks for proper delimiters. Normalizes whitespace. """ try: @@ -93,6 +156,9 @@ def _html_to_text(html_bytes): for el in doc.iter(tag): el.drop_tree() + # Pre-process tree: tables -> pipe-delimited, br -> newlines, li -> dashes + _preprocess_tree(doc) + # Extract text text = doc.text_content() From 83a21854c39feeb16425f35da38fc3f3c1bf3091 Mon Sep 17 00:00:00 2001 From: Matt Date: Thu, 7 May 2026 01:36:23 +0000 Subject: [PATCH 02/41] =?UTF-8?q?fix:=20PDF=20extraction=20quality=20?= =?UTF-8?q?=E2=80=94=20word-boundary=20checks=20and=20layout=20mode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds _text_quality_ok() gate that replaces the bare 50-char length check at each stage of the extraction fallback chain. Checks: - Word-boundary ratio (≥60% of tokens must be real words) - Concatenation ratio (lc→UC transitions must be <10% of word count) When PyPDF2 default extraction fails quality check, retries with space_width=100 for tighter word-boundary detection. This fixes Haynes/workshop manuals where tight kerning produces concatenated words like 'byMike' and 'oftheGuild'. Also adds -layout flag to pdftotext subprocess calls for better spatial awareness in the poppler fallback stage. Note: PyPDF2 3.0.1 does not support layout=True parameter. The space_width parameter serves the same purpose. Co-Authored-By: Claude Opus 4.6 --- lib/extractor.py | 71 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 62 insertions(+), 9 deletions(-) diff --git a/lib/extractor.py b/lib/extractor.py index 13159c9..bc236ab 100644 --- a/lib/extractor.py +++ b/lib/extractor.py @@ -21,6 +21,7 @@ Config: processing.extract_workers, processing.max_pdf_size_mb, processing.extract_timeout, processing.page_timeout """ import base64 +import re import json import os import random @@ -99,6 +100,40 @@ def _is_transient(error_str): return any(sig in s for sig in transient_signals) +def _text_quality_ok(text, min_length=50): + """Check if extracted text meets quality thresholds. + + Beyond the basic length check, validates: + - Word-boundary ratio: at least 60% of tokens should be real words (2+ alpha chars) + - Concatenation ratio: lowercase-immediately-followed-by-uppercase shouldn't exceed 10% of word count + + Returns True if text passes all checks. + """ + text = text.strip() + if len(text) < min_length: + return False + + words = text.split() + if not words: + return False + + # Word-like ratio: tokens with 2+ alphabetic characters + word_like = sum(1 for w in words if len(re.findall(r'[a-zA-Z]', w)) >= 2) + word_ratio = word_like / len(words) + if word_ratio < 0.60: + return False + + # Concatenation detector: lowercase immediately followed by uppercase + # Filter out common camelCase patterns in code (short tokens) + concat_hits = len(re.findall(r'[a-z][A-Z]', text)) + concat_ratio = concat_hits / len(words) if words else 0 + if concat_ratio > 0.10: + return False + + return True + + + def _render_page_to_png(pdf_path, page_num_1indexed, dpi=200, timeout=30): """Render a single PDF page to PNG bytes using pdftoppm. @@ -224,7 +259,7 @@ def _extract_page_without_reader(pdf_path, page_num_0indexed, page_timeout=30): # Method 1: pdftotext (poppler) try: result = subprocess.run( - ['pdftotext', '-f', str(page_num_0indexed + 1), + ['pdftotext', '-layout', '-f', str(page_num_0indexed + 1), '-l', str(page_num_0indexed + 1), pdf_path, '-'], capture_output=True, text=True, timeout=page_timeout ) @@ -233,7 +268,7 @@ def _extract_page_without_reader(pdf_path, page_num_0indexed, page_timeout=30): except Exception: pass - if len(text.strip()) >= 50: + if _text_quality_ok(text): return text, 'pdftotext' # Method 2: pdftoppm + Tesseract OCR @@ -258,7 +293,7 @@ def _extract_page_without_reader(pdf_path, page_num_0indexed, page_timeout=30): except Exception: pass - if len(text.strip()) >= 50: + if _text_quality_ok(text): return text, 'tesseract' # Method 3: Gemini Vision (last resort) @@ -276,8 +311,26 @@ def _extract_page_without_reader(pdf_path, page_num_0indexed, page_timeout=30): # ── Core extraction functions ── def _pypdf2_extract(reader, page_num): - """Extract text from a PyPDF2 page object. Runs inside a thread for timeout.""" - return reader.pages[page_num].extract_text() or '' + """Extract text from a PyPDF2 page object. Runs inside a thread for timeout. + + Tries default extraction first (space_width=200). If quality check fails, + retries with space_width=100 which better detects word boundaries in + tightly-kerned PDFs (common in Haynes/workshop manuals). + + Note: PyPDF2 3.0.1 does not support layout=True. The space_width parameter + controls word-boundary detection tolerance. Lower values = more aggressive + space insertion between characters. + """ + text = reader.pages[page_num].extract_text() or '' + if _text_quality_ok(text): + return text + + # Retry with tighter word-boundary detection + text_tight = reader.pages[page_num].extract_text(space_width=100.0) or '' + if len(text_tight.strip()) >= len(text.strip()): + return text_tight + + return text def extract_text_from_page(reader, page_num, pdf_path, page_timeout=30): @@ -302,13 +355,13 @@ def extract_text_from_page(reader, page_num, pdf_path, page_timeout=30): except Exception: text = '' - if len(text.strip()) >= 50: + if _text_quality_ok(text): return text, 'pypdf2' # Method 2: pdftotext via subprocess (inherently timeout-safe) try: result = subprocess.run( - ['pdftotext', '-f', str(page_num + 1), '-l', str(page_num + 1), pdf_path, '-'], + ['pdftotext', '-layout', '-f', str(page_num + 1), '-l', str(page_num + 1), pdf_path, '-'], capture_output=True, text=True, timeout=page_timeout ) if result.returncode == 0 and len(result.stdout.strip()) > len(text.strip()): @@ -316,7 +369,7 @@ def extract_text_from_page(reader, page_num, pdf_path, page_timeout=30): except Exception: pass - if len(text.strip()) >= 50: + if _text_quality_ok(text): return text, 'pdftotext' # Method 3: pdftoppm + Tesseract OCR @@ -340,7 +393,7 @@ def extract_text_from_page(reader, page_num, pdf_path, page_timeout=30): except Exception: pass - if len(text.strip()) >= 50: + if _text_quality_ok(text): return text, 'tesseract' # Method 4: Gemini Vision (last resort — costs API calls but handles scanned docs) From f2a0f81580a82bcb33bf5b72420d208f9282ba63 Mon Sep 17 00:00:00 2001 From: Matt Date: Thu, 7 May 2026 23:43:56 +0000 Subject: [PATCH 03/41] =?UTF-8?q?feat(offroute):=20Phase=20O1=20foundation?= =?UTF-8?q?=20=E2=80=94=20PMTiles=20decoder,=20Tobler=20cost,=20MCP=20path?= =?UTF-8?q?finder=20prototype?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - dem.py: Terrarium-encoded PMTiles tile reader with LRU cache - Decodes WebP tiles from planet-dem.pmtiles - Stitches tiles into numpy elevation grids for arbitrary bboxes - Provides pixel-to-latlon coordinate conversion - cost.py: Tobler off-path hiking cost function - speed = 0.6 * 6.0 * exp(-3.5 * |grade + 0.05|) km/h - Max slope cutoff: 40 degrees → impassable - Returns time-to-traverse (seconds/cell) as cost metric - prototype.py: Standalone validation on Idaho test bbox - 43km × 80km bbox (~17M cells at 14m resolution) - scikit-image MCP_Geometric Dijkstra pathfinder - Outputs GeoJSON LineString with path metadata - Validated: 61.6km path, 21.3 hours effort time Co-Authored-By: Claude Opus 4.5 --- lib/offroute/__init__.py | 1 + lib/offroute/cost.py | 94 +++++++++++++ lib/offroute/dem.py | 190 ++++++++++++++++++++++++++ lib/offroute/prototype.py | 274 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 559 insertions(+) create mode 100644 lib/offroute/__init__.py create mode 100644 lib/offroute/cost.py create mode 100644 lib/offroute/dem.py create mode 100755 lib/offroute/prototype.py diff --git a/lib/offroute/__init__.py b/lib/offroute/__init__.py new file mode 100644 index 0000000..b0536cd --- /dev/null +++ b/lib/offroute/__init__.py @@ -0,0 +1 @@ +"""OFFROUTE: Off-network effort-based routing module.""" diff --git a/lib/offroute/cost.py b/lib/offroute/cost.py new file mode 100644 index 0000000..f460ab9 --- /dev/null +++ b/lib/offroute/cost.py @@ -0,0 +1,94 @@ +""" +Tobler off-path hiking cost function for OFFROUTE. + +Computes travel time cost based on terrain slope using Tobler's +hiking function with off-trail penalty. +""" +import math +import numpy as np +from typing import Tuple + +# Maximum passable slope in degrees +MAX_SLOPE_DEG = 40.0 + +# Tobler off-path parameters +TOBLER_BASE_SPEED = 6.0 +TOBLER_OFF_TRAIL_MULT = 0.6 + + +def tobler_speed(grade: float) -> float: + """ + Calculate hiking speed using Tobler's off-path function. + + speed_kmh = 0.6 * 6.0 * exp(-3.5 * |grade + 0.05|) + + Peak speed is ~3.6 km/h at grade = -0.05 (slight downhill). + """ + return TOBLER_OFF_TRAIL_MULT * TOBLER_BASE_SPEED * math.exp(-3.5 * abs(grade + 0.05)) + + +def compute_cost_grid( + elevation: np.ndarray, + cell_size_m: float, + cell_size_lat_m: float = None, + cell_size_lon_m: float = None +) -> np.ndarray: + """ + Compute isotropic travel cost grid from elevation data. + + Each cell's cost represents the time (in seconds) to traverse that cell, + based on the average slope from neighboring cells. + """ + if cell_size_lat_m is None: + cell_size_lat_m = cell_size_m + if cell_size_lon_m is None: + cell_size_lon_m = cell_size_m + + rows, cols = elevation.shape + + # Compute gradients in both directions + dy = np.zeros_like(elevation) + dx = np.zeros_like(elevation) + + # Central differences for interior, forward/backward at edges + dy[1:-1, :] = (elevation[:-2, :] - elevation[2:, :]) / (2 * cell_size_lat_m) + dy[0, :] = (elevation[0, :] - elevation[1, :]) / cell_size_lat_m + dy[-1, :] = (elevation[-2, :] - elevation[-1, :]) / cell_size_lat_m + + dx[:, 1:-1] = (elevation[:, 2:] - elevation[:, :-2]) / (2 * cell_size_lon_m) + dx[:, 0] = (elevation[:, 1] - elevation[:, 0]) / cell_size_lon_m + dx[:, -1] = (elevation[:, -1] - elevation[:, -2]) / cell_size_lon_m + + # Compute slope magnitude (grade = rise/run) + grade_magnitude = np.sqrt(dx**2 + dy**2) + + # Convert to slope angle in degrees + slope_deg = np.degrees(np.arctan(grade_magnitude)) + + # Compute speed for each cell using Tobler function + speed_kmh = TOBLER_OFF_TRAIL_MULT * TOBLER_BASE_SPEED * np.exp(-3.5 * np.abs(grade_magnitude + 0.05)) + + # Convert speed to time cost (seconds to traverse one cell) + avg_cell_size = (cell_size_lat_m + cell_size_lon_m) / 2 + cost = avg_cell_size * 3.6 / speed_kmh + + # Set impassable cells (slope > MAX_SLOPE_DEG) to infinity + cost[slope_deg > MAX_SLOPE_DEG] = np.inf + + # Handle NaN elevations (no data) + cost[np.isnan(elevation)] = np.inf + + return cost + + +if __name__ == "__main__": + print("Testing Tobler speed function:") + for grade in [-0.3, -0.1, -0.05, 0.0, 0.05, 0.1, 0.3]: + speed = tobler_speed(grade) + print(f" Grade {grade:+.2f}: {speed:.2f} km/h") + + print("\nTesting cost grid computation:") + elev = np.arange(100).reshape(10, 10).astype(np.float32) * 10 + cost = compute_cost_grid(elev, cell_size_m=30.0) + print(f" Elevation range: {elev.min():.0f} - {elev.max():.0f} m") + print(f" Cost range: {cost[~np.isinf(cost)].min():.1f} - {cost[~np.isinf(cost)].max():.1f} s") diff --git a/lib/offroute/dem.py b/lib/offroute/dem.py new file mode 100644 index 0000000..f715611 --- /dev/null +++ b/lib/offroute/dem.py @@ -0,0 +1,190 @@ +""" +DEM tile reader for OFFROUTE. + +Reads elevation tiles from planet-dem.pmtiles (Terrarium-encoded WebP), +decodes them into numpy arrays, and provides a stitched elevation grid +for a given bounding box. +""" +import math +from functools import lru_cache +from io import BytesIO +from pathlib import Path +from typing import Tuple, Optional + +import numpy as np +from PIL import Image +from pmtiles.reader import MmapSource, Reader as PMTilesReader + +# Default path to the planet DEM PMTiles file +DEFAULT_DEM_PATH = Path("/mnt/nas/nav/planet-dem.pmtiles") + +# Tile size in pixels (z12 tiles are 512x512 in this tileset) +TILE_SIZE = 512 + +# Zoom level to use for elevation data +ZOOM_LEVEL = 12 + + +def terrarium_decode(rgb_array: np.ndarray) -> np.ndarray: + """ + Decode Terrarium-encoded RGB values to elevation in meters. + + Formula: elevation = (R * 256 + G + B/256) - 32768 + """ + r = rgb_array[:, :, 0].astype(np.float32) + g = rgb_array[:, :, 1].astype(np.float32) + b = rgb_array[:, :, 2].astype(np.float32) + + elevation = (r * 256.0 + g + b / 256.0) - 32768.0 + return elevation + + +def lat_lon_to_tile(lat: float, lon: float, zoom: int) -> Tuple[int, int]: + """Convert lat/lon to tile coordinates at given zoom level.""" + n = 2 ** zoom + x = int((lon + 180.0) / 360.0 * n) + lat_rad = math.radians(lat) + y = int((1.0 - math.asinh(math.tan(lat_rad)) / math.pi) / 2.0 * n) + return x, y + + +def tile_to_lat_lon(x: int, y: int, zoom: int) -> Tuple[float, float, float, float]: + """Convert tile coordinates to bounding box (north, south, west, east).""" + n = 2 ** zoom + lon_west = x / n * 360.0 - 180.0 + lon_east = (x + 1) / n * 360.0 - 180.0 + lat_north = math.degrees(math.atan(math.sinh(math.pi * (1 - 2 * y / n)))) + lat_south = math.degrees(math.atan(math.sinh(math.pi * (1 - 2 * (y + 1) / n)))) + return lat_north, lat_south, lon_west, lon_east + + +class DEMReader: + """Reader for Terrarium-encoded DEM tiles from PMTiles.""" + + def __init__(self, pmtiles_path: Path = DEFAULT_DEM_PATH, tile_cache_size: int = 128): + self.pmtiles_path = pmtiles_path + self._source = MmapSource(open(pmtiles_path, "rb")) + self._reader = PMTilesReader(self._source) + self._header = self._reader.header() + self._decode_tile = lru_cache(maxsize=tile_cache_size)(self._decode_tile_impl) + + def _decode_tile_impl(self, z: int, x: int, y: int) -> Optional[np.ndarray]: + """Fetch and decode a single tile.""" + tile_data = self._reader.get(z, x, y) + if tile_data is None: + return None + + img = Image.open(BytesIO(tile_data)) + rgb_array = np.array(img) + + if rgb_array.shape[2] == 4: + rgb_array = rgb_array[:, :, :3] + + elevation = terrarium_decode(rgb_array) + return elevation + + def get_elevation_grid( + self, + south: float, + north: float, + west: float, + east: float, + zoom: int = ZOOM_LEVEL + ) -> Tuple[np.ndarray, dict]: + """Get a stitched elevation grid for the given bounding box.""" + x_min, y_max = lat_lon_to_tile(south, west, zoom) + x_max, y_min = lat_lon_to_tile(north, east, zoom) + + n = 2 ** zoom + x_min = max(0, x_min) + x_max = min(n - 1, x_max) + y_min = max(0, y_min) + y_max = min(n - 1, y_max) + + n_tiles_x = x_max - x_min + 1 + n_tiles_y = y_max - y_min + 1 + out_height = n_tiles_y * TILE_SIZE + out_width = n_tiles_x * TILE_SIZE + + elevation = np.full((out_height, out_width), np.nan, dtype=np.float32) + + for ty in range(y_min, y_max + 1): + for tx in range(x_min, x_max + 1): + tile_elev = self._decode_tile(zoom, tx, ty) + if tile_elev is not None: + out_y = (ty - y_min) * TILE_SIZE + out_x = (tx - x_min) * TILE_SIZE + elevation[out_y:out_y + TILE_SIZE, out_x:out_x + TILE_SIZE] = tile_elev + + grid_north, _, grid_west, _ = tile_to_lat_lon(x_min, y_min, zoom) + _, grid_south, _, grid_east = tile_to_lat_lon(x_max, y_max, zoom) + + pixel_size_lat = (grid_north - grid_south) / out_height + pixel_size_lon = (grid_east - grid_west) / out_width + + origin_lat = grid_north - pixel_size_lat / 2 + origin_lon = grid_west + pixel_size_lon / 2 + + center_lat = (south + north) / 2 + lat_m = 111320.0 + lon_m = 111320.0 * math.cos(math.radians(center_lat)) + cell_size_lat_m = abs(pixel_size_lat) * lat_m + cell_size_lon_m = abs(pixel_size_lon) * lon_m + cell_size_m = (cell_size_lat_m + cell_size_lon_m) / 2 + + row_start = int((grid_north - north) / abs(pixel_size_lat)) + row_end = int((grid_north - south) / abs(pixel_size_lat)) + col_start = int((west - grid_west) / pixel_size_lon) + col_end = int((east - grid_west) / pixel_size_lon) + + row_start = max(0, row_start) + row_end = min(out_height, row_end) + col_start = max(0, col_start) + col_end = min(out_width, col_end) + + elevation = elevation[row_start:row_end, col_start:col_end] + + origin_lat = grid_north - (row_start + 0.5) * abs(pixel_size_lat) + origin_lon = grid_west + (col_start + 0.5) * pixel_size_lon + + metadata = { + "bounds": (south, north, west, east), + "pixel_size_lat": -abs(pixel_size_lat), + "pixel_size_lon": pixel_size_lon, + "origin_lat": origin_lat, + "origin_lon": origin_lon, + "cell_size_m": cell_size_m, + "shape": elevation.shape, + } + + return elevation, metadata + + def pixel_to_latlon(self, row: int, col: int, metadata: dict) -> Tuple[float, float]: + """Convert pixel coordinates to lat/lon.""" + lat = metadata["origin_lat"] + row * metadata["pixel_size_lat"] + lon = metadata["origin_lon"] + col * metadata["pixel_size_lon"] + return lat, lon + + def latlon_to_pixel(self, lat: float, lon: float, metadata: dict) -> Tuple[int, int]: + """Convert lat/lon to pixel coordinates.""" + row = int((metadata["origin_lat"] - lat) / abs(metadata["pixel_size_lat"])) + col = int((lon - metadata["origin_lon"]) / metadata["pixel_size_lon"]) + return row, col + + def close(self): + """Close the PMTiles file.""" + pass # MmapSource handles cleanup + + +if __name__ == "__main__": + reader = DEMReader() + elevation, meta = reader.get_elevation_grid( + south=42.4, north=42.6, west=-114.5, east=-114.3 + ) + print(f"Elevation grid shape: {elevation.shape}") + print(f"Cell size: {meta['cell_size_m']:.1f} m") + print(f"Elevation range: {np.nanmin(elevation):.1f} - {np.nanmax(elevation):.1f} m") + center_row, center_col = elevation.shape[0] // 2, elevation.shape[1] // 2 + lat, lon = reader.pixel_to_latlon(center_row, center_col, meta) + print(f"Center pixel lat/lon: {lat:.4f}, {lon:.4f}") + reader.close() diff --git a/lib/offroute/prototype.py b/lib/offroute/prototype.py new file mode 100755 index 0000000..0790a32 --- /dev/null +++ b/lib/offroute/prototype.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +""" +OFFROUTE Phase O1 Prototype + +Validates the PMTiles decoder, Tobler cost function, and MCP pathfinder +on a real Idaho bounding box. + +Test bbox (four Idaho towns as corners): + SW: Rogerson, ID (~42.21, -114.60) + NW: Buhl, ID (~42.60, -114.76) + NE: Burley, ID (~42.54, -113.79) + SE: Oakley, ID (~42.24, -113.88) + Approximate bbox: south=42.21, north=42.60, west=-114.76, east=-113.79 +""" +import json +import time +import sys +from pathlib import Path + +import numpy as np +from skimage.graph import MCP_Geometric + +# Add parent to path for imports +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from lib.offroute.dem import DEMReader +from lib.offroute.cost import compute_cost_grid + +# Test bounding box +BBOX = { + "south": 42.21, + "north": 42.60, + "west": -114.76, + "east": -113.79, +} + +# Start point: wilderness area south of Twin Falls +# (in the Sawtooth National Forest foothills) +START_LAT = 42.35 +START_LON = -114.50 + +# End point: near Burley, ID (on road network) +END_LAT = 42.52 +END_LON = -113.85 + +# Output file +OUTPUT_PATH = Path("/opt/recon/data/offroute-test.geojson") + +# Memory limit in GB +MEMORY_LIMIT_GB = 12 + + +def check_memory_usage(): + """Check current memory usage and abort if over limit.""" + try: + import psutil + process = psutil.Process() + mem_gb = process.memory_info().rss / (1024**3) + if mem_gb > MEMORY_LIMIT_GB: + print(f"ERROR: Memory usage {mem_gb:.1f}GB exceeds {MEMORY_LIMIT_GB}GB limit") + sys.exit(1) + return mem_gb + except ImportError: + return 0 + + +def main(): + print("=" * 60) + print("OFFROUTE Phase O1 Prototype") + print("=" * 60) + + t0 = time.time() + + # Step 1: Load elevation data + print(f"\n[1] Loading DEM for bbox: {BBOX}") + reader = DEMReader() + + t1 = time.time() + elevation, meta = reader.get_elevation_grid( + south=BBOX["south"], + north=BBOX["north"], + west=BBOX["west"], + east=BBOX["east"], + ) + t2 = time.time() + + print(f" Elevation grid shape: {elevation.shape}") + print(f" Cell count: {elevation.size:,}") + print(f" Cell size: {meta['cell_size_m']:.1f} m") + print(f" Elevation range: {np.nanmin(elevation):.0f} - {np.nanmax(elevation):.0f} m") + print(f" Load time: {t2 - t1:.1f}s") + + mem = check_memory_usage() + if mem > 0: + print(f" Memory usage: {mem:.1f} GB") + + # Step 2: Compute cost grid + print(f"\n[2] Computing Tobler cost grid...") + t3 = time.time() + cost = compute_cost_grid(elevation, cell_size_m=meta["cell_size_m"]) + t4 = time.time() + + finite_cost = cost[~np.isinf(cost)] + print(f" Cost range: {finite_cost.min():.1f} - {finite_cost.max():.1f} s/cell") + print(f" Impassable cells: {np.sum(np.isinf(cost)):,} ({100*np.sum(np.isinf(cost))/cost.size:.1f}%)") + print(f" Compute time: {t4 - t3:.1f}s") + + mem = check_memory_usage() + if mem > 0: + print(f" Memory usage: {mem:.1f} GB") + + # Step 3: Convert start/end to pixel coordinates + print(f"\n[3] Converting coordinates...") + start_row, start_col = reader.latlon_to_pixel(START_LAT, START_LON, meta) + end_row, end_col = reader.latlon_to_pixel(END_LAT, END_LON, meta) + + print(f" Start: ({START_LAT}, {START_LON}) -> pixel ({start_row}, {start_col})") + print(f" End: ({END_LAT}, {END_LON}) -> pixel ({end_row}, {end_col})") + + # Validate coordinates are within bounds + rows, cols = elevation.shape + if not (0 <= start_row < rows and 0 <= start_col < cols): + print(f"ERROR: Start point outside grid bounds") + sys.exit(1) + if not (0 <= end_row < rows and 0 <= end_col < cols): + print(f"ERROR: End point outside grid bounds") + sys.exit(1) + + start_elev = elevation[start_row, start_col] + end_elev = elevation[end_row, end_col] + print(f" Start elevation: {start_elev:.0f} m") + print(f" End elevation: {end_elev:.0f} m") + + # Step 4: Run MCP pathfinder + print(f"\n[4] Running MCP_Geometric pathfinder...") + t5 = time.time() + + # MCP_Geometric finds minimum cost path + # It uses Dijkstra's algorithm internally + mcp = MCP_Geometric(cost, fully_connected=True) + + # Find costs from start to all reachable cells + cumulative_costs, traceback = mcp.find_costs([(start_row, start_col)]) + t6 = time.time() + + print(f" Dijkstra completed in {t6 - t5:.1f}s") + + # Get cost to reach end point + end_cost = cumulative_costs[end_row, end_col] + print(f" Total cost to endpoint: {end_cost:.0f} seconds ({end_cost/60:.1f} minutes)") + + if np.isinf(end_cost): + print("ERROR: No path found to endpoint (blocked by impassable terrain)") + sys.exit(1) + + # Trace back the path + t7 = time.time() + path_indices = mcp.traceback((end_row, end_col)) + t8 = time.time() + + print(f" Traceback completed in {t8 - t7:.2f}s") + print(f" Path length: {len(path_indices)} cells") + + mem = check_memory_usage() + if mem > 0: + print(f" Memory usage: {mem:.1f} GB") + + # Step 5: Convert path to coordinates and compute stats + print(f"\n[5] Converting path to GeoJSON...") + + coordinates = [] + elevations = [] + + for row, col in path_indices: + lat, lon = reader.pixel_to_latlon(row, col, meta) + elev = elevation[row, col] + coordinates.append([lon, lat]) # GeoJSON is [lon, lat] + elevations.append(elev) + + # Compute path distance + total_distance_m = 0 + for i in range(1, len(coordinates)): + lon1, lat1 = coordinates[i-1] + lon2, lat2 = coordinates[i] + # Haversine formula + R = 6371000 # Earth radius in meters + dlat = np.radians(lat2 - lat1) + dlon = np.radians(lon2 - lon1) + a = np.sin(dlat/2)**2 + np.cos(np.radians(lat1)) * np.cos(np.radians(lat2)) * np.sin(dlon/2)**2 + c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1-a)) + total_distance_m += R * c + + # Compute elevation gain/loss + elev_arr = np.array(elevations) + elev_diff = np.diff(elev_arr) + elev_gain = np.sum(elev_diff[elev_diff > 0]) + elev_loss = np.sum(np.abs(elev_diff[elev_diff < 0])) + + # Build GeoJSON + geojson = { + "type": "Feature", + "properties": { + "type": "offroute_prototype", + "start": {"lat": START_LAT, "lon": START_LON}, + "end": {"lat": END_LAT, "lon": END_LON}, + "total_time_seconds": float(end_cost), + "total_time_minutes": float(end_cost / 60), + "total_distance_m": float(total_distance_m), + "total_distance_km": float(total_distance_m / 1000), + "elevation_gain_m": float(elev_gain), + "elevation_loss_m": float(elev_loss), + "min_elevation_m": float(np.min(elev_arr)), + "max_elevation_m": float(np.max(elev_arr)), + "cell_count": len(path_indices), + "cell_size_m": meta["cell_size_m"], + }, + "geometry": { + "type": "LineString", + "coordinates": coordinates, + } + } + + # Write output + OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True) + with open(OUTPUT_PATH, "w") as f: + json.dump(geojson, f, indent=2) + + t_end = time.time() + + # Final report + print(f"\n" + "=" * 60) + print("RESULTS") + print("=" * 60) + print(f"Start: ({START_LAT:.4f}, {START_LON:.4f})") + print(f"End: ({END_LAT:.4f}, {END_LON:.4f})") + print(f"Total effort: {end_cost/60:.1f} minutes ({end_cost/3600:.2f} hours)") + print(f"Distance: {total_distance_m/1000:.2f} km") + print(f"Elevation gain: {elev_gain:.0f} m") + print(f"Elevation loss: {elev_loss:.0f} m") + print(f"Elevation range: {np.min(elev_arr):.0f} - {np.max(elev_arr):.0f} m") + print(f"Path cells: {len(path_indices):,}") + print(f"Wall time: {t_end - t0:.1f}s") + print(f"\nOutput saved to: {OUTPUT_PATH}") + + # Validation checks + print(f"\n" + "-" * 60) + print("VALIDATION") + print("-" * 60) + + # Check coordinates are within bbox + lons = [c[0] for c in coordinates] + lats = [c[1] for c in coordinates] + lon_ok = BBOX["west"] <= min(lons) and max(lons) <= BBOX["east"] + lat_ok = BBOX["south"] <= min(lats) and max(lats) <= BBOX["north"] + print(f"Coordinates within bbox: {'PASS' if lon_ok and lat_ok else 'FAIL'}") + + # Check path is not trivial + is_nontrivial = len(path_indices) > 10 and total_distance_m > 1000 + print(f"Path is non-trivial: {'PASS' if is_nontrivial else 'FAIL'}") + + # Check it's not a straight line (measure sinuosity) + straight_line_dist = np.sqrt( + (coordinates[-1][0] - coordinates[0][0])**2 + + (coordinates[-1][1] - coordinates[0][1])**2 + ) * 111000 # rough degrees to meters + sinuosity = total_distance_m / max(straight_line_dist, 1) + print(f"Sinuosity: {sinuosity:.2f} (>1.0 means path curves around obstacles)") + + reader.close() + print("\nPrototype completed successfully.") + + +if __name__ == "__main__": + main() From 26d4bc74784c7586ce14318a1d2cbf8d00f13b8e Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 8 May 2026 06:33:45 +0000 Subject: [PATCH 04/41] =?UTF-8?q?feat(offroute):=20Phase=20O2b=20=E2=80=94?= =?UTF-8?q?=20WorldCover=20friction=20integration,=20lake=20avoidance=20va?= =?UTF-8?q?lidated?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - New friction.py: reads WorldCover friction VRT, resamples to match elevation grid, provides point sampling for validation - Modified cost.py: accepts optional friction array, multiplies Tobler time cost by friction multiplier, inf for water/nodata (255/0) - Modified prototype.py: loads friction layer, passes to cost function, validates path avoids water cells (friction=255) Validated on Idaho test bbox: - Path avoids Murtaugh Lake (no water cells on path) - Friction along path: min=10, max=20, mean=10.2 - Effort increased 3.4% vs Phase O1 due to friction multipliers Co-Authored-By: Claude Opus 4.5 --- lib/offroute/cost.py | 226 ++++++++++++++++++++++---------------- lib/offroute/friction.py | 137 +++++++++++++++++++++++ lib/offroute/prototype.py | 190 +++++++++++++++++++++++++------- 3 files changed, 420 insertions(+), 133 deletions(-) create mode 100644 lib/offroute/friction.py diff --git a/lib/offroute/cost.py b/lib/offroute/cost.py index f460ab9..3607de6 100644 --- a/lib/offroute/cost.py +++ b/lib/offroute/cost.py @@ -1,94 +1,132 @@ -""" -Tobler off-path hiking cost function for OFFROUTE. - -Computes travel time cost based on terrain slope using Tobler's -hiking function with off-trail penalty. -""" -import math -import numpy as np -from typing import Tuple - -# Maximum passable slope in degrees -MAX_SLOPE_DEG = 40.0 - -# Tobler off-path parameters -TOBLER_BASE_SPEED = 6.0 -TOBLER_OFF_TRAIL_MULT = 0.6 - - -def tobler_speed(grade: float) -> float: - """ - Calculate hiking speed using Tobler's off-path function. - - speed_kmh = 0.6 * 6.0 * exp(-3.5 * |grade + 0.05|) - - Peak speed is ~3.6 km/h at grade = -0.05 (slight downhill). - """ - return TOBLER_OFF_TRAIL_MULT * TOBLER_BASE_SPEED * math.exp(-3.5 * abs(grade + 0.05)) - - -def compute_cost_grid( - elevation: np.ndarray, - cell_size_m: float, - cell_size_lat_m: float = None, - cell_size_lon_m: float = None -) -> np.ndarray: - """ - Compute isotropic travel cost grid from elevation data. - - Each cell's cost represents the time (in seconds) to traverse that cell, - based on the average slope from neighboring cells. - """ - if cell_size_lat_m is None: - cell_size_lat_m = cell_size_m - if cell_size_lon_m is None: - cell_size_lon_m = cell_size_m - - rows, cols = elevation.shape - - # Compute gradients in both directions - dy = np.zeros_like(elevation) - dx = np.zeros_like(elevation) - - # Central differences for interior, forward/backward at edges - dy[1:-1, :] = (elevation[:-2, :] - elevation[2:, :]) / (2 * cell_size_lat_m) - dy[0, :] = (elevation[0, :] - elevation[1, :]) / cell_size_lat_m - dy[-1, :] = (elevation[-2, :] - elevation[-1, :]) / cell_size_lat_m - - dx[:, 1:-1] = (elevation[:, 2:] - elevation[:, :-2]) / (2 * cell_size_lon_m) - dx[:, 0] = (elevation[:, 1] - elevation[:, 0]) / cell_size_lon_m - dx[:, -1] = (elevation[:, -1] - elevation[:, -2]) / cell_size_lon_m - - # Compute slope magnitude (grade = rise/run) - grade_magnitude = np.sqrt(dx**2 + dy**2) - - # Convert to slope angle in degrees - slope_deg = np.degrees(np.arctan(grade_magnitude)) - - # Compute speed for each cell using Tobler function - speed_kmh = TOBLER_OFF_TRAIL_MULT * TOBLER_BASE_SPEED * np.exp(-3.5 * np.abs(grade_magnitude + 0.05)) - - # Convert speed to time cost (seconds to traverse one cell) - avg_cell_size = (cell_size_lat_m + cell_size_lon_m) / 2 - cost = avg_cell_size * 3.6 / speed_kmh - - # Set impassable cells (slope > MAX_SLOPE_DEG) to infinity - cost[slope_deg > MAX_SLOPE_DEG] = np.inf - - # Handle NaN elevations (no data) - cost[np.isnan(elevation)] = np.inf - - return cost - - -if __name__ == "__main__": - print("Testing Tobler speed function:") - for grade in [-0.3, -0.1, -0.05, 0.0, 0.05, 0.1, 0.3]: - speed = tobler_speed(grade) - print(f" Grade {grade:+.2f}: {speed:.2f} km/h") - - print("\nTesting cost grid computation:") - elev = np.arange(100).reshape(10, 10).astype(np.float32) * 10 - cost = compute_cost_grid(elev, cell_size_m=30.0) - print(f" Elevation range: {elev.min():.0f} - {elev.max():.0f} m") - print(f" Cost range: {cost[~np.isinf(cost)].min():.1f} - {cost[~np.isinf(cost)].max():.1f} s") +""" +Tobler off-path hiking cost function for OFFROUTE. + +Computes travel time cost based on terrain slope using Tobler's +hiking function with off-trail penalty. Optionally applies friction +multipliers from land cover data. +""" +import math +import numpy as np +from typing import Optional + +# Maximum passable slope in degrees +MAX_SLOPE_DEG = 40.0 + +# Tobler off-path parameters +TOBLER_BASE_SPEED = 6.0 +TOBLER_OFF_TRAIL_MULT = 0.6 + + +def tobler_speed(grade: float) -> float: + """ + Calculate hiking speed using Tobler's off-path function. + + speed_kmh = 0.6 * 6.0 * exp(-3.5 * |grade + 0.05|) + + Peak speed is ~3.6 km/h at grade = -0.05 (slight downhill). + """ + return TOBLER_OFF_TRAIL_MULT * TOBLER_BASE_SPEED * math.exp(-3.5 * abs(grade + 0.05)) + + +def compute_cost_grid( + elevation: np.ndarray, + cell_size_m: float, + cell_size_lat_m: float = None, + cell_size_lon_m: float = None, + friction: Optional[np.ndarray] = None +) -> np.ndarray: + """ + Compute isotropic travel cost grid from elevation data. + + Each cell's cost represents the time (in seconds) to traverse that cell, + based on the average slope from neighboring cells. + + Args: + elevation: 2D array of elevation values in meters + cell_size_m: Average cell size in meters + cell_size_lat_m: Cell size in latitude direction (optional) + cell_size_lon_m: Cell size in longitude direction (optional) + friction: Optional 2D array of friction multipliers. + Values should be float (1.0 = baseline, 2.0 = 2x slower). + np.inf marks impassable cells. + If None, no friction is applied (backward compatible). + + Returns: + 2D array of travel cost in seconds per cell. + np.inf for impassable cells. + """ + if cell_size_lat_m is None: + cell_size_lat_m = cell_size_m + if cell_size_lon_m is None: + cell_size_lon_m = cell_size_m + + rows, cols = elevation.shape + + # Compute gradients in both directions + dy = np.zeros_like(elevation) + dx = np.zeros_like(elevation) + + # Central differences for interior, forward/backward at edges + dy[1:-1, :] = (elevation[:-2, :] - elevation[2:, :]) / (2 * cell_size_lat_m) + dy[0, :] = (elevation[0, :] - elevation[1, :]) / cell_size_lat_m + dy[-1, :] = (elevation[-2, :] - elevation[-1, :]) / cell_size_lat_m + + dx[:, 1:-1] = (elevation[:, 2:] - elevation[:, :-2]) / (2 * cell_size_lon_m) + dx[:, 0] = (elevation[:, 1] - elevation[:, 0]) / cell_size_lon_m + dx[:, -1] = (elevation[:, -1] - elevation[:, -2]) / cell_size_lon_m + + # Compute slope magnitude (grade = rise/run) + grade_magnitude = np.sqrt(dx**2 + dy**2) + + # Convert to slope angle in degrees + slope_deg = np.degrees(np.arctan(grade_magnitude)) + + # Compute speed for each cell using Tobler function + speed_kmh = TOBLER_OFF_TRAIL_MULT * TOBLER_BASE_SPEED * np.exp(-3.5 * np.abs(grade_magnitude + 0.05)) + + # Convert speed to time cost (seconds to traverse one cell) + avg_cell_size = (cell_size_lat_m + cell_size_lon_m) / 2 + cost = avg_cell_size * 3.6 / speed_kmh + + # Set impassable cells (slope > MAX_SLOPE_DEG) to infinity + cost[slope_deg > MAX_SLOPE_DEG] = np.inf + + # Handle NaN elevations (no data) + cost[np.isnan(elevation)] = np.inf + + # Apply friction multipliers if provided + if friction is not None: + if friction.shape != elevation.shape: + raise ValueError( + f"Friction shape {friction.shape} does not match elevation shape {elevation.shape}" + ) + # Multiply cost by friction (inf * anything = inf, which is correct) + cost = cost * friction + + return cost + + +if __name__ == "__main__": + print("Testing Tobler speed function:") + for grade in [-0.3, -0.1, -0.05, 0.0, 0.05, 0.1, 0.3]: + speed = tobler_speed(grade) + print(f" Grade {grade:+.2f}: {speed:.2f} km/h") + + print("\nTesting cost grid computation (no friction):") + elev = np.arange(100).reshape(10, 10).astype(np.float32) * 10 + cost = compute_cost_grid(elev, cell_size_m=30.0) + print(f" Elevation range: {elev.min():.0f} - {elev.max():.0f} m") + finite = cost[~np.isinf(cost)] + if len(finite) > 0: + print(f" Cost range: {finite.min():.1f} - {finite.max():.1f} s") + else: + print(f" All cells impassable (test data too steep)") + + print("\nTesting cost grid with friction:") + elev = np.ones((10, 10), dtype=np.float32) * 1000 # flat terrain + friction = np.ones((10, 10), dtype=np.float32) * 1.5 # 1.5x friction + friction[5, 5] = np.inf # one impassable cell + cost = compute_cost_grid(elev, cell_size_m=30.0, friction=friction) + print(f" Base cost (flat, 30m cell): {30 * 3.6 / (0.6 * 6.0 * np.exp(-3.5 * 0.05)):.1f} s") + print(f" With 1.5x friction: {cost[0, 0]:.1f} s") + print(f" Impassable cells: {np.sum(np.isinf(cost))}") diff --git a/lib/offroute/friction.py b/lib/offroute/friction.py new file mode 100644 index 0000000..32df0c0 --- /dev/null +++ b/lib/offroute/friction.py @@ -0,0 +1,137 @@ +""" +Friction layer reader for OFFROUTE. + +Reads friction values from the WorldCover friction VRT and resamples +to match the elevation grid dimensions. +""" +import numpy as np +from pathlib import Path +from typing import Tuple, Optional + +try: + import rasterio + from rasterio.windows import from_bounds + from rasterio.enums import Resampling +except ImportError: + raise ImportError("rasterio is required for friction layer support") + +# Default path to the friction VRT +DEFAULT_FRICTION_PATH = Path("/mnt/nav/worldcover/friction/friction_conus.vrt") + + +class FrictionReader: + """Reader for WorldCover friction raster.""" + + def __init__(self, friction_path: Path = DEFAULT_FRICTION_PATH): + self.friction_path = friction_path + self._dataset = None + + def _open(self): + """Lazy open the dataset.""" + if self._dataset is None: + self._dataset = rasterio.open(self.friction_path) + return self._dataset + + def get_friction_grid( + self, + south: float, + north: float, + west: float, + east: float, + target_shape: Tuple[int, int] + ) -> np.ndarray: + """ + Get friction values for a bounding box, resampled to target shape. + + Args: + south, north, west, east: Bounding box coordinates + target_shape: (rows, cols) to resample to (matches elevation grid) + + Returns: + np.ndarray of uint8 friction values, same shape as target_shape. + Values: 10-40 = friction multiplier (divide by 10) + 255 = impassable + 0 = nodata (treat as impassable) + """ + ds = self._open() + + # Create a window from the bounding box + window = from_bounds(west, south, east, north, ds.transform) + + # Read with resampling to target shape + # Use nearest neighbor for categorical data + friction = ds.read( + 1, + window=window, + out_shape=target_shape, + resampling=Resampling.nearest + ) + + return friction + + def sample_point(self, lat: float, lon: float) -> int: + """Sample friction value at a single point.""" + ds = self._open() + + # Get pixel coordinates + row, col = ds.index(lon, lat) + + # Check bounds + if row < 0 or row >= ds.height or col < 0 or col >= ds.width: + return 0 # Out of bounds = nodata + + # Read single pixel + window = rasterio.windows.Window(col, row, 1, 1) + value = ds.read(1, window=window) + return int(value[0, 0]) + + def close(self): + """Close the dataset.""" + if self._dataset is not None: + self._dataset.close() + self._dataset = None + + +def friction_to_multiplier(friction: np.ndarray) -> np.ndarray: + """ + Convert friction values to cost multipliers. + + Args: + friction: uint8 array of friction values + + Returns: + float32 array of multipliers. + Values 10-40 become 1.0-4.0 (divide by 10). + Values 0 or 255 become np.inf (impassable). + """ + multiplier = friction.astype(np.float32) / 10.0 + + # Mark impassable cells + multiplier[friction == 0] = np.inf # nodata + multiplier[friction == 255] = np.inf # water/impassable + + return multiplier + + +if __name__ == "__main__": + print("Testing FrictionReader...") + + reader = FrictionReader() + + # Test point sampling - Murtaugh Lake (should be water = 255) + lake_lat, lake_lon = 42.47, -114.15 + lake_friction = reader.sample_point(lake_lat, lake_lon) + print(f"Murtaugh Lake ({lake_lat}, {lake_lon}): friction = {lake_friction}") + print(f" Expected: 255 (water/impassable)") + + # Test grid read for small bbox + friction = reader.get_friction_grid( + south=42.4, north=42.5, west=-114.2, east=-114.1, + target_shape=(100, 100) + ) + print(f"\nGrid test shape: {friction.shape}") + print(f"Unique values: {np.unique(friction)}") + print(f"Water cells (255): {np.sum(friction == 255)}") + + reader.close() + print("\nFrictionReader test complete.") diff --git a/lib/offroute/prototype.py b/lib/offroute/prototype.py index 0790a32..9822021 100755 --- a/lib/offroute/prototype.py +++ b/lib/offroute/prototype.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 """ -OFFROUTE Phase O1 Prototype +OFFROUTE Phase O2b Prototype -Validates the PMTiles decoder, Tobler cost function, and MCP pathfinder -on a real Idaho bounding box. +Validates the PMTiles decoder, Tobler cost function, WorldCover friction +integration, and MCP pathfinder on a real Idaho bounding box. + +Now includes friction layer to avoid water bodies like Murtaugh Lake. Test bbox (four Idaho towns as corners): SW: Rogerson, ID (~42.21, -114.60) @@ -25,6 +27,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent.parent)) from lib.offroute.dem import DEMReader from lib.offroute.cost import compute_cost_grid +from lib.offroute.friction import FrictionReader, friction_to_multiplier # Test bounding box BBOX = { @@ -43,8 +46,18 @@ START_LON = -114.50 END_LAT = 42.52 END_LON = -113.85 -# Output file -OUTPUT_PATH = Path("/opt/recon/data/offroute-test.geojson") +# Murtaugh Lake - actual water extent from WorldCover +LAKE_BOUNDS = { + "south": 42.44, + "north": 42.50, + "west": -114.20, + "east": -114.10, +} +LAKE_CENTER = (42.465, -114.155) # Verified water in WorldCover + +# Output files +OUTPUT_PATH_O1 = Path("/opt/recon/data/offroute-test.geojson") +OUTPUT_PATH_FRICTION = Path("/opt/recon/data/offroute-test-friction.geojson") # Memory limit in GB MEMORY_LIMIT_GB = 12 @@ -64,19 +77,28 @@ def check_memory_usage(): return 0 +def path_crosses_lake(coordinates, lake_bounds): + """Check if any path coordinates fall within the lake bounding box.""" + for lon, lat in coordinates: + if (lake_bounds["south"] <= lat <= lake_bounds["north"] and + lake_bounds["west"] <= lon <= lake_bounds["east"]): + return True, (lat, lon) + return False, None + + def main(): print("=" * 60) - print("OFFROUTE Phase O1 Prototype") + print("OFFROUTE Phase O2b Prototype (with Friction)") print("=" * 60) t0 = time.time() # Step 1: Load elevation data print(f"\n[1] Loading DEM for bbox: {BBOX}") - reader = DEMReader() + dem_reader = DEMReader() t1 = time.time() - elevation, meta = reader.get_elevation_grid( + elevation, meta = dem_reader.get_elevation_grid( south=BBOX["south"], north=BBOX["north"], west=BBOX["west"], @@ -94,25 +116,67 @@ def main(): if mem > 0: print(f" Memory usage: {mem:.1f} GB") - # Step 2: Compute cost grid - print(f"\n[2] Computing Tobler cost grid...") + # Step 2: Load friction data + print(f"\n[2] Loading WorldCover friction layer...") + t2a = time.time() + + friction_reader = FrictionReader() + + # Validate lake is marked as impassable + lake_friction = friction_reader.sample_point(LAKE_CENTER[0], LAKE_CENTER[1]) + print(f" Murtaugh Lake center ({LAKE_CENTER[0]}, {LAKE_CENTER[1]}): friction = {lake_friction}") + if lake_friction != 255: + print(f" WARNING: Lake not marked as water (expected 255, got {lake_friction})") + else: + print(f" Lake correctly marked as impassable (255)") + + # Load friction grid matching elevation shape + friction_raw = friction_reader.get_friction_grid( + south=BBOX["south"], + north=BBOX["north"], + west=BBOX["west"], + east=BBOX["east"], + target_shape=elevation.shape + ) + t2b = time.time() + + # Convert to multipliers + friction_mult = friction_to_multiplier(friction_raw) + + impassable_count = np.sum(np.isinf(friction_mult)) + print(f" Friction grid shape: {friction_raw.shape}") + print(f" Unique friction values: {np.unique(friction_raw[friction_raw > 0])}") + print(f" Impassable cells (water/nodata): {impassable_count:,} ({100*impassable_count/friction_raw.size:.1f}%)") + print(f" Load time: {t2b - t2a:.1f}s") + + mem = check_memory_usage() + if mem > 0: + print(f" Memory usage: {mem:.1f} GB") + + # Step 3: Compute cost grid with friction + print(f"\n[3] Computing Tobler cost grid with friction...") t3 = time.time() - cost = compute_cost_grid(elevation, cell_size_m=meta["cell_size_m"]) + cost = compute_cost_grid( + elevation, + cell_size_m=meta["cell_size_m"], + friction=friction_mult + ) t4 = time.time() finite_cost = cost[~np.isinf(cost)] + total_impassable = np.sum(np.isinf(cost)) print(f" Cost range: {finite_cost.min():.1f} - {finite_cost.max():.1f} s/cell") - print(f" Impassable cells: {np.sum(np.isinf(cost)):,} ({100*np.sum(np.isinf(cost))/cost.size:.1f}%)") + print(f" Total impassable cells: {total_impassable:,} ({100*total_impassable/cost.size:.1f}%)") print(f" Compute time: {t4 - t3:.1f}s") mem = check_memory_usage() if mem > 0: print(f" Memory usage: {mem:.1f} GB") - # Step 3: Convert start/end to pixel coordinates - print(f"\n[3] Converting coordinates...") - start_row, start_col = reader.latlon_to_pixel(START_LAT, START_LON, meta) - end_row, end_col = reader.latlon_to_pixel(END_LAT, END_LON, meta) + # Step 4: Convert start/end to pixel coordinates + print(f"\n[4] Converting coordinates...") + start_row, start_col = dem_reader.latlon_to_pixel(START_LAT, START_LON, meta) + end_row, end_col = dem_reader.latlon_to_pixel(END_LAT, END_LON, meta) print(f" Start: ({START_LAT}, {START_LON}) -> pixel ({start_row}, {start_col})") print(f" End: ({END_LAT}, {END_LON}) -> pixel ({end_row}, {end_col})") @@ -131,21 +195,16 @@ def main(): print(f" Start elevation: {start_elev:.0f} m") print(f" End elevation: {end_elev:.0f} m") - # Step 4: Run MCP pathfinder - print(f"\n[4] Running MCP_Geometric pathfinder...") + # Step 5: Run MCP pathfinder + print(f"\n[5] Running MCP_Geometric pathfinder...") t5 = time.time() - # MCP_Geometric finds minimum cost path - # It uses Dijkstra's algorithm internally mcp = MCP_Geometric(cost, fully_connected=True) - - # Find costs from start to all reachable cells cumulative_costs, traceback = mcp.find_costs([(start_row, start_col)]) t6 = time.time() print(f" Dijkstra completed in {t6 - t5:.1f}s") - # Get cost to reach end point end_cost = cumulative_costs[end_row, end_col] print(f" Total cost to endpoint: {end_cost:.0f} seconds ({end_cost/60:.1f} minutes)") @@ -153,7 +212,6 @@ def main(): print("ERROR: No path found to endpoint (blocked by impassable terrain)") sys.exit(1) - # Trace back the path t7 = time.time() path_indices = mcp.traceback((end_row, end_col)) t8 = time.time() @@ -165,25 +223,27 @@ def main(): if mem > 0: print(f" Memory usage: {mem:.1f} GB") - # Step 5: Convert path to coordinates and compute stats - print(f"\n[5] Converting path to GeoJSON...") + # Step 6: Convert path to coordinates and compute stats + print(f"\n[6] Converting path to GeoJSON...") coordinates = [] elevations = [] + friction_values = [] for row, col in path_indices: - lat, lon = reader.pixel_to_latlon(row, col, meta) + lat, lon = dem_reader.pixel_to_latlon(row, col, meta) elev = elevation[row, col] - coordinates.append([lon, lat]) # GeoJSON is [lon, lat] + fric = friction_raw[row, col] + coordinates.append([lon, lat]) elevations.append(elev) + friction_values.append(fric) # Compute path distance total_distance_m = 0 for i in range(1, len(coordinates)): lon1, lat1 = coordinates[i-1] lon2, lat2 = coordinates[i] - # Haversine formula - R = 6371000 # Earth radius in meters + R = 6371000 dlat = np.radians(lat2 - lat1) dlon = np.radians(lon2 - lon1) a = np.sin(dlat/2)**2 + np.cos(np.radians(lat1)) * np.cos(np.radians(lat2)) * np.sin(dlon/2)**2 @@ -196,11 +256,16 @@ def main(): elev_gain = np.sum(elev_diff[elev_diff > 0]) elev_loss = np.sum(np.abs(elev_diff[elev_diff < 0])) + # Friction stats along path + fric_arr = np.array(friction_values) + valid_fric = fric_arr[(fric_arr > 0) & (fric_arr < 255)] + # Build GeoJSON geojson = { "type": "Feature", "properties": { - "type": "offroute_prototype", + "type": "offroute_prototype_friction", + "phase": "O2b", "start": {"lat": START_LAT, "lon": START_LON}, "end": {"lat": END_LAT, "lon": END_LON}, "total_time_seconds": float(end_cost), @@ -211,6 +276,9 @@ def main(): "elevation_loss_m": float(elev_loss), "min_elevation_m": float(np.min(elev_arr)), "max_elevation_m": float(np.max(elev_arr)), + "friction_min": int(valid_fric.min()) if len(valid_fric) > 0 else 0, + "friction_max": int(valid_fric.max()) if len(valid_fric) > 0 else 0, + "friction_mean": float(valid_fric.mean()) if len(valid_fric) > 0 else 0, "cell_count": len(path_indices), "cell_size_m": meta["cell_size_m"], }, @@ -221,15 +289,15 @@ def main(): } # Write output - OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True) - with open(OUTPUT_PATH, "w") as f: + OUTPUT_PATH_FRICTION.parent.mkdir(parents=True, exist_ok=True) + with open(OUTPUT_PATH_FRICTION, "w") as f: json.dump(geojson, f, indent=2) t_end = time.time() # Final report print(f"\n" + "=" * 60) - print("RESULTS") + print("RESULTS (Phase O2b with Friction)") print("=" * 60) print(f"Start: ({START_LAT:.4f}, {START_LON:.4f})") print(f"End: ({END_LAT:.4f}, {END_LON:.4f})") @@ -238,11 +306,13 @@ def main(): print(f"Elevation gain: {elev_gain:.0f} m") print(f"Elevation loss: {elev_loss:.0f} m") print(f"Elevation range: {np.min(elev_arr):.0f} - {np.max(elev_arr):.0f} m") + if len(valid_fric) > 0: + print(f"Friction (path): min={valid_fric.min()}, max={valid_fric.max()}, mean={valid_fric.mean():.1f}") print(f"Path cells: {len(path_indices):,}") print(f"Wall time: {t_end - t0:.1f}s") - print(f"\nOutput saved to: {OUTPUT_PATH}") + print(f"\nOutput saved to: {OUTPUT_PATH_FRICTION}") - # Validation checks + # Validation print(f"\n" + "-" * 60) print("VALIDATION") print("-" * 60) @@ -258,15 +328,57 @@ def main(): is_nontrivial = len(path_indices) > 10 and total_distance_m > 1000 print(f"Path is non-trivial: {'PASS' if is_nontrivial else 'FAIL'}") - # Check it's not a straight line (measure sinuosity) + # Check sinuosity straight_line_dist = np.sqrt( (coordinates[-1][0] - coordinates[0][0])**2 + (coordinates[-1][1] - coordinates[0][1])**2 - ) * 111000 # rough degrees to meters + ) * 111000 sinuosity = total_distance_m / max(straight_line_dist, 1) print(f"Sinuosity: {sinuosity:.2f} (>1.0 means path curves around obstacles)") - reader.close() + # CRITICAL: Check no water cells (friction=255) on path + # This is the authoritative test - friction layer prevents water crossings + print(f"\n--- Water Avoidance Check ---") + water_on_path = np.sum(fric_arr == 255) + if water_on_path > 0: + print(f"FAIL: Path crosses {water_on_path} water cells (friction=255)") + sys.exit(1) + else: + print(f"PASS: No water cells (friction=255) on path") + + # Informational: Check if path goes through lake bounding box + # Path may go through land cells within the bbox, which is fine + print(f"\n--- Lake Bounding Box Check (informational) ---") + print(f"Murtaugh Lake bounds: {LAKE_BOUNDS}") + crosses_lake, crossing_point = path_crosses_lake(coordinates, LAKE_BOUNDS) + if crosses_lake: + print(f"INFO: Path passes through lake bbox at {crossing_point}") + print(f" (This is OK if friction check passed - path uses land cells)") + else: + print(f"PASS: Path does not enter lake bounding box") + + # Compare with Phase O1 if available + print(f"\n" + "-" * 60) + print("COMPARISON: Phase O1 vs O2b") + print("-" * 60) + + if OUTPUT_PATH_O1.exists(): + with open(OUTPUT_PATH_O1) as f: + o1_data = json.load(f) + o1_props = o1_data["properties"] + + print(f"{'Metric':<20} {'O1 (no friction)':<20} {'O2b (with friction)':<20}") + print("-" * 60) + print(f"{'Distance (km)':<20} {o1_props['total_distance_km']:<20.2f} {total_distance_m/1000:<20.2f}") + print(f"{'Effort (min)':<20} {o1_props['total_time_minutes']:<20.1f} {end_cost/60:<20.1f}") + print(f"{'Cell count':<20} {o1_props['cell_count']:<20} {len(path_indices):<20}") + print(f"{'Elev gain (m)':<20} {o1_props['elevation_gain_m']:<20.0f} {elev_gain:<20.0f}") + else: + print(f"Phase O1 output not found at {OUTPUT_PATH_O1}") + print(f"Run the O1 prototype first to enable comparison.") + + dem_reader.close() + friction_reader.close() print("\nPrototype completed successfully.") From e0eedcedfde5672fa3a61efe286ceb0082a6a838 Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 8 May 2026 06:53:11 +0000 Subject: [PATCH 05/41] =?UTF-8?q?feat(offroute):=20Phase=20O2c=20=E2=80=94?= =?UTF-8?q?=20PAD-US=20barriers=20with=20three-mode=20boundary=20respect?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add barriers.py: PAD-US raster reader + build_barriers_raster() function - Rasterize PAD-US Pub_Access=XA (Closed) polygons to CONUS GeoTIFF - Modify cost.py: boundary_mode parameter (strict/pragmatic/emergency) - strict: private land = impassable (np.inf) - pragmatic: private land = 5x friction penalty (default) - emergency: private land barriers ignored - Modify prototype.py: three-way comparison output - Output: padus_barriers.tif at /mnt/nav/worldcover/ (144MB, ~33m resolution) Co-Authored-By: Claude Opus 4.5 --- lib/offroute/barriers.py | 266 ++++++++++++++++++++ lib/offroute/cost.py | 310 ++++++++++++++---------- lib/offroute/prototype.py | 494 +++++++++++++++++++------------------- 3 files changed, 694 insertions(+), 376 deletions(-) create mode 100644 lib/offroute/barriers.py diff --git a/lib/offroute/barriers.py b/lib/offroute/barriers.py new file mode 100644 index 0000000..7fcad75 --- /dev/null +++ b/lib/offroute/barriers.py @@ -0,0 +1,266 @@ +""" +PAD-US barrier layer for OFFROUTE. + +Provides access to the PAD-US land ownership raster for routing decisions. +Cells with value 255 represent closed/restricted areas (Pub_Access = XA). + +Build function rasterizes PAD-US geodatabase to aligned GeoTIFF. +Runtime functions read the raster and resample to match elevation grids. +""" +import numpy as np +from pathlib import Path +from typing import Tuple, Optional +import subprocess +import tempfile +import os + +try: + import rasterio + from rasterio.windows import from_bounds + from rasterio.enums import Resampling +except ImportError: + raise ImportError("rasterio is required for barriers layer support") + +# Paths +DEFAULT_BARRIERS_PATH = Path("/mnt/nav/worldcover/padus_barriers.tif") +PADUS_GDB_PATH = Path("/mnt/nav/padus/PADUS4_0_Geodatabase.gdb") +PADUS_LAYER = "PADUS4_0Combined_Proclamation_Marine_Fee_Designation_Easement" + +# CONUS bounding box in WGS84 +CONUS_BOUNDS = { + "west": -125.0, + "east": -66.0, + "south": 24.0, + "north": 50.0, +} + +# Resolution in degrees (~30m at mid-latitudes) +PIXEL_SIZE = 0.0003 # ~33m + + +class BarrierReader: + """Reader for PAD-US barrier raster.""" + + def __init__(self, barrier_path: Path = DEFAULT_BARRIERS_PATH): + self.barrier_path = barrier_path + self._dataset = None + + def _open(self): + """Lazy open the dataset.""" + if self._dataset is None: + if not self.barrier_path.exists(): + raise FileNotFoundError( + f"Barrier raster not found at {self.barrier_path}. " + f"Run build_barriers_raster() first." + ) + self._dataset = rasterio.open(self.barrier_path) + return self._dataset + + def get_barrier_grid( + self, + south: float, + north: float, + west: float, + east: float, + target_shape: Tuple[int, int] + ) -> np.ndarray: + """ + Get barrier values for a bounding box, resampled to target shape. + + Args: + south, north, west, east: Bounding box coordinates (WGS84) + target_shape: (rows, cols) to resample to (matches elevation grid) + + Returns: + np.ndarray of uint8 barrier values: + 255 = closed/restricted (impassable when respect_boundaries=True) + 0 = public/accessible + """ + ds = self._open() + + # Create a window from the bounding box + window = from_bounds(west, south, east, north, ds.transform) + + # Read with resampling to target shape + barriers = ds.read( + 1, + window=window, + out_shape=target_shape, + resampling=Resampling.nearest + ) + + return barriers + + def sample_point(self, lat: float, lon: float) -> int: + """Sample barrier value at a single point.""" + ds = self._open() + + # Get pixel coordinates + row, col = ds.index(lon, lat) + + # Check bounds + if row < 0 or row >= ds.height or col < 0 or col >= ds.width: + return 0 # Out of bounds = accessible + + # Read single pixel + window = rasterio.windows.Window(col, row, 1, 1) + value = ds.read(1, window=window) + return int(value[0, 0]) + + def close(self): + """Close the dataset.""" + if self._dataset is not None: + self._dataset.close() + self._dataset = None + + +def build_barriers_raster( + output_path: Path = DEFAULT_BARRIERS_PATH, + gdb_path: Path = PADUS_GDB_PATH, + pixel_size: float = PIXEL_SIZE, + bounds: dict = CONUS_BOUNDS, +) -> Path: + """ + Build the PAD-US barriers raster from the source geodatabase. + + Extracts polygons where Pub_Access = 'XA' (Closed) and rasterizes them. + + Args: + output_path: Output GeoTIFF path + gdb_path: Path to PAD-US geodatabase + pixel_size: Pixel size in degrees + bounds: CONUS bounding box + + Returns: + Path to the created raster + """ + import shutil + + if not gdb_path.exists(): + raise FileNotFoundError(f"PAD-US geodatabase not found at {gdb_path}") + + # Check for required tools + if not shutil.which('ogr2ogr'): + raise RuntimeError("ogr2ogr not found. Install GDAL.") + if not shutil.which('gdal_rasterize'): + raise RuntimeError("gdal_rasterize not found. Install GDAL.") + + output_path.parent.mkdir(parents=True, exist_ok=True) + + print(f"Building PAD-US barriers raster...") + print(f" Source: {gdb_path}") + print(f" Output: {output_path}") + print(f" Pixel size: {pixel_size} degrees (~{pixel_size * 111000:.0f}m)") + print(f" Bounds: {bounds}") + + with tempfile.TemporaryDirectory() as tmpdir: + # Step 1: Extract closed areas and reproject to WGS84 + closed_gpkg = Path(tmpdir) / "closed_areas.gpkg" + + print(f"\n[1/3] Extracting closed areas (Pub_Access = 'XA')...") + + ogr_cmd = [ + "ogr2ogr", + "-f", "GPKG", + str(closed_gpkg), + str(gdb_path), + PADUS_LAYER, + "-where", "Pub_Access = 'XA'", + "-t_srs", "EPSG:4326", + "-nlt", "MULTIPOLYGON", + "-nln", "closed_areas", + ] + + result = subprocess.run(ogr_cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"STDERR: {result.stderr}") + raise RuntimeError(f"ogr2ogr failed: {result.stderr}") + + # Check feature count + info_cmd = ["ogrinfo", "-so", str(closed_gpkg), "closed_areas"] + info_result = subprocess.run(info_cmd, capture_output=True, text=True) + print(f" Extraction result:\n{info_result.stdout}") + + # Step 2: Create empty raster + print(f"\n[2/3] Creating raster grid...") + + width = int((bounds['east'] - bounds['west']) / pixel_size) + height = int((bounds['north'] - bounds['south']) / pixel_size) + + print(f" Grid size: {width} x {height} pixels") + print(f" Memory estimate: {width * height / 1e6:.1f} MB") + + # Step 3: Rasterize + print(f"\n[3/3] Rasterizing closed areas...") + + rasterize_cmd = [ + "gdal_rasterize", + "-burn", "255", + "-init", "0", + "-a_nodata", "0", # No nodata - 0 means accessible + "-te", str(bounds['west']), str(bounds['south']), + str(bounds['east']), str(bounds['north']), + "-tr", str(pixel_size), str(pixel_size), + "-ot", "Byte", + "-co", "COMPRESS=LZW", + "-co", "TILED=YES", + "-l", "closed_areas", + str(closed_gpkg), + str(output_path), + ] + + result = subprocess.run(rasterize_cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"STDERR: {result.stderr}") + raise RuntimeError(f"gdal_rasterize failed: {result.stderr}") + + # Verify output + print(f"\n[Done] Verifying output...") + with rasterio.open(output_path) as ds: + print(f" Size: {ds.width} x {ds.height}") + print(f" CRS: {ds.crs}") + print(f" Bounds: {ds.bounds}") + + # Sample a few tiles to check + sample = ds.read(1, window=rasterio.windows.Window(0, 0, 1000, 1000)) + closed_count = np.sum(sample == 255) + print(f" Sample (1000x1000): {closed_count} closed cells") + + file_size = output_path.stat().st_size / (1024**2) + print(f" File size: {file_size:.1f} MB") + + return output_path + + +if __name__ == "__main__": + import sys + + if len(sys.argv) > 1 and sys.argv[1] == "build": + # Build the raster + print("="*60) + print("PAD-US Barriers Raster Build") + print("="*60) + build_barriers_raster() + else: + # Test the reader + print("Testing BarrierReader...") + + if not DEFAULT_BARRIERS_PATH.exists(): + print(f"Barrier raster not found at {DEFAULT_BARRIERS_PATH}") + print(f"Run: python barriers.py build") + sys.exit(1) + + reader = BarrierReader() + + # Test grid read for Idaho area + barriers = reader.get_barrier_grid( + south=42.2, north=42.6, west=-114.8, east=-113.8, + target_shape=(400, 1000) + ) + print(f"\nGrid test shape: {barriers.shape}") + print(f"Unique values: {np.unique(barriers)}") + closed_cells = np.sum(barriers == 255) + print(f"Closed cells: {closed_cells} ({100*closed_cells/barriers.size:.2f}%)") + + reader.close() + print("\nBarrierReader test complete.") diff --git a/lib/offroute/cost.py b/lib/offroute/cost.py index 3607de6..f31b8f5 100644 --- a/lib/offroute/cost.py +++ b/lib/offroute/cost.py @@ -1,132 +1,178 @@ -""" -Tobler off-path hiking cost function for OFFROUTE. - -Computes travel time cost based on terrain slope using Tobler's -hiking function with off-trail penalty. Optionally applies friction -multipliers from land cover data. -""" -import math -import numpy as np -from typing import Optional - -# Maximum passable slope in degrees -MAX_SLOPE_DEG = 40.0 - -# Tobler off-path parameters -TOBLER_BASE_SPEED = 6.0 -TOBLER_OFF_TRAIL_MULT = 0.6 - - -def tobler_speed(grade: float) -> float: - """ - Calculate hiking speed using Tobler's off-path function. - - speed_kmh = 0.6 * 6.0 * exp(-3.5 * |grade + 0.05|) - - Peak speed is ~3.6 km/h at grade = -0.05 (slight downhill). - """ - return TOBLER_OFF_TRAIL_MULT * TOBLER_BASE_SPEED * math.exp(-3.5 * abs(grade + 0.05)) - - -def compute_cost_grid( - elevation: np.ndarray, - cell_size_m: float, - cell_size_lat_m: float = None, - cell_size_lon_m: float = None, - friction: Optional[np.ndarray] = None -) -> np.ndarray: - """ - Compute isotropic travel cost grid from elevation data. - - Each cell's cost represents the time (in seconds) to traverse that cell, - based on the average slope from neighboring cells. - - Args: - elevation: 2D array of elevation values in meters - cell_size_m: Average cell size in meters - cell_size_lat_m: Cell size in latitude direction (optional) - cell_size_lon_m: Cell size in longitude direction (optional) - friction: Optional 2D array of friction multipliers. - Values should be float (1.0 = baseline, 2.0 = 2x slower). - np.inf marks impassable cells. - If None, no friction is applied (backward compatible). - - Returns: - 2D array of travel cost in seconds per cell. - np.inf for impassable cells. - """ - if cell_size_lat_m is None: - cell_size_lat_m = cell_size_m - if cell_size_lon_m is None: - cell_size_lon_m = cell_size_m - - rows, cols = elevation.shape - - # Compute gradients in both directions - dy = np.zeros_like(elevation) - dx = np.zeros_like(elevation) - - # Central differences for interior, forward/backward at edges - dy[1:-1, :] = (elevation[:-2, :] - elevation[2:, :]) / (2 * cell_size_lat_m) - dy[0, :] = (elevation[0, :] - elevation[1, :]) / cell_size_lat_m - dy[-1, :] = (elevation[-2, :] - elevation[-1, :]) / cell_size_lat_m - - dx[:, 1:-1] = (elevation[:, 2:] - elevation[:, :-2]) / (2 * cell_size_lon_m) - dx[:, 0] = (elevation[:, 1] - elevation[:, 0]) / cell_size_lon_m - dx[:, -1] = (elevation[:, -1] - elevation[:, -2]) / cell_size_lon_m - - # Compute slope magnitude (grade = rise/run) - grade_magnitude = np.sqrt(dx**2 + dy**2) - - # Convert to slope angle in degrees - slope_deg = np.degrees(np.arctan(grade_magnitude)) - - # Compute speed for each cell using Tobler function - speed_kmh = TOBLER_OFF_TRAIL_MULT * TOBLER_BASE_SPEED * np.exp(-3.5 * np.abs(grade_magnitude + 0.05)) - - # Convert speed to time cost (seconds to traverse one cell) - avg_cell_size = (cell_size_lat_m + cell_size_lon_m) / 2 - cost = avg_cell_size * 3.6 / speed_kmh - - # Set impassable cells (slope > MAX_SLOPE_DEG) to infinity - cost[slope_deg > MAX_SLOPE_DEG] = np.inf - - # Handle NaN elevations (no data) - cost[np.isnan(elevation)] = np.inf - - # Apply friction multipliers if provided - if friction is not None: - if friction.shape != elevation.shape: - raise ValueError( - f"Friction shape {friction.shape} does not match elevation shape {elevation.shape}" - ) - # Multiply cost by friction (inf * anything = inf, which is correct) - cost = cost * friction - - return cost - - -if __name__ == "__main__": - print("Testing Tobler speed function:") - for grade in [-0.3, -0.1, -0.05, 0.0, 0.05, 0.1, 0.3]: - speed = tobler_speed(grade) - print(f" Grade {grade:+.2f}: {speed:.2f} km/h") - - print("\nTesting cost grid computation (no friction):") - elev = np.arange(100).reshape(10, 10).astype(np.float32) * 10 - cost = compute_cost_grid(elev, cell_size_m=30.0) - print(f" Elevation range: {elev.min():.0f} - {elev.max():.0f} m") - finite = cost[~np.isinf(cost)] - if len(finite) > 0: - print(f" Cost range: {finite.min():.1f} - {finite.max():.1f} s") - else: - print(f" All cells impassable (test data too steep)") - - print("\nTesting cost grid with friction:") - elev = np.ones((10, 10), dtype=np.float32) * 1000 # flat terrain - friction = np.ones((10, 10), dtype=np.float32) * 1.5 # 1.5x friction - friction[5, 5] = np.inf # one impassable cell - cost = compute_cost_grid(elev, cell_size_m=30.0, friction=friction) - print(f" Base cost (flat, 30m cell): {30 * 3.6 / (0.6 * 6.0 * np.exp(-3.5 * 0.05)):.1f} s") - print(f" With 1.5x friction: {cost[0, 0]:.1f} s") - print(f" Impassable cells: {np.sum(np.isinf(cost))}") +""" +Tobler off-path hiking cost function for OFFROUTE. + +Computes travel time cost based on terrain slope using Tobler's +hiking function with off-trail penalty. Optionally applies friction +multipliers from land cover data and barrier grids from PAD-US. +""" +import math +import numpy as np +from typing import Optional, Literal + +# Maximum passable slope in degrees +MAX_SLOPE_DEG = 40.0 + +# Tobler off-path parameters +TOBLER_BASE_SPEED = 6.0 +TOBLER_OFF_TRAIL_MULT = 0.6 + +# Pragmatic mode friction multiplier for private land +PRAGMATIC_BARRIER_MULTIPLIER = 5.0 + + +def tobler_speed(grade: float) -> float: + """ + Calculate hiking speed using Tobler's off-path function. + + speed_kmh = 0.6 * 6.0 * exp(-3.5 * |grade + 0.05|) + + Peak speed is ~3.6 km/h at grade = -0.05 (slight downhill). + """ + return TOBLER_OFF_TRAIL_MULT * TOBLER_BASE_SPEED * math.exp(-3.5 * abs(grade + 0.05)) + + +def compute_cost_grid( + elevation: np.ndarray, + cell_size_m: float, + cell_size_lat_m: float = None, + cell_size_lon_m: float = None, + friction: Optional[np.ndarray] = None, + barriers: Optional[np.ndarray] = None, + boundary_mode: Literal["strict", "pragmatic", "emergency"] = "pragmatic" +) -> np.ndarray: + """ + Compute isotropic travel cost grid from elevation data. + + Each cell's cost represents the time (in seconds) to traverse that cell, + based on the average slope from neighboring cells. + + Args: + elevation: 2D array of elevation values in meters + cell_size_m: Average cell size in meters + cell_size_lat_m: Cell size in latitude direction (optional) + cell_size_lon_m: Cell size in longitude direction (optional) + friction: Optional 2D array of friction multipliers. + Values should be float (1.0 = baseline, 2.0 = 2x slower). + np.inf marks impassable cells. + If None, no friction is applied (backward compatible). + barriers: Optional 2D array of barrier values (uint8). + 255 = closed/restricted area (from PAD-US Pub_Access = XA). + 0 = accessible. + If None, no barriers are applied. + boundary_mode: How to handle private/restricted land barriers: + "strict" - cells with barrier=255 become impassable (np.inf) + "pragmatic" - cells with barrier=255 get 5.0x friction penalty + "emergency" - barriers are ignored entirely + Default: "pragmatic" + + Returns: + 2D array of travel cost in seconds per cell. + np.inf for impassable cells. + """ + if boundary_mode not in ("strict", "pragmatic", "emergency"): + raise ValueError(f"boundary_mode must be 'strict', 'pragmatic', or 'emergency', got '{boundary_mode}'") + + if cell_size_lat_m is None: + cell_size_lat_m = cell_size_m + if cell_size_lon_m is None: + cell_size_lon_m = cell_size_m + + rows, cols = elevation.shape + + # Compute gradients in both directions + dy = np.zeros_like(elevation) + dx = np.zeros_like(elevation) + + # Central differences for interior, forward/backward at edges + dy[1:-1, :] = (elevation[:-2, :] - elevation[2:, :]) / (2 * cell_size_lat_m) + dy[0, :] = (elevation[0, :] - elevation[1, :]) / cell_size_lat_m + dy[-1, :] = (elevation[-2, :] - elevation[-1, :]) / cell_size_lat_m + + dx[:, 1:-1] = (elevation[:, 2:] - elevation[:, :-2]) / (2 * cell_size_lon_m) + dx[:, 0] = (elevation[:, 1] - elevation[:, 0]) / cell_size_lon_m + dx[:, -1] = (elevation[:, -1] - elevation[:, -2]) / cell_size_lon_m + + # Compute slope magnitude (grade = rise/run) + grade_magnitude = np.sqrt(dx**2 + dy**2) + + # Convert to slope angle in degrees + slope_deg = np.degrees(np.arctan(grade_magnitude)) + + # Compute speed for each cell using Tobler function + speed_kmh = TOBLER_OFF_TRAIL_MULT * TOBLER_BASE_SPEED * np.exp(-3.5 * np.abs(grade_magnitude + 0.05)) + + # Convert speed to time cost (seconds to traverse one cell) + avg_cell_size = (cell_size_lat_m + cell_size_lon_m) / 2 + cost = avg_cell_size * 3.6 / speed_kmh + + # Set impassable cells (slope > MAX_SLOPE_DEG) to infinity + cost[slope_deg > MAX_SLOPE_DEG] = np.inf + + # Handle NaN elevations (no data) + cost[np.isnan(elevation)] = np.inf + + # Apply friction multipliers if provided + if friction is not None: + if friction.shape != elevation.shape: + raise ValueError( + f"Friction shape {friction.shape} does not match elevation shape {elevation.shape}" + ) + # Multiply cost by friction (inf * anything = inf, which is correct) + cost = cost * friction + + # Apply barriers based on boundary_mode + if barriers is not None and boundary_mode != "emergency": + if barriers.shape != elevation.shape: + raise ValueError( + f"Barriers shape {barriers.shape} does not match elevation shape {elevation.shape}" + ) + + barrier_mask = barriers == 255 + + if boundary_mode == "strict": + # Mark closed/restricted areas as impassable + cost[barrier_mask] = np.inf + elif boundary_mode == "pragmatic": + # Apply friction penalty to closed/restricted areas + cost[barrier_mask] = cost[barrier_mask] * PRAGMATIC_BARRIER_MULTIPLIER + + return cost + + +if __name__ == "__main__": + print("Testing Tobler speed function:") + for grade in [-0.3, -0.1, -0.05, 0.0, 0.05, 0.1, 0.3]: + speed = tobler_speed(grade) + print(f" Grade {grade:+.2f}: {speed:.2f} km/h") + + print("\nTesting cost grid computation (no friction, no barriers):") + elev = np.arange(100).reshape(10, 10).astype(np.float32) * 10 + cost = compute_cost_grid(elev, cell_size_m=30.0) + print(f" Elevation range: {elev.min():.0f} - {elev.max():.0f} m") + finite = cost[~np.isinf(cost)] + if len(finite) > 0: + print(f" Cost range: {finite.min():.1f} - {finite.max():.1f} s") + else: + print(f" All cells impassable (test data too steep)") + + print("\nTesting cost grid with friction:") + elev = np.ones((10, 10), dtype=np.float32) * 1000 # flat terrain + friction = np.ones((10, 10), dtype=np.float32) * 1.5 # 1.5x friction + friction[5, 5] = np.inf # one impassable cell + cost = compute_cost_grid(elev, cell_size_m=30.0, friction=friction) + print(f" Base cost (flat, 30m cell): {30 * 3.6 / (0.6 * 6.0 * np.exp(-3.5 * 0.05)):.1f} s") + print(f" With 1.5x friction: {cost[0, 0]:.1f} s") + print(f" Impassable cells: {np.sum(np.isinf(cost))}") + + print("\nTesting cost grid with barriers (three modes):") + elev = np.ones((10, 10), dtype=np.float32) * 1000 # flat terrain + barriers = np.zeros((10, 10), dtype=np.uint8) + barriers[3:7, 3:7] = 255 # 4x4 closed area in center + + base_cost = 30 * 3.6 / (0.6 * 6.0 * np.exp(-3.5 * 0.05)) + + for mode in ["strict", "pragmatic", "emergency"]: + cost = compute_cost_grid(elev, cell_size_m=30.0, barriers=barriers, boundary_mode=mode) + impassable = np.sum(np.isinf(cost)) + barrier_cost = cost[5, 5] if not np.isinf(cost[5, 5]) else "inf" + print(f" {mode:10s}: {impassable} impassable, barrier cell cost = {barrier_cost}") diff --git a/lib/offroute/prototype.py b/lib/offroute/prototype.py index 9822021..b5caf86 100755 --- a/lib/offroute/prototype.py +++ b/lib/offroute/prototype.py @@ -1,18 +1,16 @@ #!/usr/bin/env python3 """ -OFFROUTE Phase O2b Prototype +OFFROUTE Phase O2c Prototype -Validates the PMTiles decoder, Tobler cost function, WorldCover friction -integration, and MCP pathfinder on a real Idaho bounding box. +Validates the PMTiles decoder, Tobler cost function, WorldCover friction, +PAD-US barriers integration, and MCP pathfinder on a real Idaho bounding box. -Now includes friction layer to avoid water bodies like Murtaugh Lake. +Runs THREE pathfinding passes with different boundary modes: + 1. boundary_mode="strict" - private land is impassable + 2. boundary_mode="pragmatic" - private land has 5x friction penalty + 3. boundary_mode="emergency" - private land barriers ignored -Test bbox (four Idaho towns as corners): - SW: Rogerson, ID (~42.21, -114.60) - NW: Buhl, ID (~42.60, -114.76) - NE: Burley, ID (~42.54, -113.79) - SE: Oakley, ID (~42.24, -113.88) - Approximate bbox: south=42.21, north=42.60, west=-114.76, east=-113.79 +Outputs comparison showing impact of boundary mode on routing. """ import json import time @@ -28,8 +26,9 @@ sys.path.insert(0, str(Path(__file__).parent.parent.parent)) from lib.offroute.dem import DEMReader from lib.offroute.cost import compute_cost_grid from lib.offroute.friction import FrictionReader, friction_to_multiplier +from lib.offroute.barriers import BarrierReader, DEFAULT_BARRIERS_PATH -# Test bounding box +# Test bounding box - Idaho area known to have mixed public/private land BBOX = { "south": 42.21, "north": 42.60, @@ -38,26 +37,25 @@ BBOX = { } # Start point: wilderness area south of Twin Falls -# (in the Sawtooth National Forest foothills) -START_LAT = 42.35 -START_LON = -114.50 +START_LAT = 42.36 +START_LON = -114.55 # End point: near Burley, ID (on road network) -END_LAT = 42.52 -END_LON = -113.85 - -# Murtaugh Lake - actual water extent from WorldCover -LAKE_BOUNDS = { - "south": 42.44, - "north": 42.50, - "west": -114.20, - "east": -114.10, -} -LAKE_CENTER = (42.465, -114.155) # Verified water in WorldCover +END_LAT = 42.55 +END_LON = -114.25 # Output files -OUTPUT_PATH_O1 = Path("/opt/recon/data/offroute-test.geojson") -OUTPUT_PATH_FRICTION = Path("/opt/recon/data/offroute-test-friction.geojson") +OUTPUT_PATHS = { + "strict": Path("/opt/recon/data/offroute-test-strict.geojson"), + "pragmatic": Path("/opt/recon/data/offroute-test-pragmatic.geojson"), + "emergency": Path("/opt/recon/data/offroute-test-emergency.geojson"), +} + +# Old files to delete +OLD_FILES = [ + Path("/opt/recon/data/offroute-test-barriers-on.geojson"), + Path("/opt/recon/data/offroute-test-barriers-off.geojson"), +] # Memory limit in GB MEMORY_LIMIT_GB = 12 @@ -77,40 +75,139 @@ def check_memory_usage(): return 0 -def path_crosses_lake(coordinates, lake_bounds): - """Check if any path coordinates fall within the lake bounding box.""" - for lon, lat in coordinates: - if (lake_bounds["south"] <= lat <= lake_bounds["north"] and - lake_bounds["west"] <= lon <= lake_bounds["east"]): - return True, (lat, lon) - return False, None +def run_pathfinder( + elevation: np.ndarray, + meta: dict, + friction_mult: np.ndarray, + barriers: np.ndarray, + boundary_mode: str, + start_row: int, + start_col: int, + end_row: int, + end_col: int, + dem_reader: DEMReader, +) -> dict: + """ + Run the MCP pathfinder with given parameters. + + Returns dict with path info and stats. + """ + # Compute cost grid + cost = compute_cost_grid( + elevation, + cell_size_m=meta["cell_size_m"], + friction=friction_mult, + barriers=barriers, + boundary_mode=boundary_mode, + ) + + # Count impassable cells + impassable_count = np.sum(np.isinf(cost)) + barrier_count = np.sum(barriers == 255) if barriers is not None else 0 + + # Run MCP + mcp = MCP_Geometric(cost, fully_connected=True) + cumulative_costs, traceback = mcp.find_costs([(start_row, start_col)]) + + end_cost = cumulative_costs[end_row, end_col] + + if np.isinf(end_cost): + return { + "success": False, + "reason": "No path found (blocked by impassable terrain)", + "impassable_cells": int(impassable_count), + "barrier_cells": int(barrier_count), + } + + # Traceback path + path_indices = mcp.traceback((end_row, end_col)) + + # Convert to coordinates + coordinates = [] + elevations = [] + barrier_values = [] + + for row, col in path_indices: + lat, lon = dem_reader.pixel_to_latlon(row, col, meta) + elev = elevation[row, col] + barr = barriers[row, col] if barriers is not None else 0 + coordinates.append([lon, lat]) + elevations.append(elev) + barrier_values.append(barr) + + # Compute distance + total_distance_m = 0 + for i in range(1, len(coordinates)): + lon1, lat1 = coordinates[i-1] + lon2, lat2 = coordinates[i] + R = 6371000 + dlat = np.radians(lat2 - lat1) + dlon = np.radians(lon2 - lon1) + a = np.sin(dlat/2)**2 + np.cos(np.radians(lat1)) * np.cos(np.radians(lat2)) * np.sin(dlon/2)**2 + c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1-a)) + total_distance_m += R * c + + # Elevation stats + elev_arr = np.array(elevations) + elev_diff = np.diff(elev_arr) + elev_gain = np.sum(elev_diff[elev_diff > 0]) + elev_loss = np.sum(np.abs(elev_diff[elev_diff < 0])) + + # Barrier crossings on path + barr_arr = np.array(barrier_values) + barrier_crossings = np.sum(barr_arr == 255) + + return { + "success": True, + "coordinates": coordinates, + "total_time_seconds": float(end_cost), + "total_time_minutes": float(end_cost / 60), + "total_distance_m": float(total_distance_m), + "total_distance_km": float(total_distance_m / 1000), + "elevation_gain_m": float(elev_gain), + "elevation_loss_m": float(elev_loss), + "min_elevation_m": float(np.min(elev_arr)), + "max_elevation_m": float(np.max(elev_arr)), + "cell_count": len(path_indices), + "impassable_cells": int(impassable_count), + "barrier_cells": int(barrier_count), + "barrier_crossings": int(barrier_crossings), + } def main(): - print("=" * 60) - print("OFFROUTE Phase O2b Prototype (with Friction)") - print("=" * 60) + print("=" * 80) + print("OFFROUTE Phase O2c Prototype (Three-Mode Boundary Respect)") + print("=" * 80) t0 = time.time() + # Delete old output files + for old_file in OLD_FILES: + if old_file.exists(): + old_file.unlink() + print(f"Deleted old file: {old_file}") + + # Check if barrier raster exists + if not DEFAULT_BARRIERS_PATH.exists(): + print(f"\nERROR: Barrier raster not found at {DEFAULT_BARRIERS_PATH}") + print(f"Run first: python /opt/recon/lib/offroute/barriers.py build") + sys.exit(1) + # Step 1: Load elevation data print(f"\n[1] Loading DEM for bbox: {BBOX}") dem_reader = DEMReader() - t1 = time.time() elevation, meta = dem_reader.get_elevation_grid( south=BBOX["south"], north=BBOX["north"], west=BBOX["west"], east=BBOX["east"], ) - t2 = time.time() print(f" Elevation grid shape: {elevation.shape}") print(f" Cell count: {elevation.size:,}") print(f" Cell size: {meta['cell_size_m']:.1f} m") - print(f" Elevation range: {np.nanmin(elevation):.0f} - {np.nanmax(elevation):.0f} m") - print(f" Load time: {t2 - t1:.1f}s") mem = check_memory_usage() if mem > 0: @@ -118,19 +215,8 @@ def main(): # Step 2: Load friction data print(f"\n[2] Loading WorldCover friction layer...") - t2a = time.time() - friction_reader = FrictionReader() - # Validate lake is marked as impassable - lake_friction = friction_reader.sample_point(LAKE_CENTER[0], LAKE_CENTER[1]) - print(f" Murtaugh Lake center ({LAKE_CENTER[0]}, {LAKE_CENTER[1]}): friction = {lake_friction}") - if lake_friction != 255: - print(f" WARNING: Lake not marked as water (expected 255, got {lake_friction})") - else: - print(f" Lake correctly marked as impassable (255)") - - # Load friction grid matching elevation shape friction_raw = friction_reader.get_friction_grid( south=BBOX["south"], north=BBOX["north"], @@ -138,36 +224,30 @@ def main(): east=BBOX["east"], target_shape=elevation.shape ) - t2b = time.time() - - # Convert to multipliers friction_mult = friction_to_multiplier(friction_raw) - impassable_count = np.sum(np.isinf(friction_mult)) print(f" Friction grid shape: {friction_raw.shape}") - print(f" Unique friction values: {np.unique(friction_raw[friction_raw > 0])}") - print(f" Impassable cells (water/nodata): {impassable_count:,} ({100*impassable_count/friction_raw.size:.1f}%)") - print(f" Load time: {t2b - t2a:.1f}s") + print(f" Water/impassable cells: {np.sum(np.isinf(friction_mult)):,}") - mem = check_memory_usage() - if mem > 0: - print(f" Memory usage: {mem:.1f} GB") + # Step 3: Load barrier data + print(f"\n[3] Loading PAD-US barrier layer...") + barrier_reader = BarrierReader() - # Step 3: Compute cost grid with friction - print(f"\n[3] Computing Tobler cost grid with friction...") - t3 = time.time() - cost = compute_cost_grid( - elevation, - cell_size_m=meta["cell_size_m"], - friction=friction_mult + barriers = barrier_reader.get_barrier_grid( + south=BBOX["south"], + north=BBOX["north"], + west=BBOX["west"], + east=BBOX["east"], + target_shape=elevation.shape ) - t4 = time.time() - finite_cost = cost[~np.isinf(cost)] - total_impassable = np.sum(np.isinf(cost)) - print(f" Cost range: {finite_cost.min():.1f} - {finite_cost.max():.1f} s/cell") - print(f" Total impassable cells: {total_impassable:,} ({100*total_impassable/cost.size:.1f}%)") - print(f" Compute time: {t4 - t3:.1f}s") + closed_cells = np.sum(barriers == 255) + print(f" Barrier grid shape: {barriers.shape}") + print(f" Closed/restricted cells: {closed_cells:,} ({100*closed_cells/barriers.size:.2f}%)") + + if closed_cells == 0: + print("\n WARNING: No closed/restricted areas in this bbox.") + print(" The test may not show meaningful differences between modes.") mem = check_memory_usage() if mem > 0: @@ -190,196 +270,122 @@ def main(): print(f"ERROR: End point outside grid bounds") sys.exit(1) - start_elev = elevation[start_row, start_col] - end_elev = elevation[end_row, end_col] - print(f" Start elevation: {start_elev:.0f} m") - print(f" End elevation: {end_elev:.0f} m") + # Step 5: Run pathfinder THREE times + results = {} + modes = ["strict", "pragmatic", "emergency"] - # Step 5: Run MCP pathfinder - print(f"\n[5] Running MCP_Geometric pathfinder...") - t5 = time.time() + for i, mode in enumerate(modes, start=5): + print(f"\n[{i}] Running pathfinder (boundary_mode=\"{mode}\")...") + t_start = time.time() + results[mode] = run_pathfinder( + elevation, meta, friction_mult, barriers, + boundary_mode=mode, + start_row=start_row, start_col=start_col, + end_row=end_row, end_col=end_col, + dem_reader=dem_reader, + ) + t_end = time.time() + print(f" Completed in {t_end - t_start:.1f}s") - mcp = MCP_Geometric(cost, fully_connected=True) - cumulative_costs, traceback = mcp.find_costs([(start_row, start_col)]) - t6 = time.time() + # Step 6: Save GeoJSON outputs + print(f"\n[8] Saving GeoJSON outputs...") - print(f" Dijkstra completed in {t6 - t5:.1f}s") + OUTPUT_PATHS["strict"].parent.mkdir(parents=True, exist_ok=True) - end_cost = cumulative_costs[end_row, end_col] - print(f" Total cost to endpoint: {end_cost:.0f} seconds ({end_cost/60:.1f} minutes)") + for mode, result in results.items(): + output_path = OUTPUT_PATHS[mode] + if result["success"]: + geojson = { + "type": "Feature", + "properties": { + "type": f"offroute_{mode}", + "phase": "O2c", + "boundary_mode": mode, + "start": {"lat": START_LAT, "lon": START_LON}, + "end": {"lat": END_LAT, "lon": END_LON}, + **{k: v for k, v in result.items() if k not in ["success", "coordinates"]}, + }, + "geometry": { + "type": "LineString", + "coordinates": result["coordinates"], + } + } + with open(output_path, "w") as f: + json.dump(geojson, f, indent=2) + print(f" Saved: {output_path}") + else: + print(f" SKIPPED ({mode}): {result['reason']}") - if np.isinf(end_cost): - print("ERROR: No path found to endpoint (blocked by impassable terrain)") - sys.exit(1) + t_total = time.time() - t7 = time.time() - path_indices = mcp.traceback((end_row, end_col)) - t8 = time.time() + # Final report - three-way comparison + print(f"\n" + "=" * 80) + print("THREE-WAY COMPARISON") + print("=" * 80) - print(f" Traceback completed in {t8 - t7:.2f}s") - print(f" Path length: {len(path_indices)} cells") + # Check how many succeeded + success_count = sum(1 for r in results.values() if r["success"]) - mem = check_memory_usage() - if mem > 0: - print(f" Memory usage: {mem:.1f} GB") + if success_count == 3: + print(f"{'Metric':<22} {'STRICT':<18} {'PRAGMATIC':<18} {'EMERGENCY':<18}") + print("-" * 80) - # Step 6: Convert path to coordinates and compute stats - print(f"\n[6] Converting path to GeoJSON...") + metrics = [ + ("Distance (km)", "total_distance_km", ".2f"), + ("Effort time (min)", "total_time_minutes", ".1f"), + ("Cell count", "cell_count", "d"), + ("Elevation gain (m)", "elevation_gain_m", ".0f"), + ("Elevation loss (m)", "elevation_loss_m", ".0f"), + ("Barrier crossings", "barrier_crossings", "d"), + ("Impassable cells", "impassable_cells", ",d"), + ] - coordinates = [] - elevations = [] - friction_values = [] + for label, key, fmt in metrics: + vals = [results[m][key] for m in modes] + print(f"{label:<22} {vals[0]:<18{fmt}} {vals[1]:<18{fmt}} {vals[2]:<18{fmt}}") - for row, col in path_indices: - lat, lon = dem_reader.pixel_to_latlon(row, col, meta) - elev = elevation[row, col] - fric = friction_raw[row, col] - coordinates.append([lon, lat]) - elevations.append(elev) - friction_values.append(fric) + # Analysis + print(f"\n" + "-" * 80) + print("ANALYSIS") + print("-" * 80) - # Compute path distance - total_distance_m = 0 - for i in range(1, len(coordinates)): - lon1, lat1 = coordinates[i-1] - lon2, lat2 = coordinates[i] - R = 6371000 - dlat = np.radians(lat2 - lat1) - dlon = np.radians(lon2 - lon1) - a = np.sin(dlat/2)**2 + np.cos(np.radians(lat1)) * np.cos(np.radians(lat2)) * np.sin(dlon/2)**2 - c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1-a)) - total_distance_m += R * c + strict_crossings = results["strict"]["barrier_crossings"] + pragmatic_crossings = results["pragmatic"]["barrier_crossings"] + emergency_crossings = results["emergency"]["barrier_crossings"] - # Compute elevation gain/loss - elev_arr = np.array(elevations) - elev_diff = np.diff(elev_arr) - elev_gain = np.sum(elev_diff[elev_diff > 0]) - elev_loss = np.sum(np.abs(elev_diff[elev_diff < 0])) + print(f"Barrier crossings: strict={strict_crossings}, pragmatic={pragmatic_crossings}, emergency={emergency_crossings}") - # Friction stats along path - fric_arr = np.array(friction_values) - valid_fric = fric_arr[(fric_arr > 0) & (fric_arr < 255)] + if strict_crossings == 0 and pragmatic_crossings == 0 and emergency_crossings == 0: + print("No path crosses private land - terrain naturally avoids barriers.") + else: + if emergency_crossings > pragmatic_crossings: + print(f"Pragmatic mode reduces barrier crossings vs emergency: {emergency_crossings} -> {pragmatic_crossings}") + if pragmatic_crossings > 0 and strict_crossings == 0: + print(f"Strict mode completely avoids private land (pragmatic crosses {pragmatic_crossings} cells)") - # Build GeoJSON - geojson = { - "type": "Feature", - "properties": { - "type": "offroute_prototype_friction", - "phase": "O2b", - "start": {"lat": START_LAT, "lon": START_LON}, - "end": {"lat": END_LAT, "lon": END_LON}, - "total_time_seconds": float(end_cost), - "total_time_minutes": float(end_cost / 60), - "total_distance_m": float(total_distance_m), - "total_distance_km": float(total_distance_m / 1000), - "elevation_gain_m": float(elev_gain), - "elevation_loss_m": float(elev_loss), - "min_elevation_m": float(np.min(elev_arr)), - "max_elevation_m": float(np.max(elev_arr)), - "friction_min": int(valid_fric.min()) if len(valid_fric) > 0 else 0, - "friction_max": int(valid_fric.max()) if len(valid_fric) > 0 else 0, - "friction_mean": float(valid_fric.mean()) if len(valid_fric) > 0 else 0, - "cell_count": len(path_indices), - "cell_size_m": meta["cell_size_m"], - }, - "geometry": { - "type": "LineString", - "coordinates": coordinates, - } - } + # Time/distance comparison + if results["strict"]["total_time_minutes"] > results["emergency"]["total_time_minutes"]: + time_penalty = results["strict"]["total_time_minutes"] - results["emergency"]["total_time_minutes"] + print(f"Time cost of strict boundary respect: +{time_penalty:.1f} min") - # Write output - OUTPUT_PATH_FRICTION.parent.mkdir(parents=True, exist_ok=True) - with open(OUTPUT_PATH_FRICTION, "w") as f: - json.dump(geojson, f, indent=2) - - t_end = time.time() - - # Final report - print(f"\n" + "=" * 60) - print("RESULTS (Phase O2b with Friction)") - print("=" * 60) - print(f"Start: ({START_LAT:.4f}, {START_LON:.4f})") - print(f"End: ({END_LAT:.4f}, {END_LON:.4f})") - print(f"Total effort: {end_cost/60:.1f} minutes ({end_cost/3600:.2f} hours)") - print(f"Distance: {total_distance_m/1000:.2f} km") - print(f"Elevation gain: {elev_gain:.0f} m") - print(f"Elevation loss: {elev_loss:.0f} m") - print(f"Elevation range: {np.min(elev_arr):.0f} - {np.max(elev_arr):.0f} m") - if len(valid_fric) > 0: - print(f"Friction (path): min={valid_fric.min()}, max={valid_fric.max()}, mean={valid_fric.mean():.1f}") - print(f"Path cells: {len(path_indices):,}") - print(f"Wall time: {t_end - t0:.1f}s") - print(f"\nOutput saved to: {OUTPUT_PATH_FRICTION}") - - # Validation - print(f"\n" + "-" * 60) - print("VALIDATION") - print("-" * 60) - - # Check coordinates are within bbox - lons = [c[0] for c in coordinates] - lats = [c[1] for c in coordinates] - lon_ok = BBOX["west"] <= min(lons) and max(lons) <= BBOX["east"] - lat_ok = BBOX["south"] <= min(lats) and max(lats) <= BBOX["north"] - print(f"Coordinates within bbox: {'PASS' if lon_ok and lat_ok else 'FAIL'}") - - # Check path is not trivial - is_nontrivial = len(path_indices) > 10 and total_distance_m > 1000 - print(f"Path is non-trivial: {'PASS' if is_nontrivial else 'FAIL'}") - - # Check sinuosity - straight_line_dist = np.sqrt( - (coordinates[-1][0] - coordinates[0][0])**2 + - (coordinates[-1][1] - coordinates[0][1])**2 - ) * 111000 - sinuosity = total_distance_m / max(straight_line_dist, 1) - print(f"Sinuosity: {sinuosity:.2f} (>1.0 means path curves around obstacles)") - - # CRITICAL: Check no water cells (friction=255) on path - # This is the authoritative test - friction layer prevents water crossings - print(f"\n--- Water Avoidance Check ---") - water_on_path = np.sum(fric_arr == 255) - if water_on_path > 0: - print(f"FAIL: Path crosses {water_on_path} water cells (friction=255)") - sys.exit(1) else: - print(f"PASS: No water cells (friction=255) on path") + print(f"Only {success_count}/3 modes found a path:") + for mode, result in results.items(): + if result["success"]: + print(f" {mode}: {result['total_distance_km']:.2f} km, {result['total_time_minutes']:.1f} min") + else: + print(f" {mode}: FAILED - {result.get('reason', 'unknown')}") - # Informational: Check if path goes through lake bounding box - # Path may go through land cells within the bbox, which is fine - print(f"\n--- Lake Bounding Box Check (informational) ---") - print(f"Murtaugh Lake bounds: {LAKE_BOUNDS}") - crosses_lake, crossing_point = path_crosses_lake(coordinates, LAKE_BOUNDS) - if crosses_lake: - print(f"INFO: Path passes through lake bbox at {crossing_point}") - print(f" (This is OK if friction check passed - path uses land cells)") - else: - print(f"PASS: Path does not enter lake bounding box") - - # Compare with Phase O1 if available - print(f"\n" + "-" * 60) - print("COMPARISON: Phase O1 vs O2b") - print("-" * 60) - - if OUTPUT_PATH_O1.exists(): - with open(OUTPUT_PATH_O1) as f: - o1_data = json.load(f) - o1_props = o1_data["properties"] - - print(f"{'Metric':<20} {'O1 (no friction)':<20} {'O2b (with friction)':<20}") - print("-" * 60) - print(f"{'Distance (km)':<20} {o1_props['total_distance_km']:<20.2f} {total_distance_m/1000:<20.2f}") - print(f"{'Effort (min)':<20} {o1_props['total_time_minutes']:<20.1f} {end_cost/60:<20.1f}") - print(f"{'Cell count':<20} {o1_props['cell_count']:<20} {len(path_indices):<20}") - print(f"{'Elev gain (m)':<20} {o1_props['elevation_gain_m']:<20.0f} {elev_gain:<20.0f}") - else: - print(f"Phase O1 output not found at {OUTPUT_PATH_O1}") - print(f"Run the O1 prototype first to enable comparison.") + print(f"\n" + "-" * 80) + print(f"Total wall time: {t_total - t0:.1f}s") + print(f"Closed cells in bbox: {closed_cells:,}") + # Cleanup dem_reader.close() friction_reader.close() - print("\nPrototype completed successfully.") + barrier_reader.close() + + print("\nPrototype completed.") if __name__ == "__main__": From 3293cb4238a1ed489b9d6eb2259a7a4094b5331c Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 8 May 2026 07:26:25 +0000 Subject: [PATCH 06/41] =?UTF-8?q?feat(offroute):=20Phase=20O3a=20=E2=80=94?= =?UTF-8?q?=20trail=20burn-in,=20pathfinder=20seeks=20trail=20corridors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Trail friction REPLACES land cover friction where trails exist: - Road (value 5): 0.1× friction - Track (value 15): 0.3× friction - Foot trail (value 25): 0.5× friction TrailReader loads /mnt/nav/worldcover/trails.tif rasterized from OSM highways. Validation shows trail-seeking behavior: - On-trail travel: 17.3% → 98.7% - Effort time: 1047 min → 155 min (-85.2%) - Path travels farther but stays on roads for speed Co-Authored-By: Claude Opus 4.5 --- lib/offroute/cost.py | 70 ++++++--- lib/offroute/prototype.py | 302 ++++++++++++++++++++------------------ lib/offroute/trails.py | 174 ++++++++++++++++++++++ 3 files changed, 389 insertions(+), 157 deletions(-) create mode 100644 lib/offroute/trails.py diff --git a/lib/offroute/cost.py b/lib/offroute/cost.py index f31b8f5..5f3618c 100644 --- a/lib/offroute/cost.py +++ b/lib/offroute/cost.py @@ -3,7 +3,7 @@ Tobler off-path hiking cost function for OFFROUTE. Computes travel time cost based on terrain slope using Tobler's hiking function with off-trail penalty. Optionally applies friction -multipliers from land cover data and barrier grids from PAD-US. +multipliers from land cover data, trail corridors, and barrier grids. """ import math import numpy as np @@ -19,6 +19,14 @@ TOBLER_OFF_TRAIL_MULT = 0.6 # Pragmatic mode friction multiplier for private land PRAGMATIC_BARRIER_MULTIPLIER = 5.0 +# Trail value to friction multiplier mapping +# Trail friction REPLACES land cover friction (a road through forest is still easy) +TRAIL_FRICTION_MAP = { + 5: 0.1, # road + 15: 0.3, # track + 25: 0.5, # foot trail +} + def tobler_speed(grade: float) -> float: """ @@ -37,6 +45,7 @@ def compute_cost_grid( cell_size_lat_m: float = None, cell_size_lon_m: float = None, friction: Optional[np.ndarray] = None, + trails: Optional[np.ndarray] = None, barriers: Optional[np.ndarray] = None, boundary_mode: Literal["strict", "pragmatic", "emergency"] = "pragmatic" ) -> np.ndarray: @@ -51,10 +60,17 @@ def compute_cost_grid( cell_size_m: Average cell size in meters cell_size_lat_m: Cell size in latitude direction (optional) cell_size_lon_m: Cell size in longitude direction (optional) - friction: Optional 2D array of friction multipliers. + friction: Optional 2D array of friction multipliers (WorldCover). Values should be float (1.0 = baseline, 2.0 = 2x slower). np.inf marks impassable cells. If None, no friction is applied (backward compatible). + trails: Optional 2D array of trail values (uint8). + 0 = no trail (use friction) + 5 = road (0.1× friction, replaces WorldCover) + 15 = track (0.3× friction, replaces WorldCover) + 25 = foot trail (0.5× friction, replaces WorldCover) + Trail friction REPLACES land cover friction where trails exist. + If None, no trail burn-in is applied. barriers: Optional 2D array of barrier values (uint8). 255 = closed/restricted area (from PAD-US Pub_Access = XA). 0 = accessible. @@ -111,14 +127,30 @@ def compute_cost_grid( # Handle NaN elevations (no data) cost[np.isnan(elevation)] = np.inf - # Apply friction multipliers if provided + # Build effective friction array + # Start with WorldCover friction if provided, else 1.0 if friction is not None: if friction.shape != elevation.shape: raise ValueError( f"Friction shape {friction.shape} does not match elevation shape {elevation.shape}" ) - # Multiply cost by friction (inf * anything = inf, which is correct) - cost = cost * friction + effective_friction = friction.copy() + else: + effective_friction = np.ones(elevation.shape, dtype=np.float32) + + # Apply trail burn-in: trails REPLACE land cover friction + if trails is not None: + if trails.shape != elevation.shape: + raise ValueError( + f"Trails shape {trails.shape} does not match elevation shape {elevation.shape}" + ) + # Replace friction where trails exist + for trail_value, trail_friction in TRAIL_FRICTION_MAP.items(): + trail_mask = trails == trail_value + effective_friction[trail_mask] = trail_friction + + # Apply friction to cost + cost = cost * effective_friction # Apply barriers based on boundary_mode if barriers is not None and boundary_mode != "emergency": @@ -145,7 +177,7 @@ if __name__ == "__main__": speed = tobler_speed(grade) print(f" Grade {grade:+.2f}: {speed:.2f} km/h") - print("\nTesting cost grid computation (no friction, no barriers):") + print("\nTesting cost grid computation (no friction, no trails):") elev = np.arange(100).reshape(10, 10).astype(np.float32) * 10 cost = compute_cost_grid(elev, cell_size_m=30.0) print(f" Elevation range: {elev.min():.0f} - {elev.max():.0f} m") @@ -155,21 +187,25 @@ if __name__ == "__main__": else: print(f" All cells impassable (test data too steep)") - print("\nTesting cost grid with friction:") + print("\nTesting cost grid with friction and trails:") elev = np.ones((10, 10), dtype=np.float32) * 1000 # flat terrain - friction = np.ones((10, 10), dtype=np.float32) * 1.5 # 1.5x friction - friction[5, 5] = np.inf # one impassable cell - cost = compute_cost_grid(elev, cell_size_m=30.0, friction=friction) - print(f" Base cost (flat, 30m cell): {30 * 3.6 / (0.6 * 6.0 * np.exp(-3.5 * 0.05)):.1f} s") - print(f" With 1.5x friction: {cost[0, 0]:.1f} s") - print(f" Impassable cells: {np.sum(np.isinf(cost))}") + friction = np.ones((10, 10), dtype=np.float32) * 2.0 # 2.0x friction (forest) + trails = np.zeros((10, 10), dtype=np.uint8) + trails[5, :] = 5 # road across middle row - print("\nTesting cost grid with barriers (three modes):") - elev = np.ones((10, 10), dtype=np.float32) * 1000 # flat terrain - barriers = np.zeros((10, 10), dtype=np.uint8) - barriers[3:7, 3:7] = 255 # 4x4 closed area in center + cost_no_trail = compute_cost_grid(elev, cell_size_m=30.0, friction=friction) + cost_with_trail = compute_cost_grid(elev, cell_size_m=30.0, friction=friction, trails=trails) base_cost = 30 * 3.6 / (0.6 * 6.0 * np.exp(-3.5 * 0.05)) + print(f" Base cost (flat, 30m cell): {base_cost:.1f} s") + print(f" Forest cell (2.0x friction): {cost_no_trail[0, 0]:.1f} s") + print(f" Road cell (0.1x friction, replaces forest): {cost_with_trail[5, 0]:.1f} s") + print(f" Road friction advantage: {cost_no_trail[0, 0] / cost_with_trail[5, 0]:.1f}x faster") + + print("\nTesting cost grid with barriers (three modes):") + elev = np.ones((10, 10), dtype=np.float32) * 1000 + barriers = np.zeros((10, 10), dtype=np.uint8) + barriers[3:7, 3:7] = 255 for mode in ["strict", "pragmatic", "emergency"]: cost = compute_cost_grid(elev, cell_size_m=30.0, barriers=barriers, boundary_mode=mode) diff --git a/lib/offroute/prototype.py b/lib/offroute/prototype.py index b5caf86..c9b78f0 100755 --- a/lib/offroute/prototype.py +++ b/lib/offroute/prototype.py @@ -1,16 +1,12 @@ #!/usr/bin/env python3 """ -OFFROUTE Phase O2c Prototype +OFFROUTE Phase O3a Prototype -Validates the PMTiles decoder, Tobler cost function, WorldCover friction, -PAD-US barriers integration, and MCP pathfinder on a real Idaho bounding box. +Validates trail burn-in integration with the MCP pathfinder. +The path should actively seek out trails and roads when nearby. -Runs THREE pathfinding passes with different boundary modes: - 1. boundary_mode="strict" - private land is impassable - 2. boundary_mode="pragmatic" - private land has 5x friction penalty - 3. boundary_mode="emergency" - private land barriers ignored - -Outputs comparison showing impact of boundary mode on routing. +Compares paths with and without trail burn-in to show the benefit +of trail-seeking behavior. """ import json import time @@ -27,8 +23,9 @@ from lib.offroute.dem import DEMReader from lib.offroute.cost import compute_cost_grid from lib.offroute.friction import FrictionReader, friction_to_multiplier from lib.offroute.barriers import BarrierReader, DEFAULT_BARRIERS_PATH +from lib.offroute.trails import TrailReader, DEFAULT_TRAILS_PATH -# Test bounding box - Idaho area known to have mixed public/private land +# Test bounding box - Idaho area BBOX = { "south": 42.21, "north": 42.60, @@ -36,26 +33,17 @@ BBOX = { "east": -113.79, } -# Start point: wilderness area south of Twin Falls -START_LAT = 42.36 -START_LON = -114.55 +# Start point: wilderness area away from roads +START_LAT = 42.35 +START_LON = -114.60 -# End point: near Burley, ID (on road network) +# End point: near Twin Falls (has roads/trails) END_LAT = 42.55 -END_LON = -114.25 +END_LON = -114.20 # Output files -OUTPUT_PATHS = { - "strict": Path("/opt/recon/data/offroute-test-strict.geojson"), - "pragmatic": Path("/opt/recon/data/offroute-test-pragmatic.geojson"), - "emergency": Path("/opt/recon/data/offroute-test-emergency.geojson"), -} - -# Old files to delete -OLD_FILES = [ - Path("/opt/recon/data/offroute-test-barriers-on.geojson"), - Path("/opt/recon/data/offroute-test-barriers-off.geojson"), -] +OUTPUT_PATH_WITH_TRAILS = Path("/opt/recon/data/offroute-test-trails.geojson") +OUTPUT_PATH_NO_TRAILS = Path("/opt/recon/data/offroute-test-no-trails.geojson") # Memory limit in GB MEMORY_LIMIT_GB = 12 @@ -79,32 +67,26 @@ def run_pathfinder( elevation: np.ndarray, meta: dict, friction_mult: np.ndarray, + trails: np.ndarray, barriers: np.ndarray, - boundary_mode: str, + use_trails: bool, start_row: int, start_col: int, end_row: int, end_col: int, dem_reader: DEMReader, ) -> dict: - """ - Run the MCP pathfinder with given parameters. - - Returns dict with path info and stats. - """ + """Run the MCP pathfinder with given parameters.""" # Compute cost grid cost = compute_cost_grid( elevation, cell_size_m=meta["cell_size_m"], friction=friction_mult, + trails=trails if use_trails else None, barriers=barriers, - boundary_mode=boundary_mode, + boundary_mode="pragmatic", ) - # Count impassable cells - impassable_count = np.sum(np.isinf(cost)) - barrier_count = np.sum(barriers == 255) if barriers is not None else 0 - # Run MCP mcp = MCP_Geometric(cost, fully_connected=True) cumulative_costs, traceback = mcp.find_costs([(start_row, start_col)]) @@ -115,25 +97,23 @@ def run_pathfinder( return { "success": False, "reason": "No path found (blocked by impassable terrain)", - "impassable_cells": int(impassable_count), - "barrier_cells": int(barrier_count), } # Traceback path path_indices = mcp.traceback((end_row, end_col)) - # Convert to coordinates + # Convert to coordinates and collect stats coordinates = [] elevations = [] - barrier_values = [] + trail_values = [] for row, col in path_indices: lat, lon = dem_reader.pixel_to_latlon(row, col, meta) elev = elevation[row, col] - barr = barriers[row, col] if barriers is not None else 0 + trail_val = trails[row, col] if trails is not None else 0 coordinates.append([lon, lat]) elevations.append(elev) - barrier_values.append(barr) + trail_values.append(trail_val) # Compute distance total_distance_m = 0 @@ -153,9 +133,14 @@ def run_pathfinder( elev_gain = np.sum(elev_diff[elev_diff > 0]) elev_loss = np.sum(np.abs(elev_diff[elev_diff < 0])) - # Barrier crossings on path - barr_arr = np.array(barrier_values) - barrier_crossings = np.sum(barr_arr == 255) + # Trail stats + trail_arr = np.array(trail_values) + road_cells = np.sum(trail_arr == 5) + track_cells = np.sum(trail_arr == 15) + trail_cells = np.sum(trail_arr == 25) + off_trail_cells = np.sum(trail_arr == 0) + on_trail_cells = road_cells + track_cells + trail_cells + total_cells = len(trail_arr) return { "success": True, @@ -168,30 +153,28 @@ def run_pathfinder( "elevation_loss_m": float(elev_loss), "min_elevation_m": float(np.min(elev_arr)), "max_elevation_m": float(np.max(elev_arr)), - "cell_count": len(path_indices), - "impassable_cells": int(impassable_count), - "barrier_cells": int(barrier_count), - "barrier_crossings": int(barrier_crossings), + "cell_count": total_cells, + "road_cells": int(road_cells), + "track_cells": int(track_cells), + "trail_cells": int(trail_cells), + "off_trail_cells": int(off_trail_cells), + "on_trail_pct": float(100 * on_trail_cells / total_cells) if total_cells > 0 else 0, } def main(): print("=" * 80) - print("OFFROUTE Phase O2c Prototype (Three-Mode Boundary Respect)") + print("OFFROUTE Phase O3a Prototype (Trail Burn-In)") print("=" * 80) t0 = time.time() - # Delete old output files - for old_file in OLD_FILES: - if old_file.exists(): - old_file.unlink() - print(f"Deleted old file: {old_file}") - - # Check if barrier raster exists + # Check for required rasters if not DEFAULT_BARRIERS_PATH.exists(): print(f"\nERROR: Barrier raster not found at {DEFAULT_BARRIERS_PATH}") - print(f"Run first: python /opt/recon/lib/offroute/barriers.py build") + sys.exit(1) + if not DEFAULT_TRAILS_PATH.exists(): + print(f"\nERROR: Trails raster not found at {DEFAULT_TRAILS_PATH}") sys.exit(1) # Step 1: Load elevation data @@ -243,25 +226,42 @@ def main(): closed_cells = np.sum(barriers == 255) print(f" Barrier grid shape: {barriers.shape}") - print(f" Closed/restricted cells: {closed_cells:,} ({100*closed_cells/barriers.size:.2f}%)") + print(f" Closed/restricted cells: {closed_cells:,}") - if closed_cells == 0: - print("\n WARNING: No closed/restricted areas in this bbox.") - print(" The test may not show meaningful differences between modes.") + # Step 4: Load trails data + print(f"\n[4] Loading OSM trails layer...") + trail_reader = TrailReader() + + trails = trail_reader.get_trails_grid( + south=BBOX["south"], + north=BBOX["north"], + west=BBOX["west"], + east=BBOX["east"], + target_shape=elevation.shape + ) + + road_cells = np.sum(trails == 5) + track_cells = np.sum(trails == 15) + trail_cells = np.sum(trails == 25) + print(f" Trails grid shape: {trails.shape}") + print(f" Road cells: {road_cells:,}") + print(f" Track cells: {track_cells:,}") + print(f" Trail cells: {trail_cells:,}") + print(f" Total trail coverage: {100*(road_cells+track_cells+trail_cells)/trails.size:.2f}%") mem = check_memory_usage() if mem > 0: print(f" Memory usage: {mem:.1f} GB") - # Step 4: Convert start/end to pixel coordinates - print(f"\n[4] Converting coordinates...") + # Step 5: Convert start/end to pixel coordinates + print(f"\n[5] Converting coordinates...") start_row, start_col = dem_reader.latlon_to_pixel(START_LAT, START_LON, meta) end_row, end_col = dem_reader.latlon_to_pixel(END_LAT, END_LON, meta) print(f" Start: ({START_LAT}, {START_LON}) -> pixel ({start_row}, {start_col})") print(f" End: ({END_LAT}, {END_LON}) -> pixel ({end_row}, {end_col})") - # Validate coordinates are within bounds + # Validate coordinates rows, cols = elevation.shape if not (0 <= start_row < rows and 0 <= start_col < cols): print(f"ERROR: Start point outside grid bounds") @@ -270,64 +270,86 @@ def main(): print(f"ERROR: End point outside grid bounds") sys.exit(1) - # Step 5: Run pathfinder THREE times - results = {} - modes = ["strict", "pragmatic", "emergency"] + # Step 6: Run pathfinder WITH trails + print(f"\n[6] Running pathfinder WITH trail burn-in...") + t6a = time.time() + result_trails = run_pathfinder( + elevation, meta, friction_mult, trails, barriers, + use_trails=True, + start_row=start_row, start_col=start_col, + end_row=end_row, end_col=end_col, + dem_reader=dem_reader, + ) + t6b = time.time() + print(f" Completed in {t6b - t6a:.1f}s") - for i, mode in enumerate(modes, start=5): - print(f"\n[{i}] Running pathfinder (boundary_mode=\"{mode}\")...") - t_start = time.time() - results[mode] = run_pathfinder( - elevation, meta, friction_mult, barriers, - boundary_mode=mode, - start_row=start_row, start_col=start_col, - end_row=end_row, end_col=end_col, - dem_reader=dem_reader, - ) - t_end = time.time() - print(f" Completed in {t_end - t_start:.1f}s") + # Step 7: Run pathfinder WITHOUT trails + print(f"\n[7] Running pathfinder WITHOUT trail burn-in...") + t7a = time.time() + result_no_trails = run_pathfinder( + elevation, meta, friction_mult, trails, barriers, + use_trails=False, + start_row=start_row, start_col=start_col, + end_row=end_row, end_col=end_col, + dem_reader=dem_reader, + ) + t7b = time.time() + print(f" Completed in {t7b - t7a:.1f}s") - # Step 6: Save GeoJSON outputs + # Step 8: Save GeoJSON outputs print(f"\n[8] Saving GeoJSON outputs...") - OUTPUT_PATHS["strict"].parent.mkdir(parents=True, exist_ok=True) + OUTPUT_PATH_WITH_TRAILS.parent.mkdir(parents=True, exist_ok=True) - for mode, result in results.items(): - output_path = OUTPUT_PATHS[mode] - if result["success"]: - geojson = { - "type": "Feature", - "properties": { - "type": f"offroute_{mode}", - "phase": "O2c", - "boundary_mode": mode, - "start": {"lat": START_LAT, "lon": START_LON}, - "end": {"lat": END_LAT, "lon": END_LON}, - **{k: v for k, v in result.items() if k not in ["success", "coordinates"]}, - }, - "geometry": { - "type": "LineString", - "coordinates": result["coordinates"], - } + if result_trails["success"]: + geojson = { + "type": "Feature", + "properties": { + "type": "offroute_with_trails", + "phase": "O3a", + "trail_burn_in": True, + "start": {"lat": START_LAT, "lon": START_LON}, + "end": {"lat": END_LAT, "lon": END_LON}, + **{k: v for k, v in result_trails.items() if k not in ["success", "coordinates"]}, + }, + "geometry": { + "type": "LineString", + "coordinates": result_trails["coordinates"], } - with open(output_path, "w") as f: - json.dump(geojson, f, indent=2) - print(f" Saved: {output_path}") - else: - print(f" SKIPPED ({mode}): {result['reason']}") + } + with open(OUTPUT_PATH_WITH_TRAILS, "w") as f: + json.dump(geojson, f, indent=2) + print(f" Saved: {OUTPUT_PATH_WITH_TRAILS}") + + if result_no_trails["success"]: + geojson = { + "type": "Feature", + "properties": { + "type": "offroute_no_trails", + "phase": "O3a", + "trail_burn_in": False, + "start": {"lat": START_LAT, "lon": START_LON}, + "end": {"lat": END_LAT, "lon": END_LON}, + **{k: v for k, v in result_no_trails.items() if k not in ["success", "coordinates"]}, + }, + "geometry": { + "type": "LineString", + "coordinates": result_no_trails["coordinates"], + } + } + with open(OUTPUT_PATH_NO_TRAILS, "w") as f: + json.dump(geojson, f, indent=2) + print(f" Saved: {OUTPUT_PATH_NO_TRAILS}") t_total = time.time() - # Final report - three-way comparison + # Final report print(f"\n" + "=" * 80) - print("THREE-WAY COMPARISON") + print("SIDE-BY-SIDE COMPARISON: Trail Burn-In Effect") print("=" * 80) - # Check how many succeeded - success_count = sum(1 for r in results.values() if r["success"]) - - if success_count == 3: - print(f"{'Metric':<22} {'STRICT':<18} {'PRAGMATIC':<18} {'EMERGENCY':<18}") + if result_trails["success"] and result_no_trails["success"]: + print(f"{'Metric':<25} {'WITH TRAILS':<20} {'WITHOUT TRAILS':<20} {'Delta':<15}") print("-" * 80) metrics = [ @@ -335,55 +357,55 @@ def main(): ("Effort time (min)", "total_time_minutes", ".1f"), ("Cell count", "cell_count", "d"), ("Elevation gain (m)", "elevation_gain_m", ".0f"), - ("Elevation loss (m)", "elevation_loss_m", ".0f"), - ("Barrier crossings", "barrier_crossings", "d"), - ("Impassable cells", "impassable_cells", ",d"), + ("On-trail %", "on_trail_pct", ".1f"), + ("Road cells", "road_cells", "d"), + ("Track cells", "track_cells", "d"), + ("Trail cells", "trail_cells", "d"), ] for label, key, fmt in metrics: - vals = [results[m][key] for m in modes] - print(f"{label:<22} {vals[0]:<18{fmt}} {vals[1]:<18{fmt}} {vals[2]:<18{fmt}}") + val_with = result_trails[key] + val_without = result_no_trails[key] + if isinstance(val_with, int): + delta = val_with - val_without + delta_str = f"{delta:+d}" + else: + delta = val_with - val_without + delta_str = f"{delta:+.2f}" + print(f"{label:<25} {val_with:<20{fmt}} {val_without:<20{fmt}} {delta_str:<15}") # Analysis print(f"\n" + "-" * 80) print("ANALYSIS") print("-" * 80) - strict_crossings = results["strict"]["barrier_crossings"] - pragmatic_crossings = results["pragmatic"]["barrier_crossings"] - emergency_crossings = results["emergency"]["barrier_crossings"] + time_saved = result_no_trails["total_time_minutes"] - result_trails["total_time_minutes"] + if time_saved > 0: + print(f"Trail burn-in saves {time_saved:.1f} minutes ({100*time_saved/result_no_trails['total_time_minutes']:.1f}% faster)") + elif time_saved < 0: + print(f"Trail burn-in adds {-time_saved:.1f} minutes (path seeks trails even if longer)") - print(f"Barrier crossings: strict={strict_crossings}, pragmatic={pragmatic_crossings}, emergency={emergency_crossings}") - - if strict_crossings == 0 and pragmatic_crossings == 0 and emergency_crossings == 0: - print("No path crosses private land - terrain naturally avoids barriers.") + on_trail_with = result_trails["on_trail_pct"] + on_trail_without = result_no_trails["on_trail_pct"] + if on_trail_with > on_trail_without: + print(f"Trail burn-in increases on-trail travel: {on_trail_without:.1f}% → {on_trail_with:.1f}%") else: - if emergency_crossings > pragmatic_crossings: - print(f"Pragmatic mode reduces barrier crossings vs emergency: {emergency_crossings} -> {pragmatic_crossings}") - if pragmatic_crossings > 0 and strict_crossings == 0: - print(f"Strict mode completely avoids private land (pragmatic crosses {pragmatic_crossings} cells)") - - # Time/distance comparison - if results["strict"]["total_time_minutes"] > results["emergency"]["total_time_minutes"]: - time_penalty = results["strict"]["total_time_minutes"] - results["emergency"]["total_time_minutes"] - print(f"Time cost of strict boundary respect: +{time_penalty:.1f} min") + print(f"Both paths have similar on-trail percentage") else: - print(f"Only {success_count}/3 modes found a path:") - for mode, result in results.items(): - if result["success"]: - print(f" {mode}: {result['total_distance_km']:.2f} km, {result['total_time_minutes']:.1f} min") - else: - print(f" {mode}: FAILED - {result.get('reason', 'unknown')}") + if not result_trails["success"]: + print(f"WITH TRAILS: FAILED - {result_trails.get('reason', 'unknown')}") + if not result_no_trails["success"]: + print(f"WITHOUT TRAILS: FAILED - {result_no_trails.get('reason', 'unknown')}") print(f"\n" + "-" * 80) print(f"Total wall time: {t_total - t0:.1f}s") - print(f"Closed cells in bbox: {closed_cells:,}") # Cleanup dem_reader.close() friction_reader.close() barrier_reader.close() + trail_reader.close() print("\nPrototype completed.") diff --git a/lib/offroute/trails.py b/lib/offroute/trails.py new file mode 100644 index 0000000..9d9185e --- /dev/null +++ b/lib/offroute/trails.py @@ -0,0 +1,174 @@ +""" +Trail corridor reader for OFFROUTE. + +Provides access to the OSM-derived trail raster for pathfinding. +Trail values replace WorldCover friction where trails exist. + +Raster values: + 0 = no trail (use WorldCover friction) + 5 = road (0.1× friction) + 15 = track (0.3× friction) + 25 = foot trail (0.5× friction) +""" +import numpy as np +from pathlib import Path +from typing import Tuple, Optional + +try: + import rasterio + from rasterio.windows import from_bounds + from rasterio.enums import Resampling +except ImportError: + raise ImportError("rasterio is required for trails layer support") + +# Default path to the trails raster +DEFAULT_TRAILS_PATH = Path("/mnt/nav/worldcover/trails.tif") + +# Trail value to friction multiplier mapping +TRAIL_FRICTION_MAP = { + 5: 0.1, # road + 15: 0.3, # track + 25: 0.5, # foot trail +} + + +class TrailReader: + """Reader for OSM-derived trail corridor raster.""" + + def __init__(self, trails_path: Path = DEFAULT_TRAILS_PATH): + self.trails_path = trails_path + self._dataset = None + + def _open(self): + """Lazy open the dataset.""" + if self._dataset is None: + if not self.trails_path.exists(): + raise FileNotFoundError( + f"Trails raster not found at {self.trails_path}. " + f"Run the Phase B rasterization script first." + ) + self._dataset = rasterio.open(self.trails_path) + return self._dataset + + def get_trails_grid( + self, + south: float, + north: float, + west: float, + east: float, + target_shape: Tuple[int, int] + ) -> np.ndarray: + """ + Get trail values for a bounding box, resampled to target shape. + + Args: + south, north, west, east: Bounding box coordinates (WGS84) + target_shape: (rows, cols) to resample to (matches elevation grid) + + Returns: + np.ndarray of uint8 trail values: + 0 = no trail + 5 = road (0.1× friction) + 15 = track (0.3× friction) + 25 = foot trail (0.5× friction) + """ + ds = self._open() + + # Create a window from the bounding box + window = from_bounds(west, south, east, north, ds.transform) + + # Read with resampling to target shape + # Use nearest neighbor to preserve discrete values + trails = ds.read( + 1, + window=window, + out_shape=target_shape, + resampling=Resampling.nearest + ) + + return trails + + def sample_point(self, lat: float, lon: float) -> int: + """Sample trail value at a single point.""" + ds = self._open() + + # Get pixel coordinates + row, col = ds.index(lon, lat) + + # Check bounds + if row < 0 or row >= ds.height or col < 0 or col >= ds.width: + return 0 # Out of bounds = no trail + + # Read single pixel + window = rasterio.windows.Window(col, row, 1, 1) + value = ds.read(1, window=window) + return int(value[0, 0]) + + def close(self): + """Close the dataset.""" + if self._dataset is not None: + self._dataset.close() + self._dataset = None + + +def trails_to_friction(trails: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + """ + Convert trail values to friction multipliers. + + Args: + trails: uint8 array of trail values (0, 5, 15, or 25) + + Returns: + Tuple of: + - friction: float32 array of friction multipliers + - has_trail: bool array indicating where trails exist + """ + friction = np.ones_like(trails, dtype=np.float32) + has_trail = trails > 0 + + # Apply friction values where trails exist + friction[trails == 5] = 0.1 # road + friction[trails == 15] = 0.3 # track + friction[trails == 25] = 0.5 # foot trail + + return friction, has_trail + + +if __name__ == "__main__": + print("Testing TrailReader...") + + if not DEFAULT_TRAILS_PATH.exists(): + print(f"Trails raster not found at {DEFAULT_TRAILS_PATH}") + print("Run Phase B rasterization first.") + exit(1) + + reader = TrailReader() + + # Test point sampling - Twin Falls downtown (should have roads) + test_lat, test_lon = 42.563, -114.461 + trail_value = reader.sample_point(test_lat, test_lon) + print(f"\nTwin Falls ({test_lat}, {test_lon}): trail value = {trail_value}") + label = {0: "no trail", 5: "road", 15: "track", 25: "trail"}.get(trail_value, "unknown") + print(f" Type: {label}") + + # Test grid read for test bbox + trails = reader.get_trails_grid( + south=42.21, north=42.60, west=-114.76, east=-113.79, + target_shape=(400, 1000) + ) + print(f"\nGrid test shape: {trails.shape}") + + unique, counts = np.unique(trails, return_counts=True) + print("Value distribution:") + for v, c in zip(unique, counts): + pct = 100 * c / trails.size + label = {0: "no trail", 5: "road", 15: "track", 25: "trail"}.get(v, f"unknown({v})") + print(f" {label}: {c:,} pixels ({pct:.2f}%)") + + # Test conversion to friction + friction, has_trail = trails_to_friction(trails) + print(f"\nTrail coverage: {100 * np.sum(has_trail) / trails.size:.2f}%") + print(f"Friction range (on trails): {friction[has_trail].min():.1f} - {friction[has_trail].max():.1f}") + + reader.close() + print("\nTrailReader test complete.") From 1a9dfc8f8d78fb5c7a9798b58fa94cd6d648f6a8 Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 8 May 2026 13:44:34 +0000 Subject: [PATCH 07/41] =?UTF-8?q?feat(offroute):=20Phase=20O3b=20=E2=80=94?= =?UTF-8?q?=20trail=20entry=20index,=20Valhalla=20stitching,=20/api/offrou?= =?UTF-8?q?te=20endpoint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase A: Trail Entry Point Index - Extract highway endpoints from idaho-latest.osm.pbf using osmium + ogr2ogr - Store 740,430 entry points in /mnt/nav/navi.db (SQLite with spatial index) - Entry points by class: service (271k), footway (152k), residential (146k), track (111k), path (26k), unclassified (16k), tertiary (9k), secondary (4k), primary (4k), bridleway (15) Phase B: Pathfinder → Valhalla Stitching (router.py) - OffrouteRouter orchestrates wilderness pathfinding + Valhalla on-network routing - Queries entry points within 50km (expanding to 100km if needed) - MCP pathfinder routes to nearest reachable entry point - Calls Valhalla pedestrian/bicycle/auto costing for on-network segment - Returns GeoJSON FeatureCollection with wilderness + network + combined segments Phase C: Flask Endpoint - POST /api/offroute with start/end coordinates, mode, boundary_mode - Returns GeoJSON route with per-segment metadata and turn-by-turn maneuvers Validated: 42.35,-114.30 → Twin Falls downtown - Wilderness: 0.5km, 9min | Network: 36km, 413min | Total: ~421min - 21 turn-by-turn instructions, segments connect at entry point Co-Authored-By: Claude Opus 4.5 --- lib/api.py | 77 +++++ lib/offroute/router.py | 752 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 829 insertions(+) create mode 100644 lib/offroute/router.py diff --git a/lib/api.py b/lib/api.py index 8a1f383..cd32e33 100644 --- a/lib/api.py +++ b/lib/api.py @@ -2722,3 +2722,80 @@ def api_auth_whoami(): 'authenticated': False, 'username': None, }) + + +# ── OFFROUTE API ── + +@app.route("/api/offroute", methods=["POST"]) +def api_offroute(): + """ + Off-network routing from wilderness to destination. + + Request body: + { + "start": [lat, lon], + "end": [lat, lon], + "mode": "foot" | "mtb" | "atv", (default: "foot") + "boundary_mode": "strict" | "pragmatic" | "emergency" (default: "pragmatic") + } + + Response: + { + "status": "ok", + "route": { GeoJSON FeatureCollection with wilderness + network segments }, + "summary": { total_distance_km, total_effort_minutes, ... } + } + """ + try: + data = request.get_json() + if not data: + return jsonify({"status": "error", "message": "No JSON body provided"}), 400 + + # Parse coordinates + start = data.get("start") + end = data.get("end") + + if not start or not end: + return jsonify({"status": "error", "message": "Missing start or end coordinates"}), 400 + + if not isinstance(start, (list, tuple)) or len(start) != 2: + return jsonify({"status": "error", "message": "start must be [lat, lon]"}), 400 + if not isinstance(end, (list, tuple)) or len(end) != 2: + return jsonify({"status": "error", "message": "end must be [lat, lon]"}), 400 + + start_lat, start_lon = float(start[0]), float(start[1]) + end_lat, end_lon = float(end[0]), float(end[1]) + + # Parse options + mode = data.get("mode", "foot") + if mode not in ("foot", "mtb", "atv"): + return jsonify({"status": "error", "message": "mode must be foot, mtb, or atv"}), 400 + + boundary_mode = data.get("boundary_mode", "pragmatic") + if boundary_mode not in ("strict", "pragmatic", "emergency"): + return jsonify({"status": "error", "message": "boundary_mode must be strict, pragmatic, or emergency"}), 400 + + # Import and run router + from .offroute.router import OffrouteRouter + + router = OffrouteRouter() + try: + result = router.route( + start_lat=start_lat, + start_lon=start_lon, + end_lat=end_lat, + end_lon=end_lon, + mode=mode, + boundary_mode=boundary_mode + ) + finally: + router.close() + + if result.get("status") == "error": + return jsonify(result), 400 + + return jsonify(result) + + except Exception as e: + logger.exception("Offroute error") + return jsonify({"status": "error", "message": str(e)}), 500 diff --git a/lib/offroute/router.py b/lib/offroute/router.py new file mode 100644 index 0000000..57d6ce5 --- /dev/null +++ b/lib/offroute/router.py @@ -0,0 +1,752 @@ +""" +OFFROUTE Router — Wilderness to network path orchestration. + +Connects the raster pathfinder (wilderness segment) to Valhalla (on-network segment). + +Entry points are extracted from OSM highways and stored in /mnt/nav/navi.db. +The pathfinder routes from a wilderness start to the nearest entry point, +then Valhalla completes the route to the destination. +""" +import json +import math +import sqlite3 +import subprocess +import tempfile +import time +from pathlib import Path +from typing import Dict, List, Optional, Tuple, Literal + +import numpy as np +import requests +from skimage.graph import MCP_Geometric + +from .dem import DEMReader +from .cost import compute_cost_grid +from .friction import FrictionReader, friction_to_multiplier +from .barriers import BarrierReader +from .trails import TrailReader + +# Paths +NAVI_DB_PATH = Path("/mnt/nav/navi.db") +OSM_PBF_PATH = Path("/mnt/nav/sources/idaho-latest.osm.pbf") + +# Valhalla endpoint +VALHALLA_URL = "http://localhost:8002" + +# Search radius for entry points (km) +DEFAULT_SEARCH_RADIUS_KM = 50 +EXPANDED_SEARCH_RADIUS_KM = 100 + +# Memory limit +MEMORY_LIMIT_GB = 12 + +# Mode to Valhalla costing mapping +MODE_TO_COSTING = { + "foot": "pedestrian", + "mtb": "bicycle", + "atv": "auto", +} + + +def haversine_distance(lat1: float, lon1: float, lat2: float, lon2: float) -> float: + """Calculate distance between two points in meters.""" + R = 6371000 + dlat = math.radians(lat2 - lat1) + dlon = math.radians(lon2 - lon1) + a = math.sin(dlat/2)**2 + math.cos(math.radians(lat1)) * math.cos(math.radians(lat2)) * math.sin(dlon/2)**2 + c = 2 * math.atan2(math.sqrt(a), math.sqrt(1-a)) + return R * c + + +def check_memory_usage() -> float: + """Check current memory usage in GB.""" + try: + import psutil + process = psutil.Process() + return process.memory_info().rss / (1024**3) + except ImportError: + return 0 + + +class EntryPointIndex: + """ + Trail entry point index for wilderness-to-network handoff. + + Entry points are endpoints and intersections of OSM highways + that connect wilderness areas to the routable network. + """ + + def __init__(self, db_path: Path = NAVI_DB_PATH): + self.db_path = db_path + self._conn = None + + def _get_conn(self) -> sqlite3.Connection: + if self._conn is None: + self._conn = sqlite3.connect(str(self.db_path)) + self._conn.row_factory = sqlite3.Row + return self._conn + + def table_exists(self) -> bool: + """Check if trail_entry_points table exists.""" + if not self.db_path.exists(): + return False + conn = self._get_conn() + cur = conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name='trail_entry_points'" + ) + return cur.fetchone() is not None + + def get_entry_point_count(self) -> int: + """Get count of entry points.""" + if not self.table_exists(): + return 0 + conn = self._get_conn() + cur = conn.execute("SELECT COUNT(*) FROM trail_entry_points") + return cur.fetchone()[0] + + def query_bbox(self, south: float, north: float, west: float, east: float) -> List[Dict]: + """Query entry points within a bounding box.""" + if not self.table_exists(): + return [] + + conn = self._get_conn() + cur = conn.execute(""" + SELECT id, lat, lon, highway_class, name + FROM trail_entry_points + WHERE lat >= ? AND lat <= ? AND lon >= ? AND lon <= ? + """, (south, north, west, east)) + + return [dict(row) for row in cur.fetchall()] + + def query_radius(self, lat: float, lon: float, radius_km: float) -> List[Dict]: + """Query entry points within radius of a point.""" + # Approximate bbox for the radius + lat_delta = radius_km / 111.0 + lon_delta = radius_km / (111.0 * math.cos(math.radians(lat))) + + points = self.query_bbox( + lat - lat_delta, lat + lat_delta, + lon - lon_delta, lon + lon_delta + ) + + # Filter by actual distance and add distance field + result = [] + for p in points: + dist = haversine_distance(lat, lon, p['lat'], p['lon']) + if dist <= radius_km * 1000: + p['distance_m'] = dist + result.append(p) + + return sorted(result, key=lambda x: x['distance_m']) + + def build_index(self, osm_pbf_path: Path = OSM_PBF_PATH) -> Dict: + """ + Build the entry point index from OSM PBF. + + Extracts endpoints of highway features that connect to the network. + """ + if not osm_pbf_path.exists(): + raise FileNotFoundError(f"OSM PBF not found: {osm_pbf_path}") + + print(f"Building trail entry point index from {osm_pbf_path}...") + + # Highway types to extract (routable network entry points) + highway_types = [ + "primary", "secondary", "tertiary", "unclassified", + "residential", "service", "track", "path", "footway", "bridleway" + ] + + stats = {"total": 0, "by_class": {}} + + with tempfile.TemporaryDirectory() as tmpdir: + # Extract highways to GeoJSON + geojson_path = Path(tmpdir) / "highways.geojson" + + # Build osmium tags-filter expressions (one per highway type) + print(f" Extracting highways with osmium...") + cmd = [ + "osmium", "tags-filter", + str(osm_pbf_path), + ] + # Add each highway type as a separate filter expression + for ht in highway_types: + cmd.append(f"w/highway={ht}") + cmd.extend(["-o", str(Path(tmpdir) / "filtered.osm.pbf"), "--overwrite"]) + + subprocess.run(cmd, check=True, capture_output=True) + + # Convert to GeoJSON + print(f" Converting to GeoJSON with ogr2ogr...") + cmd = [ + "ogr2ogr", "-f", "GeoJSON", + str(geojson_path), + str(Path(tmpdir) / "filtered.osm.pbf"), + "lines", + "-t_srs", "EPSG:4326" + ] + subprocess.run(cmd, check=True, capture_output=True) + + # Parse GeoJSON and extract endpoints + print(f" Extracting entry points...") + with open(geojson_path) as f: + data = json.load(f) + + # Collect unique points (endpoints) + # Key: (lat, lon) rounded to 5 decimal places (~1m precision) + points = {} + + for feature in data.get("features", []): + props = feature.get("properties", {}) + geom = feature.get("geometry", {}) + + if geom.get("type") != "LineString": + continue + + coords = geom.get("coordinates", []) + if len(coords) < 2: + continue + + highway_class = props.get("highway", "unknown") + name = props.get("name", "") + + # Extract endpoints + for coord in [coords[0], coords[-1]]: + lon, lat = coord[0], coord[1] + key = (round(lat, 5), round(lon, 5)) + + if key not in points: + points[key] = { + "lat": lat, + "lon": lon, + "highway_class": highway_class, + "name": name + } + else: + # Keep the "best" highway class (roads > tracks > paths) + existing = points[key] + if self._highway_priority(highway_class) < self._highway_priority(existing["highway_class"]): + points[key]["highway_class"] = highway_class + if name and not existing["name"]: + points[key]["name"] = name + + # Create/update database + print(f" Writing {len(points)} entry points to {self.db_path}...") + + self.db_path.parent.mkdir(parents=True, exist_ok=True) + conn = self._get_conn() + + # Create table + conn.execute(""" + CREATE TABLE IF NOT EXISTS trail_entry_points ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + lat REAL NOT NULL, + lon REAL NOT NULL, + highway_class TEXT NOT NULL, + name TEXT + ) + """) + + # Clear existing data + conn.execute("DELETE FROM trail_entry_points") + + # Insert new points + for point in points.values(): + conn.execute(""" + INSERT INTO trail_entry_points (lat, lon, highway_class, name) + VALUES (?, ?, ?, ?) + """, (point["lat"], point["lon"], point["highway_class"], point["name"])) + + stats["total"] += 1 + hc = point["highway_class"] + stats["by_class"][hc] = stats["by_class"].get(hc, 0) + 1 + + # Create spatial index + conn.execute("CREATE INDEX IF NOT EXISTS idx_entry_lat ON trail_entry_points(lat)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_entry_lon ON trail_entry_points(lon)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_entry_latlon ON trail_entry_points(lat, lon)") + + conn.commit() + + print(f" Done. Total: {stats['total']} entry points") + for hc, count in sorted(stats["by_class"].items(), key=lambda x: -x[1]): + print(f" {hc}: {count}") + + return stats + + def _highway_priority(self, highway_class: str) -> int: + """Lower number = better priority for entry points.""" + priority = { + "primary": 1, "secondary": 2, "tertiary": 3, + "unclassified": 4, "residential": 5, "service": 6, + "track": 7, "path": 8, "footway": 9, "bridleway": 10 + } + return priority.get(highway_class, 99) + + def close(self): + if self._conn: + self._conn.close() + self._conn = None + + +class OffrouteRouter: + """ + OFFROUTE Router — orchestrates wilderness pathfinding and Valhalla stitching. + """ + + def __init__(self): + self.dem_reader = None + self.friction_reader = None + self.barrier_reader = None + self.trail_reader = None + self.entry_index = EntryPointIndex() + + def _init_readers(self): + """Lazy init readers.""" + if self.dem_reader is None: + self.dem_reader = DEMReader() + if self.friction_reader is None: + self.friction_reader = FrictionReader() + if self.barrier_reader is None: + self.barrier_reader = BarrierReader() + if self.trail_reader is None: + self.trail_reader = TrailReader() + + def route( + self, + start_lat: float, + start_lon: float, + end_lat: float, + end_lon: float, + mode: Literal["foot", "mtb", "atv"] = "foot", + boundary_mode: Literal["strict", "pragmatic", "emergency"] = "pragmatic" + ) -> Dict: + """ + Route from a wilderness start point to a destination. + + Returns a GeoJSON FeatureCollection with wilderness and network segments. + """ + t0 = time.time() + + # Ensure entry point index exists + if not self.entry_index.table_exists() or self.entry_index.get_entry_point_count() == 0: + return { + "status": "error", + "message": "Trail entry point index not built. Run build_entry_index() first." + } + + # Find entry points near start + entry_points = self.entry_index.query_radius( + start_lat, start_lon, DEFAULT_SEARCH_RADIUS_KM + ) + + if not entry_points: + # Try expanded radius + entry_points = self.entry_index.query_radius( + start_lat, start_lon, EXPANDED_SEARCH_RADIUS_KM + ) + if not entry_points: + return { + "status": "error", + "message": f"No trail entry points found within {EXPANDED_SEARCH_RADIUS_KM}km of start" + } + + # Build bbox for pathfinding grid + # Include start, end, and all entry points + all_lats = [start_lat, end_lat] + [p["lat"] for p in entry_points] + all_lons = [start_lon, end_lon] + [p["lon"] for p in entry_points] + + padding = 0.05 # ~5km padding + bbox = { + "south": min(all_lats) - padding, + "north": max(all_lats) + padding, + "west": min(all_lons) - padding, + "east": max(all_lons) + padding, + } + + # Initialize readers + self._init_readers() + + # Load elevation + try: + elevation, meta = self.dem_reader.get_elevation_grid( + south=bbox["south"], + north=bbox["north"], + west=bbox["west"], + east=bbox["east"], + ) + except Exception as e: + return {"status": "error", "message": f"Failed to load elevation: {e}"} + + # Check memory + mem = check_memory_usage() + if mem > MEMORY_LIMIT_GB: + return {"status": "error", "message": f"Memory limit exceeded: {mem:.1f}GB > {MEMORY_LIMIT_GB}GB"} + + # Load friction + friction_raw = self.friction_reader.get_friction_grid( + south=bbox["south"], + north=bbox["north"], + west=bbox["west"], + east=bbox["east"], + target_shape=elevation.shape + ) + friction_mult = friction_to_multiplier(friction_raw) + + # Load barriers + barriers = self.barrier_reader.get_barrier_grid( + south=bbox["south"], + north=bbox["north"], + west=bbox["west"], + east=bbox["east"], + target_shape=elevation.shape + ) + + # Load trails + trails = self.trail_reader.get_trails_grid( + south=bbox["south"], + north=bbox["north"], + west=bbox["west"], + east=bbox["east"], + target_shape=elevation.shape + ) + + # Compute cost grid + cost = compute_cost_grid( + elevation, + cell_size_m=meta["cell_size_m"], + friction=friction_mult, + trails=trails, + barriers=barriers, + boundary_mode=boundary_mode, + ) + + # Convert start to pixel coordinates + start_row, start_col = self.dem_reader.latlon_to_pixel(start_lat, start_lon, meta) + + # Validate start is in bounds + rows, cols = elevation.shape + if not (0 <= start_row < rows and 0 <= start_col < cols): + return {"status": "error", "message": "Start point outside grid bounds"} + + # Mark entry points on the grid + entry_pixels = [] + for ep in entry_points: + row, col = self.dem_reader.latlon_to_pixel(ep["lat"], ep["lon"], meta) + if 0 <= row < rows and 0 <= col < cols: + entry_pixels.append({ + "row": row, + "col": col, + "entry_point": ep + }) + + if not entry_pixels: + return {"status": "error", "message": "No entry points map to grid bounds"} + + # Run MCP pathfinder + mcp = MCP_Geometric(cost, fully_connected=True) + cumulative_costs, traceback = mcp.find_costs([(start_row, start_col)]) + + # Find nearest reachable entry point + best_entry = None + best_cost = np.inf + + for ep in entry_pixels: + ep_cost = cumulative_costs[ep["row"], ep["col"]] + if ep_cost < best_cost: + best_cost = ep_cost + best_entry = ep + + if best_entry is None or np.isinf(best_cost): + return { + "status": "error", + "message": "No path found to any entry point (blocked by impassable terrain)" + } + + # Traceback wilderness path + path_indices = mcp.traceback((best_entry["row"], best_entry["col"])) + + # Convert to coordinates and collect stats + wilderness_coords = [] + elevations = [] + trail_values = [] + + for row, col in path_indices: + lat, lon = self.dem_reader.pixel_to_latlon(row, col, meta) + wilderness_coords.append([lon, lat]) + elevations.append(elevation[row, col]) + trail_values.append(trails[row, col]) + + # Calculate wilderness segment stats + wilderness_distance_m = 0 + for i in range(1, len(wilderness_coords)): + lon1, lat1 = wilderness_coords[i-1] + lon2, lat2 = wilderness_coords[i] + wilderness_distance_m += haversine_distance(lat1, lon1, lat2, lon2) + + elev_arr = np.array(elevations) + elev_diff = np.diff(elev_arr) + wilderness_gain = float(np.sum(elev_diff[elev_diff > 0])) + wilderness_loss = float(np.sum(np.abs(elev_diff[elev_diff < 0]))) + + trail_arr = np.array(trail_values) + on_trail_cells = np.sum(trail_arr > 0) + total_cells = len(trail_arr) + on_trail_pct = float(100 * on_trail_cells / total_cells) if total_cells > 0 else 0 + + # Entry point reached + entry_lat = best_entry["entry_point"]["lat"] + entry_lon = best_entry["entry_point"]["lon"] + entry_class = best_entry["entry_point"]["highway_class"] + entry_name = best_entry["entry_point"].get("name", "") + + # Call Valhalla for on-network segment + valhalla_costing = MODE_TO_COSTING.get(mode, "pedestrian") + + valhalla_request = { + "locations": [ + {"lat": entry_lat, "lon": entry_lon}, + {"lat": end_lat, "lon": end_lon} + ], + "costing": valhalla_costing, + "directions_options": {"units": "kilometers"} + } + + network_segment = None + valhalla_error = None + + try: + resp = requests.post( + f"{VALHALLA_URL}/route", + json=valhalla_request, + timeout=30 + ) + + if resp.status_code == 200: + valhalla_data = resp.json() + trip = valhalla_data.get("trip", {}) + legs = trip.get("legs", []) + + if legs: + leg = legs[0] + shape = leg.get("shape", "") + + # Decode polyline6 + network_coords = self._decode_polyline(shape) + + # Extract maneuvers + maneuvers = [] + for m in leg.get("maneuvers", []): + maneuvers.append({ + "instruction": m.get("instruction", ""), + "type": m.get("type", 0), + "distance_km": m.get("length", 0), + "time_seconds": m.get("time", 0), + "street_names": m.get("street_names", []), + }) + + summary = trip.get("summary", {}) + network_segment = { + "coordinates": network_coords, + "distance_km": summary.get("length", 0), + "duration_minutes": summary.get("time", 0) / 60, + "maneuvers": maneuvers, + } + else: + valhalla_error = f"Valhalla returned {resp.status_code}: {resp.text[:200]}" + + except Exception as e: + valhalla_error = f"Valhalla request failed: {e}" + + # Build response + features = [] + + # Feature 1: Wilderness segment + wilderness_feature = { + "type": "Feature", + "properties": { + "segment_type": "wilderness", + "effort_minutes": float(best_cost / 60), + "distance_km": float(wilderness_distance_m / 1000), + "elevation_gain_m": wilderness_gain, + "elevation_loss_m": wilderness_loss, + "boundary_mode": boundary_mode, + "on_trail_pct": on_trail_pct, + "cell_count": total_cells, + }, + "geometry": { + "type": "LineString", + "coordinates": wilderness_coords, + } + } + features.append(wilderness_feature) + + # Feature 2: Network segment (if available) + if network_segment: + network_feature = { + "type": "Feature", + "properties": { + "segment_type": "network", + "distance_km": network_segment["distance_km"], + "duration_minutes": network_segment["duration_minutes"], + "maneuvers": network_segment["maneuvers"], + }, + "geometry": { + "type": "LineString", + "coordinates": network_segment["coordinates"], + } + } + features.append(network_feature) + + # Build combined route coordinates + combined_coords = wilderness_coords.copy() + if network_segment: + # Skip first point of network segment (it's the same as last wilderness point) + combined_coords.extend(network_segment["coordinates"][1:]) + + # Feature 3: Combined route + combined_feature = { + "type": "Feature", + "properties": { + "segment_type": "combined", + "mode": mode, + "boundary_mode": boundary_mode, + }, + "geometry": { + "type": "LineString", + "coordinates": combined_coords, + } + } + features.append(combined_feature) + + geojson = { + "type": "FeatureCollection", + "features": features, + } + + # Build summary + total_distance_km = wilderness_distance_m / 1000 + total_effort_minutes = best_cost / 60 + + if network_segment: + total_distance_km += network_segment["distance_km"] + total_effort_minutes += network_segment["duration_minutes"] + + summary = { + "total_distance_km": float(total_distance_km), + "total_effort_minutes": float(total_effort_minutes), + "wilderness_distance_km": float(wilderness_distance_m / 1000), + "wilderness_effort_minutes": float(best_cost / 60), + "network_distance_km": float(network_segment["distance_km"]) if network_segment else 0, + "network_duration_minutes": float(network_segment["duration_minutes"]) if network_segment else 0, + "on_trail_pct": on_trail_pct, + "boundary_mode": boundary_mode, + "mode": mode, + "entry_point": { + "lat": entry_lat, + "lon": entry_lon, + "highway_class": entry_class, + "name": entry_name, + }, + "computation_time_s": time.time() - t0, + } + + result = { + "status": "ok", + "route": geojson, + "summary": summary, + } + + if valhalla_error: + result["warning"] = f"Network segment incomplete: {valhalla_error}" + + return result + + def _decode_polyline(self, encoded: str, precision: int = 6) -> List[List[float]]: + """Decode a polyline string into coordinates [lon, lat].""" + coords = [] + index = 0 + lat = 0 + lon = 0 + + while index < len(encoded): + # Latitude + shift = 0 + result = 0 + while True: + b = ord(encoded[index]) - 63 + index += 1 + result |= (b & 0x1f) << shift + shift += 5 + if b < 0x20: + break + dlat = ~(result >> 1) if result & 1 else result >> 1 + lat += dlat + + # Longitude + shift = 0 + result = 0 + while True: + b = ord(encoded[index]) - 63 + index += 1 + result |= (b & 0x1f) << shift + shift += 5 + if b < 0x20: + break + dlon = ~(result >> 1) if result & 1 else result >> 1 + lon += dlon + + coords.append([lon / (10 ** precision), lat / (10 ** precision)]) + + return coords + + def close(self): + """Close all readers.""" + if self.dem_reader: + self.dem_reader.close() + if self.friction_reader: + self.friction_reader.close() + if self.barrier_reader: + self.barrier_reader.close() + if self.trail_reader: + self.trail_reader.close() + self.entry_index.close() + + +def build_entry_index(): + """Build the trail entry point index.""" + index = EntryPointIndex() + stats = index.build_index() + index.close() + return stats + + +if __name__ == "__main__": + import sys + + if len(sys.argv) > 1 and sys.argv[1] == "build": + print("Building trail entry point index...") + stats = build_entry_index() + print(f"\nDone. Total entry points: {stats['total']}") + + elif len(sys.argv) > 1 and sys.argv[1] == "test": + print("Testing router...") + + router = OffrouteRouter() + + # Test route: wilderness to Twin Falls + result = router.route( + start_lat=42.35, + start_lon=-114.30, + end_lat=42.5629, + end_lon=-114.4609, + mode="foot", + boundary_mode="pragmatic" + ) + + print(json.dumps(result, indent=2, default=str)) + router.close() + + else: + print("Usage:") + print(" python router.py build # Build entry point index") + print(" python router.py test # Test route") From bc463188d56b1cf9b3e804427ab5e0117743fc57 Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 8 May 2026 14:11:56 +0000 Subject: [PATCH 08/41] =?UTF-8?q?feat(offroute):=20Phase=20O4=20=E2=80=94?= =?UTF-8?q?=20multi-mode=20cost=20functions=20(foot/mtb/atv/vehicle)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add ModeProfile dataclass for data-driven mode configuration - Implement three speed functions: * Tobler off-path hiking (foot) * Herzog wheeled-transport polynomial (mtb/atv) * Linear speed degradation (vehicle) - Add WildernessReader for PAD-US Des_Tp=WA wilderness areas - Mode-specific terrain friction overrides: * Forest impassable for ATV/vehicle, high friction for MTB * Wetland/mangrove impassable for all wheeled modes - Trail access rules: * Foot trails (value 25) impassable for ATV/vehicle - Wilderness blocking for mtb/atv/vehicle modes - Vehicle mode allows flat grassland/cropland traversal - Memory optimization: limit entry points, constrain bbox size - Update router to pass mode and wilderness to cost function - Add vehicle to API mode validation Validated all four modes with test route: - foot: 0.46km off-network, 12.11km network, 89% on trail - mtb: 0.47km off-network, 13.13km network, 90% on trail - atv: 0.47km off-network, 12.81km network, 90% on trail - vehicle: 0.46km off-network, 12.81km network, 89% on trail Co-Authored-By: Claude Opus 4.5 --- lib/api.py | 4 +- lib/offroute/barriers.py | 274 +++++++++++++++++---- lib/offroute/cost.py | 515 +++++++++++++++++++++++++++++---------- lib/offroute/router.py | 274 ++++++++++----------- 4 files changed, 740 insertions(+), 327 deletions(-) diff --git a/lib/api.py b/lib/api.py index cd32e33..a127866 100644 --- a/lib/api.py +++ b/lib/api.py @@ -2768,8 +2768,8 @@ def api_offroute(): # Parse options mode = data.get("mode", "foot") - if mode not in ("foot", "mtb", "atv"): - return jsonify({"status": "error", "message": "mode must be foot, mtb, or atv"}), 400 + if mode not in ("foot", "mtb", "atv", "vehicle"): + return jsonify({"status": "error", "message": "mode must be foot, mtb, atv, or vehicle"}), 400 boundary_mode = data.get("boundary_mode", "pragmatic") if boundary_mode not in ("strict", "pragmatic", "emergency"): diff --git a/lib/offroute/barriers.py b/lib/offroute/barriers.py index 7fcad75..f68e892 100644 --- a/lib/offroute/barriers.py +++ b/lib/offroute/barriers.py @@ -1,11 +1,12 @@ """ -PAD-US barrier layer for OFFROUTE. +PAD-US barrier and wilderness layers for OFFROUTE. -Provides access to the PAD-US land ownership raster for routing decisions. -Cells with value 255 represent closed/restricted areas (Pub_Access = XA). +Provides access to: +1. Barrier raster (Pub_Access = 'XA' - closed/restricted areas) +2. Wilderness raster (Des_Tp = 'WA' - designated wilderness areas) -Build function rasterizes PAD-US geodatabase to aligned GeoTIFF. -Runtime functions read the raster and resample to match elevation grids. +Build functions rasterize PAD-US geodatabase to aligned GeoTIFFs. +Runtime functions read the rasters and resample to match elevation grids. """ import numpy as np from pathlib import Path @@ -23,6 +24,7 @@ except ImportError: # Paths DEFAULT_BARRIERS_PATH = Path("/mnt/nav/worldcover/padus_barriers.tif") +DEFAULT_WILDERNESS_PATH = Path("/mnt/nav/worldcover/wilderness.tif") PADUS_GDB_PATH = Path("/mnt/nav/padus/PADUS4_0_Geodatabase.gdb") PADUS_LAYER = "PADUS4_0Combined_Proclamation_Marine_Fee_Designation_Easement" @@ -39,7 +41,7 @@ PIXEL_SIZE = 0.0003 # ~33m class BarrierReader: - """Reader for PAD-US barrier raster.""" + """Reader for PAD-US barrier raster (closed/restricted areas).""" def __init__(self, barrier_path: Path = DEFAULT_BARRIERS_PATH): self.barrier_path = barrier_path @@ -77,32 +79,86 @@ class BarrierReader: 0 = public/accessible """ ds = self._open() - - # Create a window from the bounding box window = from_bounds(west, south, east, north, ds.transform) - - # Read with resampling to target shape barriers = ds.read( 1, window=window, out_shape=target_shape, resampling=Resampling.nearest ) - return barriers def sample_point(self, lat: float, lon: float) -> int: """Sample barrier value at a single point.""" ds = self._open() - - # Get pixel coordinates row, col = ds.index(lon, lat) - - # Check bounds if row < 0 or row >= ds.height or col < 0 or col >= ds.width: - return 0 # Out of bounds = accessible + return 0 + window = rasterio.windows.Window(col, row, 1, 1) + value = ds.read(1, window=window) + return int(value[0, 0]) - # Read single pixel + def close(self): + """Close the dataset.""" + if self._dataset is not None: + self._dataset.close() + self._dataset = None + + +class WildernessReader: + """Reader for PAD-US wilderness raster (designated wilderness areas).""" + + def __init__(self, wilderness_path: Path = DEFAULT_WILDERNESS_PATH): + self.wilderness_path = wilderness_path + self._dataset = None + + def _open(self): + """Lazy open the dataset.""" + if self._dataset is None: + if not self.wilderness_path.exists(): + raise FileNotFoundError( + f"Wilderness raster not found at {self.wilderness_path}. " + f"Run build_wilderness_raster() first." + ) + self._dataset = rasterio.open(self.wilderness_path) + return self._dataset + + def get_wilderness_grid( + self, + south: float, + north: float, + west: float, + east: float, + target_shape: Tuple[int, int] + ) -> np.ndarray: + """ + Get wilderness values for a bounding box, resampled to target shape. + + Args: + south, north, west, east: Bounding box coordinates (WGS84) + target_shape: (rows, cols) to resample to (matches elevation grid) + + Returns: + np.ndarray of uint8 wilderness values: + 255 = designated wilderness area + 0 = not wilderness + """ + ds = self._open() + window = from_bounds(west, south, east, north, ds.transform) + wilderness = ds.read( + 1, + window=window, + out_shape=target_shape, + resampling=Resampling.nearest + ) + return wilderness + + def sample_point(self, lat: float, lon: float) -> int: + """Sample wilderness value at a single point.""" + ds = self._open() + row, col = ds.index(lon, lat) + if row < 0 or row >= ds.height or col < 0 or col >= ds.width: + return 0 window = rasterio.windows.Window(col, row, 1, 1) value = ds.read(1, window=window) return int(value[0, 0]) @@ -124,22 +180,12 @@ def build_barriers_raster( Build the PAD-US barriers raster from the source geodatabase. Extracts polygons where Pub_Access = 'XA' (Closed) and rasterizes them. - - Args: - output_path: Output GeoTIFF path - gdb_path: Path to PAD-US geodatabase - pixel_size: Pixel size in degrees - bounds: CONUS bounding box - - Returns: - Path to the created raster """ import shutil if not gdb_path.exists(): raise FileNotFoundError(f"PAD-US geodatabase not found at {gdb_path}") - # Check for required tools if not shutil.which('ogr2ogr'): raise RuntimeError("ogr2ogr not found. Install GDAL.") if not shutil.which('gdal_rasterize'): @@ -154,7 +200,6 @@ def build_barriers_raster( print(f" Bounds: {bounds}") with tempfile.TemporaryDirectory() as tmpdir: - # Step 1: Extract closed areas and reproject to WGS84 closed_gpkg = Path(tmpdir) / "closed_areas.gpkg" print(f"\n[1/3] Extracting closed areas (Pub_Access = 'XA')...") @@ -176,28 +221,23 @@ def build_barriers_raster( print(f"STDERR: {result.stderr}") raise RuntimeError(f"ogr2ogr failed: {result.stderr}") - # Check feature count info_cmd = ["ogrinfo", "-so", str(closed_gpkg), "closed_areas"] info_result = subprocess.run(info_cmd, capture_output=True, text=True) print(f" Extraction result:\n{info_result.stdout}") - # Step 2: Create empty raster print(f"\n[2/3] Creating raster grid...") width = int((bounds['east'] - bounds['west']) / pixel_size) height = int((bounds['north'] - bounds['south']) / pixel_size) - print(f" Grid size: {width} x {height} pixels") - print(f" Memory estimate: {width * height / 1e6:.1f} MB") - # Step 3: Rasterize print(f"\n[3/3] Rasterizing closed areas...") rasterize_cmd = [ "gdal_rasterize", "-burn", "255", "-init", "0", - "-a_nodata", "0", # No nodata - 0 means accessible + "-a_nodata", "0", "-te", str(bounds['west']), str(bounds['south']), str(bounds['east']), str(bounds['north']), "-tr", str(pixel_size), str(pixel_size), @@ -214,14 +254,10 @@ def build_barriers_raster( print(f"STDERR: {result.stderr}") raise RuntimeError(f"gdal_rasterize failed: {result.stderr}") - # Verify output print(f"\n[Done] Verifying output...") with rasterio.open(output_path) as ds: print(f" Size: {ds.width} x {ds.height}") print(f" CRS: {ds.crs}") - print(f" Bounds: {ds.bounds}") - - # Sample a few tiles to check sample = ds.read(1, window=rasterio.windows.Window(0, 0, 1000, 1000)) closed_count = np.sum(sample == 255) print(f" Sample (1000x1000): {closed_count} closed cells") @@ -232,17 +268,140 @@ def build_barriers_raster( return output_path +def build_wilderness_raster( + output_path: Path = DEFAULT_WILDERNESS_PATH, + gdb_path: Path = PADUS_GDB_PATH, + pixel_size: float = PIXEL_SIZE, + bounds: dict = CONUS_BOUNDS, +) -> Path: + """ + Build the PAD-US wilderness raster from the source geodatabase. + + Extracts polygons where Des_Tp = 'WA' (Wilderness Area) and rasterizes them. + """ + import shutil + + if not gdb_path.exists(): + raise FileNotFoundError(f"PAD-US geodatabase not found at {gdb_path}") + + if not shutil.which('ogr2ogr'): + raise RuntimeError("ogr2ogr not found. Install GDAL.") + if not shutil.which('gdal_rasterize'): + raise RuntimeError("gdal_rasterize not found. Install GDAL.") + + output_path.parent.mkdir(parents=True, exist_ok=True) + + print(f"Building PAD-US wilderness raster...") + print(f" Source: {gdb_path}") + print(f" Output: {output_path}") + print(f" Pixel size: {pixel_size} degrees (~{pixel_size * 111000:.0f}m)") + print(f" Bounds: {bounds}") + + with tempfile.TemporaryDirectory() as tmpdir: + wilderness_gpkg = Path(tmpdir) / "wilderness_areas.gpkg" + + print(f"\n[1/3] Extracting wilderness areas (Des_Tp = 'WA')...") + + ogr_cmd = [ + "ogr2ogr", + "-f", "GPKG", + str(wilderness_gpkg), + str(gdb_path), + PADUS_LAYER, + "-where", "Des_Tp = 'WA'", + "-t_srs", "EPSG:4326", + "-nlt", "MULTIPOLYGON", + "-nln", "wilderness_areas", + ] + + result = subprocess.run(ogr_cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"STDERR: {result.stderr}") + raise RuntimeError(f"ogr2ogr failed: {result.stderr}") + + info_cmd = ["ogrinfo", "-so", str(wilderness_gpkg), "wilderness_areas"] + info_result = subprocess.run(info_cmd, capture_output=True, text=True) + print(f" Extraction result:\n{info_result.stdout}") + + print(f"\n[2/3] Creating raster grid...") + + width = int((bounds['east'] - bounds['west']) / pixel_size) + height = int((bounds['north'] - bounds['south']) / pixel_size) + print(f" Grid size: {width} x {height} pixels") + + print(f"\n[3/3] Rasterizing wilderness areas...") + + rasterize_cmd = [ + "gdal_rasterize", + "-burn", "255", + "-init", "0", + "-a_nodata", "0", + "-te", str(bounds['west']), str(bounds['south']), + str(bounds['east']), str(bounds['north']), + "-tr", str(pixel_size), str(pixel_size), + "-ot", "Byte", + "-co", "COMPRESS=LZW", + "-co", "TILED=YES", + "-l", "wilderness_areas", + str(wilderness_gpkg), + str(output_path), + ] + + result = subprocess.run(rasterize_cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"STDERR: {result.stderr}") + raise RuntimeError(f"gdal_rasterize failed: {result.stderr}") + + print(f"\n[Done] Verifying output...") + with rasterio.open(output_path) as ds: + print(f" Size: {ds.width} x {ds.height}") + print(f" CRS: {ds.crs}") + sample = ds.read(1, window=rasterio.windows.Window(0, 0, 1000, 1000)) + wilderness_count = np.sum(sample == 255) + print(f" Sample (1000x1000): {wilderness_count} wilderness cells") + + file_size = output_path.stat().st_size / (1024**2) + print(f" File size: {file_size:.1f} MB") + + return output_path + + if __name__ == "__main__": import sys - if len(sys.argv) > 1 and sys.argv[1] == "build": - # Build the raster - print("="*60) - print("PAD-US Barriers Raster Build") - print("="*60) - build_barriers_raster() + if len(sys.argv) > 1: + cmd = sys.argv[1] + + if cmd == "build": + print("=" * 60) + print("PAD-US Barriers Raster Build") + print("=" * 60) + build_barriers_raster() + + elif cmd == "build-wilderness": + print("=" * 60) + print("PAD-US Wilderness Raster Build") + print("=" * 60) + build_wilderness_raster() + + elif cmd == "build-all": + print("=" * 60) + print("Building all PAD-US rasters") + print("=" * 60) + build_barriers_raster() + print("\n") + build_wilderness_raster() + + else: + print(f"Unknown command: {cmd}") + print("Usage:") + print(" python barriers.py build # Build barriers raster") + print(" python barriers.py build-wilderness # Build wilderness raster") + print(" python barriers.py build-all # Build both rasters") + sys.exit(1) + else: - # Test the reader + # Test readers print("Testing BarrierReader...") if not DEFAULT_BARRIERS_PATH.exists(): @@ -251,16 +410,31 @@ if __name__ == "__main__": sys.exit(1) reader = BarrierReader() - - # Test grid read for Idaho area barriers = reader.get_barrier_grid( south=42.2, north=42.6, west=-114.8, east=-113.8, target_shape=(400, 1000) ) - print(f"\nGrid test shape: {barriers.shape}") + print(f"\nBarrier grid shape: {barriers.shape}") print(f"Unique values: {np.unique(barriers)}") closed_cells = np.sum(barriers == 255) print(f"Closed cells: {closed_cells} ({100*closed_cells/barriers.size:.2f}%)") - reader.close() - print("\nBarrierReader test complete.") + + print("\nTesting WildernessReader...") + + if not DEFAULT_WILDERNESS_PATH.exists(): + print(f"Wilderness raster not found at {DEFAULT_WILDERNESS_PATH}") + print(f"Run: python barriers.py build-wilderness") + else: + wilderness_reader = WildernessReader() + wilderness = wilderness_reader.get_wilderness_grid( + south=42.2, north=42.6, west=-114.8, east=-113.8, + target_shape=(400, 1000) + ) + print(f"Wilderness grid shape: {wilderness.shape}") + print(f"Unique values: {np.unique(wilderness)}") + wilderness_cells = np.sum(wilderness == 255) + print(f"Wilderness cells: {wilderness_cells} ({100*wilderness_cells/wilderness.size:.2f}%)") + wilderness_reader.close() + + print("\nDone.") diff --git a/lib/offroute/cost.py b/lib/offroute/cost.py index 5f3618c..c3b6a5a 100644 --- a/lib/offroute/cost.py +++ b/lib/offroute/cost.py @@ -1,43 +1,207 @@ """ -Tobler off-path hiking cost function for OFFROUTE. +Multi-mode travel cost functions for OFFROUTE. -Computes travel time cost based on terrain slope using Tobler's -hiking function with off-trail penalty. Optionally applies friction -multipliers from land cover data, trail corridors, and barrier grids. +Supports four travel modes: foot, mtb, atv, vehicle. +Each mode has its own speed function, max slope, trail access rules, +and terrain friction overrides. + +Mode profiles are data-driven — adding a new mode means adding a profile entry. """ import math import numpy as np -from typing import Optional, Literal +from dataclasses import dataclass, field +from typing import Optional, Literal, Dict, Callable -# Maximum passable slope in degrees -MAX_SLOPE_DEG = 40.0 +# ═══════════════════════════════════════════════════════════════════════════════ +# SPEED FUNCTIONS +# ═══════════════════════════════════════════════════════════════════════════════ + +def tobler_off_path_speed(grade: np.ndarray, base_speed: float = 6.0) -> np.ndarray: + """ + Tobler off-path hiking function. + + W = 0.6 * base_speed * exp(-3.5 * |S + 0.05|) + + Peak ~3.6 km/h at grade = -0.05 (slight downhill). + The 0.6 multiplier is the off-trail penalty. + """ + return 0.6 * base_speed * np.exp(-3.5 * np.abs(grade + 0.05)) + + +def herzog_wheeled_speed(grade: np.ndarray, base_speed: float = 12.0) -> np.ndarray: + """ + Herzog wheeled-transport polynomial. + + Relative speed factor: + 1 / (1337.8·S^6 + 278.19·S^5 − 517.39·S^4 − 78.199·S^3 + 93.419·S^2 + 19.825·|S| + 1.64) + + Multiply by base_speed to get km/h. + """ + S = grade + S_abs = np.abs(S) + + # Herzog polynomial (returns relative speed factor 0-1) + denom = (1337.8 * S**6 + 278.19 * S**5 - 517.39 * S**4 + - 78.199 * S**3 + 93.419 * S**2 + 19.825 * S_abs + 1.64) + + # Avoid division by zero and negative speeds + denom = np.maximum(denom, 0.1) + rel_speed = 1.0 / denom + + # Clamp relative speed to reasonable bounds (0.05 to 1.5) + rel_speed = np.clip(rel_speed, 0.05, 1.5) + + return base_speed * rel_speed + + +def linear_degrade_speed(grade: np.ndarray, base_speed: float = 40.0, max_grade: float = 0.364) -> np.ndarray: + """ + Linear speed degradation with slope. + + speed = base_speed * max(0, 1 - |grade| / max_grade) + + max_grade = tan(20°) ≈ 0.364 for 20° max slope. + """ + speed = base_speed * np.maximum(0, 1.0 - np.abs(grade) / max_grade) + return np.maximum(speed, 0.1) # Minimum crawl speed + + +# ═══════════════════════════════════════════════════════════════════════════════ +# MODE PROFILES (Data-driven configuration) +# ═══════════════════════════════════════════════════════════════════════════════ + +@dataclass +class ModeProfile: + """Configuration for a travel mode.""" + + name: str + description: str + + # Speed function parameters + speed_function: str # "tobler", "herzog", "linear" + base_speed_kmh: float + max_slope_deg: float + + # Trail access: trail_value -> friction multiplier (None = impassable) + # Trail values: 5=road, 15=track, 25=foot trail + trail_friction: Dict[int, Optional[float]] = field(default_factory=dict) + + # Off-trail terrain friction overrides (by WorldCover class) + # These MULTIPLY the base WorldCover friction + # None = use default, np.inf = impassable + # WorldCover values: 10=tree, 20=shrub, 30=grass, 40=crop, 50=urban, + # 60=bare, 80=water, 90=wetland, 95=mangrove, 100=moss + terrain_friction_override: Dict[int, Optional[float]] = field(default_factory=dict) + + # Should wilderness areas be impassable? + wilderness_impassable: bool = False + + # For vehicle mode: can traverse off-trail flat terrain? + off_trail_flat_threshold_deg: float = 0.0 # 0 = no off-trail allowed + off_trail_flat_friction: float = np.inf # friction if allowed + + +# Define all mode profiles +MODE_PROFILES: Dict[str, ModeProfile] = { + "foot": ModeProfile( + name="foot", + description="Hiking on foot (Tobler off-path model)", + speed_function="tobler", + base_speed_kmh=6.0, + max_slope_deg=40.0, + trail_friction={ + 5: 0.1, # road + 15: 0.3, # track + 25: 0.5, # foot trail + }, + terrain_friction_override={ + # Use default WorldCover friction for foot mode + }, + wilderness_impassable=False, + ), + + "mtb": ModeProfile( + name="mtb", + description="Mountain bike / dirt bike (Herzog wheeled model)", + speed_function="herzog", + base_speed_kmh=12.0, + max_slope_deg=25.0, + trail_friction={ + 5: 0.1, # road + 15: 0.2, # track + 25: 0.5, # foot trail (rideable but slow) + }, + terrain_friction_override={ + 30: 2.0, # Grassland: rideable but slow + 20: 4.0, # Shrubland: barely rideable + 10: 8.0, # Tree cover/forest: effectively impassable + 60: 3.0, # Bare/rocky + 90: np.inf, # Wetland: impassable + 95: np.inf, # Mangrove: impassable + 80: np.inf, # Water: impassable + }, + wilderness_impassable=True, + ), + + "atv": ModeProfile( + name="atv", + description="ATV / side-by-side (Herzog wheeled model, higher base speed)", + speed_function="herzog", + base_speed_kmh=25.0, + max_slope_deg=30.0, + trail_friction={ + 5: 0.1, # road + 15: 0.3, # track + 25: None, # foot trail: impassable (too narrow) + }, + terrain_friction_override={ + 30: 1.5, # Grassland: passable + 20: 3.0, # Shrubland: rough + 10: np.inf, # Forest: impassable + 60: 2.0, # Bare/rocky + 90: np.inf, # Wetland: impassable + 95: np.inf, # Mangrove: impassable + 80: np.inf, # Water: impassable + }, + wilderness_impassable=True, + ), + + "vehicle": ModeProfile( + name="vehicle", + description="4x4 truck / jeep (linear speed degradation)", + speed_function="linear", + base_speed_kmh=40.0, + max_slope_deg=20.0, + trail_friction={ + 5: 0.1, # road + 15: 0.5, # track (rough but passable) + 25: None, # foot trail: impassable + }, + terrain_friction_override={ + # All off-trail terrain is impassable by default + 10: np.inf, # Forest + 20: np.inf, # Shrubland + 30: np.inf, # Grassland (except flat - see below) + 40: np.inf, # Cropland (except flat - see below) + 60: np.inf, # Bare + 90: np.inf, # Wetland + 95: np.inf, # Mangrove + 80: np.inf, # Water + }, + wilderness_impassable=True, + off_trail_flat_threshold_deg=5.0, # Can drive on flat fields + off_trail_flat_friction=5.0, # But very slow + ), +} -# Tobler off-path parameters -TOBLER_BASE_SPEED = 6.0 -TOBLER_OFF_TRAIL_MULT = 0.6 # Pragmatic mode friction multiplier for private land PRAGMATIC_BARRIER_MULTIPLIER = 5.0 -# Trail value to friction multiplier mapping -# Trail friction REPLACES land cover friction (a road through forest is still easy) -TRAIL_FRICTION_MAP = { - 5: 0.1, # road - 15: 0.3, # track - 25: 0.5, # foot trail -} - - -def tobler_speed(grade: float) -> float: - """ - Calculate hiking speed using Tobler's off-path function. - - speed_kmh = 0.6 * 6.0 * exp(-3.5 * |grade + 0.05|) - - Peak speed is ~3.6 km/h at grade = -0.05 (slight downhill). - """ - return TOBLER_OFF_TRAIL_MULT * TOBLER_BASE_SPEED * math.exp(-3.5 * abs(grade + 0.05)) +# ═══════════════════════════════════════════════════════════════════════════════ +# COST GRID COMPUTATION +# ═══════════════════════════════════════════════════════════════════════════════ def compute_cost_grid( elevation: np.ndarray, @@ -45,16 +209,16 @@ def compute_cost_grid( cell_size_lat_m: float = None, cell_size_lon_m: float = None, friction: Optional[np.ndarray] = None, + friction_raw: Optional[np.ndarray] = None, trails: Optional[np.ndarray] = None, barriers: Optional[np.ndarray] = None, - boundary_mode: Literal["strict", "pragmatic", "emergency"] = "pragmatic" + wilderness: Optional[np.ndarray] = None, + boundary_mode: Literal["strict", "pragmatic", "emergency"] = "pragmatic", + mode: Literal["foot", "mtb", "atv", "vehicle"] = "foot" ) -> np.ndarray: """ Compute isotropic travel cost grid from elevation data. - Each cell's cost represents the time (in seconds) to traverse that cell, - based on the average slope from neighboring cells. - Args: elevation: 2D array of elevation values in meters cell_size_m: Average cell size in meters @@ -63,30 +227,29 @@ def compute_cost_grid( friction: Optional 2D array of friction multipliers (WorldCover). Values should be float (1.0 = baseline, 2.0 = 2x slower). np.inf marks impassable cells. - If None, no friction is applied (backward compatible). + friction_raw: Optional 2D array of raw WorldCover class values (uint8). + Used for mode-specific terrain overrides. + Values: 10=tree, 20=shrub, 30=grass, etc. trails: Optional 2D array of trail values (uint8). - 0 = no trail (use friction) - 5 = road (0.1× friction, replaces WorldCover) - 15 = track (0.3× friction, replaces WorldCover) - 25 = foot trail (0.5× friction, replaces WorldCover) - Trail friction REPLACES land cover friction where trails exist. - If None, no trail burn-in is applied. + 0 = no trail, 5 = road, 15 = track, 25 = foot trail barriers: Optional 2D array of barrier values (uint8). - 255 = closed/restricted area (from PAD-US Pub_Access = XA). - 0 = accessible. - If None, no barriers are applied. - boundary_mode: How to handle private/restricted land barriers: - "strict" - cells with barrier=255 become impassable (np.inf) - "pragmatic" - cells with barrier=255 get 5.0x friction penalty - "emergency" - barriers are ignored entirely - Default: "pragmatic" + 255 = closed/restricted area (PAD-US Pub_Access = XA). + wilderness: Optional[np.ndarray] of wilderness values (uint8). + 255 = designated wilderness area. + boundary_mode: How to handle barriers ("strict", "pragmatic", "emergency") + mode: Travel mode ("foot", "mtb", "atv", "vehicle") Returns: 2D array of travel cost in seconds per cell. np.inf for impassable cells. """ if boundary_mode not in ("strict", "pragmatic", "emergency"): - raise ValueError(f"boundary_mode must be 'strict', 'pragmatic', or 'emergency', got '{boundary_mode}'") + raise ValueError(f"boundary_mode must be 'strict', 'pragmatic', or 'emergency'") + + if mode not in MODE_PROFILES: + raise ValueError(f"mode must be one of {list(MODE_PROFILES.keys())}") + + profile = MODE_PROFILES[mode] if cell_size_lat_m is None: cell_size_lat_m = cell_size_m @@ -95,120 +258,212 @@ def compute_cost_grid( rows, cols = elevation.shape - # Compute gradients in both directions - dy = np.zeros_like(elevation) - dx = np.zeros_like(elevation) + # ─── Compute gradients (in-place where possible) ───────────────────────── + # Use float32 to reduce memory footprint + grade = np.zeros(elevation.shape, dtype=np.float32) - # Central differences for interior, forward/backward at edges - dy[1:-1, :] = (elevation[:-2, :] - elevation[2:, :]) / (2 * cell_size_lat_m) - dy[0, :] = (elevation[0, :] - elevation[1, :]) / cell_size_lat_m - dy[-1, :] = (elevation[-2, :] - elevation[-1, :]) / cell_size_lat_m + # Compute dy contribution to grade squared + dy_contrib = np.zeros(elevation.shape, dtype=np.float32) + dy_contrib[1:-1, :] = ((elevation[:-2, :] - elevation[2:, :]) / (2 * cell_size_lat_m)) ** 2 + dy_contrib[0, :] = ((elevation[0, :] - elevation[1, :]) / cell_size_lat_m) ** 2 + dy_contrib[-1, :] = ((elevation[-2, :] - elevation[-1, :]) / cell_size_lat_m) ** 2 - dx[:, 1:-1] = (elevation[:, 2:] - elevation[:, :-2]) / (2 * cell_size_lon_m) - dx[:, 0] = (elevation[:, 1] - elevation[:, 0]) / cell_size_lon_m - dx[:, -1] = (elevation[:, -1] - elevation[:, -2]) / cell_size_lon_m + # Compute dx contribution and add to dy_contrib in-place + dy_contrib[:, 1:-1] += ((elevation[:, 2:] - elevation[:, :-2]) / (2 * cell_size_lon_m)) ** 2 + dy_contrib[:, 0] += ((elevation[:, 1] - elevation[:, 0]) / cell_size_lon_m) ** 2 + dy_contrib[:, -1] += ((elevation[:, -1] - elevation[:, -2]) / cell_size_lon_m) ** 2 - # Compute slope magnitude (grade = rise/run) - grade_magnitude = np.sqrt(dx**2 + dy**2) + # grade = sqrt(dx^2 + dy^2) + np.sqrt(dy_contrib, out=grade) + del dy_contrib # Free memory immediately - # Convert to slope angle in degrees - slope_deg = np.degrees(np.arctan(grade_magnitude)) + # ─── Compute speed based on mode ───────────────────────────────────────── + max_grade_val = np.tan(np.radians(profile.max_slope_deg)) - # Compute speed for each cell using Tobler function - speed_kmh = TOBLER_OFF_TRAIL_MULT * TOBLER_BASE_SPEED * np.exp(-3.5 * np.abs(grade_magnitude + 0.05)) + if profile.speed_function == "tobler": + speed_kmh = tobler_off_path_speed(grade, profile.base_speed_kmh) + elif profile.speed_function == "herzog": + speed_kmh = herzog_wheeled_speed(grade, profile.base_speed_kmh) + elif profile.speed_function == "linear": + speed_kmh = linear_degrade_speed(grade, profile.base_speed_kmh, max_grade_val) + else: + raise ValueError(f"Unknown speed function: {profile.speed_function}") - # Convert speed to time cost (seconds to traverse one cell) + # ─── Base cost (seconds per cell) ───────────────────────────────────────── avg_cell_size = (cell_size_lat_m + cell_size_lon_m) / 2 - cost = avg_cell_size * 3.6 / speed_kmh + cost = (avg_cell_size * 3.6) / speed_kmh + del speed_kmh - # Set impassable cells (slope > MAX_SLOPE_DEG) to infinity - cost[slope_deg > MAX_SLOPE_DEG] = np.inf + # ─── Max slope limit ────────────────────────────────────────────────────── + cost[grade > max_grade_val] = np.inf - # Handle NaN elevations (no data) + # ─── NaN elevations ────────────────────────────────────────────────────── cost[np.isnan(elevation)] = np.inf - # Build effective friction array - # Start with WorldCover friction if provided, else 1.0 + # ─── Apply friction in-place ───────────────────────────────────────────── + # Instead of creating effective_friction copy, apply directly to cost + + # Start with base friction if friction is not None: if friction.shape != elevation.shape: - raise ValueError( - f"Friction shape {friction.shape} does not match elevation shape {elevation.shape}" - ) - effective_friction = friction.copy() - else: - effective_friction = np.ones(elevation.shape, dtype=np.float32) + raise ValueError(f"Friction shape mismatch") + np.multiply(cost, friction, out=cost) - # Apply trail burn-in: trails REPLACE land cover friction + # ─── Mode-specific terrain friction overrides (memory-efficient) ───────── + if friction_raw is not None and profile.terrain_friction_override: + if friction_raw.shape != elevation.shape: + raise ValueError(f"Friction_raw shape mismatch") + + # Process all overrides without creating large intermediate masks + for wc_class, override in profile.terrain_friction_override.items(): + if override is not None: + if override == np.inf: + # Use np.where for in-place-like behavior + np.putmask(cost, friction_raw == wc_class, np.inf) + else: + # Multiply cost where friction_raw matches + # Using a loop with putmask is more memory efficient + mask = friction_raw == wc_class + cost[mask] *= override + del mask + + # ─── Vehicle mode: allow flat grassland/cropland ───────────────────────── + if mode == "vehicle" and profile.off_trail_flat_threshold_deg > 0: + if friction_raw is not None: + # Compute slope in degrees for flat terrain check + slope_deg = np.degrees(np.arctan(grade)) + # Flat grassland or cropland - recompute cost for these cells + flat_field_mask = ( + (slope_deg <= profile.off_trail_flat_threshold_deg) & + ((friction_raw == 30) | (friction_raw == 40)) + ) + del slope_deg + # Recalculate cost for these cells with flat field friction + if np.any(flat_field_mask): + base_time = avg_cell_size * 3.6 / linear_degrade_speed( + grade[flat_field_mask], profile.base_speed_kmh, max_grade_val + ) + cost[flat_field_mask] = base_time * profile.off_trail_flat_friction + del base_time + del flat_field_mask + + # ─── Trail friction (mode-specific) ────────────────────────────────────── if trails is not None: if trails.shape != elevation.shape: - raise ValueError( - f"Trails shape {trails.shape} does not match elevation shape {elevation.shape}" - ) - # Replace friction where trails exist - for trail_value, trail_friction in TRAIL_FRICTION_MAP.items(): - trail_mask = trails == trail_value - effective_friction[trail_mask] = trail_friction + raise ValueError(f"Trails shape mismatch") - # Apply friction to cost - cost = cost * effective_friction + for trail_value, trail_friction in profile.trail_friction.items(): + if trail_friction is None: + # Impassable for this mode + np.putmask(cost, trails == trail_value, np.inf) + else: + # Trail friction REPLACES terrain friction + # Recalculate cost = base_time * trail_friction + trail_mask = trails == trail_value + if np.any(trail_mask): + # Get base travel time (without friction) + if profile.speed_function == "tobler": + trail_speed = tobler_off_path_speed(grade[trail_mask], profile.base_speed_kmh) + elif profile.speed_function == "herzog": + trail_speed = herzog_wheeled_speed(grade[trail_mask], profile.base_speed_kmh) + else: + trail_speed = linear_degrade_speed( + grade[trail_mask], profile.base_speed_kmh, max_grade_val + ) + cost[trail_mask] = (avg_cell_size * 3.6 / trail_speed) * trail_friction + del trail_speed + del trail_mask - # Apply barriers based on boundary_mode + # ─── Wilderness areas (mode-specific) ──────────────────────────────────── + if wilderness is not None and profile.wilderness_impassable: + if wilderness.shape != elevation.shape: + raise ValueError(f"Wilderness shape mismatch") + np.putmask(cost, wilderness == 255, np.inf) + + # ─── Barriers (private land) ───────────────────────────────────────────── if barriers is not None and boundary_mode != "emergency": if barriers.shape != elevation.shape: - raise ValueError( - f"Barriers shape {barriers.shape} does not match elevation shape {elevation.shape}" - ) - - barrier_mask = barriers == 255 + raise ValueError(f"Barriers shape mismatch") if boundary_mode == "strict": - # Mark closed/restricted areas as impassable - cost[barrier_mask] = np.inf + np.putmask(cost, barriers == 255, np.inf) elif boundary_mode == "pragmatic": - # Apply friction penalty to closed/restricted areas - cost[barrier_mask] = cost[barrier_mask] * PRAGMATIC_BARRIER_MULTIPLIER + barrier_mask = barriers == 255 + cost[barrier_mask] *= PRAGMATIC_BARRIER_MULTIPLIER + del barrier_mask return cost +# ═══════════════════════════════════════════════════════════════════════════════ +# LEGACY API (backward compatibility) +# ═══════════════════════════════════════════════════════════════════════════════ + +def tobler_speed(grade: float) -> float: + """Legacy single-value Tobler speed function.""" + return 0.6 * 6.0 * math.exp(-3.5 * abs(grade + 0.05)) + + +# ═══════════════════════════════════════════════════════════════════════════════ +# TESTING +# ═══════════════════════════════════════════════════════════════════════════════ + if __name__ == "__main__": - print("Testing Tobler speed function:") - for grade in [-0.3, -0.1, -0.05, 0.0, 0.05, 0.1, 0.3]: - speed = tobler_speed(grade) - print(f" Grade {grade:+.2f}: {speed:.2f} km/h") + print("=" * 70) + print("OFFROUTE Multi-Mode Cost Function Tests") + print("=" * 70) - print("\nTesting cost grid computation (no friction, no trails):") - elev = np.arange(100).reshape(10, 10).astype(np.float32) * 10 - cost = compute_cost_grid(elev, cell_size_m=30.0) - print(f" Elevation range: {elev.min():.0f} - {elev.max():.0f} m") - finite = cost[~np.isinf(cost)] - if len(finite) > 0: - print(f" Cost range: {finite.min():.1f} - {finite.max():.1f} s") - else: - print(f" All cells impassable (test data too steep)") + print("\n[1] Speed functions at various grades:") + print(f"{'Grade':<10} {'Foot':<12} {'MTB':<12} {'ATV':<12} {'Vehicle':<12}") + print("-" * 60) - print("\nTesting cost grid with friction and trails:") - elev = np.ones((10, 10), dtype=np.float32) * 1000 # flat terrain - friction = np.ones((10, 10), dtype=np.float32) * 2.0 # 2.0x friction (forest) - trails = np.zeros((10, 10), dtype=np.uint8) - trails[5, :] = 5 # road across middle row + for grade_val in [-0.3, -0.1, 0.0, 0.1, 0.2, 0.3]: + grade_arr = np.array([grade_val]) + foot = tobler_off_path_speed(grade_arr, 6.0)[0] + mtb = herzog_wheeled_speed(grade_arr, 12.0)[0] + atv = herzog_wheeled_speed(grade_arr, 25.0)[0] + veh = linear_degrade_speed(grade_arr, 40.0, np.tan(np.radians(20)))[0] + print(f"{grade_val:+.2f} {foot:>6.2f} km/h {mtb:>6.2f} km/h {atv:>6.2f} km/h {veh:>6.2f} km/h") - cost_no_trail = compute_cost_grid(elev, cell_size_m=30.0, friction=friction) - cost_with_trail = compute_cost_grid(elev, cell_size_m=30.0, friction=friction, trails=trails) + print("\n[2] Mode profiles:") + for name, profile in MODE_PROFILES.items(): + print(f"\n {name.upper()}: {profile.description}") + print(f" Max slope: {profile.max_slope_deg}°") + print(f" Trail access: {profile.trail_friction}") + print(f" Wilderness blocked: {profile.wilderness_impassable}") - base_cost = 30 * 3.6 / (0.6 * 6.0 * np.exp(-3.5 * 0.05)) - print(f" Base cost (flat, 30m cell): {base_cost:.1f} s") - print(f" Forest cell (2.0x friction): {cost_no_trail[0, 0]:.1f} s") - print(f" Road cell (0.1x friction, replaces forest): {cost_with_trail[5, 0]:.1f} s") - print(f" Road friction advantage: {cost_no_trail[0, 0] / cost_with_trail[5, 0]:.1f}x faster") - - print("\nTesting cost grid with barriers (three modes):") + print("\n[3] Cost grid test (flat terrain, forest):") elev = np.ones((10, 10), dtype=np.float32) * 1000 - barriers = np.zeros((10, 10), dtype=np.uint8) - barriers[3:7, 3:7] = 255 + friction = np.ones((10, 10), dtype=np.float32) * 2.0 # Forest friction + friction_raw = np.ones((10, 10), dtype=np.uint8) * 10 # Tree cover class - for mode in ["strict", "pragmatic", "emergency"]: - cost = compute_cost_grid(elev, cell_size_m=30.0, barriers=barriers, boundary_mode=mode) + trails = np.zeros((10, 10), dtype=np.uint8) + trails[5, :] = 5 # Road across middle + + for mode_name in ["foot", "mtb", "atv", "vehicle"]: + cost = compute_cost_grid( + elev, cell_size_m=30.0, + friction=friction, + friction_raw=friction_raw, + trails=trails, + mode=mode_name + ) + off_trail_cost = cost[0, 0] + road_cost = cost[5, 0] impassable = np.sum(np.isinf(cost)) - barrier_cost = cost[5, 5] if not np.isinf(cost[5, 5]) else "inf" - print(f" {mode:10s}: {impassable} impassable, barrier cell cost = {barrier_cost}") + print(f" {mode_name:8s}: off-trail={off_trail_cost:>8.1f}s, road={road_cost:>6.1f}s, impassable={impassable}") + + print("\n[4] Wilderness blocking test:") + wilderness = np.zeros((10, 10), dtype=np.uint8) + wilderness[3:7, 3:7] = 255 + + for mode_name in ["foot", "mtb", "atv", "vehicle"]: + cost = compute_cost_grid( + elev, cell_size_m=30.0, + wilderness=wilderness, + mode=mode_name + ) + wilderness_impassable = np.sum(np.isinf(cost[3:7, 3:7])) + print(f" {mode_name:8s}: wilderness cells impassable = {wilderness_impassable}/16") + + print("\nDone.") diff --git a/lib/offroute/router.py b/lib/offroute/router.py index 57d6ce5..d44bbb4 100644 --- a/lib/offroute/router.py +++ b/lib/offroute/router.py @@ -6,6 +6,8 @@ Connects the raster pathfinder (wilderness segment) to Valhalla (on-network segm Entry points are extracted from OSM highways and stored in /mnt/nav/navi.db. The pathfinder routes from a wilderness start to the nearest entry point, then Valhalla completes the route to the destination. + +Supports four travel modes: foot, mtb, atv, vehicle. """ import json import math @@ -23,7 +25,7 @@ from skimage.graph import MCP_Geometric from .dem import DEMReader from .cost import compute_cost_grid from .friction import FrictionReader, friction_to_multiplier -from .barriers import BarrierReader +from .barriers import BarrierReader, WildernessReader, DEFAULT_WILDERNESS_PATH from .trails import TrailReader # Paths @@ -45,6 +47,7 @@ MODE_TO_COSTING = { "foot": "pedestrian", "mtb": "bicycle", "atv": "auto", + "vehicle": "auto", } @@ -120,7 +123,6 @@ class EntryPointIndex: def query_radius(self, lat: float, lon: float, radius_km: float) -> List[Dict]: """Query entry points within radius of a point.""" - # Approximate bbox for the radius lat_delta = radius_km / 111.0 lon_delta = radius_km / (111.0 * math.cos(math.radians(lat))) @@ -129,7 +131,6 @@ class EntryPointIndex: lon - lon_delta, lon + lon_delta ) - # Filter by actual distance and add distance field result = [] for p in points: dist = haversine_distance(lat, lon, p['lat'], p['lon']) @@ -140,17 +141,12 @@ class EntryPointIndex: return sorted(result, key=lambda x: x['distance_m']) def build_index(self, osm_pbf_path: Path = OSM_PBF_PATH) -> Dict: - """ - Build the entry point index from OSM PBF. - - Extracts endpoints of highway features that connect to the network. - """ + """Build the entry point index from OSM PBF.""" if not osm_pbf_path.exists(): raise FileNotFoundError(f"OSM PBF not found: {osm_pbf_path}") print(f"Building trail entry point index from {osm_pbf_path}...") - # Highway types to extract (routable network entry points) highway_types = [ "primary", "secondary", "tertiary", "unclassified", "residential", "service", "track", "path", "footway", "bridleway" @@ -159,42 +155,29 @@ class EntryPointIndex: stats = {"total": 0, "by_class": {}} with tempfile.TemporaryDirectory() as tmpdir: - # Extract highways to GeoJSON geojson_path = Path(tmpdir) / "highways.geojson" - # Build osmium tags-filter expressions (one per highway type) print(f" Extracting highways with osmium...") - cmd = [ - "osmium", "tags-filter", - str(osm_pbf_path), - ] - # Add each highway type as a separate filter expression + cmd = ["osmium", "tags-filter", str(osm_pbf_path)] for ht in highway_types: cmd.append(f"w/highway={ht}") cmd.extend(["-o", str(Path(tmpdir) / "filtered.osm.pbf"), "--overwrite"]) - subprocess.run(cmd, check=True, capture_output=True) - # Convert to GeoJSON print(f" Converting to GeoJSON with ogr2ogr...") cmd = [ "ogr2ogr", "-f", "GeoJSON", str(geojson_path), str(Path(tmpdir) / "filtered.osm.pbf"), - "lines", - "-t_srs", "EPSG:4326" + "lines", "-t_srs", "EPSG:4326" ] subprocess.run(cmd, check=True, capture_output=True) - # Parse GeoJSON and extract endpoints print(f" Extracting entry points...") with open(geojson_path) as f: data = json.load(f) - # Collect unique points (endpoints) - # Key: (lat, lon) rounded to 5 decimal places (~1m precision) points = {} - for feature in data.get("features", []): props = feature.get("properties", {}) geom = feature.get("geometry", {}) @@ -209,62 +192,48 @@ class EntryPointIndex: highway_class = props.get("highway", "unknown") name = props.get("name", "") - # Extract endpoints for coord in [coords[0], coords[-1]]: lon, lat = coord[0], coord[1] key = (round(lat, 5), round(lon, 5)) if key not in points: points[key] = { - "lat": lat, - "lon": lon, - "highway_class": highway_class, - "name": name + "lat": lat, "lon": lon, + "highway_class": highway_class, "name": name } else: - # Keep the "best" highway class (roads > tracks > paths) existing = points[key] if self._highway_priority(highway_class) < self._highway_priority(existing["highway_class"]): points[key]["highway_class"] = highway_class if name and not existing["name"]: points[key]["name"] = name - # Create/update database print(f" Writing {len(points)} entry points to {self.db_path}...") self.db_path.parent.mkdir(parents=True, exist_ok=True) conn = self._get_conn() - # Create table conn.execute(""" CREATE TABLE IF NOT EXISTS trail_entry_points ( id INTEGER PRIMARY KEY AUTOINCREMENT, - lat REAL NOT NULL, - lon REAL NOT NULL, - highway_class TEXT NOT NULL, - name TEXT + lat REAL NOT NULL, lon REAL NOT NULL, + highway_class TEXT NOT NULL, name TEXT ) """) - - # Clear existing data conn.execute("DELETE FROM trail_entry_points") - # Insert new points for point in points.values(): - conn.execute(""" - INSERT INTO trail_entry_points (lat, lon, highway_class, name) - VALUES (?, ?, ?, ?) - """, (point["lat"], point["lon"], point["highway_class"], point["name"])) - + conn.execute( + "INSERT INTO trail_entry_points (lat, lon, highway_class, name) VALUES (?, ?, ?, ?)", + (point["lat"], point["lon"], point["highway_class"], point["name"]) + ) stats["total"] += 1 hc = point["highway_class"] stats["by_class"][hc] = stats["by_class"].get(hc, 0) + 1 - # Create spatial index conn.execute("CREATE INDEX IF NOT EXISTS idx_entry_lat ON trail_entry_points(lat)") conn.execute("CREATE INDEX IF NOT EXISTS idx_entry_lon ON trail_entry_points(lon)") conn.execute("CREATE INDEX IF NOT EXISTS idx_entry_latlon ON trail_entry_points(lat, lon)") - conn.commit() print(f" Done. Total: {stats['total']} entry points") @@ -291,12 +260,15 @@ class EntryPointIndex: class OffrouteRouter: """ OFFROUTE Router — orchestrates wilderness pathfinding and Valhalla stitching. + + Supports modes: foot, mtb, atv, vehicle """ def __init__(self): self.dem_reader = None self.friction_reader = None self.barrier_reader = None + self.wilderness_reader = None self.trail_reader = None self.entry_index = EntryPointIndex() @@ -308,6 +280,8 @@ class OffrouteRouter: self.friction_reader = FrictionReader() if self.barrier_reader is None: self.barrier_reader = BarrierReader() + if self.wilderness_reader is None and DEFAULT_WILDERNESS_PATH.exists(): + self.wilderness_reader = WildernessReader() if self.trail_reader is None: self.trail_reader = TrailReader() @@ -317,16 +291,25 @@ class OffrouteRouter: start_lon: float, end_lat: float, end_lon: float, - mode: Literal["foot", "mtb", "atv"] = "foot", + mode: Literal["foot", "mtb", "atv", "vehicle"] = "foot", boundary_mode: Literal["strict", "pragmatic", "emergency"] = "pragmatic" ) -> Dict: """ Route from a wilderness start point to a destination. + Args: + start_lat, start_lon: Starting coordinates (wilderness) + end_lat, end_lon: Destination coordinates + mode: Travel mode (foot, mtb, atv, vehicle) + boundary_mode: How to handle private land (strict, pragmatic, emergency) + Returns a GeoJSON FeatureCollection with wilderness and network segments. """ t0 = time.time() + if mode not in MODE_TO_COSTING: + return {"status": "error", "message": f"Unknown mode: {mode}"} + # Ensure entry point index exists if not self.entry_index.table_exists() or self.entry_index.get_entry_point_count() == 0: return { @@ -334,28 +317,27 @@ class OffrouteRouter: "message": "Trail entry point index not built. Run build_entry_index() first." } - # Find entry points near start - entry_points = self.entry_index.query_radius( - start_lat, start_lon, DEFAULT_SEARCH_RADIUS_KM - ) + # Find entry points near start (limit to nearest 10 to control bbox size) + MAX_ENTRY_POINTS = 10 + entry_points = self.entry_index.query_radius(start_lat, start_lon, DEFAULT_SEARCH_RADIUS_KM) if not entry_points: - # Try expanded radius - entry_points = self.entry_index.query_radius( - start_lat, start_lon, EXPANDED_SEARCH_RADIUS_KM - ) + entry_points = self.entry_index.query_radius(start_lat, start_lon, EXPANDED_SEARCH_RADIUS_KM) if not entry_points: return { "status": "error", "message": f"No trail entry points found within {EXPANDED_SEARCH_RADIUS_KM}km of start" } - # Build bbox for pathfinding grid - # Include start, end, and all entry points + # Limit to nearest entry points to prevent huge bounding boxes + entry_points = entry_points[:MAX_ENTRY_POINTS] + + # Build bbox with max size limit (prevent OOM on large areas) + MAX_BBOX_DEGREES = 0.5 # ~55km at mid-latitudes all_lats = [start_lat, end_lat] + [p["lat"] for p in entry_points] all_lons = [start_lon, end_lon] + [p["lon"] for p in entry_points] - padding = 0.05 # ~5km padding + padding = 0.05 bbox = { "south": min(all_lats) - padding, "north": max(all_lats) + padding, @@ -363,16 +345,28 @@ class OffrouteRouter: "east": max(all_lons) + padding, } + # Clamp bbox size to prevent memory exhaustion + lat_span = bbox["north"] - bbox["south"] + lon_span = bbox["east"] - bbox["west"] + if lat_span > MAX_BBOX_DEGREES or lon_span > MAX_BBOX_DEGREES: + center_lat = (bbox["south"] + bbox["north"]) / 2 + center_lon = (bbox["west"] + bbox["east"]) / 2 + half_span = MAX_BBOX_DEGREES / 2 + bbox = { + "south": center_lat - half_span, + "north": center_lat + half_span, + "west": center_lon - half_span, + "east": center_lon + half_span, + } + # Initialize readers self._init_readers() # Load elevation try: elevation, meta = self.dem_reader.get_elevation_grid( - south=bbox["south"], - north=bbox["north"], - west=bbox["west"], - east=bbox["east"], + south=bbox["south"], north=bbox["north"], + west=bbox["west"], east=bbox["east"], ) except Exception as e: return {"status": "error", "message": f"Failed to load elevation: {e}"} @@ -382,62 +376,69 @@ class OffrouteRouter: if mem > MEMORY_LIMIT_GB: return {"status": "error", "message": f"Memory limit exceeded: {mem:.1f}GB > {MEMORY_LIMIT_GB}GB"} - # Load friction + # Load friction (both processed and raw for mode-specific overrides) friction_raw = self.friction_reader.get_friction_grid( - south=bbox["south"], - north=bbox["north"], - west=bbox["west"], - east=bbox["east"], + south=bbox["south"], north=bbox["north"], + west=bbox["west"], east=bbox["east"], target_shape=elevation.shape ) friction_mult = friction_to_multiplier(friction_raw) # Load barriers barriers = self.barrier_reader.get_barrier_grid( - south=bbox["south"], - north=bbox["north"], - west=bbox["west"], - east=bbox["east"], + south=bbox["south"], north=bbox["north"], + west=bbox["west"], east=bbox["east"], target_shape=elevation.shape ) + # Load wilderness (if available and mode requires it) + wilderness = None + if self.wilderness_reader is not None and mode in ("mtb", "atv", "vehicle"): + wilderness = self.wilderness_reader.get_wilderness_grid( + south=bbox["south"], north=bbox["north"], + west=bbox["west"], east=bbox["east"], + target_shape=elevation.shape + ) + # Load trails trails = self.trail_reader.get_trails_grid( - south=bbox["south"], - north=bbox["north"], - west=bbox["west"], - east=bbox["east"], + south=bbox["south"], north=bbox["north"], + west=bbox["west"], east=bbox["east"], target_shape=elevation.shape ) - # Compute cost grid + # Compute cost grid with mode-specific parameters cost = compute_cost_grid( elevation, cell_size_m=meta["cell_size_m"], friction=friction_mult, + friction_raw=friction_raw, trails=trails, barriers=barriers, + wilderness=wilderness, boundary_mode=boundary_mode, + mode=mode, ) + # Free intermediate arrays to reduce memory before MCP + # Note: Keep trails and barriers - needed for path statistics + del friction_mult, friction_raw, wilderness + import gc + gc.collect() + # Convert start to pixel coordinates start_row, start_col = self.dem_reader.latlon_to_pixel(start_lat, start_lon, meta) - # Validate start is in bounds rows, cols = elevation.shape if not (0 <= start_row < rows and 0 <= start_col < cols): return {"status": "error", "message": "Start point outside grid bounds"} - # Mark entry points on the grid + # Mark entry points on grid entry_pixels = [] for ep in entry_points: row, col = self.dem_reader.latlon_to_pixel(ep["lat"], ep["lon"], meta) if 0 <= row < rows and 0 <= col < cols: - entry_pixels.append({ - "row": row, - "col": col, - "entry_point": ep - }) + entry_pixels.append({"row": row, "col": col, "entry_point": ep}) if not entry_pixels: return {"status": "error", "message": "No entry points map to grid bounds"} @@ -465,18 +466,21 @@ class OffrouteRouter: # Traceback wilderness path path_indices = mcp.traceback((best_entry["row"], best_entry["col"])) - # Convert to coordinates and collect stats + # Convert to coordinates wilderness_coords = [] elevations = [] trail_values = [] + barrier_crossings = 0 for row, col in path_indices: lat, lon = self.dem_reader.pixel_to_latlon(row, col, meta) wilderness_coords.append([lon, lat]) elevations.append(elevation[row, col]) trail_values.append(trails[row, col]) + if barriers[row, col] == 255: + barrier_crossings += 1 - # Calculate wilderness segment stats + # Calculate stats wilderness_distance_m = 0 for i in range(1, len(wilderness_coords)): lon1, lat1 = wilderness_coords[i-1] @@ -493,13 +497,16 @@ class OffrouteRouter: total_cells = len(trail_arr) on_trail_pct = float(100 * on_trail_cells / total_cells) if total_cells > 0 else 0 - # Entry point reached + # Free trails and barriers now that path stats are computed + del trails, barriers + + # Entry point entry_lat = best_entry["entry_point"]["lat"] entry_lon = best_entry["entry_point"]["lon"] entry_class = best_entry["entry_point"]["highway_class"] entry_name = best_entry["entry_point"].get("name", "") - # Call Valhalla for on-network segment + # Call Valhalla valhalla_costing = MODE_TO_COSTING.get(mode, "pedestrian") valhalla_request = { @@ -515,11 +522,7 @@ class OffrouteRouter: valhalla_error = None try: - resp = requests.post( - f"{VALHALLA_URL}/route", - json=valhalla_request, - timeout=30 - ) + resp = requests.post(f"{VALHALLA_URL}/route", json=valhalla_request, timeout=30) if resp.status_code == 200: valhalla_data = resp.json() @@ -529,11 +532,8 @@ class OffrouteRouter: if legs: leg = legs[0] shape = leg.get("shape", "") - - # Decode polyline6 network_coords = self._decode_polyline(shape) - # Extract maneuvers maneuvers = [] for m in leg.get("maneuvers", []): maneuvers.append({ @@ -560,7 +560,6 @@ class OffrouteRouter: # Build response features = [] - # Feature 1: Wilderness segment wilderness_feature = { "type": "Feature", "properties": { @@ -572,15 +571,13 @@ class OffrouteRouter: "boundary_mode": boundary_mode, "on_trail_pct": on_trail_pct, "cell_count": total_cells, + "barrier_crossings": barrier_crossings, + "mode": mode, }, - "geometry": { - "type": "LineString", - "coordinates": wilderness_coords, - } + "geometry": {"type": "LineString", "coordinates": wilderness_coords} } features.append(wilderness_feature) - # Feature 2: Network segment (if available) if network_segment: network_feature = { "type": "Feature", @@ -590,40 +587,23 @@ class OffrouteRouter: "duration_minutes": network_segment["duration_minutes"], "maneuvers": network_segment["maneuvers"], }, - "geometry": { - "type": "LineString", - "coordinates": network_segment["coordinates"], - } + "geometry": {"type": "LineString", "coordinates": network_segment["coordinates"]} } features.append(network_feature) - # Build combined route coordinates combined_coords = wilderness_coords.copy() if network_segment: - # Skip first point of network segment (it's the same as last wilderness point) combined_coords.extend(network_segment["coordinates"][1:]) - # Feature 3: Combined route combined_feature = { "type": "Feature", - "properties": { - "segment_type": "combined", - "mode": mode, - "boundary_mode": boundary_mode, - }, - "geometry": { - "type": "LineString", - "coordinates": combined_coords, - } + "properties": {"segment_type": "combined", "mode": mode, "boundary_mode": boundary_mode}, + "geometry": {"type": "LineString", "coordinates": combined_coords} } features.append(combined_feature) - geojson = { - "type": "FeatureCollection", - "features": features, - } + geojson = {"type": "FeatureCollection", "features": features} - # Build summary total_distance_km = wilderness_distance_m / 1000 total_effort_minutes = best_cost / 60 @@ -639,22 +619,17 @@ class OffrouteRouter: "network_distance_km": float(network_segment["distance_km"]) if network_segment else 0, "network_duration_minutes": float(network_segment["duration_minutes"]) if network_segment else 0, "on_trail_pct": on_trail_pct, + "barrier_crossings": barrier_crossings, "boundary_mode": boundary_mode, "mode": mode, "entry_point": { - "lat": entry_lat, - "lon": entry_lon, - "highway_class": entry_class, - "name": entry_name, + "lat": entry_lat, "lon": entry_lon, + "highway_class": entry_class, "name": entry_name, }, "computation_time_s": time.time() - t0, } - result = { - "status": "ok", - "route": geojson, - "summary": summary, - } + result = {"status": "ok", "route": geojson, "summary": summary} if valhalla_error: result["warning"] = f"Network segment incomplete: {valhalla_error}" @@ -669,7 +644,6 @@ class OffrouteRouter: lon = 0 while index < len(encoded): - # Latitude shift = 0 result = 0 while True: @@ -682,7 +656,6 @@ class OffrouteRouter: dlat = ~(result >> 1) if result & 1 else result >> 1 lat += dlat - # Longitude shift = 0 result = 0 while True: @@ -707,6 +680,8 @@ class OffrouteRouter: self.friction_reader.close() if self.barrier_reader: self.barrier_reader.close() + if self.wilderness_reader: + self.wilderness_reader.close() if self.trail_reader: self.trail_reader.close() self.entry_index.close() @@ -729,24 +704,33 @@ if __name__ == "__main__": print(f"\nDone. Total entry points: {stats['total']}") elif len(sys.argv) > 1 and sys.argv[1] == "test": - print("Testing router...") + print("Testing router (all modes)...") router = OffrouteRouter() - # Test route: wilderness to Twin Falls - result = router.route( - start_lat=42.35, - start_lon=-114.30, - end_lat=42.5629, - end_lon=-114.4609, - mode="foot", - boundary_mode="pragmatic" - ) + for mode in ["foot", "mtb", "atv", "vehicle"]: + print(f"\n{'='*60}") + print(f"Mode: {mode}") + print("="*60) + + result = router.route( + start_lat=42.35, start_lon=-114.30, + end_lat=42.5629, end_lon=-114.4609, + mode=mode, boundary_mode="pragmatic" + ) + + if result["status"] == "ok": + s = result["summary"] + print(f" Wilderness: {s['wilderness_distance_km']:.2f} km, {s['wilderness_effort_minutes']:.1f} min") + print(f" Network: {s['network_distance_km']:.2f} km, {s['network_duration_minutes']:.1f} min") + print(f" On-trail: {s['on_trail_pct']:.1f}%") + print(f" Entry: {s['entry_point']['highway_class']}") + else: + print(f" ERROR: {result['message']}") - print(json.dumps(result, indent=2, default=str)) router.close() else: print("Usage:") print(" python router.py build # Build entry point index") - print(" python router.py test # Test route") + print(" python router.py test # Test all modes") From 2252905986ed21264a0c873f3037a25fc63543b9 Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 8 May 2026 14:26:18 +0000 Subject: [PATCH 09/41] =?UTF-8?q?feat(offroute):=20MVUM=20legal=20access?= =?UTF-8?q?=20=E2=80=94=20pathfinder=20integration=20+=20places=20panel=20?= =?UTF-8?q?API=20+=20boundary=5Fmode=20control?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MVUM Data Import: - Downloaded USFS MVUM Roads (150,636 features) and Trails (28,741 features) - Imported to navi.db as mvum_roads and mvum_trails tables - Idaho coverage: ~8,994 roads and ~4,504 trails across 7 national forests - Preserved all vehicle-class fields (ATV, MOTORCYCLE, HIGHCLEARANCEVEHICLE, etc.) - Preserved seasonal date ranges (*_DATESOPEN fields) New mvum.py module: - MVUMReader class for querying MVUM data by bbox and nearest point - parse_date_range() for seasonal date string parsing (MM/DD-MM/DD format) - check_access() for determining open/closed status with date checking - symbol_to_access() fallback when per-vehicle fields are null - get_mvum_access_grid() for rasterizing MVUM to pathfinder grid Cost function integration: - Added mvum parameter to compute_cost_grid() - MVUM closures respond to boundary_mode: * strict = impassable (np.inf) * pragmatic = 5x friction penalty * emergency = ignored entirely - Foot mode skips MVUM (motor-vehicle specific) Router integration: - Loads MVUM access grid for motorized modes (mtb, atv, vehicle) - Tracks mvum_closed_crossings in path summary Places Panel API: - GET /api/mvum?lat=XX&lon=XX&radius=50 - Returns MVUM feature with access status for all vehicle classes - Includes seasonal date ranges, maintenance level, forest/district info - GeoJSON geometry for map display Validation: - MVUM places endpoint tested with Sawtooth NF road - All four modes validated with strict/pragmatic/emergency boundary modes - Foot mode correctly ignores MVUM restrictions Co-Authored-By: Claude Opus 4.5 --- lib/api.py | 134 +++++++++ lib/offroute/cost.py | 25 ++ lib/offroute/mvum.py | 623 +++++++++++++++++++++++++++++++++++++++++ lib/offroute/router.py | 29 +- 4 files changed, 809 insertions(+), 2 deletions(-) create mode 100644 lib/offroute/mvum.py diff --git a/lib/api.py b/lib/api.py index a127866..699d09d 100644 --- a/lib/api.py +++ b/lib/api.py @@ -2799,3 +2799,137 @@ def api_offroute(): except Exception as e: logger.exception("Offroute error") return jsonify({"status": "error", "message": str(e)}), 500 + + +# ── MVUM Places Panel API ── + +@app.route("/api/mvum", methods=["GET"]) +def api_mvum(): + """ + Query MVUM (Motor Vehicle Use Map) features near a point. + + Used by the Navi frontend places panel when a user taps near a road/trail. + + Query params: + lat: Latitude + lon: Longitude + radius: Search radius in meters (default: 50) + + Response: + { + "status": "ok", + "feature": { + "id": "FR 123", + "name": "Some Forest Road", + "forest": "Sawtooth National Forest", + "district": "Ketchum Ranger District", + "surface": "NAT", + "maintenance_level": 2, + "seasonal": "Seasonal", + "symbol": 2, + "access": { + "passenger_vehicle": { "status": "Open", "dates": "06/15-10/15" }, + "high_clearance": { "status": "Open", "dates": "06/15-10/15" }, + "atv": { "status": "Open", "dates": "06/15-10/15" }, + ... + }, + "geometry": { GeoJSON LineString } + } + } + + If no MVUM feature within radius: + { "status": "ok", "feature": null } + """ + try: + lat = request.args.get("lat", type=float) + lon = request.args.get("lon", type=float) + radius = request.args.get("radius", 50, type=float) + + if lat is None or lon is None: + return jsonify({"status": "error", "message": "lat and lon required"}), 400 + + from .offroute.mvum import MVUMReader + + reader = MVUMReader() + try: + # Try roads first, then trails + feature = reader.query_nearest(lat, lon, radius, "mvum_roads") + if feature is None: + feature = reader.query_nearest(lat, lon, radius, "mvum_trails") + + if feature is None: + return jsonify({"status": "ok", "feature": None}) + + # Format access info + access = { + "passenger_vehicle": { + "status": feature.get("passengervehicle"), + "dates": feature.get("passengervehicle_datesopen") + }, + "high_clearance": { + "status": feature.get("highclearancevehicle"), + "dates": feature.get("highclearancevehicle_datesopen") + }, + "atv": { + "status": feature.get("atv"), + "dates": feature.get("atv_datesopen") + }, + "motorcycle": { + "status": feature.get("motorcycle"), + "dates": feature.get("motorcycle_datesopen") + }, + "4wd_gt50": { + "status": feature.get("fourwd_gt50inches"), + "dates": feature.get("fourwd_gt50_datesopen") + }, + "2wd_gt50": { + "status": feature.get("twowd_gt50inches"), + "dates": feature.get("twowd_gt50_datesopen") + }, + "e_bike_class1": { + "status": feature.get("e_bike_class1"), + "dates": feature.get("e_bike_class1_dur") + }, + "e_bike_class2": { + "status": feature.get("e_bike_class2"), + "dates": feature.get("e_bike_class2_dur") + }, + "e_bike_class3": { + "status": feature.get("e_bike_class3"), + "dates": feature.get("e_bike_class3_dur") + }, + } + + # Parse maintenance level + maint_level = feature.get("operationalmaintlevel", "") + maint_num = None + if maint_level: + # Extract first digit: "2 - HIGH CLEARANCE VEHICLES" -> 2 + import re + match = re.match(r"(\d+)", maint_level) + if match: + maint_num = int(match.group(1)) + + result = { + "id": feature.get("id"), + "name": feature.get("name"), + "forest": feature.get("forestname"), + "district": feature.get("districtname"), + "surface": feature.get("surfacetype"), + "maintenance_level": maint_num, + "seasonal": feature.get("seasonal"), + "symbol": feature.get("symbol"), + "trail_class": feature.get("trailclass"), + "trail_system": feature.get("trailsystem"), + "access": access, + "geometry": feature.get("geojson") + } + + return jsonify({"status": "ok", "feature": result}) + + finally: + reader.close() + + except Exception as e: + logger.exception("MVUM query error") + return jsonify({"status": "error", "message": str(e)}), 500 diff --git a/lib/offroute/cost.py b/lib/offroute/cost.py index c3b6a5a..16b8514 100644 --- a/lib/offroute/cost.py +++ b/lib/offroute/cost.py @@ -213,6 +213,7 @@ def compute_cost_grid( trails: Optional[np.ndarray] = None, barriers: Optional[np.ndarray] = None, wilderness: Optional[np.ndarray] = None, + mvum: Optional[np.ndarray] = None, boundary_mode: Literal["strict", "pragmatic", "emergency"] = "pragmatic", mode: Literal["foot", "mtb", "atv", "vehicle"] = "foot" ) -> np.ndarray: @@ -236,6 +237,10 @@ def compute_cost_grid( 255 = closed/restricted area (PAD-US Pub_Access = XA). wilderness: Optional[np.ndarray] of wilderness values (uint8). 255 = designated wilderness area. + mvum: Optional[np.ndarray] of MVUM access values (uint8). + 0 = no MVUM data, 1 = open, 255 = closed to this mode. + MVUM closures respond to boundary_mode (strict/pragmatic/emergency). + Foot mode should pass None (MVUM is motor-vehicle specific). boundary_mode: How to handle barriers ("strict", "pragmatic", "emergency") mode: Travel mode ("foot", "mtb", "atv", "vehicle") @@ -392,6 +397,26 @@ def compute_cost_grid( cost[barrier_mask] *= PRAGMATIC_BARRIER_MULTIPLIER del barrier_mask + # ─── MVUM closures (motor vehicle restrictions) ────────────────────────── + # MVUM only applies to motorized modes, not foot. Foot mode should pass mvum=None. + # MVUM closures respond to the same boundary_mode as PAD-US barriers: + # "strict" = MVUM-closed road/trail is impassable + # "pragmatic" = MVUM-closed road/trail gets 5× friction penalty + # "emergency" = MVUM closures ignored entirely + if mvum is not None and mode != "foot" and boundary_mode != "emergency": + if mvum.shape != elevation.shape: + raise ValueError(f"MVUM shape mismatch") + + # Value 255 = road/trail exists but is closed to this mode + mvum_closed_mask = mvum == 255 + + if boundary_mode == "strict": + np.putmask(cost, mvum_closed_mask, np.inf) + elif boundary_mode == "pragmatic": + cost[mvum_closed_mask] *= PRAGMATIC_BARRIER_MULTIPLIER + + del mvum_closed_mask + return cost diff --git a/lib/offroute/mvum.py b/lib/offroute/mvum.py new file mode 100644 index 0000000..31e503d --- /dev/null +++ b/lib/offroute/mvum.py @@ -0,0 +1,623 @@ +""" +MVUM (Motor Vehicle Use Map) legal access layer for OFFROUTE. + +Queries USFS MVUM data from navi.db and provides rasterized access grids +indicating which roads/trails are open or closed to specific vehicle modes. + +MVUM is motor-vehicle specific — foot mode should skip this layer entirely. +""" +import re +import sqlite3 +import warnings +from datetime import datetime +from pathlib import Path +from typing import Dict, List, Optional, Tuple, Literal + +import numpy as np + +# Path to navi.db +NAVI_DB_PATH = Path("/mnt/nav/navi.db") + + +def parse_date_range(date_str: str) -> List[Tuple[int, int, int, int]]: + """ + Parse MVUM date range strings like "05/01-11/30" or "06/15-10/15,12/01-03/31". + + Returns list of (start_month, start_day, end_month, end_day) tuples. + Returns empty list if unparseable. + """ + if not date_str or date_str.strip() == "": + return [] + + ranges = [] + # Split by comma for multi-period strings + for part in date_str.split(","): + part = part.strip() + # Match MM/DD-MM/DD pattern + match = re.match(r"(\d{1,2})/(\d{1,2})-(\d{1,2})/(\d{1,2})", part) + if match: + try: + sm, sd, em, ed = int(match.group(1)), int(match.group(2)), int(match.group(3)), int(match.group(4)) + if 1 <= sm <= 12 and 1 <= sd <= 31 and 1 <= em <= 12 and 1 <= ed <= 31: + ranges.append((sm, sd, em, ed)) + except ValueError: + pass + + return ranges + + +def is_date_in_range(month: int, day: int, ranges: List[Tuple[int, int, int, int]]) -> bool: + """ + Check if a given month/day falls within any of the date ranges. + Handles ranges that wrap around year end (e.g., 12/01-03/31). + """ + if not ranges: + return True # No ranges = assume open + + date_num = month * 100 + day # Simple numeric comparison + + for sm, sd, em, ed in ranges: + start_num = sm * 100 + sd + end_num = em * 100 + ed + + if start_num <= end_num: + # Normal range (e.g., 05/01-11/30) + if start_num <= date_num <= end_num: + return True + else: + # Wrapping range (e.g., 12/01-03/31) + if date_num >= start_num or date_num <= end_num: + return True + + return False + + +def check_access( + status_field: Optional[str], + dates_field: Optional[str], + seasonal: Optional[str], + check_date: Optional[Tuple[int, int]] = None +) -> Optional[bool]: + """ + Determine if a road/trail is open to a vehicle type. + + Args: + status_field: Value of vehicle-class field (e.g., "open", null) + dates_field: Value of *_DATESOPEN field (e.g., "05/01-11/30") + seasonal: Value of SEASONAL field ("yearlong", "seasonal") + check_date: Optional (month, day) tuple to check against date ranges + + Returns: + True = open + False = closed + None = no data (field not populated, defer to SYMBOL) + """ + if status_field is None or status_field.strip() == "": + return None # No data + + status = status_field.strip().lower() + + if status != "open": + return False # Explicitly closed or restricted + + # Status is "open" - check seasonal restrictions + if check_date is not None: + month, day = check_date + + # Parse date ranges + if dates_field: + ranges = parse_date_range(dates_field) + if ranges: + return is_date_in_range(month, day, ranges) + + # No date field but seasonal = "yearlong" means always open + if seasonal and seasonal.strip().lower() == "yearlong": + return True + + # Seasonal with no dates - assume open (data quality issue) + if seasonal and seasonal.strip().lower() == "seasonal": + warnings.warn(f"Seasonal road/trail with no DATESOPEN, assuming open") + return True + + return True # Open with no date check + + +def get_mode_field(mode: str) -> Tuple[str, str]: + """ + Get the MVUM field names for a given travel mode. + + Returns (status_field, dates_field) tuple. + """ + mode_mapping = { + "atv": ("atv", "atv_datesopen"), + "motorcycle": ("motorcycle", "motorcycle_datesopen"), + "mtb": ("e_bike_class1", "e_bike_class1_dur"), # Closest analog for e-bikes + "vehicle": ("highclearancevehicle", "highclearancevehicle_datesopen"), + "passenger": ("passengervehicle", "passengervehicle_datesopen"), + } + + return mode_mapping.get(mode, ("highclearancevehicle", "highclearancevehicle_datesopen")) + + +def symbol_to_access(symbol: str, mode: str, maint_level: Optional[str] = None) -> Optional[bool]: + """ + Fallback: interpret SYMBOL field when per-vehicle-class fields are null. + + MVUM SYMBOL meanings (roads): + 1 = Open to all vehicles + 2 = Open to highway legal vehicles only + 3 = Road closed to motorized + 4 = Road open seasonally + 11 = Administrative use only + 12 = Decommissioned + + For trails, similar logic applies based on TRAILCLASS. + """ + if symbol is None: + return None + + sym = str(symbol).strip() + + # Symbol 1: Open to all + if sym == "1": + return True + + # Symbol 2: Highway legal only + if sym == "2": + # ATVs/motorcycles typically not highway legal + if mode in ("atv", "motorcycle"): + return False + return True + + # Symbol 3: Closed to motorized + if sym == "3": + return False + + # Symbol 4: Seasonally open (assume open if no date check) + if sym == "4": + return True + + # Symbol 11/12: Administrative/decommissioned = closed + if sym in ("11", "12"): + return False + + # Unknown symbol - defer + return None + + +class MVUMReader: + """ + Reader for MVUM data from navi.db. + + Queries roads and trails by bounding box and returns access grids. + """ + + def __init__(self, db_path: Path = NAVI_DB_PATH): + self.db_path = db_path + self._conn = None + + def _get_conn(self) -> sqlite3.Connection: + if self._conn is None: + if not self.db_path.exists(): + raise FileNotFoundError(f"navi.db not found at {self.db_path}") + self._conn = sqlite3.connect(str(self.db_path)) + self._conn.row_factory = sqlite3.Row + # Load Spatialite extension if available + try: + self._conn.enable_load_extension(True) + self._conn.load_extension("mod_spatialite") + except Exception: + pass # Spatialite not available, will use manual bbox queries + return self._conn + + def table_exists(self, table_name: str) -> bool: + """Check if an MVUM table exists.""" + conn = self._get_conn() + cur = conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name=?", + (table_name,) + ) + return cur.fetchone() is not None + + def query_roads_bbox( + self, + south: float, north: float, west: float, east: float, + mode: str = "atv", + check_date: Optional[Tuple[int, int]] = None + ) -> List[Dict]: + """ + Query MVUM roads within a bounding box. + + Returns list of dicts with access info for the given mode. + """ + if not self.table_exists("mvum_roads"): + return [] + + conn = self._get_conn() + + # Query using bbox on geometry + # Since we don't have spatialite, we'll query all and filter in Python + # For production, consider pre-computing bbox columns + cur = conn.execute(""" + SELECT ogc_fid, id, name, symbol, operationalmaintlevel, seasonal, + atv, atv_datesopen, motorcycle, motorcycle_datesopen, + highclearancevehicle, highclearancevehicle_datesopen, + passengervehicle, passengervehicle_datesopen, + e_bike_class1, e_bike_class1_dur, + shape + FROM mvum_roads + """) + + status_field, dates_field = get_mode_field(mode) + results = [] + + for row in cur: + # Parse geometry to check bbox intersection + # The shape is stored as WKB blob + shape = row["shape"] + if shape is None: + continue + + # Quick bbox check using geometry extent + # Since we don't have Spatialite functions, we'll include all + # and let the rasterization handle it + + access = check_access( + row[status_field] if status_field in row.keys() else None, + row[dates_field] if dates_field in row.keys() else None, + row["seasonal"], + check_date + ) + + # Fallback to SYMBOL if no per-vehicle data + if access is None: + access = symbol_to_access(row["symbol"], mode, row["operationalmaintlevel"]) + + if access is not None: + results.append({ + "id": row["id"], + "name": row["name"], + "access": access, + "symbol": row["symbol"], + "maint_level": row["operationalmaintlevel"], + "shape": shape, + }) + + return results + + def query_trails_bbox( + self, + south: float, north: float, west: float, east: float, + mode: str = "atv", + check_date: Optional[Tuple[int, int]] = None + ) -> List[Dict]: + """ + Query MVUM trails within a bounding box. + """ + if not self.table_exists("mvum_trails"): + return [] + + conn = self._get_conn() + + cur = conn.execute(""" + SELECT ogc_fid, id, name, symbol, seasonal, trailclass, + atv, atv_datesopen, motorcycle, motorcycle_datesopen, + highclearancevehicle, highclearancevehicle_datesopen, + passengervehicle, passengervehicle_datesopen, + e_bike_class1, e_bike_class1_dur, + shape + FROM mvum_trails + """) + + status_field, dates_field = get_mode_field(mode) + results = [] + + for row in cur: + shape = row["shape"] + if shape is None: + continue + + access = check_access( + row[status_field] if status_field in row.keys() else None, + row[dates_field] if dates_field in row.keys() else None, + row["seasonal"], + check_date + ) + + if access is None: + access = symbol_to_access(row["symbol"], mode) + + if access is not None: + results.append({ + "id": row["id"], + "name": row["name"], + "access": access, + "symbol": row["symbol"], + "trail_class": row["trailclass"], + "shape": shape, + }) + + return results + + def query_nearest( + self, + lat: float, lon: float, + radius_m: float = 50, + table: str = "mvum_roads" + ) -> Optional[Dict]: + """ + Query the nearest MVUM feature to a point. + + Used for the places panel API. + """ + if not self.table_exists(table): + return None + + conn = self._get_conn() + + # Convert radius to degrees (approximate) + radius_deg = radius_m / 111000 + + # Query features in bbox around point + if table == "mvum_roads": + cur = conn.execute(""" + SELECT ogc_fid, id, name, forestname, districtname, symbol, + operationalmaintlevel, surfacetype, seasonal, jurisdiction, + passengervehicle, passengervehicle_datesopen, + highclearancevehicle, highclearancevehicle_datesopen, + atv, atv_datesopen, motorcycle, motorcycle_datesopen, + fourwd_gt50inches, fourwd_gt50_datesopen, + twowd_gt50inches, twowd_gt50_datesopen, + e_bike_class1, e_bike_class1_dur, + e_bike_class2, e_bike_class2_dur, + e_bike_class3, e_bike_class3_dur, + shape + FROM mvum_roads + LIMIT 1000 + """) + else: + cur = conn.execute(""" + SELECT ogc_fid, id, name, forestname, districtname, symbol, + seasonal, jurisdiction, trailclass, trailsystem, + passengervehicle, passengervehicle_datesopen, + highclearancevehicle, highclearancevehicle_datesopen, + atv, atv_datesopen, motorcycle, motorcycle_datesopen, + fourwd_gt50inches, fourwd_gt50_datesopen, + twowd_gt50inches, twowd_gt50_datesopen, + e_bike_class1, e_bike_class1_dur, + e_bike_class2, e_bike_class2_dur, + e_bike_class3, e_bike_class3_dur, + shape + FROM mvum_trails + LIMIT 1000 + """) + + # Find nearest feature + # This is a simplified approach - for production, use spatial index + try: + from shapely import wkb + from shapely.geometry import Point + + query_point = Point(lon, lat) + nearest = None + min_dist = float('inf') + + for row in cur: + try: + geom = wkb.loads(row["shape"]) + dist = query_point.distance(geom) + if dist < min_dist and dist < radius_deg: + min_dist = dist + nearest = dict(row) + nearest["geometry"] = geom + except Exception: + continue + + if nearest: + # Convert geometry to GeoJSON + nearest["geojson"] = nearest["geometry"].__geo_interface__ + del nearest["geometry"] + del nearest["shape"] + return nearest + + except ImportError: + warnings.warn("shapely not available for nearest query") + + return None + + def close(self): + if self._conn: + self._conn.close() + self._conn = None + + +def get_mvum_access_grid( + south: float, north: float, west: float, east: float, + target_shape: Tuple[int, int], + mode: Literal["foot", "mtb", "atv", "vehicle"] = "atv", + check_date: Optional[str] = None, + db_path: Path = NAVI_DB_PATH +) -> np.ndarray: + """ + Get MVUM access grid for pathfinding. + + Args: + south, north, west, east: Bounding box (WGS84) + target_shape: (rows, cols) to match elevation grid + mode: Travel mode (foot skips MVUM entirely) + check_date: Optional "MM/DD" string for seasonal checking + db_path: Path to navi.db + + Returns: + np.ndarray of uint8: + 0 = no MVUM data (defer to existing trail/friction logic) + 1 = road/trail is OPEN to this vehicle mode + 255 = road/trail EXISTS but is CLOSED to this mode + """ + # Foot mode bypasses MVUM entirely + if mode == "foot": + return np.zeros(target_shape, dtype=np.uint8) + + # Parse check_date if provided + parsed_date = None + if check_date: + match = re.match(r"(\d{1,2})/(\d{1,2})", check_date) + if match: + parsed_date = (int(match.group(1)), int(match.group(2))) + + # Initialize output grid + grid = np.zeros(target_shape, dtype=np.uint8) + rows, cols = target_shape + + # Pixel size + pixel_lat = (north - south) / rows + pixel_lon = (east - west) / cols + + reader = MVUMReader(db_path) + + try: + # Query roads and trails + roads = reader.query_roads_bbox(south, north, west, east, mode, parsed_date) + trails = reader.query_trails_bbox(south, north, west, east, mode, parsed_date) + + # Rasterize features + try: + from shapely import wkb + + for features in [roads, trails]: + for feat in features: + try: + geom = wkb.loads(feat["shape"]) + + # Get geometry bounds + minx, miny, maxx, maxy = geom.bounds + + # Check if intersects our bbox + if maxx < west or minx > east or maxy < south or miny > north: + continue + + # Rasterize line + value = 1 if feat["access"] else 255 + + # Simple line rasterization + if geom.geom_type in ("LineString", "MultiLineString"): + if geom.geom_type == "MultiLineString": + coords_list = [list(line.coords) for line in geom.geoms] + else: + coords_list = [list(geom.coords)] + + for coords in coords_list: + for i in range(len(coords) - 1): + x1, y1 = coords[i] + x2, y2 = coords[i + 1] + + # Convert to pixel coordinates + col1 = int((x1 - west) / pixel_lon) + row1 = int((north - y1) / pixel_lat) + col2 = int((x2 - west) / pixel_lon) + row2 = int((north - y2) / pixel_lat) + + # Bresenham's line algorithm + _draw_line(grid, row1, col1, row2, col2, value) + + except Exception as e: + continue + + except ImportError: + warnings.warn("shapely not available, MVUM rasterization skipped") + + finally: + reader.close() + + return grid + + +def _draw_line(grid: np.ndarray, r1: int, c1: int, r2: int, c2: int, value: int): + """Draw a line on the grid using Bresenham's algorithm.""" + rows, cols = grid.shape + + dr = abs(r2 - r1) + dc = abs(c2 - c1) + sr = 1 if r1 < r2 else -1 + sc = 1 if c1 < c2 else -1 + err = dr - dc + + r, c = r1, c1 + + while True: + if 0 <= r < rows and 0 <= c < cols: + # Only overwrite if current value is 0 (no data) or we're marking closed + if grid[r, c] == 0 or value == 255: + grid[r, c] = value + + if r == r2 and c == c2: + break + + e2 = 2 * err + if e2 > -dc: + err -= dc + r += sr + if e2 < dr: + err += dr + c += sc + + +if __name__ == "__main__": + import sys + + print("=" * 60) + print("MVUM Reader Test") + print("=" * 60) + + reader = MVUMReader() + + if not reader.table_exists("mvum_roads"): + print("ERROR: mvum_roads table not found in navi.db") + sys.exit(1) + + # Test bbox query (Sawtooth NF area) + print("\n[1] Testing bbox query (Sawtooth NF area)...") + roads = reader.query_roads_bbox( + south=43.5, north=44.0, west=-115.0, east=-114.0, + mode="atv" + ) + print(f" Found {len(roads)} roads") + + open_count = sum(1 for r in roads if r["access"]) + closed_count = sum(1 for r in roads if not r["access"]) + print(f" Open to ATV: {open_count}") + print(f" Closed to ATV: {closed_count}") + + # Test with seasonal date + print("\n[2] Testing with date check (July 15)...") + roads_summer = reader.query_roads_bbox( + south=43.5, north=44.0, west=-115.0, east=-114.0, + mode="atv", + check_date=(7, 15) + ) + open_summer = sum(1 for r in roads_summer if r["access"]) + print(f" Open to ATV on 07/15: {open_summer}") + + print("\n[3] Testing with date check (January 15)...") + roads_winter = reader.query_roads_bbox( + south=43.5, north=44.0, west=-115.0, east=-114.0, + mode="atv", + check_date=(1, 15) + ) + open_winter = sum(1 for r in roads_winter if r["access"]) + print(f" Open to ATV on 01/15: {open_winter}") + + # Test grid generation + print("\n[4] Testing grid generation...") + grid = get_mvum_access_grid( + south=43.5, north=44.0, west=-115.0, east=-114.0, + target_shape=(500, 1000), + mode="atv" + ) + print(f" Grid shape: {grid.shape}") + print(f" No data (0): {np.sum(grid == 0)}") + print(f" Open (1): {np.sum(grid == 1)}") + print(f" Closed (255): {np.sum(grid == 255)}") + + reader.close() + print("\nDone.") diff --git a/lib/offroute/router.py b/lib/offroute/router.py index d44bbb4..c37bddf 100644 --- a/lib/offroute/router.py +++ b/lib/offroute/router.py @@ -27,6 +27,7 @@ from .cost import compute_cost_grid from .friction import FrictionReader, friction_to_multiplier from .barriers import BarrierReader, WildernessReader, DEFAULT_WILDERNESS_PATH from .trails import TrailReader +from .mvum import get_mvum_access_grid # Paths NAVI_DB_PATH = Path("/mnt/nav/navi.db") @@ -407,6 +408,22 @@ class OffrouteRouter: target_shape=elevation.shape ) + # Load MVUM access data (only for motorized modes) + # MVUM is motor-vehicle specific — foot mode skips entirely + mvum = None + if mode in ("mtb", "atv", "vehicle"): + try: + mvum = get_mvum_access_grid( + south=bbox["south"], north=bbox["north"], + west=bbox["west"], east=bbox["east"], + target_shape=elevation.shape, + mode=mode, + check_date=None, # TODO: accept date parameter + ) + except Exception as e: + # MVUM data may not be available - continue without it + pass + # Compute cost grid with mode-specific parameters cost = compute_cost_grid( elevation, @@ -416,12 +433,13 @@ class OffrouteRouter: trails=trails, barriers=barriers, wilderness=wilderness, + mvum=mvum, boundary_mode=boundary_mode, mode=mode, ) # Free intermediate arrays to reduce memory before MCP - # Note: Keep trails and barriers - needed for path statistics + # Note: Keep trails, barriers, and mvum - needed for path statistics del friction_mult, friction_raw, wilderness import gc gc.collect() @@ -471,6 +489,7 @@ class OffrouteRouter: elevations = [] trail_values = [] barrier_crossings = 0 + mvum_closed_crossings = 0 for row, col in path_indices: lat, lon = self.dem_reader.pixel_to_latlon(row, col, meta) @@ -479,6 +498,8 @@ class OffrouteRouter: trail_values.append(trails[row, col]) if barriers[row, col] == 255: barrier_crossings += 1 + if mvum is not None and mvum[row, col] == 255: + mvum_closed_crossings += 1 # Calculate stats wilderness_distance_m = 0 @@ -497,8 +518,10 @@ class OffrouteRouter: total_cells = len(trail_arr) on_trail_pct = float(100 * on_trail_cells / total_cells) if total_cells > 0 else 0 - # Free trails and barriers now that path stats are computed + # Free trails, barriers, and mvum now that path stats are computed del trails, barriers + if mvum is not None: + del mvum # Entry point entry_lat = best_entry["entry_point"]["lat"] @@ -572,6 +595,7 @@ class OffrouteRouter: "on_trail_pct": on_trail_pct, "cell_count": total_cells, "barrier_crossings": barrier_crossings, + "mvum_closed_crossings": mvum_closed_crossings, "mode": mode, }, "geometry": {"type": "LineString", "coordinates": wilderness_coords} @@ -620,6 +644,7 @@ class OffrouteRouter: "network_duration_minutes": float(network_segment["duration_minutes"]) if network_segment else 0, "on_trail_pct": on_trail_pct, "barrier_crossings": barrier_crossings, + "mvum_closed_crossings": mvum_closed_crossings, "boundary_mode": boundary_mode, "mode": mode, "entry_point": { From ff0721c23ef00a978478c2708c9e0055a63be549 Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 8 May 2026 19:03:31 +0000 Subject: [PATCH 10/41] offroute: wilderness always uses foot mode for pathfinding The wilderness segment now ALWAYS uses foot mode for MCP pathfinding. The user's selected mode only affects: 1. Entry point selection (MODE_TO_VALID_HIGHWAYS filtering) 2. Valhalla costing for the network segment This ensures vehicles can navigate through wilderness (on foot) to reach roads, rather than failing when no vehicle-accessible path exists. Co-Authored-By: Claude Opus 4.5 --- lib/offroute/router.py | 137 ++++++++++++++++++++++++----------------- 1 file changed, 81 insertions(+), 56 deletions(-) diff --git a/lib/offroute/router.py b/lib/offroute/router.py index c37bddf..e1883f7 100644 --- a/lib/offroute/router.py +++ b/lib/offroute/router.py @@ -7,7 +7,10 @@ Entry points are extracted from OSM highways and stored in /mnt/nav/navi.db. The pathfinder routes from a wilderness start to the nearest entry point, then Valhalla completes the route to the destination. -Supports four travel modes: foot, mtb, atv, vehicle. +IMPORTANT: The wilderness segment ALWAYS uses foot mode for pathfinding. +The user's selected mode affects: + 1. Which entry points are valid (foot=any, mtb=tracks+roads, vehicle=roads only) + 2. The Valhalla costing profile for the network segment """ import json import math @@ -51,6 +54,19 @@ MODE_TO_COSTING = { "vehicle": "auto", } +# Mode to valid entry point highway classes +# foot = any trail/track/road, mtb = tracks and roads, vehicle = roads only +MODE_TO_VALID_HIGHWAYS = { + "foot": {"primary", "secondary", "tertiary", "unclassified", "residential", + "service", "track", "path", "footway", "bridleway"}, + "mtb": {"primary", "secondary", "tertiary", "unclassified", "residential", + "service", "track"}, + "atv": {"primary", "secondary", "tertiary", "unclassified", "residential", + "service", "track"}, + "vehicle": {"primary", "secondary", "tertiary", "unclassified", "residential", + "service"}, +} + def haversine_distance(lat1: float, lon1: float, lat2: float, lon2: float) -> float: """Calculate distance between two points in meters.""" @@ -122,8 +138,16 @@ class EntryPointIndex: return [dict(row) for row in cur.fetchall()] - def query_radius(self, lat: float, lon: float, radius_km: float) -> List[Dict]: - """Query entry points within radius of a point.""" + def query_radius(self, lat: float, lon: float, radius_km: float, + valid_highways: Optional[set] = None) -> List[Dict]: + """ + Query entry points within radius of a point. + + Args: + lat, lon: Center point + radius_km: Search radius in kilometers + valid_highways: Optional set of valid highway classes to filter by + """ lat_delta = radius_km / 111.0 lon_delta = radius_km / (111.0 * math.cos(math.radians(lat))) @@ -134,6 +158,10 @@ class EntryPointIndex: result = [] for p in points: + # Filter by highway class if specified + if valid_highways and p['highway_class'] not in valid_highways: + continue + dist = haversine_distance(lat, lon, p['lat'], p['lon']) if dist <= radius_km * 1000: p['distance_m'] = dist @@ -262,7 +290,8 @@ class OffrouteRouter: """ OFFROUTE Router — orchestrates wilderness pathfinding and Valhalla stitching. - Supports modes: foot, mtb, atv, vehicle + IMPORTANT: Wilderness segment ALWAYS uses foot mode for pathfinding. + User's mode affects entry point selection and Valhalla costing only. """ def __init__(self): @@ -301,9 +330,14 @@ class OffrouteRouter: Args: start_lat, start_lon: Starting coordinates (wilderness) end_lat, end_lon: Destination coordinates - mode: Travel mode (foot, mtb, atv, vehicle) + mode: Travel mode (foot, mtb, atv, vehicle) - affects entry points and network routing boundary_mode: How to handle private land (strict, pragmatic, emergency) + IMPORTANT: Wilderness pathfinding ALWAYS uses foot mode. + The user's mode only affects: + 1. Which entry points are valid targets + 2. The Valhalla costing for the network segment + Returns a GeoJSON FeatureCollection with wilderness and network segments. """ t0 = time.time() @@ -318,17 +352,29 @@ class OffrouteRouter: "message": "Trail entry point index not built. Run build_entry_index() first." } - # Find entry points near start (limit to nearest 10 to control bbox size) + # Get valid highway classes for this mode + valid_highways = MODE_TO_VALID_HIGHWAYS.get(mode) + + # Find entry points near start, filtered by mode MAX_ENTRY_POINTS = 10 - entry_points = self.entry_index.query_radius(start_lat, start_lon, DEFAULT_SEARCH_RADIUS_KM) + entry_points = self.entry_index.query_radius( + start_lat, start_lon, DEFAULT_SEARCH_RADIUS_KM, valid_highways + ) if not entry_points: - entry_points = self.entry_index.query_radius(start_lat, start_lon, EXPANDED_SEARCH_RADIUS_KM) + # Try expanded radius + entry_points = self.entry_index.query_radius( + start_lat, start_lon, EXPANDED_SEARCH_RADIUS_KM, valid_highways + ) if not entry_points: - return { - "status": "error", - "message": f"No trail entry points found within {EXPANDED_SEARCH_RADIUS_KM}km of start" - } + # For non-foot modes, the error is about no suitable roads/trails + if mode == "vehicle": + msg = f"No roads found within {EXPANDED_SEARCH_RADIUS_KM}km. Try a different mode." + elif mode in ("mtb", "atv"): + msg = f"No tracks or roads found within {EXPANDED_SEARCH_RADIUS_KM}km. Try foot mode." + else: + msg = f"No trail entry points found within {EXPANDED_SEARCH_RADIUS_KM}km of start." + return {"status": "error", "message": msg} # Limit to nearest entry points to prevent huge bounding boxes entry_points = entry_points[:MAX_ENTRY_POINTS] @@ -392,15 +438,6 @@ class OffrouteRouter: target_shape=elevation.shape ) - # Load wilderness (if available and mode requires it) - wilderness = None - if self.wilderness_reader is not None and mode in ("mtb", "atv", "vehicle"): - wilderness = self.wilderness_reader.get_wilderness_grid( - south=bbox["south"], north=bbox["north"], - west=bbox["west"], east=bbox["east"], - target_shape=elevation.shape - ) - # Load trails trails = self.trail_reader.get_trails_grid( south=bbox["south"], north=bbox["north"], @@ -408,23 +445,11 @@ class OffrouteRouter: target_shape=elevation.shape ) - # Load MVUM access data (only for motorized modes) - # MVUM is motor-vehicle specific — foot mode skips entirely - mvum = None - if mode in ("mtb", "atv", "vehicle"): - try: - mvum = get_mvum_access_grid( - south=bbox["south"], north=bbox["north"], - west=bbox["west"], east=bbox["east"], - target_shape=elevation.shape, - mode=mode, - check_date=None, # TODO: accept date parameter - ) - except Exception as e: - # MVUM data may not be available - continue without it - pass + # WILDERNESS PATHFINDING ALWAYS USES FOOT MODE + # This is the key change: we don't load wilderness grid or MVUM for pathfinding + # because foot mode can traverse wilderness and doesn't need motor-vehicle access - # Compute cost grid with mode-specific parameters + # Compute cost grid with FOOT MODE (always for wilderness segment) cost = compute_cost_grid( elevation, cell_size_m=meta["cell_size_m"], @@ -432,15 +457,14 @@ class OffrouteRouter: friction_raw=friction_raw, trails=trails, barriers=barriers, - wilderness=wilderness, - mvum=mvum, + wilderness=None, # Foot mode ignores wilderness restrictions + mvum=None, # Foot mode doesn't use MVUM boundary_mode=boundary_mode, - mode=mode, + mode="foot", # ALWAYS foot for wilderness pathfinding ) # Free intermediate arrays to reduce memory before MCP - # Note: Keep trails, barriers, and mvum - needed for path statistics - del friction_mult, friction_raw, wilderness + del friction_mult, friction_raw import gc gc.collect() @@ -489,7 +513,6 @@ class OffrouteRouter: elevations = [] trail_values = [] barrier_crossings = 0 - mvum_closed_crossings = 0 for row, col in path_indices: lat, lon = self.dem_reader.pixel_to_latlon(row, col, meta) @@ -498,8 +521,6 @@ class OffrouteRouter: trail_values.append(trails[row, col]) if barriers[row, col] == 255: barrier_crossings += 1 - if mvum is not None and mvum[row, col] == 255: - mvum_closed_crossings += 1 # Calculate stats wilderness_distance_m = 0 @@ -518,10 +539,8 @@ class OffrouteRouter: total_cells = len(trail_arr) on_trail_pct = float(100 * on_trail_cells / total_cells) if total_cells > 0 else 0 - # Free trails, barriers, and mvum now that path stats are computed + # Free trails and barriers del trails, barriers - if mvum is not None: - del mvum # Entry point entry_lat = best_entry["entry_point"]["lat"] @@ -529,7 +548,7 @@ class OffrouteRouter: entry_class = best_entry["entry_point"]["highway_class"] entry_name = best_entry["entry_point"].get("name", "") - # Call Valhalla + # Call Valhalla with USER'S SELECTED MODE (not foot) valhalla_costing = MODE_TO_COSTING.get(mode, "pedestrian") valhalla_request = { @@ -595,8 +614,7 @@ class OffrouteRouter: "on_trail_pct": on_trail_pct, "cell_count": total_cells, "barrier_crossings": barrier_crossings, - "mvum_closed_crossings": mvum_closed_crossings, - "mode": mode, + "wilderness_mode": "foot", # Always foot for wilderness }, "geometry": {"type": "LineString", "coordinates": wilderness_coords} } @@ -610,6 +628,7 @@ class OffrouteRouter: "distance_km": network_segment["distance_km"], "duration_minutes": network_segment["duration_minutes"], "maneuvers": network_segment["maneuvers"], + "network_mode": mode, # User's selected mode }, "geometry": {"type": "LineString", "coordinates": network_segment["coordinates"]} } @@ -621,7 +640,12 @@ class OffrouteRouter: combined_feature = { "type": "Feature", - "properties": {"segment_type": "combined", "mode": mode, "boundary_mode": boundary_mode}, + "properties": { + "segment_type": "combined", + "wilderness_mode": "foot", + "network_mode": mode, + "boundary_mode": boundary_mode + }, "geometry": {"type": "LineString", "coordinates": combined_coords} } features.append(combined_feature) @@ -644,9 +668,9 @@ class OffrouteRouter: "network_duration_minutes": float(network_segment["duration_minutes"]) if network_segment else 0, "on_trail_pct": on_trail_pct, "barrier_crossings": barrier_crossings, - "mvum_closed_crossings": mvum_closed_crossings, "boundary_mode": boundary_mode, - "mode": mode, + "wilderness_mode": "foot", # Always foot + "network_mode": mode, # User's selection "entry_point": { "lat": entry_lat, "lon": entry_lon, "highway_class": entry_class, "name": entry_name, @@ -730,6 +754,7 @@ if __name__ == "__main__": elif len(sys.argv) > 1 and sys.argv[1] == "test": print("Testing router (all modes)...") + print("NOTE: Wilderness always uses foot mode. User mode affects entry points + network.") router = OffrouteRouter() @@ -746,8 +771,8 @@ if __name__ == "__main__": if result["status"] == "ok": s = result["summary"] - print(f" Wilderness: {s['wilderness_distance_km']:.2f} km, {s['wilderness_effort_minutes']:.1f} min") - print(f" Network: {s['network_distance_km']:.2f} km, {s['network_duration_minutes']:.1f} min") + print(f" Wilderness: {s['wilderness_distance_km']:.2f} km, {s['wilderness_effort_minutes']:.1f} min (foot)") + print(f" Network: {s['network_distance_km']:.2f} km, {s['network_duration_minutes']:.1f} min ({mode})") print(f" On-trail: {s['on_trail_pct']:.1f}%") print(f" Entry: {s['entry_point']['highway_class']}") else: From 58347415bcd36d44e39653a6ed6af541f30faef9 Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 8 May 2026 21:11:53 +0000 Subject: [PATCH 11/41] offroute: bidirectional wilderness routing (all 4 scenarios) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Support all four routing scenarios: A: off-network → on-network (wilderness then Valhalla) B: off-network → off-network (wilderness, Valhalla, wilderness) C: on-network → off-network (Valhalla then wilderness) D: on-network → on-network (pure Valhalla passthrough) Off-network detection via Valhalla /locate endpoint: - Snap distance > 500m = off-network Key implementation details: - _locate_on_network() helper for network detection - route() dispatches to scenario-specific handlers - _pathfind_wilderness() extracted for reuse (runs MCP) - _valhalla_route() helper for network segments - _build_response() unifies GeoJSON output format Memory management: - Sequential MCP runs for scenario B (not parallel) - gc.collect() after each MCP run - Bbox centered on wilderness origin, not distant destination Co-Authored-By: Claude Opus 4.5 --- lib/offroute/router.py | 843 +++++++++++++++++++++++++++++++++-------- 1 file changed, 693 insertions(+), 150 deletions(-) diff --git a/lib/offroute/router.py b/lib/offroute/router.py index e1883f7..79c9089 100644 --- a/lib/offroute/router.py +++ b/lib/offroute/router.py @@ -1,17 +1,20 @@ """ -OFFROUTE Router — Wilderness to network path orchestration. +OFFROUTE Router — Bidirectional wilderness-to-network path orchestration. -Connects the raster pathfinder (wilderness segment) to Valhalla (on-network segment). +Supports four routing scenarios: + A: off-network start → on-network end (wilderness then Valhalla) + B: off-network start → off-network end (wilderness, Valhalla, wilderness) + C: on-network start → off-network end (Valhalla then wilderness) + D: on-network start → on-network end (pure Valhalla passthrough) -Entry points are extracted from OSM highways and stored in /mnt/nav/navi.db. -The pathfinder routes from a wilderness start to the nearest entry point, -then Valhalla completes the route to the destination. +Off-network detection: Valhalla /locate snap distance > 500m = off-network. IMPORTANT: The wilderness segment ALWAYS uses foot mode for pathfinding. The user's selected mode affects: 1. Which entry points are valid (foot=any, mtb=tracks+roads, vehicle=roads only) 2. The Valhalla costing profile for the network segment """ +import gc import json import math import sqlite3 @@ -46,6 +49,9 @@ EXPANDED_SEARCH_RADIUS_KM = 100 # Memory limit MEMORY_LIMIT_GB = 12 +# Off-network detection threshold (meters) +OFF_NETWORK_THRESHOLD_M = 500 + # Mode to Valhalla costing mapping MODE_TO_COSTING = { "foot": "pedestrian", @@ -290,6 +296,12 @@ class OffrouteRouter: """ OFFROUTE Router — orchestrates wilderness pathfinding and Valhalla stitching. + Supports four scenarios: + A: off-network start → on-network end + B: off-network start → off-network end + C: on-network start → off-network end + D: on-network start → on-network end (pure Valhalla) + IMPORTANT: Wilderness segment ALWAYS uses foot mode for pathfinding. User's mode affects entry point selection and Valhalla costing only. """ @@ -315,6 +327,49 @@ class OffrouteRouter: if self.trail_reader is None: self.trail_reader = TrailReader() + def _locate_on_network(self, lat: float, lon: float, mode: str) -> Dict: + """ + Check if a point is on the routable network using Valhalla's /locate. + + Returns: + { + "on_network": bool, + "snap_distance_m": float, + "snapped_lat": float, + "snapped_lon": float + } + """ + costing = MODE_TO_COSTING.get(mode, "pedestrian") + try: + resp = requests.post( + f"{VALHALLA_URL}/locate", + json={"locations": [{"lat": lat, "lon": lon}], "costing": costing}, + timeout=10 + ) + + if resp.status_code == 200: + data = resp.json() + if data and len(data) > 0 and data[0].get("edges"): + edge = data[0]["edges"][0] + snap_lat = edge.get("correlated_lat", lat) + snap_lon = edge.get("correlated_lon", lon) + snap_dist = haversine_distance(lat, lon, snap_lat, snap_lon) + return { + "on_network": snap_dist <= OFF_NETWORK_THRESHOLD_M, + "snap_distance_m": snap_dist, + "snapped_lat": snap_lat, + "snapped_lon": snap_lon + } + except Exception: + pass + + return { + "on_network": False, + "snap_distance_m": float('inf'), + "snapped_lat": lat, + "snapped_lon": lon + } + def route( self, start_lat: float, @@ -325,26 +380,167 @@ class OffrouteRouter: boundary_mode: Literal["strict", "pragmatic", "emergency"] = "pragmatic" ) -> Dict: """ - Route from a wilderness start point to a destination. + Route between two points, handling all four scenarios. + + Scenarios: + A: off-network start → on-network end (wilderness then network) + B: off-network start → off-network end (wilderness, network, wilderness) + C: on-network start → off-network end (network then wilderness) + D: on-network start → on-network end (pure network) Args: - start_lat, start_lon: Starting coordinates (wilderness) + start_lat, start_lon: Starting coordinates end_lat, end_lon: Destination coordinates - mode: Travel mode (foot, mtb, atv, vehicle) - affects entry points and network routing + mode: Travel mode (foot, mtb, atv, vehicle) boundary_mode: How to handle private land (strict, pragmatic, emergency) - IMPORTANT: Wilderness pathfinding ALWAYS uses foot mode. - The user's mode only affects: - 1. Which entry points are valid targets - 2. The Valhalla costing for the network segment - - Returns a GeoJSON FeatureCollection with wilderness and network segments. + Returns a GeoJSON FeatureCollection with route segments. """ - t0 = time.time() - if mode not in MODE_TO_COSTING: return {"status": "error", "message": f"Unknown mode: {mode}"} + # Detect network status for both endpoints + start_status = self._locate_on_network(start_lat, start_lon, mode) + end_status = self._locate_on_network(end_lat, end_lon, mode) + + start_off_network = not start_status["on_network"] + end_off_network = not end_status["on_network"] + + # Dispatch to appropriate handler + if not start_off_network and not end_off_network: + # Scenario D: on-network → on-network (pure Valhalla) + return self._route_D_network_only( + start_lat, start_lon, end_lat, end_lon, mode + ) + elif not start_off_network and end_off_network: + # Scenario C: on-network → off-network + return self._route_C_network_to_wilderness( + start_lat, start_lon, end_lat, end_lon, mode, boundary_mode + ) + elif start_off_network and not end_off_network: + # Scenario A: off-network → on-network + return self._route_A_wilderness_to_network( + start_lat, start_lon, end_lat, end_lon, mode, boundary_mode + ) + else: + # Scenario B: off-network → off-network + return self._route_B_wilderness_both( + start_lat, start_lon, end_lat, end_lon, mode, boundary_mode + ) + + def _route_D_network_only( + self, + start_lat: float, start_lon: float, + end_lat: float, end_lon: float, + mode: str + ) -> Dict: + """ + Scenario D: Both endpoints on-network. Pure Valhalla routing. + """ + t0 = time.time() + costing = MODE_TO_COSTING.get(mode, "pedestrian") + + valhalla_request = { + "locations": [ + {"lat": start_lat, "lon": start_lon}, + {"lat": end_lat, "lon": end_lon} + ], + "costing": costing, + "directions_options": {"units": "kilometers"} + } + + try: + resp = requests.post(f"{VALHALLA_URL}/route", json=valhalla_request, timeout=30) + + if resp.status_code != 200: + return { + "status": "error", + "message": f"Network routing failed: {resp.text[:200]}" + } + + valhalla_data = resp.json() + trip = valhalla_data.get("trip", {}) + legs = trip.get("legs", []) + + if not legs: + return {"status": "error", "message": "No route found"} + + leg = legs[0] + shape = leg.get("shape", "") + network_coords = self._decode_polyline(shape) + + maneuvers = [] + for m in leg.get("maneuvers", []): + maneuvers.append({ + "instruction": m.get("instruction", ""), + "type": m.get("type", 0), + "distance_km": m.get("length", 0), + "time_seconds": m.get("time", 0), + "street_names": m.get("street_names", []), + }) + + summary = trip.get("summary", {}) + distance_km = summary.get("length", 0) + duration_min = summary.get("time", 0) / 60 + + # Build response in same format as wilderness routes + network_feature = { + "type": "Feature", + "properties": { + "segment_type": "network", + "distance_km": distance_km, + "duration_minutes": duration_min, + "maneuvers": maneuvers, + "network_mode": mode, + }, + "geometry": {"type": "LineString", "coordinates": network_coords} + } + + combined_feature = { + "type": "Feature", + "properties": { + "segment_type": "combined", + "network_mode": mode, + }, + "geometry": {"type": "LineString", "coordinates": network_coords} + } + + geojson = {"type": "FeatureCollection", "features": [network_feature, combined_feature]} + + result = { + "status": "ok", + "route": geojson, + "summary": { + "total_distance_km": float(distance_km), + "total_effort_minutes": float(duration_min), + "wilderness_distance_km": 0.0, + "wilderness_effort_minutes": 0.0, + "network_distance_km": float(distance_km), + "network_duration_minutes": float(duration_min), + "on_trail_pct": 100.0, + "barrier_crossings": 0, + "network_mode": mode, + "scenario": "D", + "computation_time_s": time.time() - t0, + } + } + return result + + except Exception as e: + return {"status": "error", "message": f"Network routing failed: {e}"} + + def _route_A_wilderness_to_network( + self, + start_lat: float, start_lon: float, + end_lat: float, end_lon: float, + mode: str, boundary_mode: str + ) -> Dict: + """ + Scenario A: Off-network start → on-network end. + Wilderness pathfinding from start to entry point, then Valhalla to end. + """ + t0 = time.time() + # Ensure entry point index exists if not self.entry_index.table_exists() or self.entry_index.get_entry_point_count() == 0: return { @@ -362,12 +558,10 @@ class OffrouteRouter: ) if not entry_points: - # Try expanded radius entry_points = self.entry_index.query_radius( start_lat, start_lon, EXPANDED_SEARCH_RADIUS_KM, valid_highways ) if not entry_points: - # For non-foot modes, the error is about no suitable roads/trails if mode == "vehicle": msg = f"No roads found within {EXPANDED_SEARCH_RADIUS_KM}km. Try a different mode." elif mode in ("mtb", "atv"): @@ -376,13 +570,243 @@ class OffrouteRouter: msg = f"No trail entry points found within {EXPANDED_SEARCH_RADIUS_KM}km of start." return {"status": "error", "message": msg} - # Limit to nearest entry points to prevent huge bounding boxes entry_points = entry_points[:MAX_ENTRY_POINTS] - # Build bbox with max size limit (prevent OOM on large areas) - MAX_BBOX_DEGREES = 0.5 # ~55km at mid-latitudes - all_lats = [start_lat, end_lat] + [p["lat"] for p in entry_points] - all_lons = [start_lon, end_lon] + [p["lon"] for p in entry_points] + # Run wilderness pathfinding + wilderness_result = self._pathfind_wilderness( + start_lat, start_lon, end_lat, end_lon, + entry_points, boundary_mode, "start" + ) + + if wilderness_result.get("status") == "error": + return wilderness_result + + # Extract results + wilderness_coords = wilderness_result["coords"] + wilderness_stats = wilderness_result["stats"] + best_entry = wilderness_result["entry_point"] + + entry_lat = best_entry["lat"] + entry_lon = best_entry["lon"] + + # Call Valhalla from entry point to destination + network_result = self._valhalla_route(entry_lat, entry_lon, end_lat, end_lon, mode) + + # Build response + return self._build_response( + wilderness_start=wilderness_coords, + wilderness_start_stats=wilderness_stats, + network_segment=network_result.get("segment"), + wilderness_end=None, + wilderness_end_stats=None, + mode=mode, + boundary_mode=boundary_mode, + entry_start=best_entry, + entry_end=None, + scenario="A", + t0=t0, + valhalla_error=network_result.get("error") + ) + + def _route_C_network_to_wilderness( + self, + start_lat: float, start_lon: float, + end_lat: float, end_lon: float, + mode: str, boundary_mode: str + ) -> Dict: + """ + Scenario C: On-network start → off-network end. + Valhalla from start to entry point, then wilderness pathfinding to end. + """ + t0 = time.time() + + if not self.entry_index.table_exists() or self.entry_index.get_entry_point_count() == 0: + return { + "status": "error", + "message": "Trail entry point index not built. Run build_entry_index() first." + } + + valid_highways = MODE_TO_VALID_HIGHWAYS.get(mode) + + # Find entry points near END (destination) + MAX_ENTRY_POINTS = 10 + entry_points = self.entry_index.query_radius( + end_lat, end_lon, DEFAULT_SEARCH_RADIUS_KM, valid_highways + ) + + if not entry_points: + entry_points = self.entry_index.query_radius( + end_lat, end_lon, EXPANDED_SEARCH_RADIUS_KM, valid_highways + ) + if not entry_points: + if mode == "vehicle": + msg = f"No roads found within {EXPANDED_SEARCH_RADIUS_KM}km of destination. Try a different mode." + elif mode in ("mtb", "atv"): + msg = f"No tracks or roads found within {EXPANDED_SEARCH_RADIUS_KM}km of destination. Try foot mode." + else: + msg = f"No trail entry points found within {EXPANDED_SEARCH_RADIUS_KM}km of destination." + return {"status": "error", "message": msg} + + entry_points = entry_points[:MAX_ENTRY_POINTS] + + # Run wilderness pathfinding FROM END toward entry points + wilderness_result = self._pathfind_wilderness( + end_lat, end_lon, start_lat, start_lon, + entry_points, boundary_mode, "end" + ) + + if wilderness_result.get("status") == "error": + return wilderness_result + + # The path is from end→entry, reverse it for display (entry→end) + wilderness_coords = list(reversed(wilderness_result["coords"])) + wilderness_stats = wilderness_result["stats"] + best_entry = wilderness_result["entry_point"] + + entry_lat = best_entry["lat"] + entry_lon = best_entry["lon"] + + # Call Valhalla from start to entry point + network_result = self._valhalla_route(start_lat, start_lon, entry_lat, entry_lon, mode) + + # Build response (network first, then wilderness) + return self._build_response( + wilderness_start=None, + wilderness_start_stats=None, + network_segment=network_result.get("segment"), + wilderness_end=wilderness_coords, + wilderness_end_stats=wilderness_stats, + mode=mode, + boundary_mode=boundary_mode, + entry_start=None, + entry_end=best_entry, + scenario="C", + t0=t0, + valhalla_error=network_result.get("error") + ) + + def _route_B_wilderness_both( + self, + start_lat: float, start_lon: float, + end_lat: float, end_lon: float, + mode: str, boundary_mode: str + ) -> Dict: + """ + Scenario B: Off-network start → off-network end. + Wilderness from start to entry_A, Valhalla entry_A to entry_B, wilderness from entry_B to end. + """ + t0 = time.time() + + if not self.entry_index.table_exists() or self.entry_index.get_entry_point_count() == 0: + return { + "status": "error", + "message": "Trail entry point index not built. Run build_entry_index() first." + } + + valid_highways = MODE_TO_VALID_HIGHWAYS.get(mode) + MAX_ENTRY_POINTS = 10 + + # Find entry points near START + entry_points_start = self.entry_index.query_radius( + start_lat, start_lon, DEFAULT_SEARCH_RADIUS_KM, valid_highways + ) + if not entry_points_start: + entry_points_start = self.entry_index.query_radius( + start_lat, start_lon, EXPANDED_SEARCH_RADIUS_KM, valid_highways + ) + if not entry_points_start: + return {"status": "error", "message": f"No entry points found near start within {EXPANDED_SEARCH_RADIUS_KM}km."} + entry_points_start = entry_points_start[:MAX_ENTRY_POINTS] + + # Find entry points near END + entry_points_end = self.entry_index.query_radius( + end_lat, end_lon, DEFAULT_SEARCH_RADIUS_KM, valid_highways + ) + if not entry_points_end: + entry_points_end = self.entry_index.query_radius( + end_lat, end_lon, EXPANDED_SEARCH_RADIUS_KM, valid_highways + ) + if not entry_points_end: + return {"status": "error", "message": f"No entry points found near destination within {EXPANDED_SEARCH_RADIUS_KM}km."} + entry_points_end = entry_points_end[:MAX_ENTRY_POINTS] + + # Phase 1: Wilderness pathfinding from START + wilderness_start_result = self._pathfind_wilderness( + start_lat, start_lon, end_lat, end_lon, + entry_points_start, boundary_mode, "start" + ) + + if wilderness_start_result.get("status") == "error": + return wilderness_start_result + + wilderness_start_coords = wilderness_start_result["coords"] + wilderness_start_stats = wilderness_start_result["stats"] + entry_A = wilderness_start_result["entry_point"] + + # Phase 2: Wilderness pathfinding from END (run after freeing phase 1 memory) + wilderness_end_result = self._pathfind_wilderness( + end_lat, end_lon, start_lat, start_lon, + entry_points_end, boundary_mode, "end" + ) + + if wilderness_end_result.get("status") == "error": + return wilderness_end_result + + # Reverse the end wilderness path (it's end→entry, we want entry→end for display) + wilderness_end_coords = list(reversed(wilderness_end_result["coords"])) + wilderness_end_stats = wilderness_end_result["stats"] + entry_B = wilderness_end_result["entry_point"] + + # Phase 3: Valhalla from entry_A to entry_B + network_result = self._valhalla_route( + entry_A["lat"], entry_A["lon"], + entry_B["lat"], entry_B["lon"], + mode + ) + + # Build response + return self._build_response( + wilderness_start=wilderness_start_coords, + wilderness_start_stats=wilderness_start_stats, + network_segment=network_result.get("segment"), + wilderness_end=wilderness_end_coords, + wilderness_end_stats=wilderness_end_stats, + mode=mode, + boundary_mode=boundary_mode, + entry_start=entry_A, + entry_end=entry_B, + scenario="B", + t0=t0, + valhalla_error=network_result.get("error") + ) + + def _pathfind_wilderness( + self, + origin_lat: float, origin_lon: float, + dest_lat: float, dest_lon: float, + entry_points: List[Dict], + boundary_mode: str, + label: str + ) -> Dict: + """ + Run MCP wilderness pathfinding from origin toward entry points. + + Args: + origin_lat, origin_lon: Starting point for pathfinding + dest_lat, dest_lon: Ultimate destination (for bbox calculation) + entry_points: List of candidate entry points + boundary_mode: How to handle barriers + label: "start" or "end" for error messages + + Returns: + {"status": "ok", "coords": [...], "stats": {...}, "entry_point": {...}} + or {"status": "error", "message": "..."} + """ + # Build bbox - only include origin and entry points, NOT distant destination + # The destination is handled by Valhalla, wilderness only needs to reach entry points + MAX_BBOX_DEGREES = 0.5 + all_lats = [origin_lat] + [p["lat"] for p in entry_points] + all_lons = [origin_lon] + [p["lon"] for p in entry_points] padding = 0.05 bbox = { @@ -392,18 +816,16 @@ class OffrouteRouter: "east": max(all_lons) + padding, } - # Clamp bbox size to prevent memory exhaustion + # Clamp bbox size, centering on origin lat_span = bbox["north"] - bbox["south"] lon_span = bbox["east"] - bbox["west"] if lat_span > MAX_BBOX_DEGREES or lon_span > MAX_BBOX_DEGREES: - center_lat = (bbox["south"] + bbox["north"]) / 2 - center_lon = (bbox["west"] + bbox["east"]) / 2 half_span = MAX_BBOX_DEGREES / 2 bbox = { - "south": center_lat - half_span, - "north": center_lat + half_span, - "west": center_lon - half_span, - "east": center_lon + half_span, + "south": origin_lat - half_span, + "north": origin_lat + half_span, + "west": origin_lon - half_span, + "east": origin_lon + half_span, } # Initialize readers @@ -416,14 +838,14 @@ class OffrouteRouter: west=bbox["west"], east=bbox["east"], ) except Exception as e: - return {"status": "error", "message": f"Failed to load elevation: {e}"} + return {"status": "error", "message": f"Failed to load elevation for {label}: {e}"} # Check memory mem = check_memory_usage() if mem > MEMORY_LIMIT_GB: return {"status": "error", "message": f"Memory limit exceeded: {mem:.1f}GB > {MEMORY_LIMIT_GB}GB"} - # Load friction (both processed and raw for mode-specific overrides) + # Load friction friction_raw = self.friction_reader.get_friction_grid( south=bbox["south"], north=bbox["north"], west=bbox["west"], east=bbox["east"], @@ -445,11 +867,7 @@ class OffrouteRouter: target_shape=elevation.shape ) - # WILDERNESS PATHFINDING ALWAYS USES FOOT MODE - # This is the key change: we don't load wilderness grid or MVUM for pathfinding - # because foot mode can traverse wilderness and doesn't need motor-vehicle access - - # Compute cost grid with FOOT MODE (always for wilderness segment) + # Compute cost grid (ALWAYS foot mode for wilderness) cost = compute_cost_grid( elevation, cell_size_m=meta["cell_size_m"], @@ -457,25 +875,24 @@ class OffrouteRouter: friction_raw=friction_raw, trails=trails, barriers=barriers, - wilderness=None, # Foot mode ignores wilderness restrictions - mvum=None, # Foot mode doesn't use MVUM + wilderness=None, + mvum=None, boundary_mode=boundary_mode, - mode="foot", # ALWAYS foot for wilderness pathfinding + mode="foot", ) - # Free intermediate arrays to reduce memory before MCP + # Free intermediate arrays del friction_mult, friction_raw - import gc gc.collect() - # Convert start to pixel coordinates - start_row, start_col = self.dem_reader.latlon_to_pixel(start_lat, start_lon, meta) + # Convert origin to pixel coordinates + origin_row, origin_col = self.dem_reader.latlon_to_pixel(origin_lat, origin_lon, meta) rows, cols = elevation.shape - if not (0 <= start_row < rows and 0 <= start_col < cols): - return {"status": "error", "message": "Start point outside grid bounds"} + if not (0 <= origin_row < rows and 0 <= origin_col < cols): + return {"status": "error", "message": f"{label.capitalize()} point outside grid bounds"} - # Mark entry points on grid + # Map entry points to pixels entry_pixels = [] for ep in entry_points: row, col = self.dem_reader.latlon_to_pixel(ep["lat"], ep["lon"], meta) @@ -483,11 +900,11 @@ class OffrouteRouter: entry_pixels.append({"row": row, "col": col, "entry_point": ep}) if not entry_pixels: - return {"status": "error", "message": "No entry points map to grid bounds"} + return {"status": "error", "message": f"No entry points map to grid bounds for {label}"} - # Run MCP pathfinder + # Run MCP mcp = MCP_Geometric(cost, fully_connected=True) - cumulative_costs, traceback = mcp.find_costs([(start_row, start_col)]) + cumulative_costs, traceback = mcp.find_costs([(origin_row, origin_col)]) # Find nearest reachable entry point best_entry = None @@ -502,67 +919,88 @@ class OffrouteRouter: if best_entry is None or np.isinf(best_cost): return { "status": "error", - "message": "No path found to any entry point (blocked by impassable terrain)" + "message": f"No path found from {label} to any entry point (blocked by impassable terrain)" } - # Traceback wilderness path + # Traceback path path_indices = mcp.traceback((best_entry["row"], best_entry["col"])) - # Convert to coordinates - wilderness_coords = [] + # Convert to coordinates and collect stats + coords = [] elevations = [] trail_values = [] barrier_crossings = 0 for row, col in path_indices: lat, lon = self.dem_reader.pixel_to_latlon(row, col, meta) - wilderness_coords.append([lon, lat]) + coords.append([lon, lat]) elevations.append(elevation[row, col]) trail_values.append(trails[row, col]) if barriers[row, col] == 255: barrier_crossings += 1 - # Calculate stats - wilderness_distance_m = 0 - for i in range(1, len(wilderness_coords)): - lon1, lat1 = wilderness_coords[i-1] - lon2, lat2 = wilderness_coords[i] - wilderness_distance_m += haversine_distance(lat1, lon1, lat2, lon2) + # Calculate distance + distance_m = 0 + for i in range(1, len(coords)): + lon1, lat1 = coords[i-1] + lon2, lat2 = coords[i] + distance_m += haversine_distance(lat1, lon1, lat2, lon2) + # Elevation stats elev_arr = np.array(elevations) elev_diff = np.diff(elev_arr) - wilderness_gain = float(np.sum(elev_diff[elev_diff > 0])) - wilderness_loss = float(np.sum(np.abs(elev_diff[elev_diff < 0]))) + elev_gain = float(np.sum(elev_diff[elev_diff > 0])) + elev_loss = float(np.sum(np.abs(elev_diff[elev_diff < 0]))) + # Trail stats trail_arr = np.array(trail_values) on_trail_cells = np.sum(trail_arr > 0) total_cells = len(trail_arr) on_trail_pct = float(100 * on_trail_cells / total_cells) if total_cells > 0 else 0 - # Free trails and barriers - del trails, barriers + # Free memory + del mcp, cumulative_costs, traceback, cost, trails, barriers, elevation + gc.collect() - # Entry point - entry_lat = best_entry["entry_point"]["lat"] - entry_lon = best_entry["entry_point"]["lon"] - entry_class = best_entry["entry_point"]["highway_class"] - entry_name = best_entry["entry_point"].get("name", "") + return { + "status": "ok", + "coords": coords, + "stats": { + "distance_km": distance_m / 1000, + "effort_minutes": best_cost / 60, + "elevation_gain_m": elev_gain, + "elevation_loss_m": elev_loss, + "on_trail_pct": on_trail_pct, + "barrier_crossings": barrier_crossings, + "cell_count": total_cells, + }, + "entry_point": best_entry["entry_point"] + } - # Call Valhalla with USER'S SELECTED MODE (not foot) - valhalla_costing = MODE_TO_COSTING.get(mode, "pedestrian") + def _valhalla_route( + self, + start_lat: float, start_lon: float, + end_lat: float, end_lon: float, + mode: str + ) -> Dict: + """ + Call Valhalla for network routing. + + Returns: + {"segment": {...}, "error": None} on success + {"segment": None, "error": "..."} on failure + """ + costing = MODE_TO_COSTING.get(mode, "pedestrian") valhalla_request = { "locations": [ - {"lat": entry_lat, "lon": entry_lon}, + {"lat": start_lat, "lon": start_lon}, {"lat": end_lat, "lon": end_lon} ], - "costing": valhalla_costing, + "costing": costing, "directions_options": {"units": "kilometers"} } - network_segment = None - valhalla_error = None - try: resp = requests.post(f"{VALHALLA_URL}/route", json=valhalla_request, timeout=30) @@ -574,7 +1012,7 @@ class OffrouteRouter: if legs: leg = legs[0] shape = leg.get("shape", "") - network_coords = self._decode_polyline(shape) + coords = self._decode_polyline(shape) maneuvers = [] for m in leg.get("maneuvers", []): @@ -587,97 +1025,184 @@ class OffrouteRouter: }) summary = trip.get("summary", {}) - network_segment = { - "coordinates": network_coords, - "distance_km": summary.get("length", 0), - "duration_minutes": summary.get("time", 0) / 60, - "maneuvers": maneuvers, + return { + "segment": { + "coordinates": coords, + "distance_km": summary.get("length", 0), + "duration_minutes": summary.get("time", 0) / 60, + "maneuvers": maneuvers, + }, + "error": None } - else: - valhalla_error = f"Valhalla returned {resp.status_code}: {resp.text[:200]}" + + return {"segment": None, "error": f"Valhalla returned {resp.status_code}: {resp.text[:200]}"} except Exception as e: - valhalla_error = f"Valhalla request failed: {e}" + return {"segment": None, "error": f"Valhalla request failed: {e}"} - # Build response + def _build_response( + self, + wilderness_start: Optional[List], + wilderness_start_stats: Optional[Dict], + network_segment: Optional[Dict], + wilderness_end: Optional[List], + wilderness_end_stats: Optional[Dict], + mode: str, + boundary_mode: str, + entry_start: Optional[Dict], + entry_end: Optional[Dict], + scenario: str, + t0: float, + valhalla_error: Optional[str] + ) -> Dict: + """Build the final GeoJSON response.""" features = [] - wilderness_feature = { - "type": "Feature", - "properties": { - "segment_type": "wilderness", - "effort_minutes": float(best_cost / 60), - "distance_km": float(wilderness_distance_m / 1000), - "elevation_gain_m": wilderness_gain, - "elevation_loss_m": wilderness_loss, - "boundary_mode": boundary_mode, - "on_trail_pct": on_trail_pct, - "cell_count": total_cells, - "barrier_crossings": barrier_crossings, - "wilderness_mode": "foot", # Always foot for wilderness - }, - "geometry": {"type": "LineString", "coordinates": wilderness_coords} - } - features.append(wilderness_feature) + # Wilderness start segment + if wilderness_start and wilderness_start_stats: + features.append({ + "type": "Feature", + "properties": { + "segment_type": "wilderness", + "segment_position": "start", + "effort_minutes": float(wilderness_start_stats["effort_minutes"]), + "distance_km": float(wilderness_start_stats["distance_km"]), + "elevation_gain_m": wilderness_start_stats["elevation_gain_m"], + "elevation_loss_m": wilderness_start_stats["elevation_loss_m"], + "boundary_mode": boundary_mode, + "on_trail_pct": wilderness_start_stats["on_trail_pct"], + "barrier_crossings": wilderness_start_stats["barrier_crossings"], + "wilderness_mode": "foot", + }, + "geometry": {"type": "LineString", "coordinates": wilderness_start} + }) + # Network segment if network_segment: - network_feature = { + features.append({ "type": "Feature", "properties": { "segment_type": "network", "distance_km": network_segment["distance_km"], "duration_minutes": network_segment["duration_minutes"], "maneuvers": network_segment["maneuvers"], - "network_mode": mode, # User's selected mode + "network_mode": mode, }, "geometry": {"type": "LineString", "coordinates": network_segment["coordinates"]} - } - features.append(network_feature) + }) - combined_coords = wilderness_coords.copy() + # Wilderness end segment + if wilderness_end and wilderness_end_stats: + features.append({ + "type": "Feature", + "properties": { + "segment_type": "wilderness", + "segment_position": "end", + "effort_minutes": float(wilderness_end_stats["effort_minutes"]), + "distance_km": float(wilderness_end_stats["distance_km"]), + "elevation_gain_m": wilderness_end_stats["elevation_gain_m"], + "elevation_loss_m": wilderness_end_stats["elevation_loss_m"], + "boundary_mode": boundary_mode, + "on_trail_pct": wilderness_end_stats["on_trail_pct"], + "barrier_crossings": wilderness_end_stats["barrier_crossings"], + "wilderness_mode": "foot", + }, + "geometry": {"type": "LineString", "coordinates": wilderness_end} + }) + + # Combined path + combined_coords = [] + if wilderness_start: + combined_coords.extend(wilderness_start) if network_segment: - combined_coords.extend(network_segment["coordinates"][1:]) + # Skip first coord if we already have wilderness_start (avoid duplicate) + start_idx = 1 if wilderness_start else 0 + combined_coords.extend(network_segment["coordinates"][start_idx:]) + if wilderness_end: + # Skip first coord (avoid duplicate with network end) + start_idx = 1 if (wilderness_start or network_segment) else 0 + combined_coords.extend(wilderness_end[start_idx:]) - combined_feature = { - "type": "Feature", - "properties": { - "segment_type": "combined", - "wilderness_mode": "foot", - "network_mode": mode, - "boundary_mode": boundary_mode - }, - "geometry": {"type": "LineString", "coordinates": combined_coords} - } - features.append(combined_feature) + if combined_coords: + features.append({ + "type": "Feature", + "properties": { + "segment_type": "combined", + "wilderness_mode": "foot", + "network_mode": mode, + "boundary_mode": boundary_mode, + "scenario": scenario, + }, + "geometry": {"type": "LineString", "coordinates": combined_coords} + }) geojson = {"type": "FeatureCollection", "features": features} - total_distance_km = wilderness_distance_m / 1000 - total_effort_minutes = best_cost / 60 + # Calculate totals + total_distance_km = 0.0 + total_effort_minutes = 0.0 + wilderness_distance_km = 0.0 + wilderness_effort_minutes = 0.0 + network_distance_km = 0.0 + network_duration_minutes = 0.0 + barrier_crossings = 0 + on_trail_pct = 0.0 + + if wilderness_start_stats: + wilderness_distance_km += wilderness_start_stats["distance_km"] + wilderness_effort_minutes += wilderness_start_stats["effort_minutes"] + barrier_crossings += wilderness_start_stats["barrier_crossings"] + on_trail_pct = wilderness_start_stats["on_trail_pct"] + + if wilderness_end_stats: + wilderness_distance_km += wilderness_end_stats["distance_km"] + wilderness_effort_minutes += wilderness_end_stats["effort_minutes"] + barrier_crossings += wilderness_end_stats["barrier_crossings"] + # Average on-trail percentage if we have both + if wilderness_start_stats: + on_trail_pct = (on_trail_pct + wilderness_end_stats["on_trail_pct"]) / 2 + else: + on_trail_pct = wilderness_end_stats["on_trail_pct"] if network_segment: - total_distance_km += network_segment["distance_km"] - total_effort_minutes += network_segment["duration_minutes"] + network_distance_km = network_segment["distance_km"] + network_duration_minutes = network_segment["duration_minutes"] + + total_distance_km = wilderness_distance_km + network_distance_km + total_effort_minutes = wilderness_effort_minutes + network_duration_minutes summary = { "total_distance_km": float(total_distance_km), "total_effort_minutes": float(total_effort_minutes), - "wilderness_distance_km": float(wilderness_distance_m / 1000), - "wilderness_effort_minutes": float(best_cost / 60), - "network_distance_km": float(network_segment["distance_km"]) if network_segment else 0, - "network_duration_minutes": float(network_segment["duration_minutes"]) if network_segment else 0, - "on_trail_pct": on_trail_pct, + "wilderness_distance_km": float(wilderness_distance_km), + "wilderness_effort_minutes": float(wilderness_effort_minutes), + "network_distance_km": float(network_distance_km), + "network_duration_minutes": float(network_duration_minutes), + "on_trail_pct": float(on_trail_pct), "barrier_crossings": barrier_crossings, "boundary_mode": boundary_mode, - "wilderness_mode": "foot", # Always foot - "network_mode": mode, # User's selection - "entry_point": { - "lat": entry_lat, "lon": entry_lon, - "highway_class": entry_class, "name": entry_name, - }, + "wilderness_mode": "foot", + "network_mode": mode, + "scenario": scenario, "computation_time_s": time.time() - t0, } + if entry_start: + summary["entry_point_start"] = { + "lat": entry_start["lat"], + "lon": entry_start["lon"], + "highway_class": entry_start["highway_class"], + "name": entry_start.get("name", ""), + } + + if entry_end: + summary["entry_point_end"] = { + "lat": entry_end["lat"], + "lon": entry_end["lon"], + "highway_class": entry_end["highway_class"], + "name": entry_end.get("name", ""), + } + result = {"status": "ok", "route": geojson, "summary": summary} if valhalla_error: @@ -753,28 +1278,46 @@ if __name__ == "__main__": print(f"\nDone. Total entry points: {stats['total']}") elif len(sys.argv) > 1 and sys.argv[1] == "test": - print("Testing router (all modes)...") - print("NOTE: Wilderness always uses foot mode. User mode affects entry points + network.") + print("Testing router (all scenarios)...") + print("=" * 60) router = OffrouteRouter() - for mode in ["foot", "mtb", "atv", "vehicle"]: - print(f"\n{'='*60}") - print(f"Mode: {mode}") - print("="*60) + # Test points + wilderness_start = (44.0543, -115.4237) # Off-network + wilderness_end = (45.2, -115.5) # Deep wilderness (Frank Church) + road_start = (43.6150, -116.2023) # Boise downtown (on-network) + road_end = (43.5867, -116.5625) # Nampa (on-network) + + tests = [ + ("A: wilderness→road", wilderness_start, (44.0814, -115.5021)), + ("B: wilderness→wilderness", wilderness_start, wilderness_end), + ("C: road→wilderness", road_start, wilderness_start), + ("D: road→road", road_start, road_end), + ] + + for label, (slat, slon), (elat, elon) in tests: + print(f"\n{label}") + print("-" * 40) result = router.route( - start_lat=42.35, start_lon=-114.30, - end_lat=42.5629, end_lon=-114.4609, - mode=mode, boundary_mode="pragmatic" + start_lat=slat, start_lon=slon, + end_lat=elat, end_lon=elon, + mode="foot", boundary_mode="pragmatic" ) if result["status"] == "ok": s = result["summary"] - print(f" Wilderness: {s['wilderness_distance_km']:.2f} km, {s['wilderness_effort_minutes']:.1f} min (foot)") - print(f" Network: {s['network_distance_km']:.2f} km, {s['network_duration_minutes']:.1f} min ({mode})") - print(f" On-trail: {s['on_trail_pct']:.1f}%") - print(f" Entry: {s['entry_point']['highway_class']}") + print(f" Scenario: {s.get('scenario', '?')}") + print(f" Total: {s['total_distance_km']:.2f} km, {s['total_effort_minutes']:.1f} min") + print(f" Wilderness: {s['wilderness_distance_km']:.2f} km") + print(f" Network: {s['network_distance_km']:.2f} km") + if s.get('entry_point_start'): + ep = s['entry_point_start'] + print(f" Entry (start): {ep['highway_class']} at {ep['lat']:.4f}, {ep['lon']:.4f}") + if s.get('entry_point_end'): + ep = s['entry_point_end'] + print(f" Entry (end): {ep['highway_class']} at {ep['lat']:.4f}, {ep['lon']:.4f}") else: print(f" ERROR: {result['message']}") @@ -783,4 +1326,4 @@ if __name__ == "__main__": else: print("Usage:") print(" python router.py build # Build entry point index") - print(" python router.py test # Test all modes") + print(" python router.py test # Test all scenarios") From 87a4741b8d4ebecfc276256d7f73d4481bd8a216 Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 8 May 2026 21:19:04 +0000 Subject: [PATCH 12/41] =?UTF-8?q?offroute:=20raise=20bbox=20limit=20to=202?= =?UTF-8?q?.0=C2=B0=20(~220km=20coverage)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.5 --- lib/offroute/router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/offroute/router.py b/lib/offroute/router.py index 79c9089..ccc956d 100644 --- a/lib/offroute/router.py +++ b/lib/offroute/router.py @@ -804,7 +804,7 @@ class OffrouteRouter: """ # Build bbox - only include origin and entry points, NOT distant destination # The destination is handled by Valhalla, wilderness only needs to reach entry points - MAX_BBOX_DEGREES = 0.5 + MAX_BBOX_DEGREES = 2.0 all_lats = [origin_lat] + [p["lat"] for p in entry_points] all_lons = [origin_lon] + [p["lon"] for p in entry_points] From cf758476b4d44be559664a2e92211566dd7d92aa Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 8 May 2026 21:55:31 +0000 Subject: [PATCH 13/41] offroute: add auto mode for standard driving routes Co-Authored-By: Claude Opus 4.5 --- lib/offroute/router.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/offroute/router.py b/lib/offroute/router.py index ccc956d..6981e89 100644 --- a/lib/offroute/router.py +++ b/lib/offroute/router.py @@ -53,7 +53,8 @@ MEMORY_LIMIT_GB = 12 OFF_NETWORK_THRESHOLD_M = 500 # Mode to Valhalla costing mapping -MODE_TO_COSTING = { +MODE_TO_COSTING = { + "auto": "auto", "foot": "pedestrian", "mtb": "bicycle", "atv": "auto", @@ -62,7 +63,9 @@ MODE_TO_COSTING = { # Mode to valid entry point highway classes # foot = any trail/track/road, mtb = tracks and roads, vehicle = roads only -MODE_TO_VALID_HIGHWAYS = { +MODE_TO_VALID_HIGHWAYS = { + "auto": {"primary", "secondary", "tertiary", "unclassified", "residential", + "service"}, "foot": {"primary", "secondary", "tertiary", "unclassified", "residential", "service", "track", "path", "footway", "bridleway"}, "mtb": {"primary", "secondary", "tertiary", "unclassified", "residential", From 686b35710a55b369d8eafe6f554f58e795a36289 Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 8 May 2026 22:37:49 +0000 Subject: [PATCH 14/41] api: add auto mode to offroute endpoint validation Co-Authored-By: Claude Opus 4.5 --- lib/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/api.py b/lib/api.py index 699d09d..949a0cc 100644 --- a/lib/api.py +++ b/lib/api.py @@ -2768,8 +2768,8 @@ def api_offroute(): # Parse options mode = data.get("mode", "foot") - if mode not in ("foot", "mtb", "atv", "vehicle"): - return jsonify({"status": "error", "message": "mode must be foot, mtb, atv, or vehicle"}), 400 + if mode not in ("auto", "foot", "mtb", "atv", "vehicle"): + return jsonify({"status": "error", "message": "mode must be auto, foot, mtb, atv, or vehicle"}), 400 boundary_mode = data.get("boundary_mode", "pragmatic") if boundary_mode not in ("strict", "pragmatic", "emergency"): From 05c24f95f635bc856894f8f869448716042124ab Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 8 May 2026 23:27:06 +0000 Subject: [PATCH 15/41] offroute: tighten off-network threshold to 10m Co-Authored-By: Claude Opus 4.5 --- lib/offroute/router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/offroute/router.py b/lib/offroute/router.py index 6981e89..0cc3ccd 100644 --- a/lib/offroute/router.py +++ b/lib/offroute/router.py @@ -50,7 +50,7 @@ EXPANDED_SEARCH_RADIUS_KM = 100 MEMORY_LIMIT_GB = 12 # Off-network detection threshold (meters) -OFF_NETWORK_THRESHOLD_M = 500 +OFF_NETWORK_THRESHOLD_M = 10 # Mode to Valhalla costing mapping MODE_TO_COSTING = { From b4e33eb0484c99bcf53d76e71a92a1d3dd43ab40 Mon Sep 17 00:00:00 2001 From: Matt Date: Sat, 9 May 2026 03:28:58 +0000 Subject: [PATCH 16/41] offroute: PostGIS entry points with 100m densification and land_status tagging - Migrate EntryPointIndex from SQLite to PostGIS (padus database) - Densify highway LineStrings at 100m intervals via Shapely interpolate - 2.94M entry points from 476k lines (4x more coverage) - Tag each entry point with land_status via ST_Intersects against padus_sub - 1.64M public (56%), 1.30M unknown (44%) - Add geography GIST index for fast radius queries (~25ms) - Increase OFF_NETWORK_THRESHOLD_M from 10m to 50m for GPS accuracy - PBF path and PostGIS DSN configurable via home.yaml Co-Authored-By: Claude Opus 4.5 --- config/profiles/home.yaml | 17 +- lib/offroute/router.py | 2799 +++++++++++++++++++------------------ 2 files changed, 1483 insertions(+), 1333 deletions(-) diff --git a/config/profiles/home.yaml b/config/profiles/home.yaml index 5269812..474ffb2 100644 --- a/config/profiles/home.yaml +++ b/config/profiles/home.yaml @@ -6,13 +6,13 @@ profile: home region_name: "North America" tileset: - url: "/tiles/na.pmtiles" + url: "/tiles/planet/current.pmtiles" bounds: [-168, 14, -52, 72] max_zoom: 15 attribution: "Protomaps © OSM" tileset_hillshade: - url: "/tiles/hillshade-na.pmtiles" + url: "/tiles/planet-dem.pmtiles" encoding: "terrarium" max_zoom: 12 @@ -33,14 +33,14 @@ services: features: has_nominatim_details: true - has_kiwix_wiki: false + has_kiwix_wiki: true has_hillshade: true has_3d_terrain: false has_traffic_overlay: true has_landclass: true has_public_lands_layer: true has_contours: true - has_contours_test: true + has_contours_test: false has_contours_test_10ft: false has_address_book_write: false has_overture_enrichment: true @@ -48,7 +48,16 @@ features: has_contacts: true has_wiki_rewriting: true has_wiki_discovery: false + has_usfs_trails: true + has_blm_trails: true defaults: center: [42.5736, -114.6066] zoom: 10 + +# Offroute wilderness routing +offroute: + osm_pbf_path: "/mnt/nav/sources/idaho-latest.osm.pbf" + densify_interval_m: 100 + postgis_dsn: "dbname=padus" + diff --git a/lib/offroute/router.py b/lib/offroute/router.py index 0cc3ccd..4b988ab 100644 --- a/lib/offroute/router.py +++ b/lib/offroute/router.py @@ -1,1332 +1,1473 @@ -""" -OFFROUTE Router — Bidirectional wilderness-to-network path orchestration. - -Supports four routing scenarios: - A: off-network start → on-network end (wilderness then Valhalla) - B: off-network start → off-network end (wilderness, Valhalla, wilderness) - C: on-network start → off-network end (Valhalla then wilderness) - D: on-network start → on-network end (pure Valhalla passthrough) - -Off-network detection: Valhalla /locate snap distance > 500m = off-network. - -IMPORTANT: The wilderness segment ALWAYS uses foot mode for pathfinding. -The user's selected mode affects: - 1. Which entry points are valid (foot=any, mtb=tracks+roads, vehicle=roads only) - 2. The Valhalla costing profile for the network segment -""" -import gc -import json -import math -import sqlite3 -import subprocess -import tempfile -import time -from pathlib import Path -from typing import Dict, List, Optional, Tuple, Literal - -import numpy as np -import requests -from skimage.graph import MCP_Geometric - -from .dem import DEMReader -from .cost import compute_cost_grid -from .friction import FrictionReader, friction_to_multiplier -from .barriers import BarrierReader, WildernessReader, DEFAULT_WILDERNESS_PATH -from .trails import TrailReader -from .mvum import get_mvum_access_grid - -# Paths -NAVI_DB_PATH = Path("/mnt/nav/navi.db") -OSM_PBF_PATH = Path("/mnt/nav/sources/idaho-latest.osm.pbf") - -# Valhalla endpoint -VALHALLA_URL = "http://localhost:8002" - -# Search radius for entry points (km) -DEFAULT_SEARCH_RADIUS_KM = 50 -EXPANDED_SEARCH_RADIUS_KM = 100 - -# Memory limit -MEMORY_LIMIT_GB = 12 - -# Off-network detection threshold (meters) -OFF_NETWORK_THRESHOLD_M = 10 - -# Mode to Valhalla costing mapping +""" +OFFROUTE Router — Bidirectional wilderness-to-network path orchestration. + +Supports four routing scenarios: + A: off-network start → on-network end (wilderness then Valhalla) + B: off-network start → off-network end (wilderness, Valhalla, wilderness) + C: on-network start → off-network end (Valhalla then wilderness) + D: on-network start → on-network end (pure Valhalla passthrough) + +Off-network detection: Valhalla /locate snap distance > 500m = off-network. + +IMPORTANT: The wilderness segment ALWAYS uses foot mode for pathfinding. +The user's selected mode affects: + 1. Which entry points are valid (foot=any, mtb=tracks+roads, vehicle=roads only) + 2. The Valhalla costing profile for the network segment +""" +import gc +import json +import math +import subprocess +import tempfile +import time +from pathlib import Path +from typing import Dict, List, Optional, Tuple, Literal, Set + +import numpy as np +import requests +import psycopg2 +import psycopg2.extras +from shapely.geometry import LineString +from skimage.graph import MCP_Geometric + +from .dem import DEMReader +from .cost import compute_cost_grid +from .friction import FrictionReader, friction_to_multiplier +from .barriers import BarrierReader, WildernessReader, DEFAULT_WILDERNESS_PATH +from .trails import TrailReader +from .mvum import get_mvum_access_grid +from ..deployment_config import get_deployment_config + +# Load configuration +_deploy_config = get_deployment_config() +_offroute_config = _deploy_config.get("offroute", {}) + +# Paths (configurable via home.yaml) +OSM_PBF_PATH = Path(_offroute_config.get("osm_pbf_path", "/mnt/nav/sources/idaho-latest.osm.pbf")) +DENSIFY_INTERVAL_M = _offroute_config.get("densify_interval_m", 100) +POSTGIS_DSN = _offroute_config.get("postgis_dsn", "dbname=padus user=postgres") + +# Legacy SQLite path (still used by MVUM) +NAVI_DB_PATH = Path("/mnt/nav/navi.db") + +# Valhalla endpoint +VALHALLA_URL = "http://localhost:8002" + +# Search radius for entry points (km) +DEFAULT_SEARCH_RADIUS_KM = 50 +EXPANDED_SEARCH_RADIUS_KM = 100 + +# Memory limit +MEMORY_LIMIT_GB = 12 + +# Off-network detection threshold (meters) +OFF_NETWORK_THRESHOLD_M = 50 + +# Mode to Valhalla costing mapping MODE_TO_COSTING = { - "auto": "auto", - "foot": "pedestrian", - "mtb": "bicycle", - "atv": "auto", - "vehicle": "auto", -} - -# Mode to valid entry point highway classes -# foot = any trail/track/road, mtb = tracks and roads, vehicle = roads only + "auto": "auto", + "foot": "pedestrian", + "mtb": "bicycle", + "atv": "auto", + "vehicle": "auto", +} + +# Mode to valid entry point highway classes +# foot = any trail/track/road, mtb = tracks and roads, vehicle = roads only MODE_TO_VALID_HIGHWAYS = { "auto": {"primary", "secondary", "tertiary", "unclassified", "residential", - "service"}, - "foot": {"primary", "secondary", "tertiary", "unclassified", "residential", - "service", "track", "path", "footway", "bridleway"}, - "mtb": {"primary", "secondary", "tertiary", "unclassified", "residential", - "service", "track"}, - "atv": {"primary", "secondary", "tertiary", "unclassified", "residential", - "service", "track"}, - "vehicle": {"primary", "secondary", "tertiary", "unclassified", "residential", - "service"}, -} - - -def haversine_distance(lat1: float, lon1: float, lat2: float, lon2: float) -> float: - """Calculate distance between two points in meters.""" - R = 6371000 - dlat = math.radians(lat2 - lat1) - dlon = math.radians(lon2 - lon1) - a = math.sin(dlat/2)**2 + math.cos(math.radians(lat1)) * math.cos(math.radians(lat2)) * math.sin(dlon/2)**2 - c = 2 * math.atan2(math.sqrt(a), math.sqrt(1-a)) - return R * c - - -def check_memory_usage() -> float: - """Check current memory usage in GB.""" - try: - import psutil - process = psutil.Process() - return process.memory_info().rss / (1024**3) - except ImportError: - return 0 - - -class EntryPointIndex: - """ - Trail entry point index for wilderness-to-network handoff. - - Entry points are endpoints and intersections of OSM highways - that connect wilderness areas to the routable network. - """ - - def __init__(self, db_path: Path = NAVI_DB_PATH): - self.db_path = db_path - self._conn = None - - def _get_conn(self) -> sqlite3.Connection: - if self._conn is None: - self._conn = sqlite3.connect(str(self.db_path)) - self._conn.row_factory = sqlite3.Row - return self._conn - - def table_exists(self) -> bool: - """Check if trail_entry_points table exists.""" - if not self.db_path.exists(): - return False - conn = self._get_conn() - cur = conn.execute( - "SELECT name FROM sqlite_master WHERE type='table' AND name='trail_entry_points'" - ) - return cur.fetchone() is not None - - def get_entry_point_count(self) -> int: - """Get count of entry points.""" - if not self.table_exists(): - return 0 - conn = self._get_conn() - cur = conn.execute("SELECT COUNT(*) FROM trail_entry_points") - return cur.fetchone()[0] - - def query_bbox(self, south: float, north: float, west: float, east: float) -> List[Dict]: - """Query entry points within a bounding box.""" - if not self.table_exists(): - return [] - - conn = self._get_conn() - cur = conn.execute(""" - SELECT id, lat, lon, highway_class, name - FROM trail_entry_points - WHERE lat >= ? AND lat <= ? AND lon >= ? AND lon <= ? - """, (south, north, west, east)) - - return [dict(row) for row in cur.fetchall()] - - def query_radius(self, lat: float, lon: float, radius_km: float, - valid_highways: Optional[set] = None) -> List[Dict]: - """ - Query entry points within radius of a point. - - Args: - lat, lon: Center point - radius_km: Search radius in kilometers - valid_highways: Optional set of valid highway classes to filter by - """ - lat_delta = radius_km / 111.0 - lon_delta = radius_km / (111.0 * math.cos(math.radians(lat))) - - points = self.query_bbox( - lat - lat_delta, lat + lat_delta, - lon - lon_delta, lon + lon_delta - ) - - result = [] - for p in points: - # Filter by highway class if specified - if valid_highways and p['highway_class'] not in valid_highways: - continue - - dist = haversine_distance(lat, lon, p['lat'], p['lon']) - if dist <= radius_km * 1000: - p['distance_m'] = dist - result.append(p) - - return sorted(result, key=lambda x: x['distance_m']) - - def build_index(self, osm_pbf_path: Path = OSM_PBF_PATH) -> Dict: - """Build the entry point index from OSM PBF.""" - if not osm_pbf_path.exists(): - raise FileNotFoundError(f"OSM PBF not found: {osm_pbf_path}") - - print(f"Building trail entry point index from {osm_pbf_path}...") - - highway_types = [ - "primary", "secondary", "tertiary", "unclassified", - "residential", "service", "track", "path", "footway", "bridleway" - ] - - stats = {"total": 0, "by_class": {}} - - with tempfile.TemporaryDirectory() as tmpdir: - geojson_path = Path(tmpdir) / "highways.geojson" - - print(f" Extracting highways with osmium...") - cmd = ["osmium", "tags-filter", str(osm_pbf_path)] - for ht in highway_types: - cmd.append(f"w/highway={ht}") - cmd.extend(["-o", str(Path(tmpdir) / "filtered.osm.pbf"), "--overwrite"]) - subprocess.run(cmd, check=True, capture_output=True) - - print(f" Converting to GeoJSON with ogr2ogr...") - cmd = [ - "ogr2ogr", "-f", "GeoJSON", - str(geojson_path), - str(Path(tmpdir) / "filtered.osm.pbf"), - "lines", "-t_srs", "EPSG:4326" - ] - subprocess.run(cmd, check=True, capture_output=True) - - print(f" Extracting entry points...") - with open(geojson_path) as f: - data = json.load(f) - - points = {} - for feature in data.get("features", []): - props = feature.get("properties", {}) - geom = feature.get("geometry", {}) - - if geom.get("type") != "LineString": - continue - - coords = geom.get("coordinates", []) - if len(coords) < 2: - continue - - highway_class = props.get("highway", "unknown") - name = props.get("name", "") - - for coord in [coords[0], coords[-1]]: - lon, lat = coord[0], coord[1] - key = (round(lat, 5), round(lon, 5)) - - if key not in points: - points[key] = { - "lat": lat, "lon": lon, - "highway_class": highway_class, "name": name - } - else: - existing = points[key] - if self._highway_priority(highway_class) < self._highway_priority(existing["highway_class"]): - points[key]["highway_class"] = highway_class - if name and not existing["name"]: - points[key]["name"] = name - - print(f" Writing {len(points)} entry points to {self.db_path}...") - - self.db_path.parent.mkdir(parents=True, exist_ok=True) - conn = self._get_conn() - - conn.execute(""" - CREATE TABLE IF NOT EXISTS trail_entry_points ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - lat REAL NOT NULL, lon REAL NOT NULL, - highway_class TEXT NOT NULL, name TEXT - ) - """) - conn.execute("DELETE FROM trail_entry_points") - - for point in points.values(): - conn.execute( - "INSERT INTO trail_entry_points (lat, lon, highway_class, name) VALUES (?, ?, ?, ?)", - (point["lat"], point["lon"], point["highway_class"], point["name"]) - ) - stats["total"] += 1 - hc = point["highway_class"] - stats["by_class"][hc] = stats["by_class"].get(hc, 0) + 1 - - conn.execute("CREATE INDEX IF NOT EXISTS idx_entry_lat ON trail_entry_points(lat)") - conn.execute("CREATE INDEX IF NOT EXISTS idx_entry_lon ON trail_entry_points(lon)") - conn.execute("CREATE INDEX IF NOT EXISTS idx_entry_latlon ON trail_entry_points(lat, lon)") - conn.commit() - - print(f" Done. Total: {stats['total']} entry points") - for hc, count in sorted(stats["by_class"].items(), key=lambda x: -x[1]): - print(f" {hc}: {count}") - - return stats - - def _highway_priority(self, highway_class: str) -> int: - """Lower number = better priority for entry points.""" - priority = { - "primary": 1, "secondary": 2, "tertiary": 3, - "unclassified": 4, "residential": 5, "service": 6, - "track": 7, "path": 8, "footway": 9, "bridleway": 10 - } - return priority.get(highway_class, 99) - - def close(self): - if self._conn: - self._conn.close() - self._conn = None - - -class OffrouteRouter: - """ - OFFROUTE Router — orchestrates wilderness pathfinding and Valhalla stitching. - - Supports four scenarios: - A: off-network start → on-network end - B: off-network start → off-network end - C: on-network start → off-network end - D: on-network start → on-network end (pure Valhalla) - - IMPORTANT: Wilderness segment ALWAYS uses foot mode for pathfinding. - User's mode affects entry point selection and Valhalla costing only. - """ - - def __init__(self): - self.dem_reader = None - self.friction_reader = None - self.barrier_reader = None - self.wilderness_reader = None - self.trail_reader = None - self.entry_index = EntryPointIndex() - - def _init_readers(self): - """Lazy init readers.""" - if self.dem_reader is None: - self.dem_reader = DEMReader() - if self.friction_reader is None: - self.friction_reader = FrictionReader() - if self.barrier_reader is None: - self.barrier_reader = BarrierReader() - if self.wilderness_reader is None and DEFAULT_WILDERNESS_PATH.exists(): - self.wilderness_reader = WildernessReader() - if self.trail_reader is None: - self.trail_reader = TrailReader() - - def _locate_on_network(self, lat: float, lon: float, mode: str) -> Dict: - """ - Check if a point is on the routable network using Valhalla's /locate. - - Returns: - { - "on_network": bool, - "snap_distance_m": float, - "snapped_lat": float, - "snapped_lon": float - } - """ - costing = MODE_TO_COSTING.get(mode, "pedestrian") - try: - resp = requests.post( - f"{VALHALLA_URL}/locate", - json={"locations": [{"lat": lat, "lon": lon}], "costing": costing}, - timeout=10 - ) - - if resp.status_code == 200: - data = resp.json() - if data and len(data) > 0 and data[0].get("edges"): - edge = data[0]["edges"][0] - snap_lat = edge.get("correlated_lat", lat) - snap_lon = edge.get("correlated_lon", lon) - snap_dist = haversine_distance(lat, lon, snap_lat, snap_lon) - return { - "on_network": snap_dist <= OFF_NETWORK_THRESHOLD_M, - "snap_distance_m": snap_dist, - "snapped_lat": snap_lat, - "snapped_lon": snap_lon - } - except Exception: - pass - - return { - "on_network": False, - "snap_distance_m": float('inf'), - "snapped_lat": lat, - "snapped_lon": lon - } - - def route( - self, - start_lat: float, - start_lon: float, - end_lat: float, - end_lon: float, - mode: Literal["foot", "mtb", "atv", "vehicle"] = "foot", - boundary_mode: Literal["strict", "pragmatic", "emergency"] = "pragmatic" - ) -> Dict: - """ - Route between two points, handling all four scenarios. - - Scenarios: - A: off-network start → on-network end (wilderness then network) - B: off-network start → off-network end (wilderness, network, wilderness) - C: on-network start → off-network end (network then wilderness) - D: on-network start → on-network end (pure network) - - Args: - start_lat, start_lon: Starting coordinates - end_lat, end_lon: Destination coordinates - mode: Travel mode (foot, mtb, atv, vehicle) - boundary_mode: How to handle private land (strict, pragmatic, emergency) - - Returns a GeoJSON FeatureCollection with route segments. - """ - if mode not in MODE_TO_COSTING: - return {"status": "error", "message": f"Unknown mode: {mode}"} - - # Detect network status for both endpoints - start_status = self._locate_on_network(start_lat, start_lon, mode) - end_status = self._locate_on_network(end_lat, end_lon, mode) - - start_off_network = not start_status["on_network"] - end_off_network = not end_status["on_network"] - - # Dispatch to appropriate handler - if not start_off_network and not end_off_network: - # Scenario D: on-network → on-network (pure Valhalla) - return self._route_D_network_only( - start_lat, start_lon, end_lat, end_lon, mode - ) - elif not start_off_network and end_off_network: - # Scenario C: on-network → off-network - return self._route_C_network_to_wilderness( - start_lat, start_lon, end_lat, end_lon, mode, boundary_mode - ) - elif start_off_network and not end_off_network: - # Scenario A: off-network → on-network - return self._route_A_wilderness_to_network( - start_lat, start_lon, end_lat, end_lon, mode, boundary_mode - ) - else: - # Scenario B: off-network → off-network - return self._route_B_wilderness_both( - start_lat, start_lon, end_lat, end_lon, mode, boundary_mode - ) - - def _route_D_network_only( - self, - start_lat: float, start_lon: float, - end_lat: float, end_lon: float, - mode: str - ) -> Dict: - """ - Scenario D: Both endpoints on-network. Pure Valhalla routing. - """ - t0 = time.time() - costing = MODE_TO_COSTING.get(mode, "pedestrian") - - valhalla_request = { - "locations": [ - {"lat": start_lat, "lon": start_lon}, - {"lat": end_lat, "lon": end_lon} - ], - "costing": costing, - "directions_options": {"units": "kilometers"} - } - - try: - resp = requests.post(f"{VALHALLA_URL}/route", json=valhalla_request, timeout=30) - - if resp.status_code != 200: - return { - "status": "error", - "message": f"Network routing failed: {resp.text[:200]}" - } - - valhalla_data = resp.json() - trip = valhalla_data.get("trip", {}) - legs = trip.get("legs", []) - - if not legs: - return {"status": "error", "message": "No route found"} - - leg = legs[0] - shape = leg.get("shape", "") - network_coords = self._decode_polyline(shape) - - maneuvers = [] - for m in leg.get("maneuvers", []): - maneuvers.append({ - "instruction": m.get("instruction", ""), - "type": m.get("type", 0), - "distance_km": m.get("length", 0), - "time_seconds": m.get("time", 0), - "street_names": m.get("street_names", []), - }) - - summary = trip.get("summary", {}) - distance_km = summary.get("length", 0) - duration_min = summary.get("time", 0) / 60 - - # Build response in same format as wilderness routes - network_feature = { - "type": "Feature", - "properties": { - "segment_type": "network", - "distance_km": distance_km, - "duration_minutes": duration_min, - "maneuvers": maneuvers, - "network_mode": mode, - }, - "geometry": {"type": "LineString", "coordinates": network_coords} - } - - combined_feature = { - "type": "Feature", - "properties": { - "segment_type": "combined", - "network_mode": mode, - }, - "geometry": {"type": "LineString", "coordinates": network_coords} - } - - geojson = {"type": "FeatureCollection", "features": [network_feature, combined_feature]} - - result = { - "status": "ok", - "route": geojson, - "summary": { - "total_distance_km": float(distance_km), - "total_effort_minutes": float(duration_min), - "wilderness_distance_km": 0.0, - "wilderness_effort_minutes": 0.0, - "network_distance_km": float(distance_km), - "network_duration_minutes": float(duration_min), - "on_trail_pct": 100.0, - "barrier_crossings": 0, - "network_mode": mode, - "scenario": "D", - "computation_time_s": time.time() - t0, - } - } - return result - - except Exception as e: - return {"status": "error", "message": f"Network routing failed: {e}"} - - def _route_A_wilderness_to_network( - self, - start_lat: float, start_lon: float, - end_lat: float, end_lon: float, - mode: str, boundary_mode: str - ) -> Dict: - """ - Scenario A: Off-network start → on-network end. - Wilderness pathfinding from start to entry point, then Valhalla to end. - """ - t0 = time.time() - - # Ensure entry point index exists - if not self.entry_index.table_exists() or self.entry_index.get_entry_point_count() == 0: - return { - "status": "error", - "message": "Trail entry point index not built. Run build_entry_index() first." - } - - # Get valid highway classes for this mode - valid_highways = MODE_TO_VALID_HIGHWAYS.get(mode) - - # Find entry points near start, filtered by mode - MAX_ENTRY_POINTS = 10 - entry_points = self.entry_index.query_radius( - start_lat, start_lon, DEFAULT_SEARCH_RADIUS_KM, valid_highways - ) - - if not entry_points: - entry_points = self.entry_index.query_radius( - start_lat, start_lon, EXPANDED_SEARCH_RADIUS_KM, valid_highways - ) - if not entry_points: - if mode == "vehicle": - msg = f"No roads found within {EXPANDED_SEARCH_RADIUS_KM}km. Try a different mode." - elif mode in ("mtb", "atv"): - msg = f"No tracks or roads found within {EXPANDED_SEARCH_RADIUS_KM}km. Try foot mode." - else: - msg = f"No trail entry points found within {EXPANDED_SEARCH_RADIUS_KM}km of start." - return {"status": "error", "message": msg} - - entry_points = entry_points[:MAX_ENTRY_POINTS] - - # Run wilderness pathfinding - wilderness_result = self._pathfind_wilderness( - start_lat, start_lon, end_lat, end_lon, - entry_points, boundary_mode, "start" - ) - - if wilderness_result.get("status") == "error": - return wilderness_result - - # Extract results - wilderness_coords = wilderness_result["coords"] - wilderness_stats = wilderness_result["stats"] - best_entry = wilderness_result["entry_point"] - - entry_lat = best_entry["lat"] - entry_lon = best_entry["lon"] - - # Call Valhalla from entry point to destination - network_result = self._valhalla_route(entry_lat, entry_lon, end_lat, end_lon, mode) - - # Build response - return self._build_response( - wilderness_start=wilderness_coords, - wilderness_start_stats=wilderness_stats, - network_segment=network_result.get("segment"), - wilderness_end=None, - wilderness_end_stats=None, - mode=mode, - boundary_mode=boundary_mode, - entry_start=best_entry, - entry_end=None, - scenario="A", - t0=t0, - valhalla_error=network_result.get("error") - ) - - def _route_C_network_to_wilderness( - self, - start_lat: float, start_lon: float, - end_lat: float, end_lon: float, - mode: str, boundary_mode: str - ) -> Dict: - """ - Scenario C: On-network start → off-network end. - Valhalla from start to entry point, then wilderness pathfinding to end. - """ - t0 = time.time() - - if not self.entry_index.table_exists() or self.entry_index.get_entry_point_count() == 0: - return { - "status": "error", - "message": "Trail entry point index not built. Run build_entry_index() first." - } - - valid_highways = MODE_TO_VALID_HIGHWAYS.get(mode) - - # Find entry points near END (destination) - MAX_ENTRY_POINTS = 10 - entry_points = self.entry_index.query_radius( - end_lat, end_lon, DEFAULT_SEARCH_RADIUS_KM, valid_highways - ) - - if not entry_points: - entry_points = self.entry_index.query_radius( - end_lat, end_lon, EXPANDED_SEARCH_RADIUS_KM, valid_highways - ) - if not entry_points: - if mode == "vehicle": - msg = f"No roads found within {EXPANDED_SEARCH_RADIUS_KM}km of destination. Try a different mode." - elif mode in ("mtb", "atv"): - msg = f"No tracks or roads found within {EXPANDED_SEARCH_RADIUS_KM}km of destination. Try foot mode." - else: - msg = f"No trail entry points found within {EXPANDED_SEARCH_RADIUS_KM}km of destination." - return {"status": "error", "message": msg} - - entry_points = entry_points[:MAX_ENTRY_POINTS] - - # Run wilderness pathfinding FROM END toward entry points - wilderness_result = self._pathfind_wilderness( - end_lat, end_lon, start_lat, start_lon, - entry_points, boundary_mode, "end" - ) - - if wilderness_result.get("status") == "error": - return wilderness_result - - # The path is from end→entry, reverse it for display (entry→end) - wilderness_coords = list(reversed(wilderness_result["coords"])) - wilderness_stats = wilderness_result["stats"] - best_entry = wilderness_result["entry_point"] - - entry_lat = best_entry["lat"] - entry_lon = best_entry["lon"] - - # Call Valhalla from start to entry point - network_result = self._valhalla_route(start_lat, start_lon, entry_lat, entry_lon, mode) - - # Build response (network first, then wilderness) - return self._build_response( - wilderness_start=None, - wilderness_start_stats=None, - network_segment=network_result.get("segment"), - wilderness_end=wilderness_coords, - wilderness_end_stats=wilderness_stats, - mode=mode, - boundary_mode=boundary_mode, - entry_start=None, - entry_end=best_entry, - scenario="C", - t0=t0, - valhalla_error=network_result.get("error") - ) - - def _route_B_wilderness_both( - self, - start_lat: float, start_lon: float, - end_lat: float, end_lon: float, - mode: str, boundary_mode: str - ) -> Dict: - """ - Scenario B: Off-network start → off-network end. - Wilderness from start to entry_A, Valhalla entry_A to entry_B, wilderness from entry_B to end. - """ - t0 = time.time() - - if not self.entry_index.table_exists() or self.entry_index.get_entry_point_count() == 0: - return { - "status": "error", - "message": "Trail entry point index not built. Run build_entry_index() first." - } - - valid_highways = MODE_TO_VALID_HIGHWAYS.get(mode) - MAX_ENTRY_POINTS = 10 - - # Find entry points near START - entry_points_start = self.entry_index.query_radius( - start_lat, start_lon, DEFAULT_SEARCH_RADIUS_KM, valid_highways - ) - if not entry_points_start: - entry_points_start = self.entry_index.query_radius( - start_lat, start_lon, EXPANDED_SEARCH_RADIUS_KM, valid_highways - ) - if not entry_points_start: - return {"status": "error", "message": f"No entry points found near start within {EXPANDED_SEARCH_RADIUS_KM}km."} - entry_points_start = entry_points_start[:MAX_ENTRY_POINTS] - - # Find entry points near END - entry_points_end = self.entry_index.query_radius( - end_lat, end_lon, DEFAULT_SEARCH_RADIUS_KM, valid_highways - ) - if not entry_points_end: - entry_points_end = self.entry_index.query_radius( - end_lat, end_lon, EXPANDED_SEARCH_RADIUS_KM, valid_highways - ) - if not entry_points_end: - return {"status": "error", "message": f"No entry points found near destination within {EXPANDED_SEARCH_RADIUS_KM}km."} - entry_points_end = entry_points_end[:MAX_ENTRY_POINTS] - - # Phase 1: Wilderness pathfinding from START - wilderness_start_result = self._pathfind_wilderness( - start_lat, start_lon, end_lat, end_lon, - entry_points_start, boundary_mode, "start" - ) - - if wilderness_start_result.get("status") == "error": - return wilderness_start_result - - wilderness_start_coords = wilderness_start_result["coords"] - wilderness_start_stats = wilderness_start_result["stats"] - entry_A = wilderness_start_result["entry_point"] - - # Phase 2: Wilderness pathfinding from END (run after freeing phase 1 memory) - wilderness_end_result = self._pathfind_wilderness( - end_lat, end_lon, start_lat, start_lon, - entry_points_end, boundary_mode, "end" - ) - - if wilderness_end_result.get("status") == "error": - return wilderness_end_result - - # Reverse the end wilderness path (it's end→entry, we want entry→end for display) - wilderness_end_coords = list(reversed(wilderness_end_result["coords"])) - wilderness_end_stats = wilderness_end_result["stats"] - entry_B = wilderness_end_result["entry_point"] - - # Phase 3: Valhalla from entry_A to entry_B - network_result = self._valhalla_route( - entry_A["lat"], entry_A["lon"], - entry_B["lat"], entry_B["lon"], - mode - ) - - # Build response - return self._build_response( - wilderness_start=wilderness_start_coords, - wilderness_start_stats=wilderness_start_stats, - network_segment=network_result.get("segment"), - wilderness_end=wilderness_end_coords, - wilderness_end_stats=wilderness_end_stats, - mode=mode, - boundary_mode=boundary_mode, - entry_start=entry_A, - entry_end=entry_B, - scenario="B", - t0=t0, - valhalla_error=network_result.get("error") - ) - - def _pathfind_wilderness( - self, - origin_lat: float, origin_lon: float, - dest_lat: float, dest_lon: float, - entry_points: List[Dict], - boundary_mode: str, - label: str - ) -> Dict: - """ - Run MCP wilderness pathfinding from origin toward entry points. - - Args: - origin_lat, origin_lon: Starting point for pathfinding - dest_lat, dest_lon: Ultimate destination (for bbox calculation) - entry_points: List of candidate entry points - boundary_mode: How to handle barriers - label: "start" or "end" for error messages - - Returns: - {"status": "ok", "coords": [...], "stats": {...}, "entry_point": {...}} - or {"status": "error", "message": "..."} - """ - # Build bbox - only include origin and entry points, NOT distant destination - # The destination is handled by Valhalla, wilderness only needs to reach entry points - MAX_BBOX_DEGREES = 2.0 - all_lats = [origin_lat] + [p["lat"] for p in entry_points] - all_lons = [origin_lon] + [p["lon"] for p in entry_points] - - padding = 0.05 - bbox = { - "south": min(all_lats) - padding, - "north": max(all_lats) + padding, - "west": min(all_lons) - padding, - "east": max(all_lons) + padding, - } - - # Clamp bbox size, centering on origin - lat_span = bbox["north"] - bbox["south"] - lon_span = bbox["east"] - bbox["west"] - if lat_span > MAX_BBOX_DEGREES or lon_span > MAX_BBOX_DEGREES: - half_span = MAX_BBOX_DEGREES / 2 - bbox = { - "south": origin_lat - half_span, - "north": origin_lat + half_span, - "west": origin_lon - half_span, - "east": origin_lon + half_span, - } - - # Initialize readers - self._init_readers() - - # Load elevation - try: - elevation, meta = self.dem_reader.get_elevation_grid( - south=bbox["south"], north=bbox["north"], - west=bbox["west"], east=bbox["east"], - ) - except Exception as e: - return {"status": "error", "message": f"Failed to load elevation for {label}: {e}"} - - # Check memory - mem = check_memory_usage() - if mem > MEMORY_LIMIT_GB: - return {"status": "error", "message": f"Memory limit exceeded: {mem:.1f}GB > {MEMORY_LIMIT_GB}GB"} - - # Load friction - friction_raw = self.friction_reader.get_friction_grid( - south=bbox["south"], north=bbox["north"], - west=bbox["west"], east=bbox["east"], - target_shape=elevation.shape - ) - friction_mult = friction_to_multiplier(friction_raw) - - # Load barriers - barriers = self.barrier_reader.get_barrier_grid( - south=bbox["south"], north=bbox["north"], - west=bbox["west"], east=bbox["east"], - target_shape=elevation.shape - ) - - # Load trails - trails = self.trail_reader.get_trails_grid( - south=bbox["south"], north=bbox["north"], - west=bbox["west"], east=bbox["east"], - target_shape=elevation.shape - ) - - # Compute cost grid (ALWAYS foot mode for wilderness) - cost = compute_cost_grid( - elevation, - cell_size_m=meta["cell_size_m"], - friction=friction_mult, - friction_raw=friction_raw, - trails=trails, - barriers=barriers, - wilderness=None, - mvum=None, - boundary_mode=boundary_mode, - mode="foot", - ) - - # Free intermediate arrays - del friction_mult, friction_raw - gc.collect() - - # Convert origin to pixel coordinates - origin_row, origin_col = self.dem_reader.latlon_to_pixel(origin_lat, origin_lon, meta) - - rows, cols = elevation.shape - if not (0 <= origin_row < rows and 0 <= origin_col < cols): - return {"status": "error", "message": f"{label.capitalize()} point outside grid bounds"} - - # Map entry points to pixels - entry_pixels = [] - for ep in entry_points: - row, col = self.dem_reader.latlon_to_pixel(ep["lat"], ep["lon"], meta) - if 0 <= row < rows and 0 <= col < cols: - entry_pixels.append({"row": row, "col": col, "entry_point": ep}) - - if not entry_pixels: - return {"status": "error", "message": f"No entry points map to grid bounds for {label}"} - - # Run MCP - mcp = MCP_Geometric(cost, fully_connected=True) - cumulative_costs, traceback = mcp.find_costs([(origin_row, origin_col)]) - - # Find nearest reachable entry point - best_entry = None - best_cost = np.inf - - for ep in entry_pixels: - ep_cost = cumulative_costs[ep["row"], ep["col"]] - if ep_cost < best_cost: - best_cost = ep_cost - best_entry = ep - - if best_entry is None or np.isinf(best_cost): - return { - "status": "error", - "message": f"No path found from {label} to any entry point (blocked by impassable terrain)" - } - - # Traceback path - path_indices = mcp.traceback((best_entry["row"], best_entry["col"])) - - # Convert to coordinates and collect stats - coords = [] - elevations = [] - trail_values = [] - barrier_crossings = 0 - - for row, col in path_indices: - lat, lon = self.dem_reader.pixel_to_latlon(row, col, meta) - coords.append([lon, lat]) - elevations.append(elevation[row, col]) - trail_values.append(trails[row, col]) - if barriers[row, col] == 255: - barrier_crossings += 1 - - # Calculate distance - distance_m = 0 - for i in range(1, len(coords)): - lon1, lat1 = coords[i-1] - lon2, lat2 = coords[i] - distance_m += haversine_distance(lat1, lon1, lat2, lon2) - - # Elevation stats - elev_arr = np.array(elevations) - elev_diff = np.diff(elev_arr) - elev_gain = float(np.sum(elev_diff[elev_diff > 0])) - elev_loss = float(np.sum(np.abs(elev_diff[elev_diff < 0]))) - - # Trail stats - trail_arr = np.array(trail_values) - on_trail_cells = np.sum(trail_arr > 0) - total_cells = len(trail_arr) - on_trail_pct = float(100 * on_trail_cells / total_cells) if total_cells > 0 else 0 - - # Free memory - del mcp, cumulative_costs, traceback, cost, trails, barriers, elevation - gc.collect() - - return { - "status": "ok", - "coords": coords, - "stats": { - "distance_km": distance_m / 1000, - "effort_minutes": best_cost / 60, - "elevation_gain_m": elev_gain, - "elevation_loss_m": elev_loss, - "on_trail_pct": on_trail_pct, - "barrier_crossings": barrier_crossings, - "cell_count": total_cells, - }, - "entry_point": best_entry["entry_point"] - } - - def _valhalla_route( - self, - start_lat: float, start_lon: float, - end_lat: float, end_lon: float, - mode: str - ) -> Dict: - """ - Call Valhalla for network routing. - - Returns: - {"segment": {...}, "error": None} on success - {"segment": None, "error": "..."} on failure - """ - costing = MODE_TO_COSTING.get(mode, "pedestrian") - - valhalla_request = { - "locations": [ - {"lat": start_lat, "lon": start_lon}, - {"lat": end_lat, "lon": end_lon} - ], - "costing": costing, - "directions_options": {"units": "kilometers"} - } - - try: - resp = requests.post(f"{VALHALLA_URL}/route", json=valhalla_request, timeout=30) - - if resp.status_code == 200: - valhalla_data = resp.json() - trip = valhalla_data.get("trip", {}) - legs = trip.get("legs", []) - - if legs: - leg = legs[0] - shape = leg.get("shape", "") - coords = self._decode_polyline(shape) - - maneuvers = [] - for m in leg.get("maneuvers", []): - maneuvers.append({ - "instruction": m.get("instruction", ""), - "type": m.get("type", 0), - "distance_km": m.get("length", 0), - "time_seconds": m.get("time", 0), - "street_names": m.get("street_names", []), - }) - - summary = trip.get("summary", {}) - return { - "segment": { - "coordinates": coords, - "distance_km": summary.get("length", 0), - "duration_minutes": summary.get("time", 0) / 60, - "maneuvers": maneuvers, - }, - "error": None - } - - return {"segment": None, "error": f"Valhalla returned {resp.status_code}: {resp.text[:200]}"} - - except Exception as e: - return {"segment": None, "error": f"Valhalla request failed: {e}"} - - def _build_response( - self, - wilderness_start: Optional[List], - wilderness_start_stats: Optional[Dict], - network_segment: Optional[Dict], - wilderness_end: Optional[List], - wilderness_end_stats: Optional[Dict], - mode: str, - boundary_mode: str, - entry_start: Optional[Dict], - entry_end: Optional[Dict], - scenario: str, - t0: float, - valhalla_error: Optional[str] - ) -> Dict: - """Build the final GeoJSON response.""" - features = [] - - # Wilderness start segment - if wilderness_start and wilderness_start_stats: - features.append({ - "type": "Feature", - "properties": { - "segment_type": "wilderness", - "segment_position": "start", - "effort_minutes": float(wilderness_start_stats["effort_minutes"]), - "distance_km": float(wilderness_start_stats["distance_km"]), - "elevation_gain_m": wilderness_start_stats["elevation_gain_m"], - "elevation_loss_m": wilderness_start_stats["elevation_loss_m"], - "boundary_mode": boundary_mode, - "on_trail_pct": wilderness_start_stats["on_trail_pct"], - "barrier_crossings": wilderness_start_stats["barrier_crossings"], - "wilderness_mode": "foot", - }, - "geometry": {"type": "LineString", "coordinates": wilderness_start} - }) - - # Network segment - if network_segment: - features.append({ - "type": "Feature", - "properties": { - "segment_type": "network", - "distance_km": network_segment["distance_km"], - "duration_minutes": network_segment["duration_minutes"], - "maneuvers": network_segment["maneuvers"], - "network_mode": mode, - }, - "geometry": {"type": "LineString", "coordinates": network_segment["coordinates"]} - }) - - # Wilderness end segment - if wilderness_end and wilderness_end_stats: - features.append({ - "type": "Feature", - "properties": { - "segment_type": "wilderness", - "segment_position": "end", - "effort_minutes": float(wilderness_end_stats["effort_minutes"]), - "distance_km": float(wilderness_end_stats["distance_km"]), - "elevation_gain_m": wilderness_end_stats["elevation_gain_m"], - "elevation_loss_m": wilderness_end_stats["elevation_loss_m"], - "boundary_mode": boundary_mode, - "on_trail_pct": wilderness_end_stats["on_trail_pct"], - "barrier_crossings": wilderness_end_stats["barrier_crossings"], - "wilderness_mode": "foot", - }, - "geometry": {"type": "LineString", "coordinates": wilderness_end} - }) - - # Combined path - combined_coords = [] - if wilderness_start: - combined_coords.extend(wilderness_start) - if network_segment: - # Skip first coord if we already have wilderness_start (avoid duplicate) - start_idx = 1 if wilderness_start else 0 - combined_coords.extend(network_segment["coordinates"][start_idx:]) - if wilderness_end: - # Skip first coord (avoid duplicate with network end) - start_idx = 1 if (wilderness_start or network_segment) else 0 - combined_coords.extend(wilderness_end[start_idx:]) - - if combined_coords: - features.append({ - "type": "Feature", - "properties": { - "segment_type": "combined", - "wilderness_mode": "foot", - "network_mode": mode, - "boundary_mode": boundary_mode, - "scenario": scenario, - }, - "geometry": {"type": "LineString", "coordinates": combined_coords} - }) - - geojson = {"type": "FeatureCollection", "features": features} - - # Calculate totals - total_distance_km = 0.0 - total_effort_minutes = 0.0 - wilderness_distance_km = 0.0 - wilderness_effort_minutes = 0.0 - network_distance_km = 0.0 - network_duration_minutes = 0.0 - barrier_crossings = 0 - on_trail_pct = 0.0 - - if wilderness_start_stats: - wilderness_distance_km += wilderness_start_stats["distance_km"] - wilderness_effort_minutes += wilderness_start_stats["effort_minutes"] - barrier_crossings += wilderness_start_stats["barrier_crossings"] - on_trail_pct = wilderness_start_stats["on_trail_pct"] - - if wilderness_end_stats: - wilderness_distance_km += wilderness_end_stats["distance_km"] - wilderness_effort_minutes += wilderness_end_stats["effort_minutes"] - barrier_crossings += wilderness_end_stats["barrier_crossings"] - # Average on-trail percentage if we have both - if wilderness_start_stats: - on_trail_pct = (on_trail_pct + wilderness_end_stats["on_trail_pct"]) / 2 - else: - on_trail_pct = wilderness_end_stats["on_trail_pct"] - - if network_segment: - network_distance_km = network_segment["distance_km"] - network_duration_minutes = network_segment["duration_minutes"] - - total_distance_km = wilderness_distance_km + network_distance_km - total_effort_minutes = wilderness_effort_minutes + network_duration_minutes - - summary = { - "total_distance_km": float(total_distance_km), - "total_effort_minutes": float(total_effort_minutes), - "wilderness_distance_km": float(wilderness_distance_km), - "wilderness_effort_minutes": float(wilderness_effort_minutes), - "network_distance_km": float(network_distance_km), - "network_duration_minutes": float(network_duration_minutes), - "on_trail_pct": float(on_trail_pct), - "barrier_crossings": barrier_crossings, - "boundary_mode": boundary_mode, - "wilderness_mode": "foot", - "network_mode": mode, - "scenario": scenario, - "computation_time_s": time.time() - t0, - } - - if entry_start: - summary["entry_point_start"] = { - "lat": entry_start["lat"], - "lon": entry_start["lon"], - "highway_class": entry_start["highway_class"], - "name": entry_start.get("name", ""), - } - - if entry_end: - summary["entry_point_end"] = { - "lat": entry_end["lat"], - "lon": entry_end["lon"], - "highway_class": entry_end["highway_class"], - "name": entry_end.get("name", ""), - } - - result = {"status": "ok", "route": geojson, "summary": summary} - - if valhalla_error: - result["warning"] = f"Network segment incomplete: {valhalla_error}" - - return result - - def _decode_polyline(self, encoded: str, precision: int = 6) -> List[List[float]]: - """Decode a polyline string into coordinates [lon, lat].""" - coords = [] - index = 0 - lat = 0 - lon = 0 - - while index < len(encoded): - shift = 0 - result = 0 - while True: - b = ord(encoded[index]) - 63 - index += 1 - result |= (b & 0x1f) << shift - shift += 5 - if b < 0x20: - break - dlat = ~(result >> 1) if result & 1 else result >> 1 - lat += dlat - - shift = 0 - result = 0 - while True: - b = ord(encoded[index]) - 63 - index += 1 - result |= (b & 0x1f) << shift - shift += 5 - if b < 0x20: - break - dlon = ~(result >> 1) if result & 1 else result >> 1 - lon += dlon - - coords.append([lon / (10 ** precision), lat / (10 ** precision)]) - - return coords - - def close(self): - """Close all readers.""" - if self.dem_reader: - self.dem_reader.close() - if self.friction_reader: - self.friction_reader.close() - if self.barrier_reader: - self.barrier_reader.close() - if self.wilderness_reader: - self.wilderness_reader.close() - if self.trail_reader: - self.trail_reader.close() - self.entry_index.close() - - -def build_entry_index(): - """Build the trail entry point index.""" - index = EntryPointIndex() - stats = index.build_index() - index.close() - return stats - - -if __name__ == "__main__": - import sys - - if len(sys.argv) > 1 and sys.argv[1] == "build": - print("Building trail entry point index...") - stats = build_entry_index() - print(f"\nDone. Total entry points: {stats['total']}") - - elif len(sys.argv) > 1 and sys.argv[1] == "test": - print("Testing router (all scenarios)...") - print("=" * 60) - - router = OffrouteRouter() - - # Test points - wilderness_start = (44.0543, -115.4237) # Off-network - wilderness_end = (45.2, -115.5) # Deep wilderness (Frank Church) - road_start = (43.6150, -116.2023) # Boise downtown (on-network) - road_end = (43.5867, -116.5625) # Nampa (on-network) - - tests = [ - ("A: wilderness→road", wilderness_start, (44.0814, -115.5021)), - ("B: wilderness→wilderness", wilderness_start, wilderness_end), - ("C: road→wilderness", road_start, wilderness_start), - ("D: road→road", road_start, road_end), - ] - - for label, (slat, slon), (elat, elon) in tests: - print(f"\n{label}") - print("-" * 40) - - result = router.route( - start_lat=slat, start_lon=slon, - end_lat=elat, end_lon=elon, - mode="foot", boundary_mode="pragmatic" - ) - - if result["status"] == "ok": - s = result["summary"] - print(f" Scenario: {s.get('scenario', '?')}") - print(f" Total: {s['total_distance_km']:.2f} km, {s['total_effort_minutes']:.1f} min") - print(f" Wilderness: {s['wilderness_distance_km']:.2f} km") - print(f" Network: {s['network_distance_km']:.2f} km") - if s.get('entry_point_start'): - ep = s['entry_point_start'] - print(f" Entry (start): {ep['highway_class']} at {ep['lat']:.4f}, {ep['lon']:.4f}") - if s.get('entry_point_end'): - ep = s['entry_point_end'] - print(f" Entry (end): {ep['highway_class']} at {ep['lat']:.4f}, {ep['lon']:.4f}") - else: - print(f" ERROR: {result['message']}") - - router.close() - - else: - print("Usage:") - print(" python router.py build # Build entry point index") - print(" python router.py test # Test all scenarios") + "service"}, + "foot": {"primary", "secondary", "tertiary", "unclassified", "residential", + "service", "track", "path", "footway", "bridleway"}, + "mtb": {"primary", "secondary", "tertiary", "unclassified", "residential", + "service", "track"}, + "atv": {"primary", "secondary", "tertiary", "unclassified", "residential", + "service", "track"}, + "vehicle": {"primary", "secondary", "tertiary", "unclassified", "residential", + "service"}, +} + + +def haversine_distance(lat1: float, lon1: float, lat2: float, lon2: float) -> float: + """Calculate distance between two points in meters.""" + R = 6371000 + dlat = math.radians(lat2 - lat1) + dlon = math.radians(lon2 - lon1) + a = math.sin(dlat/2)**2 + math.cos(math.radians(lat1)) * math.cos(math.radians(lat2)) * math.sin(dlon/2)**2 + c = 2 * math.atan2(math.sqrt(a), math.sqrt(1-a)) + return R * c + + +def check_memory_usage() -> float: + """Check current memory usage in GB.""" + try: + import psutil + process = psutil.Process() + return process.memory_info().rss / (1024**3) + except ImportError: + return 0 + + +class EntryPointIndex: + """ + PostGIS-backed spatial index of road/trail entry points. + Uses ST_DWithin for fast radius queries with meter-accurate distances. + Densifies highway LineStrings at 100m intervals for better coverage. + """ + + def __init__(self, dsn: str = None): + self.dsn = dsn or POSTGIS_DSN + self._conn: Optional[psycopg2.extensions.connection] = None + + def _get_conn(self) -> psycopg2.extensions.connection: + if self._conn is None or self._conn.closed: + self._conn = psycopg2.connect(self.dsn) + return self._conn + + def table_exists(self) -> bool: + """Check if entry_points table exists.""" + conn = self._get_conn() + with conn.cursor() as cur: + cur.execute(""" + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_name = 'entry_points' + ) + """) + return cur.fetchone()[0] + + def get_entry_point_count(self) -> int: + """Return the number of entry points in the index.""" + if not self.table_exists(): + return 0 + conn = self._get_conn() + with conn.cursor() as cur: + cur.execute("SELECT COUNT(*) FROM entry_points") + return cur.fetchone()[0] + + def query_bbox( + self, + south: float, + north: float, + west: float, + east: float, + valid_highways: Optional[Set[str]] = None + ) -> List[Dict]: + """Find entry points within a bounding box.""" + if not self.table_exists(): + return [] + + conn = self._get_conn() + + highway_filter = "" + params = [west, south, east, north] + if valid_highways: + placeholders = ','.join(['%s'] * len(valid_highways)) + highway_filter = f"AND highway_class IN ({placeholders})" + params.extend(list(valid_highways)) + + query = f""" + SELECT + id, + ST_Y(geom) as lat, + ST_X(geom) as lon, + highway_class, + name, + land_status + FROM entry_points + WHERE geom && ST_MakeEnvelope(%s, %s, %s, %s, 4326) + {highway_filter} + """ + + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + cur.execute(query, params) + return [dict(row) for row in cur.fetchall()] + + def query_radius( + self, + lat: float, + lon: float, + radius_km: float, + valid_highways: Optional[Set[str]] = None, + limit: int = 50 + ) -> List[Dict]: + """ + Find entry points within radius_km of (lat, lon). + Uses PostGIS ST_DWithin with geography cast for meter-accurate distance. + """ + if not self.table_exists(): + return [] + + conn = self._get_conn() + radius_m = radius_km * 1000 + + # Build query with optional highway filter + highway_filter = "" + params = [lon, lat, lon, lat, radius_m] + if valid_highways: + placeholders = ','.join(['%s'] * len(valid_highways)) + highway_filter = f"AND highway_class IN ({placeholders})" + params.extend(list(valid_highways)) + params.append(limit) + + query = f""" + SELECT + id, + ST_Y(geom) as lat, + ST_X(geom) as lon, + highway_class, + name, + land_status, + ST_Distance( + geom::geography, + ST_SetSRID(ST_Point(%s, %s), 4326)::geography + ) as distance_m + FROM entry_points + WHERE ST_DWithin( + geom::geography, + ST_SetSRID(ST_Point(%s, %s), 4326)::geography, + %s + ) + {highway_filter} + ORDER BY distance_m + LIMIT %s + """ + + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + cur.execute(query, params) + return [dict(row) for row in cur.fetchall()] + + def build_index(self, osm_pbf_path: Path = None) -> Dict: + """ + Build the entry point index from OSM PBF. + Densifies LineStrings to sample points every 100m. + Tags points with land_status from PAD-US. + """ + if osm_pbf_path is None: + osm_pbf_path = OSM_PBF_PATH + + if not osm_pbf_path.exists(): + raise FileNotFoundError(f"OSM PBF not found: {osm_pbf_path}") + + print(f"Building entry point index from {osm_pbf_path}...") + start_time = time.time() + + highway_types = [ + "primary", "secondary", "tertiary", "unclassified", + "residential", "service", "track", "path", "footway", "bridleway" + ] + + stats = {"total": 0, "by_class": {}, "lines_processed": 0} + + with tempfile.TemporaryDirectory() as tmpdir: + geojson_path = Path(tmpdir) / "highways.geojson" + + # Extract highways with osmium + print(" Extracting highways with osmium...") + cmd = ["osmium", "tags-filter", str(osm_pbf_path)] + for ht in highway_types: + cmd.append(f"w/highway={ht}") + cmd.extend(["-o", str(Path(tmpdir) / "filtered.osm.pbf"), "--overwrite"]) + subprocess.run(cmd, check=True, capture_output=True) + + # Convert to GeoJSON + print(" Converting to GeoJSON with ogr2ogr...") + cmd = [ + "ogr2ogr", "-f", "GeoJSON", + str(geojson_path), + str(Path(tmpdir) / "filtered.osm.pbf"), + "lines", "-t_srs", "EPSG:4326" + ] + subprocess.run(cmd, check=True, capture_output=True) + + # Load GeoJSON + print(" Loading GeoJSON...") + with open(geojson_path) as f: + data = json.load(f) + + # Process features and densify + print(f" Densifying LineStrings at {DENSIFY_INTERVAL_M}m intervals...") + points_to_insert = [] + seen_keys = set() + + features = data.get("features", []) + total_features = len(features) + + for idx, feature in enumerate(features): + if idx > 0 and idx % 100000 == 0: + print(f" Processed {idx}/{total_features} features...") + + props = feature.get("properties", {}) + geom = feature.get("geometry", {}) + + if geom.get("type") != "LineString": + continue + + coords = geom.get("coordinates", []) + if len(coords) < 2: + continue + + highway_class = props.get("highway", "unknown") + name = props.get("name", "") + stats["lines_processed"] += 1 + + # Densify this LineString + densified = self._densify_line(coords, DENSIFY_INTERVAL_M) + + for lon, lat in densified: + # Deduplicate by rounding to 5 decimal places (~1m precision) + key = (round(lat, 5), round(lon, 5)) + if key in seen_keys: + continue + seen_keys.add(key) + + points_to_insert.append((lon, lat, highway_class, name)) + + # Insert into PostGIS + print(f" Inserting {len(points_to_insert)} entry points into PostGIS...") + conn = self._get_conn() + + with conn.cursor() as cur: + # Truncate existing data + cur.execute("TRUNCATE entry_points RESTART IDENTITY") + + # Batch insert with execute_values for speed + batch_size = 50000 + for i in range(0, len(points_to_insert), batch_size): + batch = points_to_insert[i:i+batch_size] + psycopg2.extras.execute_values( + cur, + """ + INSERT INTO entry_points (geom, highway_class, name) + VALUES %s + """, + batch, + template="(ST_SetSRID(ST_Point(%s, %s), 4326), %s, %s)", + page_size=10000 + ) + if i > 0 and i % 500000 == 0: + print(f" Inserted {i}/{len(points_to_insert)} points...") + + conn.commit() + + # Tag land_status from PAD-US + print(" Tagging land_status from PAD-US subdivided polygons...") + with conn.cursor() as cur: + cur.execute(""" + UPDATE entry_points e + SET land_status = 'public' + FROM padus_sub p + WHERE ST_Intersects(e.geom, p.geom) + """) + public_count = cur.rowcount + print(f" Tagged {public_count} points as public land") + + conn.commit() + + # Gather stats + elapsed = time.time() - start_time + stats["total"] = len(points_to_insert) + stats["build_time_sec"] = round(elapsed, 1) + + for lon, lat, hc, name in points_to_insert: + stats["by_class"][hc] = stats["by_class"].get(hc, 0) + 1 + + print(f" Done in {elapsed:.1f}s. Total: {stats['total']} entry points from {stats['lines_processed']} lines") + for hc, count in sorted(stats["by_class"].items(), key=lambda x: -x[1]): + print(f" {hc}: {count}") + + return stats + + def _densify_line(self, coords: List[List[float]], interval_m: float) -> List[tuple]: + """ + Sample points along a LineString at regular intervals. + coords: [[lon, lat], ...] in GeoJSON order + Returns: [(lon, lat), ...] sampled points including first and last + """ + if len(coords) < 2: + return [(coords[0][0], coords[0][1])] if coords else [] + + # Calculate line length in meters using haversine on segments + total_m = 0 + for i in range(len(coords) - 1): + lon1, lat1 = coords[i] + lon2, lat2 = coords[i + 1] + total_m += haversine_distance(lat1, lon1, lat2, lon2) + + if total_m == 0: + return [(coords[0][0], coords[0][1])] + + # Create Shapely LineString + line = LineString(coords) + + # Calculate number of points needed + n_points = max(2, int(total_m / interval_m) + 1) + + # Sample using normalized interpolation + result = [] + for i in range(n_points): + fraction = min(i / (n_points - 1), 1.0) if n_points > 1 else 0 + point = line.interpolate(fraction, normalized=True) + result.append((point.x, point.y)) # (lon, lat) + + # Always ensure first and last original coordinates are included + first_coord = (coords[0][0], coords[0][1]) + last_coord = (coords[-1][0], coords[-1][1]) + + if result[0] != first_coord: + result[0] = first_coord + if result[-1] != last_coord: + result[-1] = last_coord + + return result + + def _highway_priority(self, highway_class: str) -> int: + """Lower number = better priority for entry points.""" + priority = { + "primary": 1, "secondary": 2, "tertiary": 3, + "unclassified": 4, "residential": 5, "service": 6, + "track": 7, "path": 8, "footway": 9, "bridleway": 10 + } + return priority.get(highway_class, 99) + + def close(self): + if self._conn and not self._conn.closed: + self._conn.close() + self._conn = None + + +class OffrouteRouter: + """ + OFFROUTE Router — orchestrates wilderness pathfinding and Valhalla stitching. + + Supports four scenarios: + A: off-network start → on-network end + B: off-network start → off-network end + C: on-network start → off-network end + D: on-network start → on-network end (pure Valhalla) + + IMPORTANT: Wilderness segment ALWAYS uses foot mode for pathfinding. + User's mode affects entry point selection and Valhalla costing only. + """ + + def __init__(self): + self.dem_reader = None + self.friction_reader = None + self.barrier_reader = None + self.wilderness_reader = None + self.trail_reader = None + self.entry_index = EntryPointIndex() + + def _init_readers(self): + """Lazy init readers.""" + if self.dem_reader is None: + self.dem_reader = DEMReader() + if self.friction_reader is None: + self.friction_reader = FrictionReader() + if self.barrier_reader is None: + self.barrier_reader = BarrierReader() + if self.wilderness_reader is None and DEFAULT_WILDERNESS_PATH.exists(): + self.wilderness_reader = WildernessReader() + if self.trail_reader is None: + self.trail_reader = TrailReader() + + def _locate_on_network(self, lat: float, lon: float, mode: str) -> Dict: + """ + Check if a point is on the routable network using Valhalla's /locate. + + Returns: + { + "on_network": bool, + "snap_distance_m": float, + "snapped_lat": float, + "snapped_lon": float + } + """ + costing = MODE_TO_COSTING.get(mode, "pedestrian") + try: + resp = requests.post( + f"{VALHALLA_URL}/locate", + json={"locations": [{"lat": lat, "lon": lon}], "costing": costing}, + timeout=10 + ) + + if resp.status_code == 200: + data = resp.json() + if data and len(data) > 0 and data[0].get("edges"): + edge = data[0]["edges"][0] + snap_lat = edge.get("correlated_lat", lat) + snap_lon = edge.get("correlated_lon", lon) + snap_dist = haversine_distance(lat, lon, snap_lat, snap_lon) + return { + "on_network": snap_dist <= OFF_NETWORK_THRESHOLD_M, + "snap_distance_m": snap_dist, + "snapped_lat": snap_lat, + "snapped_lon": snap_lon + } + except Exception: + pass + + return { + "on_network": False, + "snap_distance_m": float('inf'), + "snapped_lat": lat, + "snapped_lon": lon + } + + def route( + self, + start_lat: float, + start_lon: float, + end_lat: float, + end_lon: float, + mode: Literal["foot", "mtb", "atv", "vehicle"] = "foot", + boundary_mode: Literal["strict", "pragmatic", "emergency"] = "pragmatic" + ) -> Dict: + """ + Route between two points, handling all four scenarios. + + Scenarios: + A: off-network start → on-network end (wilderness then network) + B: off-network start → off-network end (wilderness, network, wilderness) + C: on-network start → off-network end (network then wilderness) + D: on-network start → on-network end (pure network) + + Args: + start_lat, start_lon: Starting coordinates + end_lat, end_lon: Destination coordinates + mode: Travel mode (foot, mtb, atv, vehicle) + boundary_mode: How to handle private land (strict, pragmatic, emergency) + + Returns a GeoJSON FeatureCollection with route segments. + """ + if mode not in MODE_TO_COSTING: + return {"status": "error", "message": f"Unknown mode: {mode}"} + + # Detect network status for both endpoints + start_status = self._locate_on_network(start_lat, start_lon, mode) + end_status = self._locate_on_network(end_lat, end_lon, mode) + + start_off_network = not start_status["on_network"] + end_off_network = not end_status["on_network"] + + # Dispatch to appropriate handler + if not start_off_network and not end_off_network: + # Scenario D: on-network → on-network (pure Valhalla) + return self._route_D_network_only( + start_lat, start_lon, end_lat, end_lon, mode + ) + elif not start_off_network and end_off_network: + # Scenario C: on-network → off-network + return self._route_C_network_to_wilderness( + start_lat, start_lon, end_lat, end_lon, mode, boundary_mode + ) + elif start_off_network and not end_off_network: + # Scenario A: off-network → on-network + return self._route_A_wilderness_to_network( + start_lat, start_lon, end_lat, end_lon, mode, boundary_mode + ) + else: + # Scenario B: off-network → off-network + return self._route_B_wilderness_both( + start_lat, start_lon, end_lat, end_lon, mode, boundary_mode + ) + + def _route_D_network_only( + self, + start_lat: float, start_lon: float, + end_lat: float, end_lon: float, + mode: str + ) -> Dict: + """ + Scenario D: Both endpoints on-network. Pure Valhalla routing. + """ + t0 = time.time() + costing = MODE_TO_COSTING.get(mode, "pedestrian") + + valhalla_request = { + "locations": [ + {"lat": start_lat, "lon": start_lon}, + {"lat": end_lat, "lon": end_lon} + ], + "costing": costing, + "directions_options": {"units": "kilometers"} + } + + try: + resp = requests.post(f"{VALHALLA_URL}/route", json=valhalla_request, timeout=30) + + if resp.status_code != 200: + return { + "status": "error", + "message": f"Network routing failed: {resp.text[:200]}" + } + + valhalla_data = resp.json() + trip = valhalla_data.get("trip", {}) + legs = trip.get("legs", []) + + if not legs: + return {"status": "error", "message": "No route found"} + + leg = legs[0] + shape = leg.get("shape", "") + network_coords = self._decode_polyline(shape) + + maneuvers = [] + for m in leg.get("maneuvers", []): + maneuvers.append({ + "instruction": m.get("instruction", ""), + "type": m.get("type", 0), + "distance_km": m.get("length", 0), + "time_seconds": m.get("time", 0), + "street_names": m.get("street_names", []), + }) + + summary = trip.get("summary", {}) + distance_km = summary.get("length", 0) + duration_min = summary.get("time", 0) / 60 + + # Build response in same format as wilderness routes + network_feature = { + "type": "Feature", + "properties": { + "segment_type": "network", + "distance_km": distance_km, + "duration_minutes": duration_min, + "maneuvers": maneuvers, + "network_mode": mode, + }, + "geometry": {"type": "LineString", "coordinates": network_coords} + } + + combined_feature = { + "type": "Feature", + "properties": { + "segment_type": "combined", + "network_mode": mode, + }, + "geometry": {"type": "LineString", "coordinates": network_coords} + } + + geojson = {"type": "FeatureCollection", "features": [network_feature, combined_feature]} + + result = { + "status": "ok", + "route": geojson, + "summary": { + "total_distance_km": float(distance_km), + "total_effort_minutes": float(duration_min), + "wilderness_distance_km": 0.0, + "wilderness_effort_minutes": 0.0, + "network_distance_km": float(distance_km), + "network_duration_minutes": float(duration_min), + "on_trail_pct": 100.0, + "barrier_crossings": 0, + "network_mode": mode, + "scenario": "D", + "computation_time_s": time.time() - t0, + } + } + return result + + except Exception as e: + return {"status": "error", "message": f"Network routing failed: {e}"} + + def _route_A_wilderness_to_network( + self, + start_lat: float, start_lon: float, + end_lat: float, end_lon: float, + mode: str, boundary_mode: str + ) -> Dict: + """ + Scenario A: Off-network start → on-network end. + Wilderness pathfinding from start to entry point, then Valhalla to end. + """ + t0 = time.time() + + # Ensure entry point index exists + if not self.entry_index.table_exists() or self.entry_index.get_entry_point_count() == 0: + return { + "status": "error", + "message": "Trail entry point index not built. Run build_entry_index() first." + } + + # Get valid highway classes for this mode + valid_highways = MODE_TO_VALID_HIGHWAYS.get(mode) + + # Find entry points near start, filtered by mode + MAX_ENTRY_POINTS = 10 + entry_points = self.entry_index.query_radius( + start_lat, start_lon, DEFAULT_SEARCH_RADIUS_KM, valid_highways + ) + + if not entry_points: + entry_points = self.entry_index.query_radius( + start_lat, start_lon, EXPANDED_SEARCH_RADIUS_KM, valid_highways + ) + if not entry_points: + if mode == "vehicle": + msg = f"No roads found within {EXPANDED_SEARCH_RADIUS_KM}km. Try a different mode." + elif mode in ("mtb", "atv"): + msg = f"No tracks or roads found within {EXPANDED_SEARCH_RADIUS_KM}km. Try foot mode." + else: + msg = f"No trail entry points found within {EXPANDED_SEARCH_RADIUS_KM}km of start." + return {"status": "error", "message": msg} + + entry_points = entry_points[:MAX_ENTRY_POINTS] + + # Run wilderness pathfinding + wilderness_result = self._pathfind_wilderness( + start_lat, start_lon, end_lat, end_lon, + entry_points, boundary_mode, "start" + ) + + if wilderness_result.get("status") == "error": + return wilderness_result + + # Extract results + wilderness_coords = wilderness_result["coords"] + wilderness_stats = wilderness_result["stats"] + best_entry = wilderness_result["entry_point"] + + entry_lat = best_entry["lat"] + entry_lon = best_entry["lon"] + + # Call Valhalla from entry point to destination + network_result = self._valhalla_route(entry_lat, entry_lon, end_lat, end_lon, mode) + + # Build response + return self._build_response( + wilderness_start=wilderness_coords, + wilderness_start_stats=wilderness_stats, + network_segment=network_result.get("segment"), + wilderness_end=None, + wilderness_end_stats=None, + mode=mode, + boundary_mode=boundary_mode, + entry_start=best_entry, + entry_end=None, + scenario="A", + t0=t0, + valhalla_error=network_result.get("error") + ) + + def _route_C_network_to_wilderness( + self, + start_lat: float, start_lon: float, + end_lat: float, end_lon: float, + mode: str, boundary_mode: str + ) -> Dict: + """ + Scenario C: On-network start → off-network end. + Valhalla from start to entry point, then wilderness pathfinding to end. + """ + t0 = time.time() + + if not self.entry_index.table_exists() or self.entry_index.get_entry_point_count() == 0: + return { + "status": "error", + "message": "Trail entry point index not built. Run build_entry_index() first." + } + + valid_highways = MODE_TO_VALID_HIGHWAYS.get(mode) + + # Find entry points near END (destination) + MAX_ENTRY_POINTS = 10 + entry_points = self.entry_index.query_radius( + end_lat, end_lon, DEFAULT_SEARCH_RADIUS_KM, valid_highways + ) + + if not entry_points: + entry_points = self.entry_index.query_radius( + end_lat, end_lon, EXPANDED_SEARCH_RADIUS_KM, valid_highways + ) + if not entry_points: + if mode == "vehicle": + msg = f"No roads found within {EXPANDED_SEARCH_RADIUS_KM}km of destination. Try a different mode." + elif mode in ("mtb", "atv"): + msg = f"No tracks or roads found within {EXPANDED_SEARCH_RADIUS_KM}km of destination. Try foot mode." + else: + msg = f"No trail entry points found within {EXPANDED_SEARCH_RADIUS_KM}km of destination." + return {"status": "error", "message": msg} + + entry_points = entry_points[:MAX_ENTRY_POINTS] + + # Run wilderness pathfinding FROM END toward entry points + wilderness_result = self._pathfind_wilderness( + end_lat, end_lon, start_lat, start_lon, + entry_points, boundary_mode, "end" + ) + + if wilderness_result.get("status") == "error": + return wilderness_result + + # The path is from end→entry, reverse it for display (entry→end) + wilderness_coords = list(reversed(wilderness_result["coords"])) + wilderness_stats = wilderness_result["stats"] + best_entry = wilderness_result["entry_point"] + + entry_lat = best_entry["lat"] + entry_lon = best_entry["lon"] + + # Call Valhalla from start to entry point + network_result = self._valhalla_route(start_lat, start_lon, entry_lat, entry_lon, mode) + + # Build response (network first, then wilderness) + return self._build_response( + wilderness_start=None, + wilderness_start_stats=None, + network_segment=network_result.get("segment"), + wilderness_end=wilderness_coords, + wilderness_end_stats=wilderness_stats, + mode=mode, + boundary_mode=boundary_mode, + entry_start=None, + entry_end=best_entry, + scenario="C", + t0=t0, + valhalla_error=network_result.get("error") + ) + + def _route_B_wilderness_both( + self, + start_lat: float, start_lon: float, + end_lat: float, end_lon: float, + mode: str, boundary_mode: str + ) -> Dict: + """ + Scenario B: Off-network start → off-network end. + Wilderness from start to entry_A, Valhalla entry_A to entry_B, wilderness from entry_B to end. + """ + t0 = time.time() + + if not self.entry_index.table_exists() or self.entry_index.get_entry_point_count() == 0: + return { + "status": "error", + "message": "Trail entry point index not built. Run build_entry_index() first." + } + + valid_highways = MODE_TO_VALID_HIGHWAYS.get(mode) + MAX_ENTRY_POINTS = 10 + + # Find entry points near START + entry_points_start = self.entry_index.query_radius( + start_lat, start_lon, DEFAULT_SEARCH_RADIUS_KM, valid_highways + ) + if not entry_points_start: + entry_points_start = self.entry_index.query_radius( + start_lat, start_lon, EXPANDED_SEARCH_RADIUS_KM, valid_highways + ) + if not entry_points_start: + return {"status": "error", "message": f"No entry points found near start within {EXPANDED_SEARCH_RADIUS_KM}km."} + entry_points_start = entry_points_start[:MAX_ENTRY_POINTS] + + # Find entry points near END + entry_points_end = self.entry_index.query_radius( + end_lat, end_lon, DEFAULT_SEARCH_RADIUS_KM, valid_highways + ) + if not entry_points_end: + entry_points_end = self.entry_index.query_radius( + end_lat, end_lon, EXPANDED_SEARCH_RADIUS_KM, valid_highways + ) + if not entry_points_end: + return {"status": "error", "message": f"No entry points found near destination within {EXPANDED_SEARCH_RADIUS_KM}km."} + entry_points_end = entry_points_end[:MAX_ENTRY_POINTS] + + # Phase 1: Wilderness pathfinding from START + wilderness_start_result = self._pathfind_wilderness( + start_lat, start_lon, end_lat, end_lon, + entry_points_start, boundary_mode, "start" + ) + + if wilderness_start_result.get("status") == "error": + return wilderness_start_result + + wilderness_start_coords = wilderness_start_result["coords"] + wilderness_start_stats = wilderness_start_result["stats"] + entry_A = wilderness_start_result["entry_point"] + + # Phase 2: Wilderness pathfinding from END (run after freeing phase 1 memory) + wilderness_end_result = self._pathfind_wilderness( + end_lat, end_lon, start_lat, start_lon, + entry_points_end, boundary_mode, "end" + ) + + if wilderness_end_result.get("status") == "error": + return wilderness_end_result + + # Reverse the end wilderness path (it's end→entry, we want entry→end for display) + wilderness_end_coords = list(reversed(wilderness_end_result["coords"])) + wilderness_end_stats = wilderness_end_result["stats"] + entry_B = wilderness_end_result["entry_point"] + + # Phase 3: Valhalla from entry_A to entry_B + network_result = self._valhalla_route( + entry_A["lat"], entry_A["lon"], + entry_B["lat"], entry_B["lon"], + mode + ) + + # Build response + return self._build_response( + wilderness_start=wilderness_start_coords, + wilderness_start_stats=wilderness_start_stats, + network_segment=network_result.get("segment"), + wilderness_end=wilderness_end_coords, + wilderness_end_stats=wilderness_end_stats, + mode=mode, + boundary_mode=boundary_mode, + entry_start=entry_A, + entry_end=entry_B, + scenario="B", + t0=t0, + valhalla_error=network_result.get("error") + ) + + def _pathfind_wilderness( + self, + origin_lat: float, origin_lon: float, + dest_lat: float, dest_lon: float, + entry_points: List[Dict], + boundary_mode: str, + label: str + ) -> Dict: + """ + Run MCP wilderness pathfinding from origin toward entry points. + + Args: + origin_lat, origin_lon: Starting point for pathfinding + dest_lat, dest_lon: Ultimate destination (for bbox calculation) + entry_points: List of candidate entry points + boundary_mode: How to handle barriers + label: "start" or "end" for error messages + + Returns: + {"status": "ok", "coords": [...], "stats": {...}, "entry_point": {...}} + or {"status": "error", "message": "..."} + """ + # Build bbox - only include origin and entry points, NOT distant destination + # The destination is handled by Valhalla, wilderness only needs to reach entry points + MAX_BBOX_DEGREES = 2.0 + all_lats = [origin_lat] + [p["lat"] for p in entry_points] + all_lons = [origin_lon] + [p["lon"] for p in entry_points] + + padding = 0.05 + bbox = { + "south": min(all_lats) - padding, + "north": max(all_lats) + padding, + "west": min(all_lons) - padding, + "east": max(all_lons) + padding, + } + + # Clamp bbox size, centering on origin + lat_span = bbox["north"] - bbox["south"] + lon_span = bbox["east"] - bbox["west"] + if lat_span > MAX_BBOX_DEGREES or lon_span > MAX_BBOX_DEGREES: + half_span = MAX_BBOX_DEGREES / 2 + bbox = { + "south": origin_lat - half_span, + "north": origin_lat + half_span, + "west": origin_lon - half_span, + "east": origin_lon + half_span, + } + + # Initialize readers + self._init_readers() + + # Load elevation + try: + elevation, meta = self.dem_reader.get_elevation_grid( + south=bbox["south"], north=bbox["north"], + west=bbox["west"], east=bbox["east"], + ) + except Exception as e: + return {"status": "error", "message": f"Failed to load elevation for {label}: {e}"} + + # Check memory + mem = check_memory_usage() + if mem > MEMORY_LIMIT_GB: + return {"status": "error", "message": f"Memory limit exceeded: {mem:.1f}GB > {MEMORY_LIMIT_GB}GB"} + + # Load friction + friction_raw = self.friction_reader.get_friction_grid( + south=bbox["south"], north=bbox["north"], + west=bbox["west"], east=bbox["east"], + target_shape=elevation.shape + ) + friction_mult = friction_to_multiplier(friction_raw) + + # Load barriers + barriers = self.barrier_reader.get_barrier_grid( + south=bbox["south"], north=bbox["north"], + west=bbox["west"], east=bbox["east"], + target_shape=elevation.shape + ) + + # Load trails + trails = self.trail_reader.get_trails_grid( + south=bbox["south"], north=bbox["north"], + west=bbox["west"], east=bbox["east"], + target_shape=elevation.shape + ) + + # Compute cost grid (ALWAYS foot mode for wilderness) + cost = compute_cost_grid( + elevation, + cell_size_m=meta["cell_size_m"], + friction=friction_mult, + friction_raw=friction_raw, + trails=trails, + barriers=barriers, + wilderness=None, + mvum=None, + boundary_mode=boundary_mode, + mode="foot", + ) + + # Free intermediate arrays + del friction_mult, friction_raw + gc.collect() + + # Convert origin to pixel coordinates + origin_row, origin_col = self.dem_reader.latlon_to_pixel(origin_lat, origin_lon, meta) + + rows, cols = elevation.shape + if not (0 <= origin_row < rows and 0 <= origin_col < cols): + return {"status": "error", "message": f"{label.capitalize()} point outside grid bounds"} + + # Map entry points to pixels + entry_pixels = [] + for ep in entry_points: + row, col = self.dem_reader.latlon_to_pixel(ep["lat"], ep["lon"], meta) + if 0 <= row < rows and 0 <= col < cols: + entry_pixels.append({"row": row, "col": col, "entry_point": ep}) + + if not entry_pixels: + return {"status": "error", "message": f"No entry points map to grid bounds for {label}"} + + # Run MCP + mcp = MCP_Geometric(cost, fully_connected=True) + cumulative_costs, traceback = mcp.find_costs([(origin_row, origin_col)]) + + # Find nearest reachable entry point + best_entry = None + best_cost = np.inf + + for ep in entry_pixels: + ep_cost = cumulative_costs[ep["row"], ep["col"]] + if ep_cost < best_cost: + best_cost = ep_cost + best_entry = ep + + if best_entry is None or np.isinf(best_cost): + return { + "status": "error", + "message": f"No path found from {label} to any entry point (blocked by impassable terrain)" + } + + # Traceback path + path_indices = mcp.traceback((best_entry["row"], best_entry["col"])) + + # Convert to coordinates and collect stats + coords = [] + elevations = [] + trail_values = [] + barrier_crossings = 0 + + for row, col in path_indices: + lat, lon = self.dem_reader.pixel_to_latlon(row, col, meta) + coords.append([lon, lat]) + elevations.append(elevation[row, col]) + trail_values.append(trails[row, col]) + if barriers[row, col] == 255: + barrier_crossings += 1 + + # Calculate distance + distance_m = 0 + for i in range(1, len(coords)): + lon1, lat1 = coords[i-1] + lon2, lat2 = coords[i] + distance_m += haversine_distance(lat1, lon1, lat2, lon2) + + # Elevation stats + elev_arr = np.array(elevations) + elev_diff = np.diff(elev_arr) + elev_gain = float(np.sum(elev_diff[elev_diff > 0])) + elev_loss = float(np.sum(np.abs(elev_diff[elev_diff < 0]))) + + # Trail stats + trail_arr = np.array(trail_values) + on_trail_cells = np.sum(trail_arr > 0) + total_cells = len(trail_arr) + on_trail_pct = float(100 * on_trail_cells / total_cells) if total_cells > 0 else 0 + + # Free memory + del mcp, cumulative_costs, traceback, cost, trails, barriers, elevation + gc.collect() + + return { + "status": "ok", + "coords": coords, + "stats": { + "distance_km": distance_m / 1000, + "effort_minutes": best_cost / 60, + "elevation_gain_m": elev_gain, + "elevation_loss_m": elev_loss, + "on_trail_pct": on_trail_pct, + "barrier_crossings": barrier_crossings, + "cell_count": total_cells, + }, + "entry_point": best_entry["entry_point"] + } + + def _valhalla_route( + self, + start_lat: float, start_lon: float, + end_lat: float, end_lon: float, + mode: str + ) -> Dict: + """ + Call Valhalla for network routing. + + Returns: + {"segment": {...}, "error": None} on success + {"segment": None, "error": "..."} on failure + """ + costing = MODE_TO_COSTING.get(mode, "pedestrian") + + valhalla_request = { + "locations": [ + {"lat": start_lat, "lon": start_lon}, + {"lat": end_lat, "lon": end_lon} + ], + "costing": costing, + "directions_options": {"units": "kilometers"} + } + + try: + resp = requests.post(f"{VALHALLA_URL}/route", json=valhalla_request, timeout=30) + + if resp.status_code == 200: + valhalla_data = resp.json() + trip = valhalla_data.get("trip", {}) + legs = trip.get("legs", []) + + if legs: + leg = legs[0] + shape = leg.get("shape", "") + coords = self._decode_polyline(shape) + + maneuvers = [] + for m in leg.get("maneuvers", []): + maneuvers.append({ + "instruction": m.get("instruction", ""), + "type": m.get("type", 0), + "distance_km": m.get("length", 0), + "time_seconds": m.get("time", 0), + "street_names": m.get("street_names", []), + }) + + summary = trip.get("summary", {}) + return { + "segment": { + "coordinates": coords, + "distance_km": summary.get("length", 0), + "duration_minutes": summary.get("time", 0) / 60, + "maneuvers": maneuvers, + }, + "error": None + } + + return {"segment": None, "error": f"Valhalla returned {resp.status_code}: {resp.text[:200]}"} + + except Exception as e: + return {"segment": None, "error": f"Valhalla request failed: {e}"} + + def _build_response( + self, + wilderness_start: Optional[List], + wilderness_start_stats: Optional[Dict], + network_segment: Optional[Dict], + wilderness_end: Optional[List], + wilderness_end_stats: Optional[Dict], + mode: str, + boundary_mode: str, + entry_start: Optional[Dict], + entry_end: Optional[Dict], + scenario: str, + t0: float, + valhalla_error: Optional[str] + ) -> Dict: + """Build the final GeoJSON response.""" + features = [] + + # Wilderness start segment + if wilderness_start and wilderness_start_stats: + features.append({ + "type": "Feature", + "properties": { + "segment_type": "wilderness", + "segment_position": "start", + "effort_minutes": float(wilderness_start_stats["effort_minutes"]), + "distance_km": float(wilderness_start_stats["distance_km"]), + "elevation_gain_m": wilderness_start_stats["elevation_gain_m"], + "elevation_loss_m": wilderness_start_stats["elevation_loss_m"], + "boundary_mode": boundary_mode, + "on_trail_pct": wilderness_start_stats["on_trail_pct"], + "barrier_crossings": wilderness_start_stats["barrier_crossings"], + "wilderness_mode": "foot", + }, + "geometry": {"type": "LineString", "coordinates": wilderness_start} + }) + + # Network segment + if network_segment: + features.append({ + "type": "Feature", + "properties": { + "segment_type": "network", + "distance_km": network_segment["distance_km"], + "duration_minutes": network_segment["duration_minutes"], + "maneuvers": network_segment["maneuvers"], + "network_mode": mode, + }, + "geometry": {"type": "LineString", "coordinates": network_segment["coordinates"]} + }) + + # Wilderness end segment + if wilderness_end and wilderness_end_stats: + features.append({ + "type": "Feature", + "properties": { + "segment_type": "wilderness", + "segment_position": "end", + "effort_minutes": float(wilderness_end_stats["effort_minutes"]), + "distance_km": float(wilderness_end_stats["distance_km"]), + "elevation_gain_m": wilderness_end_stats["elevation_gain_m"], + "elevation_loss_m": wilderness_end_stats["elevation_loss_m"], + "boundary_mode": boundary_mode, + "on_trail_pct": wilderness_end_stats["on_trail_pct"], + "barrier_crossings": wilderness_end_stats["barrier_crossings"], + "wilderness_mode": "foot", + }, + "geometry": {"type": "LineString", "coordinates": wilderness_end} + }) + + # Combined path + combined_coords = [] + if wilderness_start: + combined_coords.extend(wilderness_start) + if network_segment: + # Skip first coord if we already have wilderness_start (avoid duplicate) + start_idx = 1 if wilderness_start else 0 + combined_coords.extend(network_segment["coordinates"][start_idx:]) + if wilderness_end: + # Skip first coord (avoid duplicate with network end) + start_idx = 1 if (wilderness_start or network_segment) else 0 + combined_coords.extend(wilderness_end[start_idx:]) + + if combined_coords: + features.append({ + "type": "Feature", + "properties": { + "segment_type": "combined", + "wilderness_mode": "foot", + "network_mode": mode, + "boundary_mode": boundary_mode, + "scenario": scenario, + }, + "geometry": {"type": "LineString", "coordinates": combined_coords} + }) + + geojson = {"type": "FeatureCollection", "features": features} + + # Calculate totals + total_distance_km = 0.0 + total_effort_minutes = 0.0 + wilderness_distance_km = 0.0 + wilderness_effort_minutes = 0.0 + network_distance_km = 0.0 + network_duration_minutes = 0.0 + barrier_crossings = 0 + on_trail_pct = 0.0 + + if wilderness_start_stats: + wilderness_distance_km += wilderness_start_stats["distance_km"] + wilderness_effort_minutes += wilderness_start_stats["effort_minutes"] + barrier_crossings += wilderness_start_stats["barrier_crossings"] + on_trail_pct = wilderness_start_stats["on_trail_pct"] + + if wilderness_end_stats: + wilderness_distance_km += wilderness_end_stats["distance_km"] + wilderness_effort_minutes += wilderness_end_stats["effort_minutes"] + barrier_crossings += wilderness_end_stats["barrier_crossings"] + # Average on-trail percentage if we have both + if wilderness_start_stats: + on_trail_pct = (on_trail_pct + wilderness_end_stats["on_trail_pct"]) / 2 + else: + on_trail_pct = wilderness_end_stats["on_trail_pct"] + + if network_segment: + network_distance_km = network_segment["distance_km"] + network_duration_minutes = network_segment["duration_minutes"] + + total_distance_km = wilderness_distance_km + network_distance_km + total_effort_minutes = wilderness_effort_minutes + network_duration_minutes + + summary = { + "total_distance_km": float(total_distance_km), + "total_effort_minutes": float(total_effort_minutes), + "wilderness_distance_km": float(wilderness_distance_km), + "wilderness_effort_minutes": float(wilderness_effort_minutes), + "network_distance_km": float(network_distance_km), + "network_duration_minutes": float(network_duration_minutes), + "on_trail_pct": float(on_trail_pct), + "barrier_crossings": barrier_crossings, + "boundary_mode": boundary_mode, + "wilderness_mode": "foot", + "network_mode": mode, + "scenario": scenario, + "computation_time_s": time.time() - t0, + } + + if entry_start: + summary["entry_point_start"] = { + "lat": entry_start["lat"], + "lon": entry_start["lon"], + "highway_class": entry_start["highway_class"], + "name": entry_start.get("name", ""), + } + + if entry_end: + summary["entry_point_end"] = { + "lat": entry_end["lat"], + "lon": entry_end["lon"], + "highway_class": entry_end["highway_class"], + "name": entry_end.get("name", ""), + } + + result = {"status": "ok", "route": geojson, "summary": summary} + + if valhalla_error: + result["warning"] = f"Network segment incomplete: {valhalla_error}" + + return result + + def _decode_polyline(self, encoded: str, precision: int = 6) -> List[List[float]]: + """Decode a polyline string into coordinates [lon, lat].""" + coords = [] + index = 0 + lat = 0 + lon = 0 + + while index < len(encoded): + shift = 0 + result = 0 + while True: + b = ord(encoded[index]) - 63 + index += 1 + result |= (b & 0x1f) << shift + shift += 5 + if b < 0x20: + break + dlat = ~(result >> 1) if result & 1 else result >> 1 + lat += dlat + + shift = 0 + result = 0 + while True: + b = ord(encoded[index]) - 63 + index += 1 + result |= (b & 0x1f) << shift + shift += 5 + if b < 0x20: + break + dlon = ~(result >> 1) if result & 1 else result >> 1 + lon += dlon + + coords.append([lon / (10 ** precision), lat / (10 ** precision)]) + + return coords + + def close(self): + """Close all readers.""" + if self.dem_reader: + self.dem_reader.close() + if self.friction_reader: + self.friction_reader.close() + if self.barrier_reader: + self.barrier_reader.close() + if self.wilderness_reader: + self.wilderness_reader.close() + if self.trail_reader: + self.trail_reader.close() + self.entry_index.close() + + +def build_entry_index(): + """Build the trail entry point index.""" + index = EntryPointIndex() + stats = index.build_index() + index.close() + return stats + + +if __name__ == "__main__": + import sys + + if len(sys.argv) > 1 and sys.argv[1] == "build": + print("Building trail entry point index...") + stats = build_entry_index() + print(f"\nDone. Total entry points: {stats['total']}") + + elif len(sys.argv) > 1 and sys.argv[1] == "test": + print("Testing router (all scenarios)...") + print("=" * 60) + + router = OffrouteRouter() + + # Test points + wilderness_start = (44.0543, -115.4237) # Off-network + wilderness_end = (45.2, -115.5) # Deep wilderness (Frank Church) + road_start = (43.6150, -116.2023) # Boise downtown (on-network) + road_end = (43.5867, -116.5625) # Nampa (on-network) + + tests = [ + ("A: wilderness→road", wilderness_start, (44.0814, -115.5021)), + ("B: wilderness→wilderness", wilderness_start, wilderness_end), + ("C: road→wilderness", road_start, wilderness_start), + ("D: road→road", road_start, road_end), + ] + + for label, (slat, slon), (elat, elon) in tests: + print(f"\n{label}") + print("-" * 40) + + result = router.route( + start_lat=slat, start_lon=slon, + end_lat=elat, end_lon=elon, + mode="foot", boundary_mode="pragmatic" + ) + + if result["status"] == "ok": + s = result["summary"] + print(f" Scenario: {s.get('scenario', '?')}") + print(f" Total: {s['total_distance_km']:.2f} km, {s['total_effort_minutes']:.1f} min") + print(f" Wilderness: {s['wilderness_distance_km']:.2f} km") + print(f" Network: {s['network_distance_km']:.2f} km") + if s.get('entry_point_start'): + ep = s['entry_point_start'] + print(f" Entry (start): {ep['highway_class']} at {ep['lat']:.4f}, {ep['lon']:.4f}") + if s.get('entry_point_end'): + ep = s['entry_point_end'] + print(f" Entry (end): {ep['highway_class']} at {ep['lat']:.4f}, {ep['lon']:.4f}") + else: + print(f" ERROR: {result['message']}") + + router.close() + + else: + print("Usage:") + print(" python router.py build # Build entry point index") + print(" python router.py test # Test all scenarios") From d8f84ab55aa8f6e910ed6f2a52aae2032a70c91b Mon Sep 17 00:00:00 2001 From: Matt Date: Sat, 9 May 2026 03:34:37 +0000 Subject: [PATCH 17/41] offroute: revert off-network threshold to 10m Co-Authored-By: Claude Opus 4.5 --- lib/offroute/router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/offroute/router.py b/lib/offroute/router.py index 4b988ab..bdb7894 100644 --- a/lib/offroute/router.py +++ b/lib/offroute/router.py @@ -61,7 +61,7 @@ EXPANDED_SEARCH_RADIUS_KM = 100 MEMORY_LIMIT_GB = 12 # Off-network detection threshold (meters) -OFF_NETWORK_THRESHOLD_M = 50 +OFF_NETWORK_THRESHOLD_M = 10 # Mode to Valhalla costing mapping MODE_TO_COSTING = { From a04c10ad556ba5e5d915c402bd003a6e5d6db609 Mon Sep 17 00:00:00 2001 From: Matt Date: Sat, 9 May 2026 05:05:00 +0000 Subject: [PATCH 18/41] offroute: wilderness maneuvers with bearing, elevation, grade MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Segment breaks on: bearing change >30°, grade category change, distance >0.5mi - Grade categories: flat (0-2°), gentle (2-5°), moderate (5-10°), steep (10-15°), very steep (15°+) - Distance formatting: feet with commas <1mi, miles with decimal ≥1mi - Instruction format: Head {cardinal}, gaining/descending X ft ({grade} uphill/downhill) — {dist} Co-Authored-By: Claude --- lib/offroute/router.py | 209 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 209 insertions(+) diff --git a/lib/offroute/router.py b/lib/offroute/router.py index bdb7894..bd3d379 100644 --- a/lib/offroute/router.py +++ b/lib/offroute/router.py @@ -728,6 +728,7 @@ class OffrouteRouter: # Extract results wilderness_coords = wilderness_result["coords"] wilderness_stats = wilderness_result["stats"] + wilderness_elevations = wilderness_result.get("elevations", []) best_entry = wilderness_result["entry_point"] entry_lat = best_entry["lat"] @@ -740,9 +741,11 @@ class OffrouteRouter: return self._build_response( wilderness_start=wilderness_coords, wilderness_start_stats=wilderness_stats, + wilderness_start_elevations=wilderness_elevations, network_segment=network_result.get("segment"), wilderness_end=None, wilderness_end_stats=None, + wilderness_end_elevations=None, mode=mode, boundary_mode=boundary_mode, entry_start=best_entry, @@ -805,6 +808,7 @@ class OffrouteRouter: # The path is from end→entry, reverse it for display (entry→end) wilderness_coords = list(reversed(wilderness_result["coords"])) wilderness_stats = wilderness_result["stats"] + wilderness_elevations = list(reversed(wilderness_result.get("elevations", []))) best_entry = wilderness_result["entry_point"] entry_lat = best_entry["lat"] @@ -817,9 +821,11 @@ class OffrouteRouter: return self._build_response( wilderness_start=None, wilderness_start_stats=None, + wilderness_start_elevations=None, network_segment=network_result.get("segment"), wilderness_end=wilderness_coords, wilderness_end_stats=wilderness_stats, + wilderness_end_elevations=wilderness_elevations, mode=mode, boundary_mode=boundary_mode, entry_start=None, @@ -885,6 +891,7 @@ class OffrouteRouter: wilderness_start_coords = wilderness_start_result["coords"] wilderness_start_stats = wilderness_start_result["stats"] + wilderness_start_elevations = wilderness_start_result.get("elevations", []) entry_A = wilderness_start_result["entry_point"] # Phase 2: Wilderness pathfinding from END (run after freeing phase 1 memory) @@ -899,6 +906,7 @@ class OffrouteRouter: # Reverse the end wilderness path (it's end→entry, we want entry→end for display) wilderness_end_coords = list(reversed(wilderness_end_result["coords"])) wilderness_end_stats = wilderness_end_result["stats"] + wilderness_end_elevations = list(reversed(wilderness_end_result.get("elevations", []))) entry_B = wilderness_end_result["entry_point"] # Phase 3: Valhalla from entry_A to entry_B @@ -912,9 +920,11 @@ class OffrouteRouter: return self._build_response( wilderness_start=wilderness_start_coords, wilderness_start_stats=wilderness_start_stats, + wilderness_start_elevations=wilderness_start_elevations, network_segment=network_result.get("segment"), wilderness_end=wilderness_end_coords, wilderness_end_stats=wilderness_end_stats, + wilderness_end_elevations=wilderness_end_elevations, mode=mode, boundary_mode=boundary_mode, entry_start=entry_A, @@ -1109,6 +1119,7 @@ class OffrouteRouter: return { "status": "ok", "coords": coords, + "elevations": elevations, # Raw elevation values for maneuver generation "stats": { "distance_km": distance_m / 1000, "effort_minutes": best_cost / 60, @@ -1184,13 +1195,199 @@ class OffrouteRouter: except Exception as e: return {"segment": None, "error": f"Valhalla request failed: {e}"} + def _generate_wilderness_maneuvers( + self, + coords: List[List[float]], + elevations: List[float], + position: str = "start" + ) -> List[Dict]: + """ + Generate turn-by-turn maneuvers for a wilderness segment. + + Segment breaks occur when: + - Bearing changes more than 30° from segment start + - Grade category changes (flat→steep etc) + - Distance exceeds 0.5 miles without a break + + Args: + coords: [[lon, lat], ...] coordinate list + elevations: Elevation values (meters) for each coord + position: "start" or "end" for labeling + + Returns: + List of maneuver dicts with instruction, distance, elevation, grade, bearing + """ + if not coords or len(coords) < 2: + return [] + + # Constants + COMPASS = ["N", "NNE", "NE", "ENE", "E", "ESE", "SE", "SSE", + "S", "SSW", "SW", "WSW", "W", "WNW", "NW", "NNW"] + MAX_SEGMENT_M = 804.672 # 0.5 miles in meters + BEARING_THRESHOLD = 30 # degrees + M_TO_FT = 3.28084 + M_TO_MI = 0.000621371 + + def get_bearing(lat1, lon1, lat2, lon2): + """Calculate bearing between two points (degrees 0-360).""" + dlon = math.radians(lon2 - lon1) + lat1_r, lat2_r = math.radians(lat1), math.radians(lat2) + x = math.sin(dlon) * math.cos(lat2_r) + y = math.cos(lat1_r) * math.sin(lat2_r) - math.sin(lat1_r) * math.cos(lat2_r) * math.cos(dlon) + return (math.degrees(math.atan2(x, y)) + 360) % 360 + + def bearing_to_cardinal(bearing): + """Convert bearing to 16-point compass direction.""" + return COMPASS[round(bearing / 22.5) % 16] + + def get_grade_category(grade_deg): + """Categorize grade angle: flat (0-2°), gentle (2-5°), moderate (5-10°), steep (10-15°), very steep (15°+).""" + grade_abs = abs(grade_deg) + if grade_abs < 2: + return "flat" + elif grade_abs < 5: + return "gentle" + elif grade_abs < 10: + return "moderate" + elif grade_abs < 15: + return "steep" + else: + return "very steep" + + def format_distance(meters): + """Format distance: feet with commas if under 1 mile, miles with one decimal if over.""" + miles = meters * M_TO_MI + if miles < 1.0: + feet = round(meters * M_TO_FT) + return f"{feet:,} ft" + else: + return f"{miles:.1f} mi" + + def build_instruction(cardinal, gain_ft, loss_ft, grade_cat, distance_m): + """Build instruction string per spec.""" + dist_str = format_distance(distance_m) + if grade_cat == "flat": + return f"Head {cardinal} on level ground — {dist_str}" + elif gain_ft > loss_ft: + return f"Head {cardinal}, gaining {gain_ft:,} ft ({grade_cat} uphill) — {dist_str}" + else: + return f"Head {cardinal}, descending {loss_ft:,} ft ({grade_cat} downhill) — {dist_str}" + + maneuvers = [] + i = 0 + + while i < len(coords) - 1: + seg_start_idx = i + seg_start_lon, seg_start_lat = coords[i] + seg_start_elev = elevations[i] if i < len(elevations) else 0 + + # Initial bearing for this segment + next_lon, next_lat = coords[i + 1] + seg_bearing = get_bearing(seg_start_lat, seg_start_lon, next_lat, next_lon) + + # Accumulate elevation changes within segment + seg_distance_m = 0 + seg_elev_gain = 0 + seg_elev_loss = 0 + prev_elev = seg_start_elev + + # Calculate initial grade category + step_dist = haversine_distance(seg_start_lat, seg_start_lon, next_lat, next_lon) + step_elev_change = (elevations[i + 1] if i + 1 < len(elevations) else seg_start_elev) - seg_start_elev + initial_grade = math.degrees(math.atan(step_elev_change / step_dist)) if step_dist > 0 else 0 + seg_grade_cat = get_grade_category(initial_grade) + + j = i + while j < len(coords) - 1: + lon1, lat1 = coords[j] + lon2, lat2 = coords[j + 1] + elev1 = elevations[j] if j < len(elevations) else prev_elev + elev2 = elevations[j + 1] if j + 1 < len(elevations) else elev1 + + step_dist = haversine_distance(lat1, lon1, lat2, lon2) + step_bearing = get_bearing(lat1, lon1, lat2, lon2) + step_elev_change = elev2 - elev1 + step_grade = math.degrees(math.atan(step_elev_change / step_dist)) if step_dist > 0 else 0 + step_grade_cat = get_grade_category(step_grade) + + # Check break conditions + bearing_diff = abs(step_bearing - seg_bearing) + if bearing_diff > 180: + bearing_diff = 360 - bearing_diff + + # Break if: bearing changed >30°, grade category changed, or distance >0.5mi + if seg_distance_m > 0: # Don't break on first step + if bearing_diff > BEARING_THRESHOLD: + break + if step_grade_cat != seg_grade_cat: + break + if seg_distance_m >= MAX_SEGMENT_M: + break + + # Accumulate + seg_distance_m += step_dist + if step_elev_change > 0: + seg_elev_gain += step_elev_change + else: + seg_elev_loss += abs(step_elev_change) + prev_elev = elev2 + j += 1 + + # Compute segment stats + seg_end_idx = j + gain_ft = round(seg_elev_gain * M_TO_FT) + loss_ft = round(seg_elev_loss * M_TO_FT) + + # Net elevation change for grade calculation + net_elev_change = seg_elev_gain - seg_elev_loss + grade_deg = math.degrees(math.atan(net_elev_change / seg_distance_m)) if seg_distance_m > 0 else 0 + grade_cat = get_grade_category(grade_deg) + + cardinal = bearing_to_cardinal(seg_bearing) + instruction = build_instruction(cardinal, gain_ft, loss_ft, grade_cat, seg_distance_m) + + maneuvers.append({ + "instruction": instruction, + "type": "wilderness", + "distance_m": round(seg_distance_m, 1), + "elevation_gain_ft": gain_ft, + "elevation_loss_ft": loss_ft, + "grade_degrees": round(grade_deg, 1), + "grade_category": grade_cat, + "bearing": round(seg_bearing, 1), + "cardinal": cardinal, + }) + + i = seg_end_idx + + # Add arrival maneuver + arrival_text = "Arrive at trail/road" if position == "start" else "Arrive at destination" + last_bearing = maneuvers[-1]["bearing"] if maneuvers else 0 + last_cardinal = maneuvers[-1]["cardinal"] if maneuvers else "N" + + maneuvers.append({ + "instruction": arrival_text, + "type": "arrival", + "distance_m": 0, + "elevation_gain_ft": 0, + "elevation_loss_ft": 0, + "grade_degrees": 0, + "grade_category": "flat", + "bearing": last_bearing, + "cardinal": last_cardinal, + }) + + return maneuvers + def _build_response( self, wilderness_start: Optional[List], wilderness_start_stats: Optional[Dict], + wilderness_start_elevations: Optional[List], network_segment: Optional[Dict], wilderness_end: Optional[List], wilderness_end_stats: Optional[Dict], + wilderness_end_elevations: Optional[List], mode: str, boundary_mode: str, entry_start: Optional[Dict], @@ -1204,6 +1401,11 @@ class OffrouteRouter: # Wilderness start segment if wilderness_start and wilderness_start_stats: + wild_start_maneuvers = [] + if wilderness_start_elevations: + wild_start_maneuvers = self._generate_wilderness_maneuvers( + wilderness_start, wilderness_start_elevations, position="start" + ) features.append({ "type": "Feature", "properties": { @@ -1217,6 +1419,7 @@ class OffrouteRouter: "on_trail_pct": wilderness_start_stats["on_trail_pct"], "barrier_crossings": wilderness_start_stats["barrier_crossings"], "wilderness_mode": "foot", + "maneuvers": wild_start_maneuvers, }, "geometry": {"type": "LineString", "coordinates": wilderness_start} }) @@ -1237,6 +1440,11 @@ class OffrouteRouter: # Wilderness end segment if wilderness_end and wilderness_end_stats: + wild_end_maneuvers = [] + if wilderness_end_elevations: + wild_end_maneuvers = self._generate_wilderness_maneuvers( + wilderness_end, wilderness_end_elevations, position="end" + ) features.append({ "type": "Feature", "properties": { @@ -1250,6 +1458,7 @@ class OffrouteRouter: "on_trail_pct": wilderness_end_stats["on_trail_pct"], "barrier_crossings": wilderness_end_stats["barrier_crossings"], "wilderness_mode": "foot", + "maneuvers": wild_end_maneuvers, }, "geometry": {"type": "LineString", "coordinates": wilderness_end} }) From f276b957532117c73e9efbbb2bee3f2c6d9380bb Mon Sep 17 00:00:00 2001 From: Matt Date: Wed, 20 May 2026 05:33:45 +0000 Subject: [PATCH 19/41] Add /api/reverse// localhost-sourced enrichment bundle New geocode_bp sibling to the existing /api/reverse?lat=&lon= route (which is unchanged). Returns a flat 9-field bundle for the Central enrichment framework: name, city, county, state, country, postal_code (Photon), timezone (timezones.sqlite via R-tree + shapely), landclass (in-process lookup_landclass), elevation_m (Valhalla /height). - Each component lookup is independent and wrapped in try/except: a failure logs a warning and yields null, never a 5xx. 400 only on unparseable / out-of-range coordinates. - lat/lon parsed manually rather than via Flask , which rejects negative and integer coordinates and would 404 instead of 400. - 10k-entry / 24h TTLCache keyed on coords rounded to 4 decimals. - Tests mock Photon/Valhalla/landclass; one test exercises the real timezones.sqlite. cachetools pinned in requirements.txt. Co-Authored-By: Claude Opus 4.7 (1M context) --- lib/netsyms_api.py | 161 +++++++++++++++++++++++++++++++++++++ lib/reverse_bundle_test.py | 136 +++++++++++++++++++++++++++++++ requirements.txt | 1 + 3 files changed, 298 insertions(+) create mode 100644 lib/reverse_bundle_test.py diff --git a/lib/netsyms_api.py b/lib/netsyms_api.py index 4a0847f..e530d15 100644 --- a/lib/netsyms_api.py +++ b/lib/netsyms_api.py @@ -4,13 +4,19 @@ RECON Netsyms API + Geocode — Flask Blueprints. GET /api/netsyms/lookup?q=&country= GET /api/netsyms/health GET /api/geocode?q=&limit= (Photon-first search with ranked results) +GET /api/reverse// (localhost-sourced enrichment bundle for Central) """ +import sqlite3 +import threading + +from cachetools import TTLCache from flask import Blueprint, request, jsonify from . import netsyms from . import address_book from . import nav_tools +from .geocode import PHOTON_URL from .utils import setup_logging logger = setup_logging('recon.netsyms_api') @@ -124,3 +130,158 @@ def api_reverse(): results = _parse_photon_features(features, source='photon_reverse') return jsonify({'query': query_str, 'results': results, 'count': len(results)}) + + +# ───────────────────────────────────────────────────────────────────────── +# /api/reverse// — localhost-sourced enrichment bundle (Central) +# +# Sibling to the query-string /api/reverse above; that route is unchanged. +# Every component is sourced from localhost only (Photon, timezones.sqlite, +# in-process landclass/PostGIS, Valhalla). Each lookup is independent: a +# component failure logs a warning and yields null — never a 5xx. +# ───────────────────────────────────────────────────────────────────────── + +_TZ_DB_PATH = "/mnt/nav/sources/timezones.sqlite" +_VALHALLA_HEIGHT_URL = "http://localhost:8002/height" + +# Full bundle cache: key=(round(lat,4), round(lon,4)) -> dict. ~10k entries, 24h TTL. +_REVERSE_BUNDLE_CACHE = TTLCache(maxsize=10_000, ttl=86_400) +_REVERSE_BUNDLE_LOCK = threading.Lock() + +_BUNDLE_KEYS = ('name', 'city', 'county', 'state', 'country', + 'postal_code', 'timezone', 'landclass', 'elevation_m') + + +def _spatialite_blob_to_wkb(blob): + """Recover standard WKB from a SpatiaLite geometry BLOB. + + Layout: [00][endian][srid:4][mbr:32][7C][WKB body][FE]. The body omits the + leading byte-order marker, so we re-prepend it and drop the trailing 0xFE. + """ + return bytes([blob[1]]) + blob[39:-1] + + +def _reverse_photon(lat, lon): + """Nearest-feature admin fields from local Photon. Returns the six address + fields (any value may be None). Mirrors the existing /api/reverse call.""" + import requests as http_requests + resp = http_requests.get( + f"{PHOTON_URL}/reverse", + params={"lat": lat, "lon": lon, "limit": 1}, + timeout=10, + ) + resp.raise_for_status() + features = resp.json().get("features", []) + if not features: + return {} + props = features[0].get("properties", {}) + return { + "name": props.get("name"), + "city": props.get("city"), + "county": props.get("county"), + "state": props.get("state"), + "country": props.get("country"), + "postal_code": props.get("postcode"), + } + + +def _reverse_timezone(lat, lon): + """IANA tzid for the point from local timezones.sqlite (SpatiaLite tz_world). + + Uses the table's R-tree index for an MBR prefilter, then shapely + point-in-polygon on the few candidates. Returns None if unresolved. + """ + from shapely import wkb + from shapely.geometry import Point + con = sqlite3.connect(f"file:{_TZ_DB_PATH}?mode=ro", uri=True) + try: + cur = con.cursor() + cur.execute( + "SELECT pkid FROM idx_tz_world_geom " + "WHERE xmin<=? AND xmax>=? AND ymin<=? AND ymax>=?", + (lon, lon, lat, lat), + ) + candidates = [r[0] for r in cur.fetchall()] + if not candidates: + return None + pt = Point(lon, lat) + for pk in candidates: + row = cur.execute( + "SELECT tzid, geom FROM tz_world WHERE pk_uid=?", (pk,) + ).fetchone() + if row and wkb.loads(_spatialite_blob_to_wkb(row[1])).contains(pt): + return row[0] + return None + finally: + con.close() + + +def _reverse_landclass(lat, lon): + """Most-specific PAD-US land class for the point, looked up in-process. + Returns None when there is no coverage or landclass is unavailable.""" + from .landclass import lookup_landclass, format_summary + return format_summary(lookup_landclass(lat, lon)) + + +def _reverse_elevation(lat, lon): + """Elevation in metres from local Valhalla /height. None on failure.""" + import requests as http_requests + resp = http_requests.post( + _VALHALLA_HEIGHT_URL, + json={"shape": [{"lat": lat, "lon": lon}]}, + timeout=10, + ) + resp.raise_for_status() + heights = resp.json().get("height", []) + return heights[0] if heights else None + + +@geocode_bp.route('/api/reverse//') +def api_reverse_bundle(lat, lon): + """Localhost-sourced reverse-geocode enrichment bundle for Central. + + GET /api/reverse// + + Always returns 200 with EXACTLY these keys (any may be null): + name, city, county, state, country, postal_code, timezone, landclass, elevation_m + + lat/lon are parsed manually (not via Flask's converter, which + rejects negative and integer coordinates) so out-of-range or unparseable + input yields 400 per contract; 503 is reserved for catastrophic failure. + """ + try: + lat = float(lat) + lon = float(lon) + except (ValueError, TypeError): + return jsonify({'error': 'lat and lon must be numbers'}), 400 + if not (-90 <= lat <= 90) or not (-180 <= lon <= 180): + return jsonify({'error': 'lat must be -90..90, lon must be -180..180'}), 400 + + key = (round(lat, 4), round(lon, 4)) + with _REVERSE_BUNDLE_LOCK: + cached = _REVERSE_BUNDLE_CACHE.get(key) + if cached is not None: + return jsonify(cached) + + bundle = {k: None for k in _BUNDLE_KEYS} + + try: + bundle.update(_reverse_photon(lat, lon)) + except Exception: + logger.warning("reverse-bundle: Photon lookup failed for %s,%s", lat, lon) + try: + bundle['timezone'] = _reverse_timezone(lat, lon) + except Exception: + logger.warning("reverse-bundle: timezone lookup failed for %s,%s", lat, lon) + try: + bundle['landclass'] = _reverse_landclass(lat, lon) + except Exception: + logger.warning("reverse-bundle: landclass lookup failed for %s,%s", lat, lon) + try: + bundle['elevation_m'] = _reverse_elevation(lat, lon) + except Exception: + logger.warning("reverse-bundle: elevation lookup failed for %s,%s", lat, lon) + + with _REVERSE_BUNDLE_LOCK: + _REVERSE_BUNDLE_CACHE[key] = bundle + return jsonify(bundle) diff --git a/lib/reverse_bundle_test.py b/lib/reverse_bundle_test.py new file mode 100644 index 0000000..d825b71 --- /dev/null +++ b/lib/reverse_bundle_test.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Tests for the /api/reverse// enrichment bundle (lib.netsyms_api). + +Photon/Valhalla/landclass are mocked so the suite runs without live services; +one timezone test exercises the real SpatiaLite DB when it is present. Plain +asserts + a __main__ runner, matching the rest of lib/*_test.py. +""" + +import os +import sys +from unittest import mock + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from flask import Flask +from lib import netsyms_api + +EXPECTED_KEYS = {'name', 'city', 'county', 'state', 'country', + 'postal_code', 'timezone', 'landclass', 'elevation_m'} + + +def _client(): + app = Flask(__name__) + app.register_blueprint(netsyms_api.geocode_bp) + return app.test_client() + + +def _clear_cache(): + netsyms_api._REVERSE_BUNDLE_CACHE.clear() + + +def test_happy_path(): + _clear_cache() + with mock.patch.object(netsyms_api, '_reverse_photon', return_value={ + 'name': 'Where you are', 'city': 'Boise', 'county': 'Ada', + 'state': 'Idaho', 'country': 'United States', 'postal_code': '83701'}), \ + mock.patch.object(netsyms_api, '_reverse_timezone', return_value='America/Boise'), \ + mock.patch.object(netsyms_api, '_reverse_landclass', return_value='Boise National Forest'), \ + mock.patch.object(netsyms_api, '_reverse_elevation', return_value=824): + resp = _client().get('/api/reverse/43.6150/-116.2023') + assert resp.status_code == 200, resp.status_code + data = resp.get_json() + assert set(data.keys()) == EXPECTED_KEYS, data.keys() + assert data['city'] == 'Boise' and data['timezone'] == 'America/Boise' + assert data['landclass'] == 'Boise National Forest' and data['elevation_m'] == 824 + print(" PASS: happy path — all 9 fields populated, exact key set") + + +def test_negative_and_integer_coords_parse(): + # Regression: Flask's converter would 404 these; manual parse must not. + _clear_cache() + with mock.patch.object(netsyms_api, '_reverse_photon', return_value={}), \ + mock.patch.object(netsyms_api, '_reverse_timezone', return_value=None), \ + mock.patch.object(netsyms_api, '_reverse_landclass', return_value=None), \ + mock.patch.object(netsyms_api, '_reverse_elevation', return_value=None): + for path in ('/api/reverse/43.6/-116.2', '/api/reverse/43/-116'): + resp = _client().get(path) + assert resp.status_code == 200, f"{path} -> {resp.status_code}" + assert set(resp.get_json().keys()) == EXPECTED_KEYS + print(" PASS: negative and integer coordinates parse (200, not 404)") + + +def test_partial_failure_returns_200_with_nulls(): + _clear_cache() + with mock.patch.object(netsyms_api, '_reverse_photon', + side_effect=RuntimeError('photon down')), \ + mock.patch.object(netsyms_api, '_reverse_timezone', return_value='America/Boise'), \ + mock.patch.object(netsyms_api, '_reverse_landclass', + side_effect=RuntimeError('postgis down')), \ + mock.patch.object(netsyms_api, '_reverse_elevation', return_value=824): + resp = _client().get('/api/reverse/43.6150/-116.2023') + assert resp.status_code == 200, resp.status_code + data = resp.get_json() + assert set(data.keys()) == EXPECTED_KEYS + assert data['name'] is None and data['city'] is None # photon failed -> nulls + assert data['landclass'] is None # landclass failed -> null + assert data['timezone'] == 'America/Boise' and data['elevation_m'] == 824 + print(" PASS: per-component failure -> 200 with nulls, no 5xx") + + +def test_ocean_point_mostly_null(): + _clear_cache() + with mock.patch.object(netsyms_api, '_reverse_photon', return_value={}), \ + mock.patch.object(netsyms_api, '_reverse_timezone', return_value='Etc/GMT+2'), \ + mock.patch.object(netsyms_api, '_reverse_landclass', return_value=None), \ + mock.patch.object(netsyms_api, '_reverse_elevation', return_value=0): + resp = _client().get('/api/reverse/0.0/-30.0') + assert resp.status_code == 200, resp.status_code + data = resp.get_json() + assert set(data.keys()) == EXPECTED_KEYS + assert data['city'] is None and data['country'] is None and data['landclass'] is None + print(" PASS: ocean point -> 200, mostly null") + + +def test_invalid_input_400(): + _clear_cache() + client = _client() + for path in ('/api/reverse/9999/0', '/api/reverse/0/9999', '/api/reverse/abc/0'): + resp = client.get(path) + assert resp.status_code == 400, f"{path} -> {resp.status_code}" + print(" PASS: out-of-range / unparseable input -> 400") + + +def test_cache_hit_serves_without_recompute(): + _clear_cache() + with mock.patch.object(netsyms_api, '_reverse_photon', + return_value={'name': 'X'}) as m_photon, \ + mock.patch.object(netsyms_api, '_reverse_timezone', return_value=None), \ + mock.patch.object(netsyms_api, '_reverse_landclass', return_value=None), \ + mock.patch.object(netsyms_api, '_reverse_elevation', return_value=None): + client = _client() + client.get('/api/reverse/12.3456/-65.4321') + client.get('/api/reverse/12.3456/-65.4321') # same key (rounded) -> cached + assert m_photon.call_count == 1, f"expected 1 compute, got {m_photon.call_count}" + print(" PASS: second identical request served from cache (no recompute)") + + +def test_real_timezone_db(): + if not os.path.exists(netsyms_api._TZ_DB_PATH): + print(" SKIP: real timezone test (timezones.sqlite not present)") + return + assert netsyms_api._reverse_timezone(43.6150, -116.2023) == 'America/Boise' + assert netsyms_api._reverse_timezone(40.7128, -74.0060) == 'America/New_York' + print(" PASS: real timezones.sqlite point-in-polygon") + + +if __name__ == '__main__': + print("Running reverse-bundle tests...") + test_happy_path() + test_negative_and_integer_coords_parse() + test_partial_failure_returns_200_with_nulls() + test_ocean_point_mostly_null() + test_invalid_input_400() + test_cache_hit_serves_without_recompute() + test_real_timezone_db() + print("All tests passed.") diff --git a/requirements.txt b/requirements.txt index f643cd8..1da21bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ anyio==4.12.1 babel==2.18.0 beautifulsoup4==4.14.3 blinker==1.9.0 +cachetools==7.1.3 certifi==2026.1.4 cffi==2.0.0 charset-normalizer==3.4.4 From 3d2d69cd56504af44ea54062e6dd3ac048e09345 Mon Sep 17 00:00:00 2001 From: Matt Date: Wed, 20 May 2026 15:20:35 +0000 Subject: [PATCH 20/41] Switch /api/reverse// elevation source from Valhalla to planet-DEM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per OFFROUTE-ARCHITECTURE.md §9 ("planet-dem.pmtiles as single elevation source"). The bundle endpoint previously called Valhalla /height, which only has 48 Idaho HGT tiles; it now reads the planet-scale Terrarium PMTiles that already back the frontend hillshade and contours. - dem.py: add DEMReader.sample_point(lat, lon) — one z12 tile (LRU-cached), Web-Mercator pixel index, None outside the +/-85.05 pole cap or when untiled. - netsyms_api.py: module-level DEMReader singleton (lazy mmap, None if init fails); _reverse_elevation now calls _DEM.sample_point; drop the Valhalla HTTP call and _VALHALLA_HEIGHT_URL. - tests: DEM-mock and DEM-unavailable cases; EXPECTED_KEYS derives from _BUNDLE_KEYS. All 9 tests pass. Verified live: Boise 824m, London 8m, Tokyo 35m, Yosemite 2804m, pole -> None. Co-Authored-By: Claude Opus 4.7 (1M context) --- lib/netsyms_api.py | 30 ++++++++++++++++------------ lib/offroute/dem.py | 23 ++++++++++++++++++++- lib/reverse_bundle_test.py | 41 +++++++++++++++++++++++++++++++++++--- 3 files changed, 77 insertions(+), 17 deletions(-) diff --git a/lib/netsyms_api.py b/lib/netsyms_api.py index e530d15..d217eb0 100644 --- a/lib/netsyms_api.py +++ b/lib/netsyms_api.py @@ -17,6 +17,7 @@ from . import netsyms from . import address_book from . import nav_tools from .geocode import PHOTON_URL +from .offroute.dem import DEMReader from .utils import setup_logging logger = setup_logging('recon.netsyms_api') @@ -137,12 +138,11 @@ def api_reverse(): # # Sibling to the query-string /api/reverse above; that route is unchanged. # Every component is sourced from localhost only (Photon, timezones.sqlite, -# in-process landclass/PostGIS, Valhalla). Each lookup is independent: a -# component failure logs a warning and yields null — never a 5xx. +# in-process landclass/PostGIS, planet-DEM PMTiles). Each lookup is +# independent: a component failure logs a warning and yields null — never 5xx. # ───────────────────────────────────────────────────────────────────────── _TZ_DB_PATH = "/mnt/nav/sources/timezones.sqlite" -_VALHALLA_HEIGHT_URL = "http://localhost:8002/height" # Full bundle cache: key=(round(lat,4), round(lon,4)) -> dict. ~10k entries, 24h TTL. _REVERSE_BUNDLE_CACHE = TTLCache(maxsize=10_000, ttl=86_400) @@ -151,6 +151,14 @@ _REVERSE_BUNDLE_LOCK = threading.Lock() _BUNDLE_KEYS = ('name', 'city', 'county', 'state', 'country', 'postal_code', 'timezone', 'landclass', 'elevation_m') +# planet-DEM elevation source (single PMTiles, replaces Valhalla /height). +# Instantiated once at import; the underlying mmap is lazy. None if unavailable. +try: + _DEM = DEMReader() +except Exception as e: # pragma: no cover - depends on PMTiles availability + logger.warning("DEMReader unavailable, elevation will be null: %s", e) + _DEM = None + def _spatialite_blob_to_wkb(blob): """Recover standard WKB from a SpatiaLite geometry BLOB. @@ -224,16 +232,12 @@ def _reverse_landclass(lat, lon): def _reverse_elevation(lat, lon): - """Elevation in metres from local Valhalla /height. None on failure.""" - import requests as http_requests - resp = http_requests.post( - _VALHALLA_HEIGHT_URL, - json={"shape": [{"lat": lat, "lon": lon}]}, - timeout=10, - ) - resp.raise_for_status() - heights = resp.json().get("height", []) - return heights[0] if heights else None + """Elevation in metres from the planet-DEM PMTiles — the single elevation + source per OFFROUTE-ARCHITECTURE.md §9. None on failure, on untiled points + (e.g. true ocean), or if DEMReader could not be initialized at startup.""" + if _DEM is None: + return None + return _DEM.sample_point(lat, lon) @geocode_bp.route('/api/reverse//') diff --git a/lib/offroute/dem.py b/lib/offroute/dem.py index f715611..06cfcea 100644 --- a/lib/offroute/dem.py +++ b/lib/offroute/dem.py @@ -158,7 +158,28 @@ class DEMReader: } return elevation, metadata - + + def sample_point(self, lat: float, lon: float) -> Optional[float]: + """Return elevation in meters at a single point, or None if untiled. + + Reads one z12 Terrarium tile (LRU-cached) and indexes the matching + pixel. Sub-ms warm, ~15 ms cold per tile via NFS. Returns None when the + tile is absent (e.g. true ocean nodata) or lat is outside the + Web-Mercator pole cap (~+/-85.05 deg). + """ + if not -85.05112878 <= lat <= 85.05112878: + return None + n = 2 ** ZOOM_LEVEL + fx = (lon + 180.0) / 360.0 * n + fy = (1.0 - math.asinh(math.tan(math.radians(lat))) / math.pi) / 2.0 * n + tx, ty = int(fx), int(fy) + tile = self._decode_tile(ZOOM_LEVEL, tx, ty) + if tile is None: + return None + row = min(TILE_SIZE - 1, int((fy - ty) * TILE_SIZE)) + col = min(TILE_SIZE - 1, int((fx - tx) * TILE_SIZE)) + return float(tile[row, col]) + def pixel_to_latlon(self, row: int, col: int, metadata: dict) -> Tuple[float, float]: """Convert pixel coordinates to lat/lon.""" lat = metadata["origin_lat"] + row * metadata["pixel_size_lat"] diff --git a/lib/reverse_bundle_test.py b/lib/reverse_bundle_test.py index d825b71..6defd9e 100644 --- a/lib/reverse_bundle_test.py +++ b/lib/reverse_bundle_test.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Tests for the /api/reverse// enrichment bundle (lib.netsyms_api). -Photon/Valhalla/landclass are mocked so the suite runs without live services; +Photon/DEM/landclass are mocked so the suite runs without live services; one timezone test exercises the real SpatiaLite DB when it is present. Plain asserts + a __main__ runner, matching the rest of lib/*_test.py. """ @@ -15,8 +15,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from flask import Flask from lib import netsyms_api -EXPECTED_KEYS = {'name', 'city', 'county', 'state', 'country', - 'postal_code', 'timezone', 'landclass', 'elevation_m'} +EXPECTED_KEYS = set(netsyms_api._BUNDLE_KEYS) def _client(): @@ -124,6 +123,40 @@ def test_real_timezone_db(): print(" PASS: real timezones.sqlite point-in-polygon") +def test_elevation_from_dem_reader_mock(): + # elevation_m comes from DEMReader.sample_point (not Valhalla); other + # components stubbed to null so the bundle is hermetic. + _clear_cache() + fake_dem = mock.Mock() + fake_dem.sample_point.return_value = 824 + with mock.patch.object(netsyms_api, '_DEM', fake_dem), \ + mock.patch.object(netsyms_api, '_reverse_photon', return_value={}), \ + mock.patch.object(netsyms_api, '_reverse_timezone', return_value=None), \ + mock.patch.object(netsyms_api, '_reverse_landclass', return_value=None): + resp = _client().get('/api/reverse/43.6150/-116.2023') + assert resp.status_code == 200, resp.status_code + data = resp.get_json() + assert set(data.keys()) == EXPECTED_KEYS + assert data['elevation_m'] == 824, data['elevation_m'] + fake_dem.sample_point.assert_called_once() + print(" PASS: elevation_m sourced from DEMReader.sample_point") + + +def test_elevation_dem_unavailable(): + # DEMReader failed to init at startup (_DEM is None) -> elevation_m null, 200. + _clear_cache() + with mock.patch.object(netsyms_api, '_DEM', None), \ + mock.patch.object(netsyms_api, '_reverse_photon', return_value={}), \ + mock.patch.object(netsyms_api, '_reverse_timezone', return_value=None), \ + mock.patch.object(netsyms_api, '_reverse_landclass', return_value=None): + resp = _client().get('/api/reverse/43.6150/-116.2023') + assert resp.status_code == 200, resp.status_code + data = resp.get_json() + assert set(data.keys()) == EXPECTED_KEYS + assert data['elevation_m'] is None + print(" PASS: DEMReader unavailable -> elevation_m null, still 200") + + if __name__ == '__main__': print("Running reverse-bundle tests...") test_happy_path() @@ -133,4 +166,6 @@ if __name__ == '__main__': test_invalid_input_400() test_cache_hit_serves_without_recompute() test_real_timezone_db() + test_elevation_from_dem_reader_mock() + test_elevation_dem_unavailable() print("All tests passed.") From 484dfbd1e0da7e31fc7a5dcc5e1d8d7952e1ab23 Mon Sep 17 00:00:00 2001 From: Matt Date: Wed, 20 May 2026 16:34:14 +0000 Subject: [PATCH 21/41] landclass: filter antimeridian-wrapping PAD-US records 47 PAD-US units (Aleutian/Bering-Sea BOEM marine features, all is_valid=False) are stored as antimeridian-wrapping polygons whose bbox spans ~360 deg of longitude. Their invalid planar geometry forms latitude bands that ST_Intersects false-matches for non-US points (e.g. London/Germany at ~51N matched "Rat Islands" ogc_fid 3974). Fix: add `AND (ST_XMax(geom) - ST_XMin(geom)) < 60` to the lookup_landclass SELECT. No DB writes; two cheap ST_XMax/XMin evals on the already spatial-index-filtered result set. Verified live: total 651088 rows, filtered 651041 (exactly 47 excluded); Yosemite/Grand Canyon retained, London/Germany now empty. Co-Authored-By: Claude Opus 4.7 (1M context) --- lib/landclass.py | 3 +++ lib/landclass_test.py | 44 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 lib/landclass_test.py diff --git a/lib/landclass.py b/lib/landclass.py index f581994..7760cce 100644 --- a/lib/landclass.py +++ b/lib/landclass.py @@ -214,6 +214,9 @@ def lookup_landclass(lat, lon): des_tp, gap_sts, pub_access, category, gis_acres, state_nm FROM pad_units WHERE ST_Intersects(geom, ST_SetSRID(ST_MakePoint(%s, %s), 4326)) + -- exclude antimeridian-wrapping polygons: 47 BOEM marine artifacts + -- span ~360 deg longitude and false-match non-US points at their lat band + AND (ST_XMax(geom) - ST_XMin(geom)) < 60 ORDER BY gis_acres ASC LIMIT 10""", (lon, lat) diff --git a/lib/landclass_test.py b/lib/landclass_test.py new file mode 100644 index 0000000..cba8ca7 --- /dev/null +++ b/lib/landclass_test.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Tests for lib.landclass PAD-US lookups. + +Live-PostgreSQL regression test using the skip-if-not-available pattern +(matching test_real_timezone_db in reverse_bundle_test.py). Plain asserts + +a __main__ runner, matching the rest of lib/*_test.py. + +Note: lookup_landclass swallows DB errors and returns [] (it never raises), +so PG availability is probed via a known US point (Boise) rather than by +catching an exception. +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from lib import landclass + + +def test_landclass_no_antimeridian_false_match(): + # Yosemite doubles as the liveness probe: a point on real US public land. + # (lookup_landclass returns [] when PG is unreachable AND when the point is + # off public land, so the probe must be a known-public-land point — e.g. + # downtown Boise is private and would yield [] even with PG up.) + yosemite = landclass.lookup_landclass(37.85, -119.55) + if not yosemite: + print(" SKIP: live PG not available (Yosemite returned no rows)") + return + # Filter must NOT drop legitimate (non-wrapping) US units. + assert len(yosemite) >= 1, f"Yosemite should match >=1 PAD-US unit, got {len(yosemite)}" + + # London (51.5074 N) previously false-matched the antimeridian-wrapping + # 'Rat Islands' record (ogc_fid 3974, ~360 deg lon span). The < 60 deg + # filter must now drop it -> empty result. + london = landclass.lookup_landclass(51.5074, -0.1278) + assert london == [], f"London should match no PAD-US unit, got {[r.get('unit_name') for r in london]}" + print(" PASS: antimeridian filter drops London false-match, keeps Yosemite coverage") + + +if __name__ == '__main__': + print("Running landclass tests...") + test_landclass_no_antimeridian_false_match() + print("All tests passed.") From f67f4ec9e3796caa3cde75c0e9571e3350ed6e64 Mon Sep 17 00:00:00 2001 From: Matt Date: Thu, 21 May 2026 21:47:52 +0000 Subject: [PATCH 22/41] Add wiki_index enrichment for place details Enriches place API responses with wiki_summary, wiki_url, wiki_population, and wikivoyage_url from wiki_index.db. Lookups by wikidata_id first, then falls back to name + country_code. Called from Nominatim, Overpass, and Wikidata endpoints. Gated by has_kiwix_wiki feature flag. Co-Authored-By: Claude Opus 4.5 --- lib/place_detail.py | 85 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/lib/place_detail.py b/lib/place_detail.py index e85ee54..e2515b5 100644 --- a/lib/place_detail.py +++ b/lib/place_detail.py @@ -286,6 +286,88 @@ def _apply_google_data(result, google_data, gaps): _WIKI_TAGS = ('wikipedia', 'wikidata', 'wikivoyage', 'appropedia') + +# ── Wiki Index enrichment ─────────────────────────────────────────────── + +_wiki_index_conn = None + +def _get_wiki_index_db(): + global _wiki_index_conn + if _wiki_index_conn is not None: + return _wiki_index_conn + + db_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data", "wiki_index.db") + if not os.path.exists(db_path): + logger.debug(f"wiki_index.db not found at {db_path}") + return None + + _wiki_index_conn = sqlite3.connect(db_path, check_same_thread=False) + _wiki_index_conn.row_factory = sqlite3.Row + logger.info(f"Wiki index DB ready at {db_path}") + return _wiki_index_conn + + +def _enrich_with_wiki_index(result): + try: + from .deployment_config import get_deployment_config + deploy_config = get_deployment_config() + features = deploy_config.get("features", {}) + if not features.get("has_kiwix_wiki", False): + return result + except Exception: + return result + + db = _get_wiki_index_db() + if not db: + return result + + try: + cur = db.cursor() + row = None + + extratags = result.get("extratags", {}) + wikidata_id = result.get("wikidata_id") or extratags.get("wikidata") + if wikidata_id: + if isinstance(wikidata_id, str) and wikidata_id.startswith("http"): + wikidata_id = wikidata_id.split("/")[-1] + cur.execute( + "SELECT summary, wiki_population, wikipedia_title, wikivoyage_title FROM wiki_places WHERE wikidata_id = ?", + (wikidata_id,) + ) + row = cur.fetchone() + + if not row: + name = result.get("name") + address = result.get("address") or {} + country_code = address.get("country_code") or result.get("country_code") + if name and country_code: + cur.execute( + "SELECT summary, wiki_population, wikipedia_title, wikivoyage_title FROM wiki_places WHERE place_name = ? AND country_code = ? LIMIT 1", + (name, country_code.lower()) + ) + row = cur.fetchone() + + if row: + if row["summary"]: + result["wiki_summary"] = row["summary"] + if row["wiki_population"]: + try: + result["wiki_population"] = int(row["wiki_population"]) + except (ValueError, TypeError): + result["wiki_population"] = row["wiki_population"] + if row["wikipedia_title"]: + title = row["wikipedia_title"].replace(" ", "_") + result["wiki_url"] = f"https://en.wikipedia.org/wiki/{title}" + if row["wikivoyage_title"]: + title = row["wikivoyage_title"].replace(" ", "_") + result["wikivoyage_url"] = f"https://en.wikivoyage.org/wiki/{title}" + logger.debug(f"Wiki index enrichment hit for {result.get(name)}") + + except Exception as e: + logger.debug(f"Wiki index enrichment error: {e}") + + return result + def _enrich_wiki_links(result): """ Rewrite wiki-related extratags to local Kiwix URLs where available. @@ -625,6 +707,7 @@ def get_place_detail(osm_type, osm_id): nominatim_result = _enrich_with_overture(nominatim_result, osm_type, osm_id) nominatim_result = _enrich_with_google(nominatim_result, osm_type, osm_id) nominatim_result = _enrich_wiki_links(nominatim_result) + nominatim_result = _enrich_with_wiki_index(nominatim_result) cache_put(osm_type, osm_id, nominatim_result, 'nominatim_local') return nominatim_result, 200 @@ -658,6 +741,7 @@ def get_place_detail(osm_type, osm_id): overpass_result = _enrich_with_overture(overpass_result, osm_type, osm_id) overpass_result = _enrich_with_google(overpass_result, osm_type, osm_id) overpass_result = _enrich_wiki_links(overpass_result) + overpass_result = _enrich_with_wiki_index(overpass_result) cache_put(osm_type, osm_id, overpass_result, 'overpass') return overpass_result, 200 @@ -809,6 +893,7 @@ def get_place_by_wikidata(wikidata_id): result["boundary"] = boundary + result = _enrich_with_wiki_index(result) logger.debug(f"Wikidata hit: {wikidata_id} -> {name}") return result, 200 From dcd4ddd3581cd466c48f886c7aa59b24993c9410 Mon Sep 17 00:00:00 2001 From: malice Date: Thu, 21 May 2026 16:07:54 -0600 Subject: [PATCH 23/41] Migrate TomTom flow proxy from classic to Orbis Maps API --- lib/api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/api.py b/lib/api.py index 949a0cc..d56c519 100644 --- a/lib/api.py +++ b/lib/api.py @@ -1215,7 +1215,8 @@ def api_traffic_flow(z, x, y): key = os.environ.get('TOMTOM_API_KEY') if not key: return 'Traffic service not configured', 503 - url = f'https://api.tomtom.com/traffic/map/4/tile/flow/relative/{z}/{x}/{y}.png?key={key}' + # Orbis Maps Traffic API (migrated from classic) + url = f'https://api.tomtom.com/maps/orbis/traffic/tile/flow/{z}/{x}/{y}.png?key={key}&apiVersion=1&style=light' try: resp = http_requests.get(url, timeout=10) if resp.status_code != 200: From 75664c7d02c5b11e3fe7f2a022be0b118df1aa0c Mon Sep 17 00:00:00 2001 From: malice Date: Fri, 22 May 2026 01:12:20 -0600 Subject: [PATCH 24/41] recon: remove /api/traffic/flow handler (now served by navi-traffic, extraction #1) The /api/traffic/flow///.png handler is dead code in recon. As of extraction #1 of the recon<->Navi decoupling, this path is served by the standalone navi-traffic service. Live request flow is now: Caddy (CT 101, navi.echo6.co @authed_api, forward_auth) -> nginx :8440 (location ^~ /api/traffic/ -> proxy_cache traffic_cache) -> navi-traffic gunicorn :8421 (services/navi_traffic) Cutover verified live: authenticated browser fetch to https://navi.echo6.co/api/traffic/flow/... returns 200 image/png with X-Cache-Status MISS then HIT (120s cache), Server: gunicorn. navi-backend (github.com/zvx-echo6/navi-backend): - dae54f3 Initial scaffold: navi-backend + navi-traffic - 311cb8f nginx: use ^~ prefix on /api/traffic/ to beat .png regex catch-all Caddy cutover (@authed_api upstream 8420 -> nginx 8440) applied on Utility CT 101. Also drops the now-unused make_response flask import (no other uses in lib/api.py). os and http_requests remain (used elsewhere). Co-Authored-By: Claude Opus 4.7 (1M context) --- lib/api.py | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/lib/api.py b/lib/api.py index d56c519..7aae00f 100644 --- a/lib/api.py +++ b/lib/api.py @@ -17,7 +17,7 @@ import shutil import tempfile import requests as http_requests -from flask import Flask, request, jsonify, redirect, render_template, make_response +from flask import Flask, request, jsonify, redirect, render_template from qdrant_client import QdrantClient from qdrant_client.models import Filter, FieldCondition, MatchValue from werkzeug.utils import secure_filename @@ -1208,27 +1208,6 @@ def api_knowledge_stats(): return jsonify(_cache['knowledge_stats']) - -@app.route('/api/traffic/flow///.png') -def api_traffic_flow(z, x, y): - """Proxy TomTom traffic flow tiles to hide API key from frontend.""" - key = os.environ.get('TOMTOM_API_KEY') - if not key: - return 'Traffic service not configured', 503 - # Orbis Maps Traffic API (migrated from classic) - url = f'https://api.tomtom.com/maps/orbis/traffic/tile/flow/{z}/{x}/{y}.png?key={key}&apiVersion=1&style=light' - try: - resp = http_requests.get(url, timeout=10) - if resp.status_code != 200: - return 'Upstream error', 502 - r = make_response(resp.content) - r.headers['Content-Type'] = 'image/png' - r.headers['Cache-Control'] = 'public, max-age=120' - return r - except Exception: - return 'Upstream timeout', 504 - - @app.route('/api/place//') def api_place_detail(osm_type, osm_id): """Proxy place details from local Nominatim or Overpass API.""" From bb220b7ba3d890aecc100a2540f3d143c5eb2dc5 Mon Sep 17 00:00:00 2001 From: malice Date: Fri, 22 May 2026 08:10:33 -0600 Subject: [PATCH 25/41] recon: add auth.login_url/logout_url to deployment profiles (extraction #2) Additive prep for the Navi Panel.jsx login/logout cutover. Adds an `auth` block (login_url, logout_url) to each deployment profile, placed after the existing `services` block: - home.yaml login=/outpost.goauthentik.io/start?rd=%2F logout=auth.echo6.co invalidation flow, next=navi.echo6.co - minimal_pi.yaml same, with TODO(matt) to confirm logout next= host - regional_pi.yaml same, with TODO(matt) to confirm logout next= host No Python change. /api/config returns the whole profile dict, so these keys flow through automatically; existing consumers ignore unknown keys, making this backward-safe (the frontend fallback path is simply never needed once this is live). Next steps (separate PRs): the navi-config service (:8422) mirroring this handler, and the Panel.jsx fix to read cfg.auth.login_url/logout_url with the current literals as fallback. Co-Authored-By: Claude Opus 4.7 (1M context) --- config/profiles/home.yaml | 4 ++++ config/profiles/minimal_pi.yaml | 5 +++++ config/profiles/regional_pi.yaml | 5 +++++ 3 files changed, 14 insertions(+) diff --git a/config/profiles/home.yaml b/config/profiles/home.yaml index 474ffb2..de704d9 100644 --- a/config/profiles/home.yaml +++ b/config/profiles/home.yaml @@ -31,6 +31,10 @@ services: address_book: "/api/address_book" valhalla: "/valhalla" +auth: + login_url: "/outpost.goauthentik.io/start?rd=%2F" + logout_url: "https://auth.echo6.co/if/flow/default-invalidation-flow/?next=https://navi.echo6.co/" + features: has_nominatim_details: true has_kiwix_wiki: true diff --git a/config/profiles/minimal_pi.yaml b/config/profiles/minimal_pi.yaml index e3ae0fd..c2fd90a 100644 --- a/config/profiles/minimal_pi.yaml +++ b/config/profiles/minimal_pi.yaml @@ -26,6 +26,11 @@ services: address_book: "/api/address_book" valhalla: "/valhalla" +# TODO(matt): confirm logout next= host for this profile +auth: + login_url: "/outpost.goauthentik.io/start?rd=%2F" + logout_url: "https://auth.echo6.co/if/flow/default-invalidation-flow/?next=https://navi.echo6.co/" + features: has_nominatim_details: false has_kiwix_wiki: false diff --git a/config/profiles/regional_pi.yaml b/config/profiles/regional_pi.yaml index 8e70cd6..b6f2cad 100644 --- a/config/profiles/regional_pi.yaml +++ b/config/profiles/regional_pi.yaml @@ -31,6 +31,11 @@ services: address_book: "/api/address_book" valhalla: "/valhalla" +# TODO(matt): confirm logout next= host for this profile +auth: + login_url: "/outpost.goauthentik.io/start?rd=%2F" + logout_url: "https://auth.echo6.co/if/flow/default-invalidation-flow/?next=https://navi.echo6.co/" + features: has_nominatim_details: true has_kiwix_wiki: false From f42b1fef3ba07bd5a7f3fa4b63adeb8c8ec38180 Mon Sep 17 00:00:00 2001 From: malice Date: Fri, 22 May 2026 13:23:08 -0600 Subject: [PATCH 26/41] recon: add /api/wiki-enrich endpoint (extraction #5 prep, additive) (#8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HTTP wrapper over the wiki_index lookup so the (future) navi-places service can fetch wiki enrichment over HTTP instead of reading recon's 2.1 GB data/wiki_index.db directly (Phase A option B — HTTP coupling). GET /api/wiki-enrich?wikidata= (primary key) GET /api/wiki-enrich?name=&country= (fallback key) -> 200 {wiki_summary?, wiki_population?, wiki_url?, wikivoyage_url?} -> 400 if no usable key; 404 on no match. Public (no auth, like /api/place/*). Route keys are wikidata_id / name+country — NOT osm_type/osm_id — because that is how wiki_index is actually queried (the in-process _enrich_with_wiki_index looks up by result['wikidata_id'] then name+country_code, never by OSM id; see extraction-5-wiki-enrich-investigation.md). An osm-keyed route would have forced a redundant in-recon place lookup. Changes (additive only): - lib/place_detail.py: new standalone lookup_wiki_index(wikidata_id, name, country_code) doing the same two SELECTs + field/URL mapping as the in-process path, returning a dict or None. Pure DB read, never raises. `_enrich_with_wiki_index` is LEFT UNTOUCHED — it can be DRY-refactored to delegate to this in a later PR; the in-process enrichment path is unchanged. - lib/wiki_enrich_api.py: new wiki_enrich_bp blueprint with the route. - lib/api.py: register the blueprint (one block). - lib/wiki_enrich_api_test.py: 4 tests (hit-by-wikidata + decoded fields, no-match -> 404, name+country fallback, no-key -> 400) over an in-memory fixture DB; plain-assert style + __main__ runner (recon venv has no pytest). Verified green against recon's venv (flask 3.1.2). Does NOT remove the in-process _enrich_with_wiki_index call from place_detail — that happens in a later PR once navi-places is live and serving. Co-authored-by: Claude Opus 4.7 (1M context) --- lib/api.py | 4 ++ lib/place_detail.py | 64 ++++++++++++++++++++++++++++++ lib/wiki_enrich_api.py | 31 +++++++++++++++ lib/wiki_enrich_api_test.py | 77 +++++++++++++++++++++++++++++++++++++ 4 files changed, 176 insertions(+) create mode 100644 lib/wiki_enrich_api.py create mode 100644 lib/wiki_enrich_api_test.py diff --git a/lib/api.py b/lib/api.py index 7aae00f..c9f991e 100644 --- a/lib/api.py +++ b/lib/api.py @@ -73,6 +73,10 @@ from .netsyms_api import netsyms_bp, geocode_bp app.register_blueprint(netsyms_bp) app.register_blueprint(geocode_bp) +# ── Wiki-enrich Blueprint (extraction #5 prep — HTTP wrapper over wiki_index) ── +from .wiki_enrich_api import wiki_enrich_bp +app.register_blueprint(wiki_enrich_bp) + # ── Navigation Constants ── diff --git a/lib/place_detail.py b/lib/place_detail.py index e2515b5..46aa8b0 100644 --- a/lib/place_detail.py +++ b/lib/place_detail.py @@ -307,6 +307,70 @@ def _get_wiki_index_db(): return _wiki_index_conn +def lookup_wiki_index(wikidata_id=None, name=None, country_code=None): + """Standalone wiki_index lookup, extracted for the /api/wiki-enrich endpoint + (extraction #5: navi-places HTTP-fetches wiki enrichment instead of reading + the 2.1 GB wiki_index.db directly). + + Mirrors the lookup that `_enrich_with_wiki_index` performs in-process: + by wikidata_id first, then a name + country_code fallback. Returns a dict of + wiki enrichment fields (only those present), or None if there is no match or + the wiki_index DB is unavailable. Pure DB read — no feature-flag gating + (callers decide whether to call) and never raises. + + NOTE: additive only — `_enrich_with_wiki_index` is intentionally left + untouched here; it can be DRY-refactored to delegate to this in a later PR. + """ + db = _get_wiki_index_db() + if not db: + return None + + try: + cur = db.cursor() + row = None + + if wikidata_id: + wid = wikidata_id + if isinstance(wid, str) and wid.startswith("http"): + wid = wid.split("/")[-1] + cur.execute( + "SELECT summary, wiki_population, wikipedia_title, wikivoyage_title FROM wiki_places WHERE wikidata_id = ?", + (wid,) + ) + row = cur.fetchone() + + if not row and name and country_code: + cur.execute( + "SELECT summary, wiki_population, wikipedia_title, wikivoyage_title FROM wiki_places WHERE place_name = ? AND country_code = ? LIMIT 1", + (name, country_code.lower()) + ) + row = cur.fetchone() + + if not row: + return None + + out = {} + if row["summary"]: + out["wiki_summary"] = row["summary"] + if row["wiki_population"]: + try: + out["wiki_population"] = int(row["wiki_population"]) + except (ValueError, TypeError): + out["wiki_population"] = row["wiki_population"] + if row["wikipedia_title"]: + title = row["wikipedia_title"].replace(" ", "_") + out["wiki_url"] = f"https://en.wikipedia.org/wiki/{title}" + if row["wikivoyage_title"]: + title = row["wikivoyage_title"].replace(" ", "_") + out["wikivoyage_url"] = f"https://en.wikivoyage.org/wiki/{title}" + + return out or None + + except Exception as e: + logger.debug(f"wiki_index lookup error: {e}") + return None + + def _enrich_with_wiki_index(result): try: from .deployment_config import get_deployment_config diff --git a/lib/wiki_enrich_api.py b/lib/wiki_enrich_api.py new file mode 100644 index 0000000..ff0f9c7 --- /dev/null +++ b/lib/wiki_enrich_api.py @@ -0,0 +1,31 @@ +"""Wiki-enrich API — read-only HTTP wrapper over the wiki_index lookup. + +Extraction #5 prep: lets the (future) navi-places service fetch wiki enrichment +over HTTP instead of reading recon's 2.1 GB data/wiki_index.db directly. Additive +only — does not change place_detail's in-process `_enrich_with_wiki_index` path. + + GET /api/wiki-enrich?wikidata= (primary key) + GET /api/wiki-enrich?name=&country= (fallback key) + +Public (no auth), matching /api/place/*. 400 if no usable key; 404 on no match. +""" +from flask import Blueprint, request, jsonify + +from .place_detail import lookup_wiki_index + +wiki_enrich_bp = Blueprint('wiki_enrich', __name__) + + +@wiki_enrich_bp.route('/api/wiki-enrich') +def api_wiki_enrich(): + wikidata = (request.args.get('wikidata') or '').strip() or None + name = (request.args.get('name') or '').strip() or None + country = (request.args.get('country') or '').strip() or None + + if not wikidata and not (name and country): + return jsonify({'error': 'provide ?wikidata= or ?name=&country='}), 400 + + result = lookup_wiki_index(wikidata_id=wikidata, name=name, country_code=country) + if result is None: + return jsonify({'error': 'no wiki match'}), 404 + return jsonify(result) diff --git a/lib/wiki_enrich_api_test.py b/lib/wiki_enrich_api_test.py new file mode 100644 index 0000000..681e5cb --- /dev/null +++ b/lib/wiki_enrich_api_test.py @@ -0,0 +1,77 @@ +"""Tests for the /api/wiki-enrich endpoint (extraction #5 prep). + +Plain-assert style (matching the other lib *_test.py; recon's venv has no +pytest). Builds a minimal Flask app with only wiki_enrich_bp registered (avoids +importing the full recon app) and points place_detail's lazy wiki_index +connection at an in-memory fixture DB. Run with pytest, or directly: + python -m lib.wiki_enrich_api_test +""" +import sqlite3 + +from flask import Flask + +from lib import place_detail +from lib.wiki_enrich_api import wiki_enrich_bp + + +def _client(): + """Fresh in-memory wiki_index fixture + a minimal app with just the route.""" + conn = sqlite3.connect(":memory:", check_same_thread=False) + conn.row_factory = sqlite3.Row + conn.execute( + "CREATE TABLE wiki_places (wikidata_id TEXT, place_name TEXT, country_code TEXT, " + "summary TEXT, wiki_population TEXT, wikipedia_title TEXT, wikivoyage_title TEXT)" + ) + conn.execute( + "INSERT INTO wiki_places VALUES (?,?,?,?,?,?,?)", + ("Q830149", "Filer", "us", "A city in Idaho.", "2508", "Filer, Idaho", "Filer"), + ) + conn.commit() + # Point the lazy module-level connection at the fixture so + # _get_wiki_index_db()/lookup_wiki_index() use it (bypasses the file path). + place_detail._wiki_index_conn = conn + app = Flask(__name__) + app.register_blueprint(wiki_enrich_bp) + return app.test_client() + + +def test_wiki_enrich_hit_by_wikidata(): + resp = _client().get("/api/wiki-enrich?wikidata=Q830149") + assert resp.status_code == 200, resp.status_code + d = resp.get_json() + assert d["wiki_summary"] == "A city in Idaho." + assert d["wiki_population"] == 2508 # cast to int + assert d["wiki_url"] == "https://en.wikipedia.org/wiki/Filer,_Idaho" + assert d["wikivoyage_url"] == "https://en.wikivoyage.org/wiki/Filer" + + +def test_wiki_enrich_no_match_404(): + resp = _client().get("/api/wiki-enrich?wikidata=Q9999999") + assert resp.status_code == 404, resp.status_code + + +def test_wiki_enrich_name_country_fallback(): + resp = _client().get("/api/wiki-enrich?name=Filer&country=US") + assert resp.status_code == 200, resp.status_code + assert resp.get_json()["wiki_summary"] == "A city in Idaho." + + +def test_wiki_enrich_no_key_400(): + c = _client() + assert c.get("/api/wiki-enrich").status_code == 400 + # name without country is not a usable key + assert c.get("/api/wiki-enrich?name=Filer").status_code == 400 + + +if __name__ == "__main__": + failures = 0 + for _name, _fn in sorted(globals().items()): + if _name.startswith("test_") and callable(_fn): + try: + _fn() + print(f"PASS {_name}") + except Exception as exc: # noqa: BLE001 + failures += 1 + print(f"FAIL {_name}: {exc!r}") + print("OK" if failures == 0 else f"{failures} FAILED") + raise SystemExit(1 if failures else 0) From 14ad2cd34ac6a53587103f44949bec0f052bb957 Mon Sep 17 00:00:00 2001 From: malice Date: Fri, 22 May 2026 14:08:18 -0600 Subject: [PATCH 27/41] recon: add /api/wiki-rewrite endpoint (extraction #5 prep, additive) (#9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per-tag HTTP wrapper over wiki_rewrite.rewrite_wiki_link so the (future) navi-places service can rewrite OSM wiki tags to local Kiwix URLs over HTTP instead of importing recon's wiki_rewrite module (which talks to Kiwix on localhost:8430 and the wiki_cache table in /opt/recon/data/place_cache.db). Companion to PR #8 (/api/wiki-enrich) — Matt picked option B (HTTP-couple the Kiwix offline-wiki rewriting too, since it matters in prod). GET /api/wiki-rewrite?tag=&value= -> 200 {url, status} where status is "local" | "public" | "original" -> 400 on missing value or unknown tag -> no 404 (unclassifiable value echoes back with status "original", mirroring rewrite_wiki_link) Public (no auth), like /api/place/* and /api/wiki-enrich. Changes (additive only): - lib/wiki_rewrite_api.py: new wiki_rewrite_bp blueprint. Thin route directly over the existing rewrite_wiki_link(tag, value) — no extraction needed (it's already a clean standalone function, unlike wiki-enrich's lookup). - lib/api.py: register the blueprint (one block). - lib/wiki_rewrite_api_test.py: 5 tests (local Kiwix hit, public fallback, unclassifiable -> original, missing value -> 400, unknown tag -> 400), stubbing check_kiwix_has_article (no Kiwix/DB), plain-assert + __main__ runner. Verified green against recon's venv (flask 3.1.2). Does NOT touch place_detail's in-process _enrich_wiki_links — that gets removed in a later PR once navi-places is live (same as PR #8). wiki_cache stays in recon's own place_cache.db post-cutover (harmless positive-cache duplication). Co-authored-by: Claude Opus 4.7 (1M context) --- lib/api.py | 4 ++ lib/wiki_rewrite_api.py | 34 +++++++++++++++++ lib/wiki_rewrite_api_test.py | 73 ++++++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+) create mode 100644 lib/wiki_rewrite_api.py create mode 100644 lib/wiki_rewrite_api_test.py diff --git a/lib/api.py b/lib/api.py index c9f991e..a245706 100644 --- a/lib/api.py +++ b/lib/api.py @@ -77,6 +77,10 @@ app.register_blueprint(geocode_bp) from .wiki_enrich_api import wiki_enrich_bp app.register_blueprint(wiki_enrich_bp) +# ── Wiki-rewrite Blueprint (extraction #5 prep — HTTP wrapper over rewrite_wiki_link) ── +from .wiki_rewrite_api import wiki_rewrite_bp +app.register_blueprint(wiki_rewrite_bp) + # ── Navigation Constants ── diff --git a/lib/wiki_rewrite_api.py b/lib/wiki_rewrite_api.py new file mode 100644 index 0000000..ae1d52e --- /dev/null +++ b/lib/wiki_rewrite_api.py @@ -0,0 +1,34 @@ +"""Wiki-rewrite API — read-only HTTP wrapper over wiki_rewrite.rewrite_wiki_link. + +Extraction #5 prep: lets the (future) navi-places service rewrite OSM wiki tags +to local Kiwix URLs over HTTP instead of importing recon's wiki_rewrite module +(which talks to Kiwix and the wiki_cache table in /opt/recon/data/place_cache.db). +Additive only — does not change place_detail's in-process `_enrich_wiki_links`. + + GET /api/wiki-rewrite?tag=&value= + +Public (no auth), matching /api/place/* and /api/wiki-enrich. 400 on missing +value or unknown tag. No 404 — an unclassifiable value returns the original +value with status "original" (mirrors rewrite_wiki_link). +""" +from flask import Blueprint, request, jsonify + +from .wiki_rewrite import rewrite_wiki_link + +wiki_rewrite_bp = Blueprint('wiki_rewrite', __name__) + +_KNOWN_TAGS = {'wikipedia', 'wikidata', 'wikivoyage', 'appropedia'} + + +@wiki_rewrite_bp.route('/api/wiki-rewrite') +def api_wiki_rewrite(): + tag = (request.args.get('tag') or '').strip().lower() + value = (request.args.get('value') or '').strip() + + if not value: + return jsonify({'error': 'value is required'}), 400 + if tag not in _KNOWN_TAGS: + return jsonify({'error': f"tag must be one of {sorted(_KNOWN_TAGS)}"}), 400 + + url, status = rewrite_wiki_link(tag, value) + return jsonify({'url': url, 'status': status}) diff --git a/lib/wiki_rewrite_api_test.py b/lib/wiki_rewrite_api_test.py new file mode 100644 index 0000000..2bc50f4 --- /dev/null +++ b/lib/wiki_rewrite_api_test.py @@ -0,0 +1,73 @@ +"""Tests for the /api/wiki-rewrite endpoint (extraction #5 prep). + +Plain-assert style (recon's venv has no pytest). Builds a minimal Flask app +with only wiki_rewrite_bp registered. Mocks `wiki_rewrite.check_kiwix_has_article` +to control the local-Kiwix-hit vs. fallback paths without touching Kiwix or the +wiki_cache DB. classify_wiki_link (pure regex) runs for real. Run with pytest, +or directly: python -m lib.wiki_rewrite_api_test +""" +from flask import Flask + +from lib import wiki_rewrite +from lib.wiki_rewrite_api import wiki_rewrite_bp + + +def _client(kiwix_hit): + """kiwix_hit: (found_bool, url) returned by a stubbed check_kiwix_has_article.""" + wiki_rewrite.check_kiwix_has_article = lambda source_type, article_id: kiwix_hit + app = Flask(__name__) + app.register_blueprint(wiki_rewrite_bp) + return app.test_client() + + +def test_local_kiwix_hit(): + url = "https://wiki.echo6.co/content/wikipedia/Filer,_Idaho" + c = _client((True, url)) + resp = c.get("/api/wiki-rewrite?tag=wikipedia&value=Filer, Idaho") + assert resp.status_code == 200, resp.status_code + d = resp.get_json() + assert d["status"] == "local" + assert d["url"] == url + + +def test_public_fallback_when_not_in_kiwix(): + c = _client((False, None)) # not in Kiwix -> canonical public URL + resp = c.get("/api/wiki-rewrite?tag=wikipedia&value=Filer") + assert resp.status_code == 200, resp.status_code + d = resp.get_json() + assert d["status"] == "public" + assert d["url"] == "https://en.wikipedia.org/wiki/Filer" + + +def test_unclassifiable_returns_original(): + # 'wikidata' requires a Q-id; a non-matching value -> classify None -> original. + c = _client((False, None)) + resp = c.get("/api/wiki-rewrite?tag=wikidata&value=not-a-qid") + assert resp.status_code == 200, resp.status_code + d = resp.get_json() + assert d["status"] == "original" + assert d["url"] == "not-a-qid" + + +def test_missing_value_400(): + c = _client((False, None)) + assert c.get("/api/wiki-rewrite?tag=wikipedia").status_code == 400 + + +def test_unknown_tag_400(): + c = _client((False, None)) + assert c.get("/api/wiki-rewrite?tag=facebook&value=x").status_code == 400 + + +if __name__ == "__main__": + failures = 0 + for _name, _fn in sorted(globals().items()): + if _name.startswith("test_") and callable(_fn): + try: + _fn() + print(f"PASS {_name}") + except Exception as exc: # noqa: BLE001 + failures += 1 + print(f"FAIL {_name}: {exc!r}") + print("OK" if failures == 0 else f"{failures} FAILED") + raise SystemExit(1 if failures else 0) From ed36eec85e6f68265695a6516248aac77581c65e Mon Sep 17 00:00:00 2001 From: malice Date: Sat, 23 May 2026 00:08:48 -0600 Subject: [PATCH 28/41] cleanup: remove /api/config handler (extraction #2 shadow) (#10) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cleanup: remove /api/config handler (extraction #2 shadow) recon's /api/config Flask handler (lib/api.py) is edge-shadowed since extraction #2 — navi-config (:8422) serves the route via nginx on navi.echo6.co. The recon-side handler is dead at the edge; remove it. lib/deployment_config.py is KEPT: get_deployment_config() still has many in-process consumers (lib/api.py:1237 /api/landclass has_landclass gate, google_places.py, place_detail.py x4, offroute/router.py). Only the /api/config HTTP handler is removed; the import at api.py:27 stays. Co-Authored-By: Claude Opus 4.7 (1M context) * cleanup: refresh deployment_config docstring (drop /api/config reference) The module docstring still said get_deployment_config() was "for use by the /api/config endpoint" — that handler was removed in the parent commit. Rewrite to reflect the actual 5 in-process consumers (landclass gate, google_places, place_detail ×4, offroute/router.py profile.offroute.*). Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: zvx-echo6 Co-authored-by: Claude Opus 4.7 (1M context) --- lib/api.py | 9 --------- lib/deployment_config.py | 10 +++++++++- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/lib/api.py b/lib/api.py index a245706..1fbde76 100644 --- a/lib/api.py +++ b/lib/api.py @@ -1263,15 +1263,6 @@ def api_landclass(): }) -@app.route('/api/config') -def api_config(): - """Return deployment profile config for frontend consumption.""" - config = get_deployment_config() - resp = jsonify(config) - resp.headers['Cache-Control'] = 'public, max-age=300' - return resp - - @app.route('/api/health') def api_health(): """Health check endpoint for monitoring.""" diff --git a/lib/deployment_config.py b/lib/deployment_config.py index 978b8a0..83cc864 100644 --- a/lib/deployment_config.py +++ b/lib/deployment_config.py @@ -3,7 +3,15 @@ Deployment profile loader. Reads RECON_PROFILE env var (default: "home"), loads the matching YAML from config/profiles/.yaml, and caches the parsed dict in memory. -Provides get_deployment_config() for use by the /api/config endpoint. + +Provides get_deployment_config() for in-process consumers of the profile: + - lib/api.py:api_landclass — the has_landclass feature-flag gate + - lib/google_places.py — Google Places enrichment config + - lib/place_detail.py — place-detail enrichment config (×4 call sites) + - lib/offroute/router.py — profile.offroute.* (osm_pbf_path / postgis_dsn / + densify_interval_m) +(The former /api/config HTTP endpoint that served this dict to the frontend was +removed once navi-config (:8422) took over that route.) """ import os import yaml From c968497b943ec5065139f93b5838d4906b87f605 Mon Sep 17 00:00:00 2001 From: malice Date: Sat, 23 May 2026 00:27:44 -0600 Subject: [PATCH 29/41] cleanup: remove /api/place handlers (extraction #5 shadow) (#11) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /api/place// and /api/place/wikidata/ are edge-shadowed since extraction #5 — navi-places (:8425) serves both via nginx. Removes the two recon-side handlers + the now-unused `from .place_detail import get_place_detail, get_place_by_wikidata` import. NO modules deleted. place_detail.py is KEPT — wiki_enrich_api.py (the /api/wiki-enrich endpoint, which stays; navi-places HTTP-consumes it) imports `lookup_wiki_index` from it. That transitively keeps its deps google_places.py, overture.py, osm_categories.py (all imported only by place_detail). This corrects Phase A #5 §3's "only lib/api.py imports place_detail" — the wiki-enrich endpoint (added post-#5) is a second consumer. Co-authored-by: zvx-echo6 Co-authored-by: Claude Opus 4.7 (1M context) --- lib/api.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/lib/api.py b/lib/api.py index 1fbde76..d14068c 100644 --- a/lib/api.py +++ b/lib/api.py @@ -25,7 +25,6 @@ from werkzeug.utils import secure_filename from .utils import get_config, content_hash, clean_filename_to_title, derive_source_and_category, generate_download_url, setup_logging from .status import StatusDB from .deployment_config import get_deployment_config -from .place_detail import get_place_detail, get_place_by_wikidata from .landclass import lookup_landclass, format_summary logger = setup_logging('recon.api') @@ -1216,21 +1215,6 @@ def api_knowledge_stats(): return jsonify(_cache['knowledge_stats']) -@app.route('/api/place//') -def api_place_detail(osm_type, osm_id): - """Proxy place details from local Nominatim or Overpass API.""" - result, status = get_place_detail(osm_type, osm_id) - return jsonify(result), status - - -@app.route("/api/place/wikidata/") -def api_place_wikidata(wikidata_id): - """Fetch place details from Wikidata entity.""" - result, status = get_place_by_wikidata(wikidata_id) - return jsonify(result), status - - - @app.route('/api/landclass') def api_landclass(): """PAD-US land classification lookup for a point.""" From d56b1d5f922e5d54ba3c95794833bc05da9e4096 Mon Sep 17 00:00:00 2001 From: malice Date: Sat, 23 May 2026 03:34:22 -0600 Subject: [PATCH 30/41] cleanup: remove /api/contacts + /api/address_book handlers + pull entire /nav-i/* subtree (extraction #3 shadow) (#12) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cleanup: remove /api/address_book handlers (extraction #3 shadow) Removes address_book_bp (lib/address_book_api.py: /api/address_book/lookup + /api/address_book/list) + its registration in lib/api.py. Edge-shadowed since extraction #3 — navi-contacts (:8423) serves /api/address_book/* on navi.echo6.co; no recon-side consumer (no template/JS reference). lib/address_book.py is KEPT — geocode.py (nickname short-circuit + annotation) and netsyms_api.py import it. NOT removed this PR: contacts_bp. The recon dashboard at /deleted-contacts (recon-product, stays) calls /api/contacts//{restore,restore-as,purge} via XHR, and recon.echo6.co proxies straight to recon:8420 (verified the Caddy block — no navi-contacts routing there). Removing contacts_bp would break those dashboard actions. Flagged for a decision; lib/contacts.py also stays (dashboard ContactsDB reads). See PR body. Co-Authored-By: Claude Opus 4.7 (1M context) * cleanup: deprecate /nav-i + /deleted-contacts; remove contacts_bp + lib/contacts.py Probe found recon's /deleted-contacts dashboard reads /opt/recon/data/contacts.db — frozen since extraction #3 moved write ownership to navi-contacts (/var/lib/navi-backend/contacts.db). The page has been silently rendering ~25-day stale data, and its restore/restore-as/purge XHRs hit recon's contacts_bp (the recon.echo6.co Caddy block proxies straight to recon:8420 — no navi-contacts routing there). Per Matt's decision, deprecate the pages entirely; they'll be re-surfaced later as a proper admin page consuming navi-contacts via API. Removed: - contacts_bp (lib/contacts_api.py, all 10 /api/contacts* routes) + its registration in lib/api.py — edge-shadowed by navi-contacts :8423 since #3, and now free of recon-product consumers once the dashboard goes. - /nav-i (navi_landing_page) + /deleted-contacts (deleted_contacts_page) route handlers; templates/navi/landing.html + templates/navi/deleted_contacts.html. - lib/contacts.py (ContactsDB) — the dashboard was its only non-contacts_bp consumer; both gone. - The two dead NAVI_SUBNAV entries (Overview→/nav-i, Deleted Contacts→ /deleted-contacts). Kept / adapted: - /nav-i/api-keys page (recon-product key management) stays. NAVI_SUBNAV reduced to just its API Keys entry; the base.html top-nav "Nav-I" link repointed /nav-i -> /nav-i/api-keys so the surviving section page stays reachable (minimal href change, not a nav restructure — flagged in PR). - lib/address_book.py — geocode.py + netsyms_api.py still consume it (untouched). Out-of-band follow-up after merge: delete the stale /opt/recon/data/contacts.db (frozen 2026-04-28; data, not code). Co-Authored-By: Claude Opus 4.7 (1M context) * cleanup: pull the entire /nav-i/* subtree (api-keys page is a weaker dup of /settings/keys) Completes the contacts cleanup by removing the rest of /nav-i/. The /nav-i/api-keys page was (a) a weaker duplicate of /settings/keys for Gemini (it lacked remove + reload-from-.env), and (b) a write-only-to-dead-files surface for TomTom + Google Places: it wrote /opt/recon/.env, but the live navi-traffic (:8421) and navi-places (:8425) services read their own /etc/navi-backend/.env and have ignored recon's copy since extractions #1 + #5. End state: no /nav-i/* URLs in recon. Removed: - /nav-i/api-keys route + template (templates/navi/api_keys.html) - all /api/nav-i/api-keys/* endpoints (list/update/test/restart-recon) - lib/api_keys_admin.py (its only importers were those 4 endpoints; _KEY_DEFS/ _read_env/_write_env were private to it) - the now-orphaned NAVI_SUBNAV - the "Nav-I" top-nav entry in base.html (reverses the /nav-i->/nav-i/api-keys repoint from the previous commit, now that the page itself is gone) Kept (Gemini's real home, recon-product): - /settings/keys + /api/keys/* + lib/key_manager.py (KeyManager) — they import key_manager directly, never api_keys_admin, so untouched. Note: TOMTOM_API_KEY now has zero recon .py references. GOOGLE_PLACES_API_KEY still has one (lib/google_places.py), kept in the prior /api/place cleanup as place_detail's dep; its only caller (_enrich_with_google) is unreachable since the /api/place handlers were removed — left in place pending /api/wiki-enrich retirement (out of scope here). Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: zvx-echo6 Co-authored-by: Claude Opus 4.7 (1M context) --- lib/address_book_api.py | 31 --- lib/api.py | 98 -------- lib/api_keys_admin.py | 358 --------------------------- lib/contacts.py | 230 ----------------- lib/contacts_api.py | 132 ---------- templates/base.html | 1 - templates/navi/api_keys.html | 269 -------------------- templates/navi/deleted_contacts.html | 116 --------- templates/navi/landing.html | 22 -- 9 files changed, 1257 deletions(-) delete mode 100644 lib/address_book_api.py delete mode 100644 lib/api_keys_admin.py delete mode 100644 lib/contacts.py delete mode 100644 lib/contacts_api.py delete mode 100644 templates/navi/api_keys.html delete mode 100644 templates/navi/deleted_contacts.html delete mode 100644 templates/navi/landing.html diff --git a/lib/address_book_api.py b/lib/address_book_api.py deleted file mode 100644 index 020828b..0000000 --- a/lib/address_book_api.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -RECON Address Book API — Flask Blueprint. - -GET /api/address_book/lookup?q= — best match or 404 -GET /api/address_book/list — all entries -""" - -from flask import Blueprint, request, jsonify - -from . import address_book - -address_book_bp = Blueprint('address_book', __name__) - - -@address_book_bp.route('/api/address_book/lookup') -def api_address_book_lookup(): - q = request.args.get('q', '').strip() - if not q: - return jsonify({'error': 'Missing q parameter'}), 400 - - result = address_book.lookup(q) - if result is None: - return '', 404 - - return jsonify(result) - - -@address_book_bp.route('/api/address_book/list') -def api_address_book_list(): - entries = address_book.list_all() - return jsonify(entries) diff --git a/lib/api.py b/lib/api.py index d14068c..61f25f2 100644 --- a/lib/api.py +++ b/lib/api.py @@ -59,14 +59,6 @@ class _LargeZimRequest(_FlaskRequest): return super()._get_file_stream(total_content_length, content_type, filename, content_length) app.request_class = _LargeZimRequest -# ── Address Book Blueprint ── -from .address_book_api import address_book_bp -app.register_blueprint(address_book_bp) - -# ── Contacts Blueprint ── -from .contacts_api import contacts_bp -app.register_blueprint(contacts_bp) - # ── Netsyms + Geocode Blueprints ── from .netsyms_api import netsyms_bp, geocode_bp app.register_blueprint(netsyms_bp) @@ -109,12 +101,6 @@ SETTINGS_SUBNAV = [ {'href': '/settings/health', 'label': 'Service Health'}, ] -NAVI_SUBNAV = [ - {'href': '/nav-i', 'label': 'Overview'}, - {'href': '/deleted-contacts', 'label': 'Deleted Contacts'}, - {'href': '/nav-i/api-keys', 'label': 'API Keys'}, -] - def _format_source_citation(payload): """Format a human-readable citation from a search result payload.""" @@ -341,36 +327,6 @@ def failures_page(): failures=failures) -@app.route("/deleted-contacts") -def deleted_contacts_page(): - from .auth import get_user_id - from .contacts import ContactsDB - user_id = get_user_id() or "anonymous" - db = ContactsDB() - contacts = db.list_deleted(user_id) - return render_template("navi/deleted_contacts.html", - domain="navi", subnav=NAVI_SUBNAV, active_page="/deleted-contacts", - contacts=contacts) - - -@app.route("/nav-i") -def navi_landing_page(): - from .auth import get_user_id - from .contacts import ContactsDB - user_id = get_user_id() or "anonymous" - db = ContactsDB() - deleted_count = len(db.list_deleted(user_id)) - return render_template("navi/landing.html", - domain="navi", subnav=NAVI_SUBNAV, active_page="/nav-i", - deleted_count=deleted_count) - - -@app.route("/nav-i/api-keys") -def navi_api_keys_page(): - return render_template("navi/api_keys.html", - domain="navi", subnav=NAVI_SUBNAV, active_page="/nav-i/api-keys") - - @app.route('/peertube') def peertube_dashboard(): return render_template('peertube/dashboard.html', @@ -1408,60 +1364,6 @@ def api_keys_reload(): -# ── Nav-I API Key Admin ── - -@app.route('/api/nav-i/api-keys/list', methods=['GET']) -def navi_api_keys_list(): - from .api_keys_admin import list_keys - return jsonify({'keys': list_keys()}) - - -@app.route('/api/nav-i/api-keys/update', methods=['POST']) -def navi_api_keys_update(): - from .auth import require_auth - from .api_keys_admin import update_key, update_gemini_key - data = request.get_json(force=True) - name = data.get('name', '') - new_value = data.get('new_value', '') - index = data.get('index') # optional, for Gemini key replacement - if not name or not new_value: - return jsonify({'error': 'name and new_value required'}), 400 - if name == 'GEMINI_KEY' and index is not None: - result = update_gemini_key(int(index), new_value) - else: - result = update_key(name, new_value) - if result.get('success'): - return jsonify(result) - return jsonify(result), 400 - - -@app.route('/api/nav-i/api-keys/test', methods=['POST']) -def navi_api_keys_test(): - from .api_keys_admin import test_key - data = request.get_json(force=True) - name = data.get('name', '') - index = data.get('index') # optional, for testing specific Gemini key - if not name: - return jsonify({'error': 'name required'}), 400 - result = test_key(name, index=int(index) if index is not None else None) - return jsonify(result) - - -@app.route('/api/nav-i/api-keys/restart-recon', methods=['POST']) -def navi_api_keys_restart(): - import subprocess - try: - result = subprocess.run( - ['sudo', 'systemctl', 'restart', 'recon'], - capture_output=True, text=True, timeout=30 - ) - if result.returncode == 0: - return jsonify({'success': True, 'note': 'RECON service restarted'}) - return jsonify({'success': False, 'error': result.stderr.strip()}), 500 - except subprocess.TimeoutExpired: - return jsonify({'success': False, 'error': 'Restart timed out'}), 500 - except Exception as e: - return jsonify({'success': False, 'error': str(e)}), 500 # ── YouTube Cookie Management ── diff --git a/lib/api_keys_admin.py b/lib/api_keys_admin.py deleted file mode 100644 index 3c63565..0000000 --- a/lib/api_keys_admin.py +++ /dev/null @@ -1,358 +0,0 @@ -""" -Nav-I API Keys Admin — unified view/update/test for third-party API keys. - -Manages three provider categories: - - Gemini (multiple keys via KeyManager singleton) - - TomTom (single key in .env) - - Google Places (single key in .env) - -All key values are masked in responses. Full values never leave the server -except as user-supplied input on update. -""" -import os -import re -import shutil -import tempfile -import time - -import requests as http_requests - -from .utils import setup_logging - -logger = setup_logging('recon.api_keys_admin') - -ENV_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), '.env') - -# Key definitions: env_name → display metadata -_KEY_DEFS = { - 'TOMTOM_API_KEY': { - 'display_name': 'TomTom', - 'provider': 'tomtom', - }, - 'GOOGLE_PLACES_API_KEY': { - 'display_name': 'Google Places', - 'provider': 'google_places', - }, -} - - -# ── .env read/write helpers ───────────────────────────────────────────── - -def _read_env(): - """Read .env file into a dict of key=value pairs, preserving order.""" - entries = [] # list of (key, value, raw_line) — preserves order and comments - if not os.path.exists(ENV_PATH): - return entries - with open(ENV_PATH, 'r') as f: - for line in f: - raw = line.rstrip('\n') - stripped = raw.strip() - if not stripped or stripped.startswith('#'): - entries.append((None, None, raw)) - continue - m = re.match(r'^([A-Za-z_][A-Za-z0-9_]*)=(.*)$', stripped) - if m: - entries.append((m.group(1), m.group(2).strip().strip('"').strip("'"), raw)) - else: - entries.append((None, None, raw)) - return entries - - -def _write_env(entries): - """Atomically write .env from entries list. Backs up to .env.bak first.""" - # Backup current .env - if os.path.exists(ENV_PATH): - bak_path = ENV_PATH + '.bak' - shutil.copy2(ENV_PATH, bak_path) - - # Write to temp file, then rename (atomic on same filesystem) - fd, tmp_path = tempfile.mkstemp(dir=os.path.dirname(ENV_PATH), prefix='.env.', suffix='.tmp') - try: - with os.fdopen(fd, 'w') as f: - for key, value, raw in entries: - if key is not None: - f.write(f'{key}={value}\n') - else: - f.write(raw + '\n') - os.rename(tmp_path, ENV_PATH) - except Exception: - # Clean up temp file on failure - try: - os.unlink(tmp_path) - except OSError: - pass - raise - - logger.info(f"Wrote .env atomically ({len([e for e in entries if e[0]])} keys)") - - -def _get_env_value(name): - """Get a single value from .env by key name.""" - for key, value, _ in _read_env(): - if key == name: - return value - return None - - -def _set_env_value(name, new_value): - """Set a single value in .env. Adds if not present.""" - entries = _read_env() - found = False - for i, (key, value, raw) in enumerate(entries): - if key == name: - entries[i] = (name, new_value, f'{name}={new_value}') - found = True - break - if not found: - entries.append((name, new_value, f'{name}={new_value}')) - _write_env(entries) - - -# ── Masking ───────────────────────────────────────────────────────────── - -def _mask_key(value): - """Mask a key: first 4 chars + '...' + last 4 chars. Never return full value.""" - if not value: - return None - if len(value) <= 8: - return '****' - return value[:4] + '...' + value[-4:] - - -# ── List ──────────────────────────────────────────────────────────────── - -def list_keys(): - """ - Return masked status of all managed API keys. - - Returns list of dicts with: name, display_name, provider, masked_value, - is_set, count (for multi-key providers like Gemini). - """ - result = [] - env_mtime = None - if os.path.exists(ENV_PATH): - env_mtime = time.strftime('%Y-%m-%dT%H:%M:%SZ', - time.gmtime(os.path.getmtime(ENV_PATH))) - - # Gemini keys (via KeyManager) - from .key_manager import get_key_manager - km = get_key_manager() - gemini_keys = km.get_masked_keys() - gemini_count = len(gemini_keys) - # Show a single summary entry for Gemini with count - first_masked = gemini_keys[0]['masked'] if gemini_keys else None - result.append({ - 'name': 'GEMINI_KEY', - 'display_name': 'Gemini', - 'provider': 'gemini', - 'masked_value': first_masked, - 'is_set': gemini_count > 0, - 'count': gemini_count, - 'last_modified': env_mtime, - 'keys': gemini_keys, # full list with per-key stats - }) - - # Single-value keys - for env_name, meta in _KEY_DEFS.items(): - value = _get_env_value(env_name) - result.append({ - 'name': env_name, - 'display_name': meta['display_name'], - 'provider': meta['provider'], - 'masked_value': _mask_key(value), - 'is_set': bool(value), - 'count': 1 if value else 0, - 'last_modified': env_mtime, - }) - - return result - - -# ── Update ────────────────────────────────────────────────────────────── - -def update_key(name, new_value): - """ - Update a key value. For Gemini, name should be 'GEMINI_KEY' with an - optional 'index' for replacing a specific key, or use the KeyManager API. - For TomTom/Google Places, writes directly to .env. - - Returns dict with success status and masked value. - """ - new_value = new_value.strip() - if not new_value: - return {'success': False, 'error': 'Key value cannot be empty'} - - if name == 'GEMINI_KEY': - # Use KeyManager for Gemini - from .key_manager import get_key_manager - km = get_key_manager() - try: - idx = km.add_gemini_key(new_value) - return { - 'success': True, - 'name': name, - 'masked_value': _mask_key(new_value), - 'action': 'added', - 'index': idx, - } - except ValueError as e: - return {'success': False, 'error': str(e)} - - if name in _KEY_DEFS: - _set_env_value(name, new_value) - return { - 'success': True, - 'name': name, - 'masked_value': _mask_key(new_value), - 'action': 'updated', - } - - return {'success': False, 'error': f'Unknown key: {name}'} - - -def update_gemini_key(index, new_value): - """Replace a specific Gemini key by index.""" - new_value = new_value.strip() - if not new_value: - return {'success': False, 'error': 'Key value cannot be empty'} - - from .key_manager import get_key_manager - km = get_key_manager() - try: - km.replace_gemini_key(index, new_value) - return { - 'success': True, - 'name': 'GEMINI_KEY', - 'index': index, - 'masked_value': _mask_key(new_value), - 'action': 'replaced', - } - except (ValueError, IndexError) as e: - return {'success': False, 'error': str(e)} - - -# ── Test ──────────────────────────────────────────────────────────────── - -def test_key(name, index=None): - """ - Test a key against its provider API using the current .env value. - - Returns dict with: success, latency_ms, error, note. - """ - if name == 'GEMINI_KEY': - return _test_gemini(index) - elif name == 'TOMTOM_API_KEY': - return _test_tomtom() - elif name == 'GOOGLE_PLACES_API_KEY': - return _test_google_places() - else: - return {'success': False, 'error': f'Unknown key: {name}', 'latency_ms': 0} - - -def _test_gemini(index=None): - """Test Gemini key by listing models.""" - from .key_manager import get_key_manager - km = get_key_manager() - - if index is not None: - key = km.get_gemini_key(index) - if not key: - return {'success': False, 'error': f'Gemini key index {index} not found', 'latency_ms': 0} - else: - key = km.get_gemini_key(0) - if not key: - return {'success': False, 'error': 'No Gemini keys configured', 'latency_ms': 0} - - t0 = time.time() - try: - resp = http_requests.get( - f"https://generativelanguage.googleapis.com/v1beta/models?key={key}", - timeout=10 - ) - latency = int((time.time() - t0) * 1000) - - if resp.status_code == 200 and 'models' in resp.text: - return {'success': True, 'latency_ms': latency, 'error': None, - 'note': 'Models list returned successfully'} - elif resp.status_code == 403: - return {'success': False, 'latency_ms': latency, - 'error': 'Key disabled or quota exhausted'} - elif resp.status_code == 429: - return {'success': True, 'latency_ms': latency, 'error': None, - 'note': 'Valid key — currently rate-limited'} - else: - return {'success': False, 'latency_ms': latency, - 'error': f'HTTP {resp.status_code}'} - except Exception as e: - latency = int((time.time() - t0) * 1000) - return {'success': False, 'latency_ms': latency, 'error': str(e)} - - -def _test_tomtom(): - """Test TomTom key with a minimal geocode request.""" - key = _get_env_value('TOMTOM_API_KEY') - if not key: - return {'success': False, 'error': 'TOMTOM_API_KEY not set', 'latency_ms': 0} - - t0 = time.time() - try: - resp = http_requests.get( - f"https://api.tomtom.com/search/2/geocode/Boise.json", - params={'key': key, 'limit': 1}, - timeout=10 - ) - latency = int((time.time() - t0) * 1000) - - if resp.status_code == 200: - data = resp.json() - count = data.get('summary', {}).get('totalResults', 0) - return {'success': True, 'latency_ms': latency, 'error': None, - 'note': f'Geocode returned {count} result(s)'} - elif resp.status_code == 403: - return {'success': False, 'latency_ms': latency, - 'error': 'Invalid or expired key'} - else: - return {'success': False, 'latency_ms': latency, - 'error': f'HTTP {resp.status_code}'} - except Exception as e: - latency = int((time.time() - t0) * 1000) - return {'success': False, 'latency_ms': latency, 'error': str(e)} - - -def _test_google_places(): - """Test Google Places (New) API key with a minimal searchText request.""" - key = _get_env_value('GOOGLE_PLACES_API_KEY') - if not key: - return {'success': False, 'error': 'GOOGLE_PLACES_API_KEY not set', 'latency_ms': 0} - - t0 = time.time() - try: - resp = http_requests.post( - "https://places.googleapis.com/v1/places:searchText", - json={'textQuery': 'Boise Idaho', 'maxResultCount': 1}, - headers={ - 'X-Goog-Api-Key': key, - 'X-Goog-FieldMask': 'places.displayName', - }, - timeout=10 - ) - latency = int((time.time() - t0) * 1000) - - if resp.status_code == 200: - data = resp.json() - count = len(data.get('places', [])) - return {'success': True, 'latency_ms': latency, 'error': None, - 'note': f'searchText returned {count} place(s)'} - elif resp.status_code == 403: - return {'success': False, 'latency_ms': latency, - 'error': 'Key not authorized for Places API (New)'} - elif resp.status_code == 429: - return {'success': True, 'latency_ms': latency, 'error': None, - 'note': 'Valid key — quota exceeded'} - else: - body = resp.text[:200] - return {'success': False, 'latency_ms': latency, - 'error': f'HTTP {resp.status_code}: {body}'} - except Exception as e: - latency = int((time.time() - t0) * 1000) - return {'success': False, 'latency_ms': latency, 'error': str(e)} diff --git a/lib/contacts.py b/lib/contacts.py deleted file mode 100644 index f2782db..0000000 --- a/lib/contacts.py +++ /dev/null @@ -1,230 +0,0 @@ -""" -RECON Contacts Database — per-user phone book with soft delete and proximity queries. - -Separate DB at data/contacts.db. Thread-local connections with WAL mode (StatusDB pattern). -""" -import math -import os -import sqlite3 -import threading -from datetime import datetime, timezone - -_local = threading.local() - -_SCHEMA = """ -CREATE TABLE IF NOT EXISTS contacts ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_id TEXT NOT NULL, - label TEXT NOT NULL, - name TEXT, - call_sign TEXT, - phone TEXT, - email TEXT, - category TEXT, - notes TEXT, - lat REAL, - lon REAL, - osm_type TEXT, - osm_id INTEGER, - address TEXT, - show_proximity INTEGER DEFAULT 0, - created_at TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), - updated_at TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), - deleted_at TEXT, - deleted_by TEXT -); - -CREATE INDEX IF NOT EXISTS idx_contacts_user ON contacts(user_id); -CREATE INDEX IF NOT EXISTS idx_contacts_user_category ON contacts(user_id, category); -CREATE INDEX IF NOT EXISTS idx_contacts_user_deleted ON contacts(user_id, deleted_at); -CREATE INDEX IF NOT EXISTS idx_contacts_geo ON contacts(lat, lon); -CREATE UNIQUE INDEX IF NOT EXISTS idx_contacts_home_work - ON contacts(user_id, label) - WHERE label IN ('Home', 'Work') AND deleted_at IS NULL; -""" - - -def _haversine_m(lat1, lon1, lat2, lon2): - """Haversine distance in meters.""" - R = 6_371_000 - rlat1, rlat2 = math.radians(lat1), math.radians(lat2) - dlat = math.radians(lat2 - lat1) - dlon = math.radians(lon2 - lon1) - a = math.sin(dlat / 2) ** 2 + math.cos(rlat1) * math.cos(rlat2) * math.sin(dlon / 2) ** 2 - return R * 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a)) - - -def _row_to_dict(row): - """Convert sqlite3.Row to dict, casting show_proximity to bool.""" - d = dict(row) - d['show_proximity'] = bool(d.get('show_proximity', 0)) - return d - - -class ContactsDB: - def __init__(self, db_path=None): - if db_path is None: - db_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'data', 'contacts.db') - self.db_path = db_path - os.makedirs(os.path.dirname(db_path), exist_ok=True) - self._init_db() - - def _get_conn(self): - if not hasattr(_local, 'contacts_conn') or _local.contacts_conn is None: - _local.contacts_conn = sqlite3.connect(self.db_path, timeout=30) - _local.contacts_conn.row_factory = sqlite3.Row - _local.contacts_conn.execute("PRAGMA journal_mode=WAL") - _local.contacts_conn.execute("PRAGMA busy_timeout=5000") - return _local.contacts_conn - - def _init_db(self): - conn = self._get_conn() - conn.executescript(_SCHEMA) - conn.commit() - - def list_all(self, user_id, category=None, search=None): - conn = self._get_conn() - sql = "SELECT * FROM contacts WHERE user_id = ? AND deleted_at IS NULL" - params = [user_id] - if category: - sql += " AND category = ?" - params.append(category) - if search: - sql += " AND (label LIKE ? OR name LIKE ? OR call_sign LIKE ? OR phone LIKE ?)" - like = f"%{search}%" - params.extend([like, like, like, like]) - sql += " ORDER BY label" - return [_row_to_dict(r) for r in conn.execute(sql, params).fetchall()] - - def list_deleted(self, user_id): - conn = self._get_conn() - rows = conn.execute( - "SELECT * FROM contacts WHERE user_id = ? AND deleted_at IS NOT NULL ORDER BY deleted_at DESC", - (user_id,) - ).fetchall() - return [_row_to_dict(r) for r in rows] - - def get(self, user_id, contact_id, include_deleted=False): - conn = self._get_conn() - sql = "SELECT * FROM contacts WHERE id = ? AND user_id = ?" - if not include_deleted: - sql += " AND deleted_at IS NULL" - row = conn.execute(sql, (contact_id, user_id)).fetchone() - return _row_to_dict(row) if row else None - - def create(self, user_id, **fields): - conn = self._get_conn() - fields.pop('id', None) - fields.pop('user_id', None) - fields.pop('created_at', None) - fields.pop('updated_at', None) - fields.pop('deleted_at', None) - fields.pop('deleted_by', None) - if 'show_proximity' in fields: - fields['show_proximity'] = 1 if fields['show_proximity'] else 0 - columns = ['user_id'] + list(fields.keys()) - placeholders = ', '.join(['?'] * len(columns)) - col_str = ', '.join(columns) - values = [user_id] + list(fields.values()) - try: - cur = conn.execute(f"INSERT INTO contacts ({col_str}) VALUES ({placeholders})", values) - conn.commit() - return self.get(user_id, cur.lastrowid), None - except sqlite3.IntegrityError: - return None, 'conflict' - - def update(self, user_id, contact_id, **fields): - conn = self._get_conn() - fields.pop('id', None) - fields.pop('user_id', None) - fields.pop('created_at', None) - fields.pop('deleted_at', None) - fields.pop('deleted_by', None) - if 'show_proximity' in fields: - fields['show_proximity'] = 1 if fields['show_proximity'] else 0 - fields['updated_at'] = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.%fZ') - sets = ', '.join(f"{k} = ?" for k in fields) - values = list(fields.values()) + [contact_id, user_id] - conn.execute(f"UPDATE contacts SET {sets} WHERE id = ? AND user_id = ? AND deleted_at IS NULL", values) - conn.commit() - return self.get(user_id, contact_id) - - def soft_delete(self, user_id, contact_id): - conn = self._get_conn() - now = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.%fZ') - conn.execute( - "UPDATE contacts SET deleted_at = ?, deleted_by = ? WHERE id = ? AND user_id = ? AND deleted_at IS NULL", - (now, user_id, contact_id, user_id) - ) - conn.commit() - return self.get(user_id, contact_id, include_deleted=True) - - def restore(self, user_id, contact_id): - conn = self._get_conn() - row = self.get(user_id, contact_id, include_deleted=True) - if not row or not row.get('deleted_at'): - return None, 'not_found' - if row.get('label') in ('Home', 'Work'): - existing = conn.execute( - "SELECT id FROM contacts WHERE user_id = ? AND label = ? AND deleted_at IS NULL AND id != ?", - (user_id, row['label'], contact_id) - ).fetchone() - if existing: - return None, 'conflict' - conn.execute( - "UPDATE contacts SET deleted_at = NULL, deleted_by = NULL WHERE id = ? AND user_id = ?", - (contact_id, user_id) - ) - conn.commit() - return self.get(user_id, contact_id), None - - def restore_as(self, user_id, contact_id, new_label): - """Restore a soft-deleted contact with a new label (for Home/Work conflict resolution).""" - conn = self._get_conn() - row = self.get(user_id, contact_id, include_deleted=True) - if not row or not row.get('deleted_at'): - return None, 'not_found' - if not new_label or not new_label.strip(): - return None, 'invalid_label' - now = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%fZ') - try: - conn.execute( - "UPDATE contacts SET deleted_at = NULL, deleted_by = NULL, label = ?, updated_at = ? WHERE id = ? AND user_id = ?", - (new_label.strip(), now, contact_id, user_id) - ) - conn.commit() - except sqlite3.IntegrityError: - return None, 'conflict' - return self.get(user_id, contact_id), None - - def purge(self, user_id, contact_id): - conn = self._get_conn() - row = self.get(user_id, contact_id, include_deleted=True) - if not row: - return False, 'not_found' - if not row.get('deleted_at'): - return False, 'not_deleted' - conn.execute("DELETE FROM contacts WHERE id = ? AND user_id = ?", (contact_id, user_id)) - conn.commit() - return True, None - - def find_nearby(self, user_id, lat, lon, radius_m=75): - conn = self._get_conn() - # Bounding box pre-filter (~111km per degree lat) - dlat = radius_m / 111_000 - dlon = radius_m / (111_000 * math.cos(math.radians(lat))) - rows = conn.execute( - """SELECT * FROM contacts - WHERE user_id = ? AND deleted_at IS NULL AND show_proximity = 1 - AND lat BETWEEN ? AND ? AND lon BETWEEN ? AND ?""", - (user_id, lat - dlat, lat + dlat, lon - dlon, lon + dlon) - ).fetchall() - results = [] - for r in rows: - dist = _haversine_m(lat, lon, r['lat'], r['lon']) - if dist <= radius_m: - d = _row_to_dict(r) - d['distance_m'] = round(dist, 1) - results.append(d) - results.sort(key=lambda x: x['distance_m']) - return results diff --git a/lib/contacts_api.py b/lib/contacts_api.py deleted file mode 100644 index 0e4506b..0000000 --- a/lib/contacts_api.py +++ /dev/null @@ -1,132 +0,0 @@ -""" -RECON Contacts API — Flask Blueprint. - -Per-user phone book with soft delete, restore, purge, and proximity queries. -All endpoints require Authentik forward-auth (X-Authentik-Username header). -""" -from flask import Blueprint, request, jsonify - -from .auth import require_auth -from .contacts import ContactsDB - -contacts_bp = Blueprint('contacts', __name__) - -_db = None - -def _get_db(): - global _db - if _db is None: - _db = ContactsDB() - return _db - - -@contacts_bp.route('/api/contacts', methods=['GET']) -@require_auth -def list_contacts(): - db = _get_db() - category = request.args.get('category') - search = request.args.get('search') - return jsonify(db.list_all(request.user_id, category=category, search=search)) - - -@contacts_bp.route('/api/contacts', methods=['POST']) -@require_auth -def create_contact(): - db = _get_db() - data = request.get_json(force=True) - contact, err = db.create(request.user_id, **data) - if err == 'conflict': - return jsonify({'error': 'You already have a Home/Work contact'}), 409 - return jsonify(contact), 201 - - -@contacts_bp.route('/api/contacts/nearby', methods=['GET']) -@require_auth -def nearby_contacts(): - db = _get_db() - lat = request.args.get('lat', type=float) - lon = request.args.get('lon', type=float) - radius_m = request.args.get('radius_m', 75, type=float) - if lat is None or lon is None: - return jsonify({'error': 'lat and lon required'}), 400 - return jsonify(db.find_nearby(request.user_id, lat, lon, radius_m)) - - -@contacts_bp.route('/api/contacts/deleted', methods=['GET']) -@require_auth -def list_deleted(): - db = _get_db() - return jsonify(db.list_deleted(request.user_id)) - - -@contacts_bp.route('/api/contacts/', methods=['GET']) -@require_auth -def get_contact(contact_id): - db = _get_db() - contact = db.get(request.user_id, contact_id) - if not contact: - return jsonify({'error': 'Not found'}), 404 - return jsonify(contact) - - -@contacts_bp.route('/api/contacts/', methods=['PATCH']) -@require_auth -def update_contact(contact_id): - db = _get_db() - data = request.get_json(force=True) - contact = db.update(request.user_id, contact_id, **data) - if not contact: - return jsonify({'error': 'Not found'}), 404 - return jsonify(contact) - - -@contacts_bp.route('/api/contacts/', methods=['DELETE']) -@require_auth -def delete_contact(contact_id): - db = _get_db() - contact = db.soft_delete(request.user_id, contact_id) - if not contact: - return jsonify({'error': 'Not found'}), 404 - return jsonify(contact) - - -@contacts_bp.route('/api/contacts//restore', methods=['POST']) -@require_auth -def restore_contact(contact_id): - db = _get_db() - contact, err = db.restore(request.user_id, contact_id) - if err == 'not_found': - return jsonify({'error': 'Not found'}), 404 - if err == 'conflict': - return jsonify({'error': 'You already have a Home/Work contact'}), 409 - return jsonify(contact) - - -@contacts_bp.route('/api/contacts//restore-as', methods=['POST']) -@require_auth -def restore_as_contact(contact_id): - db = _get_db() - data = request.get_json(force=True) - new_label = data.get('label', '').strip() - if not new_label: - return jsonify({'error': 'label is required'}), 400 - contact, err = db.restore_as(request.user_id, contact_id, new_label) - if err == 'not_found': - return jsonify({'error': 'Not found'}), 404 - if err == 'invalid_label': - return jsonify({'error': 'Invalid label'}), 400 - if err == 'conflict': - return jsonify({'error': 'Label conflict'}), 409 - return jsonify(contact) - - -@contacts_bp.route('/api/contacts//purge', methods=['DELETE']) -@require_auth -def purge_contact(contact_id): - db = _get_db() - ok, err = db.purge(request.user_id, contact_id) - if err == 'not_found': - return jsonify({'error': 'Not found'}), 404 - if err == 'not_deleted': - return jsonify({'error': 'Contact must be deleted before purging'}), 400 - return jsonify({'ok': True}) diff --git a/templates/base.html b/templates/base.html index 4c06892..49b1a21 100644 --- a/templates/base.html +++ b/templates/base.html @@ -21,7 +21,6 @@ PeerTube Kiwix Search - Nav-I Settings {% if subnav %} diff --git a/templates/navi/api_keys.html b/templates/navi/api_keys.html deleted file mode 100644 index abf2d16..0000000 --- a/templates/navi/api_keys.html +++ /dev/null @@ -1,269 +0,0 @@ -{% extends "base.html" %} -{% block content %} -

    API Keys

    - -
    -

    Updating keys does not restart RECON. After updates, click Restart RECON below or restart manually from terminal.

    -
    - -
    Loading keys...
    - - -
    • - - - - - - - - -
    - - -
    - - - -{% endblock %} - -{% block scripts %} - -{% endblock %} diff --git a/templates/navi/deleted_contacts.html b/templates/navi/deleted_contacts.html deleted file mode 100644 index 0847fab..0000000 --- a/templates/navi/deleted_contacts.html +++ /dev/null @@ -1,116 +0,0 @@ -{% extends "base.html" %} -{% block content %} -

    Deleted Contacts

    -{% if not contacts %} -

    No deleted contacts.

    -{% else %} - - - {% for c in contacts %} - - - - - - - - - {% endfor %} -
    LabelNameCategoryPhoneDeleted AtActions
    {{ c.label }}{{ c.name or '' }}{{ c.category or '' }}{{ c.phone or '' }}{{ c.deleted_at or '' }} - - -
    -{% endif %} - - - -{% endblock %} -{% block scripts %} - -{% endblock %} diff --git a/templates/navi/landing.html b/templates/navi/landing.html deleted file mode 100644 index 131f3af..0000000 --- a/templates/navi/landing.html +++ /dev/null @@ -1,22 +0,0 @@ -{% extends "base.html" %} -{% block content %} -

    Nav-I

    -

    Navi frontend management — contacts, API keys, and configuration.

    - -
    - -
    -
    Deleted Contacts
    -
    {{ deleted_count }}
    -
    awaiting restore or purge
    -
    -     - -
    -
    API Keys
    -
    Coming soon
    -
    per-user key management
    -
    -     -
    -{% endblock %} From d7292c4cc7c834bd5e7f4166a19bc5a1aac5f2f6 Mon Sep 17 00:00:00 2001 From: malice Date: Sat, 23 May 2026 04:04:45 -0600 Subject: [PATCH 31/41] cleanup: remove /api/geocode + /api/reverse handlers (extraction #6 shadow) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All three routes (/api/geocode, /api/reverse, /api/reverse//) are edge-shadowed since extraction #6 — navi-geo :8426 serves them via nginx. - netsyms_api.py: drop geocode_bp + its three handlers, the bundle-private helpers, and module state (TTLCache/lock/_TZ_DB_PATH/_DEM). netsyms_bp (/api/netsyms/lookup + /health) survives. - api.py: drop the geocode_bp import + register_blueprint line. - DELETE lib/geocode.py, lib/nav_tools.py (both orphaned once the handlers go). - DELETE reverse_bundle_test.py, geocode_test.py, nav_tools_test.py. Decouples netsyms_api.py from landclass.py and offroute/dem.py — prerequisite for cleanups #5 and #6. Co-authored-by: Claude Opus 4.7 (1M context) --- lib/api.py | 5 +- lib/geocode.py | 774 ------------------------------------- lib/geocode_test.py | 157 -------- lib/nav_tools.py | 168 -------- lib/nav_tools_test.py | 77 ---- lib/netsyms_api.py | 261 +------------ lib/reverse_bundle_test.py | 171 -------- 7 files changed, 3 insertions(+), 1610 deletions(-) delete mode 100644 lib/geocode.py delete mode 100644 lib/geocode_test.py delete mode 100644 lib/nav_tools.py delete mode 100644 lib/nav_tools_test.py delete mode 100644 lib/reverse_bundle_test.py diff --git a/lib/api.py b/lib/api.py index 61f25f2..e4a186f 100644 --- a/lib/api.py +++ b/lib/api.py @@ -59,10 +59,9 @@ class _LargeZimRequest(_FlaskRequest): return super()._get_file_stream(total_content_length, content_type, filename, content_length) app.request_class = _LargeZimRequest -# ── Netsyms + Geocode Blueprints ── -from .netsyms_api import netsyms_bp, geocode_bp +# ── Netsyms Blueprint ── +from .netsyms_api import netsyms_bp app.register_blueprint(netsyms_bp) -app.register_blueprint(geocode_bp) # ── Wiki-enrich Blueprint (extraction #5 prep — HTTP wrapper over wiki_index) ── from .wiki_enrich_api import wiki_enrich_bp diff --git a/lib/geocode.py b/lib/geocode.py deleted file mode 100644 index aabd37e..0000000 --- a/lib/geocode.py +++ /dev/null @@ -1,774 +0,0 @@ -""" -RECON geocode — structured preprocessing, multi-source retrieval, reranking. - -Replaces the naive Photon-only search with: - 1. usaddress parsing + intent classification (ADDRESS / POI / LOCALITY / COORD / POSTCODE) - 2. Multi-source retrieval: ADDRESS → Netsyms + Photon; POI/LOCALITY → Photon /api - 3. Python reranker with weighted signals - -Public entry point: geocode(query, limit) → {query, results, count} -""" - -import math -import re -import logging - -import requests -import usaddress -from rapidfuzz import fuzz - -from .utils import setup_logging - -logger = setup_logging('recon.geocode') - -# ── Trace logger for reranking audit ── -_trace_logger = logging.getLogger('recon.geocode.trace') -_trace_handler = logging.FileHandler('/tmp/geocode_rerank_trace.log') -_trace_handler.setFormatter(logging.Formatter('%(asctime)s %(message)s')) -_trace_logger.addHandler(_trace_handler) -_trace_logger.setLevel(logging.DEBUG) - -# ── Config constants ── -PHOTON_URL = "http://localhost:2322" -GEOCODE_BIAS_LAT = 42.5736 -GEOCODE_BIAS_LON = -114.6066 -GEOCODE_BIAS_ZOOM = 10 -ADDRESS_BOOK_ANNOTATION_RADIUS_M = 75 - -# ── Reranker weights ── -# Derived from research analysis of failure modes: -# housenumber_exact is the strongest signal because Photon's soft-boost -# lets wrong-number results bubble up. street_name_fuzz and locality_fuzz -# handle abbreviation/case variation. source_authority gives Netsyms a -# boost for US addresses since it has USPS-verified data. -W_HOUSENUMBER_EXACT = 6.0 # exact housenumber match -W_HOUSENUMBER_MISMATCH = -5.0 # housenumber present but wrong -W_STREET_NAME_FUZZ = 3.0 # fuzzy street name similarity [0..1] * weight -W_TOKEN_COVERAGE = 2.0 # fraction of query tokens found in result -W_STREET_TYPE_MATCH = 1.5 # "st" matches "street", etc. -W_LOCALITY_FUZZ = 2.0 # city/state fuzzy match -W_SOURCE_AUTHORITY = 2.0 # Netsyms for US addresses -W_LAYER_RANK = 1.0 # type-appropriate results ranked higher -W_PHOTON_POSITION_NORM = 1.0 # Photon's native ranking (normalized by position) -W_STATE_EXACT = 1.0 # exact state code match -W_POI_CLASS_BOOST = 3.0 # amenity/shop/etc boost for business-name queries -W_HIGHWAY_CLASS_PENALTY = -4.0 # highway/route penalty for business-name queries - -# ── US abbreviation expansions ── -# Applied ONLY to parsed StreetName/StreetNamePostType tokens, NOT to ordinals. -_STREET_TYPE_ABBREVS = { - 'st': 'street', 'ave': 'avenue', 'blvd': 'boulevard', 'dr': 'drive', - 'rd': 'road', 'ln': 'lane', 'ct': 'court', 'cir': 'circle', - 'pl': 'place', 'way': 'way', 'pkwy': 'parkway', 'hwy': 'highway', - 'trl': 'trail', 'ter': 'terrace', 'sq': 'square', -} -_DIRECTIONAL_ABBREVS = { - 'n': 'north', 's': 'south', 'e': 'east', 'w': 'west', - 'ne': 'northeast', 'nw': 'northwest', 'se': 'southeast', 'sw': 'southwest', -} -_ORDINAL_RE = re.compile(r'^\d+(st|nd|rd|th)$', re.IGNORECASE) - -# ── Road keywords (for detecting when query is about a road vs a business) ── -_ROAD_KEYWORDS = ( - set(_STREET_TYPE_ABBREVS.keys()) - | set(_STREET_TYPE_ABBREVS.values()) - | {'route', 'rte', 'pass'} -) - -# ── US state codes ── -_STATE_CODES = { - 'AL', 'AK', 'AZ', 'AR', 'CA', 'CO', 'CT', 'DE', 'FL', 'GA', - 'HI', 'ID', 'IL', 'IN', 'IA', 'KS', 'KY', 'LA', 'ME', 'MD', - 'MA', 'MI', 'MN', 'MS', 'MO', 'MT', 'NE', 'NV', 'NH', 'NJ', - 'NM', 'NY', 'NC', 'ND', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC', - 'SD', 'TN', 'TX', 'UT', 'VT', 'VA', 'WA', 'WV', 'WI', 'WY', 'DC', -} - -# ── Full state name → code (for intent classifier) ── -_STATE_NAME_TO_CODE = { - 'alabama': 'AL', 'alaska': 'AK', 'arizona': 'AZ', 'arkansas': 'AR', - 'california': 'CA', 'colorado': 'CO', 'connecticut': 'CT', 'delaware': 'DE', - 'florida': 'FL', 'georgia': 'GA', 'hawaii': 'HI', 'idaho': 'ID', - 'illinois': 'IL', 'indiana': 'IN', 'iowa': 'IA', 'kansas': 'KS', - 'kentucky': 'KY', 'louisiana': 'LA', 'maine': 'ME', 'maryland': 'MD', - 'massachusetts': 'MA', 'michigan': 'MI', 'minnesota': 'MN', - 'mississippi': 'MS', 'missouri': 'MO', 'montana': 'MT', 'nebraska': 'NE', - 'nevada': 'NV', 'new hampshire': 'NH', 'new jersey': 'NJ', - 'new mexico': 'NM', 'new york': 'NY', 'north carolina': 'NC', - 'north dakota': 'ND', 'ohio': 'OH', 'oklahoma': 'OK', 'oregon': 'OR', - 'pennsylvania': 'PA', 'rhode island': 'RI', 'south carolina': 'SC', - 'south dakota': 'SD', 'tennessee': 'TN', 'texas': 'TX', 'utah': 'UT', - 'vermont': 'VT', 'virginia': 'VA', 'washington': 'WA', - 'west virginia': 'WV', 'wisconsin': 'WI', 'wyoming': 'WY', -} - -# Coordinate regex -_COORD_RE = re.compile(r'^\s*(-?\d+\.?\d*)\s*[,\s]\s*(-?\d+\.?\d*)\s*$') - - -# ═══════════════════════════════════════════════════════════════════ -# STEP 1: PREPROCESSING -# ═══════════════════════════════════════════════════════════════════ - -def _parse_coords(text): - """Return (lat, lon) if text looks like coordinates with valid bounds, else None.""" - m = _COORD_RE.match(text.strip()) - if not m: - return None - lat, lon = float(m.group(1)), float(m.group(2)) - if -90 <= lat <= 90 and -180 <= lon <= 180: - return lat, lon - return None - - -def _classify_and_parse(query): - """ - Parse query with usaddress, classify intent, expand abbreviations. - - Returns (intent, parsed_dict) where: - intent: 'ADDRESS' | 'POI' | 'LOCALITY' | 'POSTCODE' | 'COORD' | 'UNKNOWN' - parsed_dict: {number, street, city, state, zipcode, raw_query, expanded_query} - """ - q = query.strip() - parsed = { - 'number': None, 'street': None, 'street_raw': None, - 'city': None, 'state': None, - 'zipcode': None, 'raw_query': q, 'expanded_query': q, - } - - # Coordinate check first - if _parse_coords(q): - return 'COORD', parsed - - # Try usaddress - try: - tagged, addr_type = usaddress.tag(q) - except usaddress.RepeatedLabelError: - # Ambiguous input — fall back to free-text Photon - return 'UNKNOWN', parsed - - # Extract components - number = tagged.get('AddressNumber', '').strip() - street_name = tagged.get('StreetName', '').strip() - street_pre_dir = tagged.get('StreetNamePreDirectional', '').strip() - street_post_type = tagged.get('StreetNamePostType', '').strip() - place = tagged.get('PlaceName', '').strip() - state = tagged.get('StateName', '').strip() - zipcode = tagged.get('ZipCode', '').strip() - - # ── Fix usaddress edge case: "214 N St Filer" ── - # usaddress reads single-letter directional + "St" as PreDirectional + empty, - # mashing "St Filer" into StreetName. Detect: PreDirectional is single letter, - # StreetName has 2+ tokens where the first is a street type. - if (street_pre_dir and len(street_pre_dir) <= 2 - and not street_name.strip().startswith(street_pre_dir) - and ' ' in street_name): - name_tokens = street_name.split() - first_lower = name_tokens[0].lower() - if first_lower in _STREET_TYPE_ABBREVS or first_lower in _STREET_TYPE_ABBREVS.values(): - # "N" is actually the street name, "St" is the post-type - street_name = street_pre_dir - street_post_type = name_tokens[0] - if len(name_tokens) > 1: - place = ' '.join(name_tokens[1:]) - street_pre_dir = '' - - # ── Expand abbreviations (guard ordinals) ── - expanded_parts = [] - - if number: - parsed['number'] = number - expanded_parts.append(number) - - if street_pre_dir: - exp = _DIRECTIONAL_ABBREVS.get(street_pre_dir.lower(), street_pre_dir) - expanded_parts.append(exp) - - if street_name: - # Don't expand ordinals: "21st" stays "21st" - if _ORDINAL_RE.match(street_name): - expanded_parts.append(street_name) - else: - # Expand directional abbreviation if it IS the street name - exp = _DIRECTIONAL_ABBREVS.get(street_name.lower(), street_name) - expanded_parts.append(exp) - parsed['street'] = street_name - - if street_post_type: - if _ORDINAL_RE.match(street_post_type): - expanded_parts.append(street_post_type) - else: - exp = _STREET_TYPE_ABBREVS.get(street_post_type.lower(), street_post_type) - expanded_parts.append(exp) - - # Build raw street (original abbreviations, for Netsyms) and expanded (for Photon) - raw_street_parts = [] - if street_pre_dir: - raw_street_parts.append(street_pre_dir) - if street_name: - raw_street_parts.append(street_name) - if street_post_type: - raw_street_parts.append(street_post_type) - parsed['street_raw'] = ' '.join(raw_street_parts) - - # Build the full expanded street - if expanded_parts: - # The street is everything after the number - street_full = ' '.join(expanded_parts[1:] if number else expanded_parts) - parsed['street'] = street_full - - if place: - parsed['city'] = place - expanded_parts.append(place) - if state: - parsed['state'] = state.upper() - expanded_parts.append(state) - if zipcode: - parsed['zipcode'] = zipcode - expanded_parts.append(zipcode) - - parsed['expanded_query'] = ' '.join(expanded_parts) - - # ── Intent classification ── - if addr_type == 'Street Address' and number: - return 'ADDRESS', parsed - elif zipcode and not number and not street_name: - return 'POSTCODE', parsed - elif addr_type == 'Ambiguous': - # Check if it looks like a locality: last token(s) are a state code or name - tokens = q.replace(',', ' ').split() - if len(tokens) >= 2: - last_upper = tokens[-1].upper() - if last_upper in _STATE_CODES: - parsed['city'] = ' '.join(tokens[:-1]) - parsed['state'] = last_upper - return 'LOCALITY', parsed - # Check full state names (single-word like "idaho" or two-word like "new york") - last_lower = tokens[-1].lower() - if last_lower in _STATE_NAME_TO_CODE: - parsed['city'] = ' '.join(tokens[:-1]) - parsed['state'] = _STATE_NAME_TO_CODE[last_lower] - return 'LOCALITY', parsed - if len(tokens) >= 3: - two_word = f"{tokens[-2].lower()} {last_lower}" - if two_word in _STATE_NAME_TO_CODE: - parsed['city'] = ' '.join(tokens[:-2]) - parsed['state'] = _STATE_NAME_TO_CODE[two_word] - return 'LOCALITY', parsed - return 'UNKNOWN', parsed - else: - return 'UNKNOWN', parsed - - -# ═══════════════════════════════════════════════════════════════════ -# STEP 2: RETRIEVAL -# ═══════════════════════════════════════════════════════════════════ - -def _retrieve_netsyms(parsed, limit=10, lat=None, lon=None): - """Query Netsyms for structured address lookup. Returns list of candidate dicts.""" - try: - from . import netsyms - except Exception: - return [] - - results = [] - number = parsed.get('number', '') - street = parsed.get('street_raw') or parsed.get('street', '') - city = parsed.get('city', '') - state = parsed.get('state', '') - zipcode = parsed.get('zipcode', '') - - # When viewport provided, fetch more results to sort from - fetch_limit = 200 if (lat is not None and lon is not None) else limit - - if number and street: - rows = netsyms.lookup_by_street( - number, street, city=city, state=state, zipcode=zipcode, limit=fetch_limit - ) - elif zipcode: - rows = netsyms.lookup_by_zipcode(zipcode, limit=fetch_limit) - else: - return [] - - for row in rows: - addr_parts = [row['number'], row['street']] - if row.get('street2'): - addr_parts.append(row['street2']) - addr_parts.extend([row['city'], row['state'], row['zipcode']]) - display = ' '.join(p for p in addr_parts if p) - results.append({ - 'name': display, - 'lat': row['lat'], - 'lon': row['lon'], - 'source': 'netsyms', - 'type': 'street_address', - 'raw': row, - '_number': row.get('number', ''), - '_street': row.get('street', ''), - '_city': row.get('city', ''), - '_state': row.get('state', ''), - }) - # Sort by viewport distance if lat/lon provided, then limit - if lat is not None and lon is not None and results: - results.sort(key=lambda r: (r["lat"] - lat)**2 + (r["lon"] - lon)**2) - results = results[:limit] - return results - - -def _retrieve_photon_structured(parsed, limit=10): - """Query Photon /structured endpoint for address lookup.""" - params = {'limit': limit, 'countrycode': 'US'} - if parsed.get('street'): - params['street'] = parsed['street'] - if parsed.get('number'): - params['housenumber'] = parsed['number'] - if parsed.get('city'): - params['city'] = parsed['city'] - if parsed.get('state'): - params['state'] = parsed['state'] - - if 'street' not in params: - return [] - - try: - resp = requests.get(f"{PHOTON_URL}/structured", params=params, timeout=5) - resp.raise_for_status() - data = resp.json() - except Exception as e: - logger.debug("Photon /structured failed: %s", e) - return [] - - return _parse_photon_features(data.get('features', []), 'photon') - - -def _retrieve_photon_freetext(query, limit=10, lat=None, lon=None, zoom=None): - """Query Photon /api for free-text search with location bias.""" - try: - params = { - 'q': query, - 'limit': limit, - 'lat': lat if lat is not None else GEOCODE_BIAS_LAT, - 'lon': lon if lon is not None else GEOCODE_BIAS_LON, - 'zoom': int(zoom) if zoom is not None else GEOCODE_BIAS_ZOOM, - } - resp = requests.get(f"{PHOTON_URL}/api", params=params, timeout=5) - resp.raise_for_status() - data = resp.json() - except Exception as e: - return [] - - return _parse_photon_features(data.get('features', []), 'photon') - - -def _parse_photon_features(features, source): - """Convert Photon GeoJSON features to candidate dicts.""" - results = [] - for i, feature in enumerate(features): - props = feature.get('properties', {}) - coords = feature.get('geometry', {}).get('coordinates', [0, 0]) - - osm_key = props.get('osm_key', '') - osm_value = props.get('osm_value', '') - feat_type = props.get('type', '') - has_hn = bool(props.get('housenumber')) - - if osm_key in ('amenity', 'shop', 'tourism', 'leisure', 'office'): - rtype = 'poi' - elif has_hn or osm_value in ('house', 'residential'): - rtype = 'street_address' - elif feat_type in ('city', 'town', 'village', 'hamlet', 'county', 'state', 'country'): - rtype = 'locality' - else: - rtype = 'poi' - - # Build display name - parts = [] - hn = props.get('housenumber') - street = props.get('street') - name = props.get('name', '') - if hn and street: - parts.append(f"{hn} {street}") - if name and name != street: - parts.append(name) - elif name: - parts.append(name) - elif street: - parts.append(street) - for key in ('city', 'county', 'state', 'country'): - v = props.get(key) - if v and (not parts or v != parts[-1]): - parts.append(v) - display = ', '.join(p for p in parts if p) or 'Unknown' - - results.append({ - 'name': display, - 'lat': coords[1], - 'lon': coords[0], - 'source': source, - 'type': rtype, - 'raw': props, - '_photon_rank': i, - '_number': props.get('housenumber', ''), - '_street': props.get('street', ''), - # For locality results, the name IS the city (Photon omits 'city' on city-type features) - '_city': props.get('city', '') or (props.get('name', '') if rtype == 'locality' else ''), - '_state': props.get('state', ''), - }) - return results - - -# ═══════════════════════════════════════════════════════════════════ -# STEP 3: RERANKER -# ═══════════════════════════════════════════════════════════════════ - -def _expand_street_type(s): - """Expand a street type abbreviation for comparison.""" - return _STREET_TYPE_ABBREVS.get(s.lower(), s.lower()) - - -def _score_candidate(candidate, parsed, intent): - """ - Score a candidate against the parsed query. - Returns (total_score, signal_breakdown_dict). - """ - signals = {} - total = 0.0 - - query_number = (parsed.get('number') or '').strip().upper() - query_street = (parsed.get('street') or '').strip().upper() - query_city = (parsed.get('city') or '').strip().upper() - query_state = (parsed.get('state') or '').strip().upper() - - cand_number = (candidate.get('_number') or '').strip().upper() - cand_street = (candidate.get('_street') or '').strip().upper() - cand_city = (candidate.get('_city') or '').strip().upper() - cand_state = (candidate.get('_state') or '').strip().upper() - - # ── Housenumber ── - if intent == 'ADDRESS' and query_number: - if cand_number == query_number: - signals['housenumber_exact'] = W_HOUSENUMBER_EXACT - total += W_HOUSENUMBER_EXACT - elif cand_number and cand_number != query_number: - signals['housenumber_mismatch'] = W_HOUSENUMBER_MISMATCH - total += W_HOUSENUMBER_MISMATCH - - # ── Street name fuzz ── - if query_street and cand_street: - # Expand both for comparison - q_expanded = ' '.join(_expand_street_type(t) for t in query_street.split()) - c_expanded = ' '.join(_expand_street_type(t) for t in cand_street.split()) - ratio = fuzz.token_sort_ratio(q_expanded, c_expanded) / 100.0 - score = ratio * W_STREET_NAME_FUZZ - signals['street_name_fuzz'] = round(score, 2) - total += score - - # ── Street type match ── - if query_street and cand_street: - q_tokens = set(_expand_street_type(t) for t in query_street.split()) - c_tokens = set(_expand_street_type(t) for t in cand_street.split()) - # Check if the street type words overlap - street_types = set(_STREET_TYPE_ABBREVS.values()) - q_types = q_tokens & street_types - c_types = c_tokens & street_types - if q_types and q_types & c_types: - signals['street_type_match'] = W_STREET_TYPE_MATCH - total += W_STREET_TYPE_MATCH - - # ── Token coverage ── - raw_q = parsed.get('raw_query', '').upper() - q_tokens = set(raw_q.replace(',', ' ').split()) - if q_tokens: - cand_text = candidate.get('name', '').upper() - matched = sum(1 for t in q_tokens if t in cand_text) - coverage = matched / len(q_tokens) - score = coverage * W_TOKEN_COVERAGE - signals['token_coverage'] = round(score, 2) - total += score - - # ── Locality fuzz ── - if query_city and cand_city: - ratio = fuzz.ratio(query_city, cand_city) / 100.0 - score = ratio * W_LOCALITY_FUZZ - signals['locality_fuzz'] = round(score, 2) - total += score - - # ── State exact ── - if query_state and cand_state: - if cand_state == query_state: - signals['state_exact'] = W_STATE_EXACT - total += W_STATE_EXACT - - # ── Source authority ── - if candidate.get('source') == 'netsyms' and intent == 'ADDRESS': - signals['source_authority'] = W_SOURCE_AUTHORITY - total += W_SOURCE_AUTHORITY - - # ── Layer rank (type-appropriate bonus) ── - cand_type = candidate.get('type', '') - if intent == 'ADDRESS' and cand_type == 'street_address': - signals['layer_rank'] = W_LAYER_RANK - total += W_LAYER_RANK - elif intent == 'LOCALITY' and cand_type == 'locality': - signals['layer_rank'] = W_LAYER_RANK - total += W_LAYER_RANK - elif intent == 'POI' and cand_type == 'poi': - signals['layer_rank'] = W_LAYER_RANK - total += W_LAYER_RANK - - # ── Photon position normalization ── - photon_rank = candidate.get('_photon_rank') - if photon_rank is not None: - # Top result gets full bonus, decays linearly - score = max(0, (1.0 - photon_rank / 10.0)) * W_PHOTON_POSITION_NORM - signals['photon_position'] = round(score, 2) - total += score - - # ── Business intent POI boost ── - # When the query has no road keywords (likely a business/POI search), - # boost amenity/shop/etc results and penalize highway/route results. - # Skipped for LOCALITY, POSTCODE, COORD queries where class is irrelevant. - if intent not in ('LOCALITY', 'POSTCODE', 'COORD'): - q_tokens_lower = set(parsed.get('raw_query', '').lower().replace(',', ' ').split()) - if not (q_tokens_lower & _ROAD_KEYWORDS): - osm_key = (candidate.get('raw') or {}).get('osm_key', '') - if osm_key in ('amenity', 'shop', 'tourism', 'leisure', 'office', 'craft'): - signals['poi_class_boost'] = W_POI_CLASS_BOOST - total += W_POI_CLASS_BOOST - elif osm_key in ('highway', 'route'): - signals['highway_class_penalty'] = W_HIGHWAY_CLASS_PENALTY - total += W_HIGHWAY_CLASS_PENALTY - - return round(total, 2), signals - - -def _build_match_code(candidate, parsed, intent): - """Build a match_code dict indicating match quality for each field.""" - mc = {} - if intent == 'ADDRESS': - q_num = (parsed.get('number') or '').strip().upper() - c_num = (candidate.get('_number') or '').strip().upper() - if q_num and c_num == q_num: - mc['housenumber'] = 'matched' - elif q_num and c_num: - mc['housenumber'] = 'unmatched' - elif q_num and not c_num: - mc['housenumber'] = 'inferred' - - q_street = (parsed.get('street') or '').strip().upper() - c_street = (candidate.get('_street') or '').strip().upper() - if q_street and c_street: - q_exp = ' '.join(_expand_street_type(t) for t in q_street.split()) - c_exp = ' '.join(_expand_street_type(t) for t in c_street.split()) - ratio = fuzz.token_sort_ratio(q_exp, c_exp) / 100.0 - mc['street'] = 'matched' if ratio > 0.8 else 'unmatched' - elif q_street: - mc['street'] = 'inferred' - - q_city = (parsed.get('city') or '').strip().upper() - c_city = (candidate.get('_city') or '').strip().upper() - if q_city and c_city: - ratio = fuzz.ratio(q_city, c_city) / 100.0 - mc['city'] = 'matched' if ratio > 0.8 else 'unmatched' - elif q_city: - mc['city'] = 'inferred' - - return mc - - -def _rerank(candidates, parsed, intent, query, limit): - """Score, sort, and trim candidates. Trace-log top 3.""" - scored = [] - for c in candidates: - total, signals = _score_candidate(c, parsed, intent) - c['_score'] = total - c['_signals'] = signals - scored.append(c) - - scored.sort(key=lambda c: c['_score'], reverse=True) - - # Trace log for audit - _trace_logger.debug("─── Query: %r intent=%s ───", query, intent) - for i, c in enumerate(scored): - osm_key = (c.get('raw') or {}).get('osm_key', '—') - osm_val = (c.get('raw') or {}).get('osm_value', '—') - _trace_logger.debug( - " #%d score=%.2f src=%s key=%s/%s name=%s", - i, c['_score'], c.get('source', '?'), osm_key, osm_val, - c.get('name', '?')[:60] - ) - _trace_logger.debug(" signals=%s", c.get('_signals', {})) - - # Clean internal fields and add match_code - result = [] - for c in scored[:limit]: - mc = _build_match_code(c, parsed, intent) - - # Assign confidence from score - score = c.get('_score', 0) - if score >= 10: - confidence = 'exact' - elif score >= 5: - confidence = 'high' - elif score >= 2: - confidence = 'medium' - else: - confidence = 'low' - - entry = { - 'name': c['name'], - 'lat': c['lat'], - 'lon': c['lon'], - 'source': c['source'], - 'confidence': confidence, - 'type': c.get('type', 'poi'), - 'raw': c.get('raw'), - } - if mc: - entry['match_code'] = mc - result.append(entry) - - return result - - -# ═══════════════════════════════════════════════════════════════════ -# STEP 4: ANNOTATION -# ═══════════════════════════════════════════════════════════════════ - -def _haversine_m(lat1, lon1, lat2, lon2): - """Haversine distance in meters.""" - R = 6_371_000 - rlat1, rlat2 = math.radians(lat1), math.radians(lat2) - dlat = math.radians(lat2 - lat1) - dlon = math.radians(lon2 - lon1) - a = math.sin(dlat / 2) ** 2 + math.cos(rlat1) * math.cos(rlat2) * math.sin(dlon / 2) ** 2 - return R * 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a)) - - -def _annotate_with_address_book(results): - """Add labeled_as to results within radius of an address book entry.""" - try: - from . import address_book - entries = address_book.load() - except Exception: - return - for result in results: - rlat, rlon = result.get('lat'), result.get('lon') - if rlat is None or rlon is None: - continue - for entry in entries: - elat, elon = entry.get('lat'), entry.get('lon') - if elat is None or elon is None: - continue - if _haversine_m(rlat, rlon, elat, elon) <= ADDRESS_BOOK_ANNOTATION_RADIUS_M: - result['labeled_as'] = entry['name'] - break - - -# ═══════════════════════════════════════════════════════════════════ -# PUBLIC API -# ═══════════════════════════════════════════════════════════════════ - -def geocode(query, limit=10, lat=None, lon=None, zoom=None): - """ - Structured geocoding with multi-source retrieval and reranking. - - Returns {query, results: [...], count} — always 200-safe. - """ - limit = max(1, min(limit, 20)) - q = (query or '').strip() - empty = {'query': q, 'results': [], 'count': 0} - - if not q: - return empty - - # ── Coordinate detection ── - coords = _parse_coords(q) - if coords: - return { - 'query': q, - 'results': [{ - 'name': q, - 'lat': coords[0], - 'lon': coords[1], - 'source': 'coordinates', - 'confidence': 'exact', - 'type': 'coordinates', - 'raw': None, - }], - 'count': 1, - } - - # ── Address book nickname short-circuit ── - normalized_q = ' '.join(q.lower().replace(',', ' ').split()) - is_single_word = ' ' not in normalized_q - try: - from . import address_book - ab_match = address_book.lookup(q) - if (ab_match - and ab_match['confidence'] == 'exact' - and ab_match.get('lat') and ab_match.get('lon') - and is_single_word): - logger.info("geocode: nickname short-circuit %r → %s", q, ab_match['name']) - return { - 'query': q, - 'results': [{ - 'name': ab_match.get('address') or ab_match['name'], - 'lat': ab_match['lat'], - 'lon': ab_match['lon'], - 'source': 'address_book', - 'confidence': 'exact', - 'type': 'nickname', - 'raw': ab_match, - }], - 'count': 1, - } - except Exception as e: - logger.debug("geocode: address_book lookup failed: %s", e) - - # ── Classify intent + parse ── - intent, parsed = _classify_and_parse(q) - logger.debug("geocode: intent=%s parsed=%s", intent, parsed) - - # ── Retrieve candidates ── - candidates = [] - - if intent == 'ADDRESS': - # Parallel: Netsyms (structured) + Photon (freetext with expanded query) - netsyms_results = _retrieve_netsyms(parsed, limit=limit, lat=lat, lon=lon) - photon_results = _retrieve_photon_freetext( - parsed.get('expanded_query', q), limit=limit, lat=lat, lon=lon, zoom=zoom - ) - # Also try Photon /structured for addresses - photon_struct = _retrieve_photon_structured(parsed, limit=5) - candidates = netsyms_results + photon_results + photon_struct - - elif intent == 'POSTCODE': - netsyms_results = _retrieve_netsyms(parsed, limit=limit, lat=lat, lon=lon) - photon_results = _retrieve_photon_freetext(q, limit=limit, lat=lat, lon=lon, zoom=zoom) - candidates = netsyms_results + photon_results - - elif intent in ('LOCALITY', 'POI', 'UNKNOWN'): - candidates = _retrieve_photon_freetext(q, limit=limit, lat=lat, lon=lon, zoom=zoom) - - # ── Deduplicate by (lat, lon) proximity ── - deduped = [] - for c in candidates: - is_dup = False - for existing in deduped: - if (_haversine_m(c['lat'], c['lon'], existing['lat'], existing['lon']) < 50 - and c.get('source') == existing.get('source')): - is_dup = True - break - if not is_dup: - deduped.append(c) - candidates = deduped - - # ── Rerank ── - results = _rerank(candidates, parsed, intent, q, limit) - - # ── Address book annotation ── - _annotate_with_address_book(results) - - logger.info("geocode: %r → intent=%s, %d results", q, intent, len(results)) - return {'query': q, 'results': results, 'count': len(results)} diff --git a/lib/geocode_test.py b/lib/geocode_test.py deleted file mode 100644 index 4717b1e..0000000 --- a/lib/geocode_test.py +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env python3 -"""Tests for RECON Photon-first geocode chain.""" -import sys -import os -import json -import urllib.request -import urllib.parse - -BASE = "http://localhost:8420" - -TESTS = [ - { - "name": "home → nickname short-circuit", - "query": "home", - "check": lambda r: ( - r["count"] == 1 - and r["results"][0]["source"] == "address_book" - and r["results"][0]["confidence"] == "exact" - and r["results"][0]["type"] == "nickname" - ), - }, - { - "name": "214 north st filer → netsyms exact match (multi-word, not nickname)", - "query": "214 north st filer", - "check": lambda r: ( - r["count"] >= 1 - and r["results"][0]["source"] == "netsyms" - and r["results"][0]["confidence"] == "exact" - and r["results"][0]["type"] == "street_address" - ), - }, - { - "name": "214 North St, Filer, ID → netsyms (case/punctuation)", - "query": "214 North St, Filer, ID", - "check": lambda r: r["count"] >= 1 and r["results"][0]["source"] == "netsyms", - }, - { - "name": "214 NORTH ST FILER ID → netsyms (uppercase)", - "query": "214 NORTH ST FILER ID", - "check": lambda r: r["count"] >= 1 and r["results"][0]["source"] == "netsyms", - }, - { - "name": "1600 Pennsylvania Ave Washington DC → White House", - "query": "1600 Pennsylvania Ave Washington DC", - "check": lambda r: ( - r["count"] >= 1 - and r["results"][0]["source"] == "photon" - ), - }, - { - "name": "1600 pennsylvania ave washington dc → lowercase", - "query": "1600 pennsylvania ave washington dc", - "check": lambda r: r["count"] >= 1 and r["results"][0]["source"] == "photon", - }, - { - "name": "starbucks filer → POI result", - "query": "starbucks filer", - "check": lambda r: r["count"] >= 1 and r["results"][0]["source"] == "photon", - }, - { - "name": "filer idaho → locality", - "query": "filer idaho", - "check": lambda r: ( - r["count"] >= 1 - and r["results"][0]["source"] == "photon" - and r["results"][0]["type"] == "locality" - ), - }, - { - "name": "filer → partial query, at least 1 result", - "query": "filer", - "check": lambda r: r["count"] >= 1 and r["results"][0]["source"] == "photon", - }, - { - "name": "42.5736, -114.6066 → coordinates (with space)", - "query": "42.5736, -114.6066", - "check": lambda r: ( - r["count"] == 1 - and r["results"][0]["source"] == "coordinates" - and r["results"][0]["confidence"] == "exact" - and r["results"][0]["type"] == "coordinates" - ), - }, - { - "name": "42.5736,-114.6066 → coordinates (no space)", - "query": "42.5736,-114.6066", - "check": lambda r: ( - r["count"] == 1 - and r["results"][0]["source"] == "coordinates" - and r["results"][0]["confidence"] == "exact" - ), - }, - { - "name": "boise → at least 1 result", - "query": "boise", - "check": lambda r: r["count"] >= 1 and r["results"][0]["source"] == "photon", - }, - { - "name": "toronto → CA canary", - "query": "toronto", - "check": lambda r: r["count"] >= 1 and r["results"][0]["source"] == "photon", - }, - { - "name": "asdfghjklqwerty → empty results, 200 OK", - "query": "asdfghjklqwerty", - "check": lambda r: r["count"] == 0 and r["results"] == [], - }, - { - "name": "empty query → empty results", - "query": "", - "check": lambda r: r["count"] == 0 and r["results"] == [], - }, -] - -passed = 0 -failed = 0 - -for t in TESTS: - q = urllib.parse.urlencode({"q": t["query"]}) if t["query"] else "q=" - url = f"{BASE}/api/geocode?{q}" - try: - req = urllib.request.Request(url) - with urllib.request.urlopen(req, timeout=10) as resp: - status = resp.status - body = json.loads(resp.read()) - except urllib.error.HTTPError as e: - status = e.code - try: - body = json.loads(e.read()) - except Exception: - body = {} - except Exception as e: - status = 0 - body = {} - print(f" [FAIL] {t['name']}") - print(f" EXCEPTION: {e}") - failed += 1 - continue - - ok = status == 200 and t["check"](body) - tag = "PASS" if ok else "FAIL" - if ok: - passed += 1 - else: - failed += 1 - - top = body.get("results", [{}])[0] if body.get("results") else {} - top_summary = f"source={top.get('source','—')} type={top.get('type','—')} conf={top.get('confidence','—')} name={top.get('name','—')[:50]}" - print(f" [{tag}] {t['name']}") - if not ok: - print(f" HTTP {status}, count={body.get('count','?')}, top: {top_summary}") - else: - labeled = f" labeled_as={top.get('labeled_as')}" if top.get('labeled_as') else "" - print(f" → {top_summary}{labeled}") - -print(f"\n{passed} passed, {failed} failed") -sys.exit(0 if failed == 0 else 1) diff --git a/lib/nav_tools.py b/lib/nav_tools.py deleted file mode 100644 index d4bb1f7..0000000 --- a/lib/nav_tools.py +++ /dev/null @@ -1,168 +0,0 @@ -"""Navigation tools: geocoding via Photon and routing via Valhalla.""" - -import math -import re -import requests - -from .utils import setup_logging - -logger = setup_logging('recon.nav_tools') - -PHOTON_URL = "http://localhost:2322" -VALHALLA_URL = "http://localhost:8002" - -# Regional bias for Photon searches (Idaho-centric for Matt's use case). -# Adjustable — Photon uses these to rank nearby results higher. -GEOCODE_BIAS_LAT = 42.5736 -GEOCODE_BIAS_LON = -114.6066 -GEOCODE_BIAS_ZOOM = 10 - -# Distance threshold (meters) for annotating Photon results with address -# book labels. 75m covers GPS jitter + geocoder imprecision. -ADDRESS_BOOK_ANNOTATION_RADIUS_M = 75 - -# Coordinate regex — handles comma-separated and space-separated forms. -_COORD_RE = re.compile( - r'^\s*(-?\d+\.\d+)\s*[,\s]\s*(-?\d+\.\d+)\s*$' -) - -VALID_MODES = {"auto", "pedestrian", "bicycle", "truck"} - - -def _parse_coords(text: str): - """Return (lat, lon) if text looks like coordinates with valid bounds, else None.""" - m = _COORD_RE.match(text.strip()) - if not m: - return None - lat, lon = float(m.group(1)), float(m.group(2)) - if -90 <= lat <= 90 and -180 <= lon <= 180: - return lat, lon - return None - - -def _haversine_m(lat1, lon1, lat2, lon2): - """Haversine distance in meters between two (lat, lon) points.""" - R = 6_371_000 # Earth radius in meters - rlat1, rlat2 = math.radians(lat1), math.radians(lat2) - dlat = math.radians(lat2 - lat1) - dlon = math.radians(lon2 - lon1) - a = math.sin(dlat / 2) ** 2 + math.cos(rlat1) * math.cos(rlat2) * math.sin(dlon / 2) ** 2 - return R * 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a)) - - -def geocode(query: str, limit: int = 10, lat=None, lon=None, zoom=None): - """Delegate to the structured geocode module. See lib/geocode.py.""" - from . import geocode as geocode_mod - return geocode_mod.geocode(query, limit=limit, lat=lat, lon=lon, zoom=zoom) - - -def _geocode(query: str): - """Internal: returns (lat, lon, display_name) tuple for route().""" - result = geocode(query, limit=1) - results = result.get('results', []) - if not results: - raise ValueError(f"Could not find location: {query}") - top = results[0] - return top['lat'], top['lon'], top['name'] - - -def reverse_geocode(lat: float, lon: float) -> str: - """Reverse geocode coordinates via Photon. Returns formatted address string.""" - try: - resp = requests.get( - f"{PHOTON_URL}/reverse", - params={"lat": lat, "lon": lon, "limit": 1}, - timeout=10, - ) - resp.raise_for_status() - except requests.RequestException: - raise RuntimeError("Navigation service unavailable") - - data = resp.json() - features = data.get("features", []) - if not features: - return f"{lat}, {lon}" - - props = features[0]["properties"] - parts = [] - for key in ("name", "housenumber", "street", "city", "state", "country", "postcode"): - v = props.get(key) - if v: - parts.append(v) - return ", ".join(parts) if parts else f"{lat}, {lon}" - - -def route(origin: str, destination: str, mode: str = "auto") -> dict: - """ - Get a route between two locations. - - Args: - origin: Starting location — address, place name, or "lat,lon" - destination: Destination — address, place name, or "lat,lon" - mode: Travel mode — auto, pedestrian, bicycle, truck - - Returns: - dict with summary, maneuvers, origin/destination info, and raw shape - """ - if mode not in VALID_MODES: - mode = "auto" - - # Geocode both endpoints - orig_lat, orig_lon, orig_name = _geocode(origin) - dest_lat, dest_lon, dest_name = _geocode(destination) - - # Query Valhalla - valhalla_req = { - "locations": [ - {"lat": orig_lat, "lon": orig_lon}, - {"lat": dest_lat, "lon": dest_lon}, - ], - "costing": mode, - "directions_options": {"units": "miles"}, - } - - try: - resp = requests.post( - f"{VALHALLA_URL}/route", - json=valhalla_req, - timeout=30, - ) - except requests.RequestException: - raise RuntimeError("Navigation service unavailable") - - if resp.status_code != 200: - try: - err = resp.json() - msg = err.get("error", "Unknown routing error") - except Exception: - msg = f"Routing error (HTTP {resp.status_code})" - raise RuntimeError(f"No route found between locations: {msg}") - - data = resp.json() - trip = data["trip"] - summary = trip["summary"] - leg = trip["legs"][0] - - # Build maneuver list - maneuvers = [] - for m in leg["maneuvers"]: - streets = m.get("street_names", []) - maneuvers.append({ - "instruction": m["instruction"], - "distance_miles": round(m.get("length", 0), 2), - "street_name": streets[0] if streets else "", - "type": m.get("type", 0), - "verbal_succinct": m.get("verbal_succinct_transition_instruction", ""), - }) - - return { - "origin": {"name": orig_name, "lat": orig_lat, "lon": orig_lon}, - "destination": {"name": dest_name, "lat": dest_lat, "lon": dest_lon}, - "summary": { - "distance_miles": round(summary["length"], 1), - "time_minutes": round(summary["time"] / 60, 1), - "mode": mode, - }, - "maneuvers": maneuvers, - "shape": leg.get("shape", ""), - } diff --git a/lib/nav_tools_test.py b/lib/nav_tools_test.py deleted file mode 100644 index b987293..0000000 --- a/lib/nav_tools_test.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Tests for nav_tools — run against live Photon + Valhalla services.""" - -import sys -import json - -from nav_tools import route, reverse_geocode - - -def test_route_named(): - """route("Buhl Idaho", "Boise Idaho", "auto") returns maneuvers.""" - print("TEST 1: route('Buhl Idaho', 'Boise Idaho', 'auto')") - r = route("Buhl Idaho", "Boise Idaho", "auto") - assert r["summary"]["distance_miles"] > 50, f"Expected >50 mi, got {r['summary']['distance_miles']}" - assert r["summary"]["time_minutes"] > 60, f"Expected >60 min, got {r['summary']['time_minutes']}" - assert len(r["maneuvers"]) > 5, f"Expected >5 maneuvers, got {len(r['maneuvers'])}" - assert r["shape"], "Missing polyline shape" - print(f" OK — {r['summary']['distance_miles']} mi, {r['summary']['time_minutes']} min, {len(r['maneuvers'])} maneuvers") - print(f" Origin: {r['origin']['name']}") - print(f" Destination: {r['destination']['name']}") - print(f" First maneuver: {r['maneuvers'][0]['instruction']}") - - -def test_route_coords(): - """route with raw lat,lon coordinates.""" - print("\nTEST 2: route('42.5991,-114.7636', '43.615,-116.2023', 'auto')") - r = route("42.5991,-114.7636", "43.615,-116.2023", "auto") - assert r["summary"]["distance_miles"] > 100, f"Expected >100 mi, got {r['summary']['distance_miles']}" - assert len(r["maneuvers"]) > 3, f"Expected >3 maneuvers" - print(f" OK — {r['summary']['distance_miles']} mi, {r['summary']['time_minutes']} min") - - -def test_route_pedestrian(): - """route with pedestrian mode.""" - print("\nTEST 3: route('Buhl Idaho', 'Boise Idaho', 'pedestrian')") - r = route("Buhl Idaho", "Boise Idaho", "pedestrian") - assert r["summary"]["mode"] == "pedestrian" - assert r["summary"]["time_minutes"] > r["summary"]["distance_miles"], "Walking should take more min than miles" - print(f" OK — {r['summary']['distance_miles']} mi, {r['summary']['time_minutes']} min (pedestrian)") - - -def test_reverse_geocode(): - """reverse_geocode near Buhl, Idaho.""" - print("\nTEST 4: reverse_geocode(42.5991, -114.7636)") - result = reverse_geocode(42.5991, -114.7636) - assert "Buhl" in result or "Twin Falls" in result or "Idaho" in result, f"Expected Buhl/Idaho, got: {result}" - print(f" OK — {result}") - - -def test_route_bad_origin(): - """route with nonexistent place returns clean error.""" - print("\nTEST 5: route('nonexistent place xyz123abc', 'Boise Idaho')") - try: - r = route("nonexistent place xyz123abc", "Boise Idaho") - print(f" FAIL — expected error, got result: {r['summary']}") - return False - except ValueError as e: - print(f" OK — clean error: {e}") - except RuntimeError as e: - print(f" OK — runtime error: {e}") - - -if __name__ == "__main__": - passed = 0 - failed = 0 - tests = [test_route_named, test_route_coords, test_route_pedestrian, test_reverse_geocode, test_route_bad_origin] - - for test in tests: - try: - test() - passed += 1 - except Exception as e: - print(f" FAIL — {e}") - failed += 1 - - print(f"\n{'='*40}") - print(f"Results: {passed} passed, {failed} failed out of {len(tests)}") - sys.exit(1 if failed else 0) diff --git a/lib/netsyms_api.py b/lib/netsyms_api.py index d217eb0..2caf47c 100644 --- a/lib/netsyms_api.py +++ b/lib/netsyms_api.py @@ -1,29 +1,19 @@ """ -RECON Netsyms API + Geocode — Flask Blueprints. +RECON Netsyms API — Flask Blueprint. GET /api/netsyms/lookup?q=&country= GET /api/netsyms/health -GET /api/geocode?q=&limit= (Photon-first search with ranked results) -GET /api/reverse// (localhost-sourced enrichment bundle for Central) """ -import sqlite3 -import threading - -from cachetools import TTLCache from flask import Blueprint, request, jsonify from . import netsyms from . import address_book -from . import nav_tools -from .geocode import PHOTON_URL -from .offroute.dem import DEMReader from .utils import setup_logging logger = setup_logging('recon.netsyms_api') netsyms_bp = Blueprint('netsyms', __name__) -geocode_bp = Blueprint('geocode', __name__) @netsyms_bp.route('/api/netsyms/lookup') @@ -40,252 +30,3 @@ def api_netsyms_lookup(): @netsyms_bp.route('/api/netsyms/health') def api_netsyms_health(): return jsonify(netsyms.health()) - - - -def _safe_float(val, lo, hi): - """Parse val as float; return None if missing, non-numeric, or out of [lo, hi].""" - if val is None: - return None - try: - f = float(val) - if lo <= f <= hi: - return f - except (ValueError, TypeError): - pass - return None - -@geocode_bp.route('/api/geocode') -def api_geocode(): - """ - Photon-first geocoding with ranked candidates. - - GET /api/geocode?q=&limit= - - Always returns 200 OK with: - {query, results: [{name, lat, lon, source, confidence, type, raw, ...}], count} - - - source: "address_book" | "coordinates" | "photon" - - confidence: "exact" | "high" | "medium" | "low" - - type: "nickname" | "coordinates" | "street_address" | "poi" | "locality" - - labeled_as: present when result is within 75m of an address book entry - - Empty results array is valid (no match). No 404s. - """ - q = request.args.get('q', '').strip() - limit = request.args.get('limit', '10') - try: - limit = max(1, min(int(limit), 20)) - except (ValueError, TypeError): - limit = 10 - - # Viewport bias parameters (optional) - lat = _safe_float(request.args.get("lat"), -90, 90) - lon = _safe_float(request.args.get("lon"), -180, 180) - zoom = _safe_float(request.args.get("zoom"), 0, 22) - - result = nav_tools.geocode(q, limit=limit, lat=lat, lon=lon, zoom=zoom) - return jsonify(result) - - -@geocode_bp.route('/api/reverse') -def api_reverse(): - """ - Reverse geocode coordinates via Photon. - - GET /api/reverse?lat=X&lon=Y - - Returns same shape as /api/geocode: - {query: "lat,lon", results: [{name, lat, lon, source, type, raw, ...}], count} - - Returns 200 OK with empty results on no match. 400 on invalid coords. - """ - try: - lat = float(request.args.get('lat', '')) - lon = float(request.args.get('lon', '')) - except (ValueError, TypeError): - return jsonify({'error': 'Missing or invalid lat/lon parameters'}), 400 - - if not (-90 <= lat <= 90) or not (-180 <= lon <= 180): - return jsonify({'error': 'Coordinates out of range'}), 400 - - query_str = f"{lat},{lon}" - - try: - import requests as http_requests - resp = http_requests.get( - "http://localhost:2322/reverse", - params={"lat": lat, "lon": lon, "limit": 1}, - timeout=10, - ) - resp.raise_for_status() - data = resp.json() - features = data.get("features", []) - except Exception: - logger.warning("Photon reverse geocode failed for %s", query_str) - return jsonify({'query': query_str, 'results': [], 'count': 0}) - - if not features: - return jsonify({'query': query_str, 'results': [], 'count': 0}) - - from .geocode import _parse_photon_features - results = _parse_photon_features(features, source='photon_reverse') - - return jsonify({'query': query_str, 'results': results, 'count': len(results)}) - - -# ───────────────────────────────────────────────────────────────────────── -# /api/reverse// — localhost-sourced enrichment bundle (Central) -# -# Sibling to the query-string /api/reverse above; that route is unchanged. -# Every component is sourced from localhost only (Photon, timezones.sqlite, -# in-process landclass/PostGIS, planet-DEM PMTiles). Each lookup is -# independent: a component failure logs a warning and yields null — never 5xx. -# ───────────────────────────────────────────────────────────────────────── - -_TZ_DB_PATH = "/mnt/nav/sources/timezones.sqlite" - -# Full bundle cache: key=(round(lat,4), round(lon,4)) -> dict. ~10k entries, 24h TTL. -_REVERSE_BUNDLE_CACHE = TTLCache(maxsize=10_000, ttl=86_400) -_REVERSE_BUNDLE_LOCK = threading.Lock() - -_BUNDLE_KEYS = ('name', 'city', 'county', 'state', 'country', - 'postal_code', 'timezone', 'landclass', 'elevation_m') - -# planet-DEM elevation source (single PMTiles, replaces Valhalla /height). -# Instantiated once at import; the underlying mmap is lazy. None if unavailable. -try: - _DEM = DEMReader() -except Exception as e: # pragma: no cover - depends on PMTiles availability - logger.warning("DEMReader unavailable, elevation will be null: %s", e) - _DEM = None - - -def _spatialite_blob_to_wkb(blob): - """Recover standard WKB from a SpatiaLite geometry BLOB. - - Layout: [00][endian][srid:4][mbr:32][7C][WKB body][FE]. The body omits the - leading byte-order marker, so we re-prepend it and drop the trailing 0xFE. - """ - return bytes([blob[1]]) + blob[39:-1] - - -def _reverse_photon(lat, lon): - """Nearest-feature admin fields from local Photon. Returns the six address - fields (any value may be None). Mirrors the existing /api/reverse call.""" - import requests as http_requests - resp = http_requests.get( - f"{PHOTON_URL}/reverse", - params={"lat": lat, "lon": lon, "limit": 1}, - timeout=10, - ) - resp.raise_for_status() - features = resp.json().get("features", []) - if not features: - return {} - props = features[0].get("properties", {}) - return { - "name": props.get("name"), - "city": props.get("city"), - "county": props.get("county"), - "state": props.get("state"), - "country": props.get("country"), - "postal_code": props.get("postcode"), - } - - -def _reverse_timezone(lat, lon): - """IANA tzid for the point from local timezones.sqlite (SpatiaLite tz_world). - - Uses the table's R-tree index for an MBR prefilter, then shapely - point-in-polygon on the few candidates. Returns None if unresolved. - """ - from shapely import wkb - from shapely.geometry import Point - con = sqlite3.connect(f"file:{_TZ_DB_PATH}?mode=ro", uri=True) - try: - cur = con.cursor() - cur.execute( - "SELECT pkid FROM idx_tz_world_geom " - "WHERE xmin<=? AND xmax>=? AND ymin<=? AND ymax>=?", - (lon, lon, lat, lat), - ) - candidates = [r[0] for r in cur.fetchall()] - if not candidates: - return None - pt = Point(lon, lat) - for pk in candidates: - row = cur.execute( - "SELECT tzid, geom FROM tz_world WHERE pk_uid=?", (pk,) - ).fetchone() - if row and wkb.loads(_spatialite_blob_to_wkb(row[1])).contains(pt): - return row[0] - return None - finally: - con.close() - - -def _reverse_landclass(lat, lon): - """Most-specific PAD-US land class for the point, looked up in-process. - Returns None when there is no coverage or landclass is unavailable.""" - from .landclass import lookup_landclass, format_summary - return format_summary(lookup_landclass(lat, lon)) - - -def _reverse_elevation(lat, lon): - """Elevation in metres from the planet-DEM PMTiles — the single elevation - source per OFFROUTE-ARCHITECTURE.md §9. None on failure, on untiled points - (e.g. true ocean), or if DEMReader could not be initialized at startup.""" - if _DEM is None: - return None - return _DEM.sample_point(lat, lon) - - -@geocode_bp.route('/api/reverse//') -def api_reverse_bundle(lat, lon): - """Localhost-sourced reverse-geocode enrichment bundle for Central. - - GET /api/reverse// - - Always returns 200 with EXACTLY these keys (any may be null): - name, city, county, state, country, postal_code, timezone, landclass, elevation_m - - lat/lon are parsed manually (not via Flask's converter, which - rejects negative and integer coordinates) so out-of-range or unparseable - input yields 400 per contract; 503 is reserved for catastrophic failure. - """ - try: - lat = float(lat) - lon = float(lon) - except (ValueError, TypeError): - return jsonify({'error': 'lat and lon must be numbers'}), 400 - if not (-90 <= lat <= 90) or not (-180 <= lon <= 180): - return jsonify({'error': 'lat must be -90..90, lon must be -180..180'}), 400 - - key = (round(lat, 4), round(lon, 4)) - with _REVERSE_BUNDLE_LOCK: - cached = _REVERSE_BUNDLE_CACHE.get(key) - if cached is not None: - return jsonify(cached) - - bundle = {k: None for k in _BUNDLE_KEYS} - - try: - bundle.update(_reverse_photon(lat, lon)) - except Exception: - logger.warning("reverse-bundle: Photon lookup failed for %s,%s", lat, lon) - try: - bundle['timezone'] = _reverse_timezone(lat, lon) - except Exception: - logger.warning("reverse-bundle: timezone lookup failed for %s,%s", lat, lon) - try: - bundle['landclass'] = _reverse_landclass(lat, lon) - except Exception: - logger.warning("reverse-bundle: landclass lookup failed for %s,%s", lat, lon) - try: - bundle['elevation_m'] = _reverse_elevation(lat, lon) - except Exception: - logger.warning("reverse-bundle: elevation lookup failed for %s,%s", lat, lon) - - with _REVERSE_BUNDLE_LOCK: - _REVERSE_BUNDLE_CACHE[key] = bundle - return jsonify(bundle) diff --git a/lib/reverse_bundle_test.py b/lib/reverse_bundle_test.py deleted file mode 100644 index 6defd9e..0000000 --- a/lib/reverse_bundle_test.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env python3 -"""Tests for the /api/reverse// enrichment bundle (lib.netsyms_api). - -Photon/DEM/landclass are mocked so the suite runs without live services; -one timezone test exercises the real SpatiaLite DB when it is present. Plain -asserts + a __main__ runner, matching the rest of lib/*_test.py. -""" - -import os -import sys -from unittest import mock - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from flask import Flask -from lib import netsyms_api - -EXPECTED_KEYS = set(netsyms_api._BUNDLE_KEYS) - - -def _client(): - app = Flask(__name__) - app.register_blueprint(netsyms_api.geocode_bp) - return app.test_client() - - -def _clear_cache(): - netsyms_api._REVERSE_BUNDLE_CACHE.clear() - - -def test_happy_path(): - _clear_cache() - with mock.patch.object(netsyms_api, '_reverse_photon', return_value={ - 'name': 'Where you are', 'city': 'Boise', 'county': 'Ada', - 'state': 'Idaho', 'country': 'United States', 'postal_code': '83701'}), \ - mock.patch.object(netsyms_api, '_reverse_timezone', return_value='America/Boise'), \ - mock.patch.object(netsyms_api, '_reverse_landclass', return_value='Boise National Forest'), \ - mock.patch.object(netsyms_api, '_reverse_elevation', return_value=824): - resp = _client().get('/api/reverse/43.6150/-116.2023') - assert resp.status_code == 200, resp.status_code - data = resp.get_json() - assert set(data.keys()) == EXPECTED_KEYS, data.keys() - assert data['city'] == 'Boise' and data['timezone'] == 'America/Boise' - assert data['landclass'] == 'Boise National Forest' and data['elevation_m'] == 824 - print(" PASS: happy path — all 9 fields populated, exact key set") - - -def test_negative_and_integer_coords_parse(): - # Regression: Flask's converter would 404 these; manual parse must not. - _clear_cache() - with mock.patch.object(netsyms_api, '_reverse_photon', return_value={}), \ - mock.patch.object(netsyms_api, '_reverse_timezone', return_value=None), \ - mock.patch.object(netsyms_api, '_reverse_landclass', return_value=None), \ - mock.patch.object(netsyms_api, '_reverse_elevation', return_value=None): - for path in ('/api/reverse/43.6/-116.2', '/api/reverse/43/-116'): - resp = _client().get(path) - assert resp.status_code == 200, f"{path} -> {resp.status_code}" - assert set(resp.get_json().keys()) == EXPECTED_KEYS - print(" PASS: negative and integer coordinates parse (200, not 404)") - - -def test_partial_failure_returns_200_with_nulls(): - _clear_cache() - with mock.patch.object(netsyms_api, '_reverse_photon', - side_effect=RuntimeError('photon down')), \ - mock.patch.object(netsyms_api, '_reverse_timezone', return_value='America/Boise'), \ - mock.patch.object(netsyms_api, '_reverse_landclass', - side_effect=RuntimeError('postgis down')), \ - mock.patch.object(netsyms_api, '_reverse_elevation', return_value=824): - resp = _client().get('/api/reverse/43.6150/-116.2023') - assert resp.status_code == 200, resp.status_code - data = resp.get_json() - assert set(data.keys()) == EXPECTED_KEYS - assert data['name'] is None and data['city'] is None # photon failed -> nulls - assert data['landclass'] is None # landclass failed -> null - assert data['timezone'] == 'America/Boise' and data['elevation_m'] == 824 - print(" PASS: per-component failure -> 200 with nulls, no 5xx") - - -def test_ocean_point_mostly_null(): - _clear_cache() - with mock.patch.object(netsyms_api, '_reverse_photon', return_value={}), \ - mock.patch.object(netsyms_api, '_reverse_timezone', return_value='Etc/GMT+2'), \ - mock.patch.object(netsyms_api, '_reverse_landclass', return_value=None), \ - mock.patch.object(netsyms_api, '_reverse_elevation', return_value=0): - resp = _client().get('/api/reverse/0.0/-30.0') - assert resp.status_code == 200, resp.status_code - data = resp.get_json() - assert set(data.keys()) == EXPECTED_KEYS - assert data['city'] is None and data['country'] is None and data['landclass'] is None - print(" PASS: ocean point -> 200, mostly null") - - -def test_invalid_input_400(): - _clear_cache() - client = _client() - for path in ('/api/reverse/9999/0', '/api/reverse/0/9999', '/api/reverse/abc/0'): - resp = client.get(path) - assert resp.status_code == 400, f"{path} -> {resp.status_code}" - print(" PASS: out-of-range / unparseable input -> 400") - - -def test_cache_hit_serves_without_recompute(): - _clear_cache() - with mock.patch.object(netsyms_api, '_reverse_photon', - return_value={'name': 'X'}) as m_photon, \ - mock.patch.object(netsyms_api, '_reverse_timezone', return_value=None), \ - mock.patch.object(netsyms_api, '_reverse_landclass', return_value=None), \ - mock.patch.object(netsyms_api, '_reverse_elevation', return_value=None): - client = _client() - client.get('/api/reverse/12.3456/-65.4321') - client.get('/api/reverse/12.3456/-65.4321') # same key (rounded) -> cached - assert m_photon.call_count == 1, f"expected 1 compute, got {m_photon.call_count}" - print(" PASS: second identical request served from cache (no recompute)") - - -def test_real_timezone_db(): - if not os.path.exists(netsyms_api._TZ_DB_PATH): - print(" SKIP: real timezone test (timezones.sqlite not present)") - return - assert netsyms_api._reverse_timezone(43.6150, -116.2023) == 'America/Boise' - assert netsyms_api._reverse_timezone(40.7128, -74.0060) == 'America/New_York' - print(" PASS: real timezones.sqlite point-in-polygon") - - -def test_elevation_from_dem_reader_mock(): - # elevation_m comes from DEMReader.sample_point (not Valhalla); other - # components stubbed to null so the bundle is hermetic. - _clear_cache() - fake_dem = mock.Mock() - fake_dem.sample_point.return_value = 824 - with mock.patch.object(netsyms_api, '_DEM', fake_dem), \ - mock.patch.object(netsyms_api, '_reverse_photon', return_value={}), \ - mock.patch.object(netsyms_api, '_reverse_timezone', return_value=None), \ - mock.patch.object(netsyms_api, '_reverse_landclass', return_value=None): - resp = _client().get('/api/reverse/43.6150/-116.2023') - assert resp.status_code == 200, resp.status_code - data = resp.get_json() - assert set(data.keys()) == EXPECTED_KEYS - assert data['elevation_m'] == 824, data['elevation_m'] - fake_dem.sample_point.assert_called_once() - print(" PASS: elevation_m sourced from DEMReader.sample_point") - - -def test_elevation_dem_unavailable(): - # DEMReader failed to init at startup (_DEM is None) -> elevation_m null, 200. - _clear_cache() - with mock.patch.object(netsyms_api, '_DEM', None), \ - mock.patch.object(netsyms_api, '_reverse_photon', return_value={}), \ - mock.patch.object(netsyms_api, '_reverse_timezone', return_value=None), \ - mock.patch.object(netsyms_api, '_reverse_landclass', return_value=None): - resp = _client().get('/api/reverse/43.6150/-116.2023') - assert resp.status_code == 200, resp.status_code - data = resp.get_json() - assert set(data.keys()) == EXPECTED_KEYS - assert data['elevation_m'] is None - print(" PASS: DEMReader unavailable -> elevation_m null, still 200") - - -if __name__ == '__main__': - print("Running reverse-bundle tests...") - test_happy_path() - test_negative_and_integer_coords_parse() - test_partial_failure_returns_200_with_nulls() - test_ocean_point_mostly_null() - test_invalid_input_400() - test_cache_hit_serves_without_recompute() - test_real_timezone_db() - test_elevation_from_dem_reader_mock() - test_elevation_dem_unavailable() - print("All tests passed.") From 1f05d4b4d6ce05f6682b9d43f5a025a7fc58e36e Mon Sep 17 00:00:00 2001 From: malice Date: Sat, 23 May 2026 04:14:47 -0600 Subject: [PATCH 32/41] cleanup: remove /api/landclass handler + lib/landclass.py (extraction #4 shadow) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /api/landclass is edge-shadowed since extraction #4 — navi-landclass :8424 serves the route via nginx. Cleanup #4 removed the last in-process consumer (netsyms_api._reverse_landclass), so lib/landclass.py is now fully orphaned. - api.py: drop the @app.route('/api/landclass') handler + the `from .landclass import lookup_landclass, format_summary` import. - DELETE lib/landclass.py (only consumer was the deleted handler). - DELETE lib/landclass_test.py (SUT gone). PADUS_DB_* vars in /opt/recon/.env are now dead in recon — flagged for an out-of-band post-merge cleanup, not touched here (data, not code). Co-authored-by: Claude Opus 4.7 (1M context) --- lib/api.py | 33 ------ lib/landclass.py | 255 ------------------------------------------ lib/landclass_test.py | 44 -------- 3 files changed, 332 deletions(-) delete mode 100644 lib/landclass.py delete mode 100644 lib/landclass_test.py diff --git a/lib/api.py b/lib/api.py index e4a186f..9d57de1 100644 --- a/lib/api.py +++ b/lib/api.py @@ -25,7 +25,6 @@ from werkzeug.utils import secure_filename from .utils import get_config, content_hash, clean_filename_to_title, derive_source_and_category, generate_download_url, setup_logging from .status import StatusDB from .deployment_config import get_deployment_config -from .landclass import lookup_landclass, format_summary logger = setup_logging('recon.api') @@ -1170,38 +1169,6 @@ def api_knowledge_stats(): return jsonify(_cache['knowledge_stats']) -@app.route('/api/landclass') -def api_landclass(): - """PAD-US land classification lookup for a point.""" - config = get_deployment_config() - if not config.get('features', {}).get('has_landclass'): - return jsonify({'error': 'Land classification not available'}), 404 - - try: - lat = float(request.args.get('lat', '')) - lon = float(request.args.get('lon', '')) - except (ValueError, TypeError): - return jsonify({'error': 'lat and lon required as numbers'}), 400 - - if not (-90 <= lat <= 90) or not (-180 <= lon <= 180): - return jsonify({'error': 'lat must be -90..90, lon must be -180..180'}), 400 - - classifications = lookup_landclass(lat, lon) - is_public = len(classifications) > 0 - is_private = len(classifications) == 0 - summary = format_summary(classifications) - - return jsonify({ - 'lat': lat, - 'lon': lon, - 'classifications': classifications, - 'count': len(classifications), - 'is_public': is_public, - 'is_private': is_private, - 'summary': summary, - }) - - @app.route('/api/health') def api_health(): """Health check endpoint for monitoring.""" diff --git a/lib/landclass.py b/lib/landclass.py deleted file mode 100644 index 7760cce..0000000 --- a/lib/landclass.py +++ /dev/null @@ -1,255 +0,0 @@ -""" -PAD-US land classification lookup. - -Provides point-in-polygon queries against the USGS Protected Areas Database -(PAD-US) stored in a local PostGIS database. Returns land ownership, -management, and public access information for any lat/lon coordinate. - -Connection pool is lazy-initialized on first call. If PostgreSQL is unreachable, -functions return empty results gracefully (feature degrades, doesn't crash). -""" -import os - -import psycopg2 -import psycopg2.pool - -from .utils import setup_logging - -logger = setup_logging('recon.landclass') - -_pool = None -_pool_failed = False - -# ── Label mappings from PAD-US domain tables ──────────────────────────── -# Extracted from PADUS4_0_Geodatabase.gdb domain lookup layers. -# ogr2ogr lowercases all column names. - -AGENCY_NAME_MAP = { - 'TVA': 'Tennessee Valley Authority', - 'BLM': 'Bureau of Land Management', - 'BOEM': 'Bureau of Ocean Energy Management', - 'USBR': 'Bureau of Reclamation', - 'FWS': 'U.S. Fish and Wildlife Service', - 'USFS': 'Forest Service', - 'DOD': 'Department of Defense', - 'USACE': 'Army Corps of Engineers', - 'DOE': 'Department of Energy', - 'NPS': 'National Park Service', - 'NRCS': 'Natural Resources Conservation Service', - 'ARS': 'Agricultural Research Service', - 'BIA': 'Bureau of Indian Affairs', - 'NOAA': 'National Oceanic and Atmospheric Administration', - 'BPA': 'Bonneville Power Administration', - 'OTHF': 'Other or Unknown Federal Land', - 'TRIB': 'American Indian Lands', - 'SPR': 'State Park and Recreation', - 'SDC': 'State Department of Conservation', - 'SLB': 'State Land Board', -} - -AGENCY_TYPE_MAP = { - 'FED': 'Federal', - 'TRIB': 'American Indian Lands', - 'STAT': 'State', - 'DIST': 'Regional Agency Special District', - 'LOC': 'Local Government', - 'NGO': 'Non-Governmental Organization', - 'PVT': 'Private', - 'JNT': 'Joint', - 'UNK': 'Unknown', - 'TERR': 'Territorial', - 'DESG': 'Designation', -} - -DESIGNATION_TYPE_MAP = { - 'NP': 'National Park', - 'NM': 'National Monument', - 'NCA': 'Conservation Area', - 'NF': 'National Forest', - 'NG': 'National Grassland', - 'PUB': 'National Public Lands', - 'NT': 'National Scenic or Historic Trail', - 'NWR': 'National Wildlife Refuge', - 'WA': 'Wilderness Area', - 'WSR': 'Wild and Scenic River', - 'WSA': 'Wilderness Study Area', - 'MPA': 'Marine Protected Area', - 'NRA': 'National Recreation Area', - 'NSBV': 'National Scenic, Botanical or Volcanic Area', - 'NLS': 'National Lakeshore or Seashore', - 'IRA': 'Inventoried Roadless Area', - 'ACEC': 'Area of Critical Environmental Concern', - 'RNA': 'Research Natural Area', - 'REC': 'Recreation Management Area', - 'RMA': 'Resource Management Area', - 'WPA': 'Watershed Protection Area', - 'REA': 'Research or Educational Area', - 'HCA': 'Historic or Cultural Area', - 'MIT': 'Mitigation Land or Bank', - 'MIL': 'Military Land', - 'ACC': 'Access Area', - 'SDA': 'Special Designation Area', - 'PROC': 'Approved or Proclamation Boundary', - 'FOTH': 'Federal Other or Unknown', - 'ND': 'Not Designated', -} - -PUBLIC_ACCESS_MAP = { - 'OA': 'Open Access', - 'RA': 'Restricted Access', - 'XA': 'Closed', - 'UK': 'Unknown', -} - -GAP_STATUS_MAP = { - '1': 'Managed for biodiversity (disturbance events proceed)', - '2': 'Managed for biodiversity (disturbance suppressed)', - '3': 'Multiple uses (extractive/OHV)', - '4': 'No known mandate for biodiversity protection', -} - -CATEGORY_MAP = { - 'Fee': 'Fee', - 'Easement': 'Easement', - 'Other': 'Other', - 'Unknown': 'Unknown', - 'Designation': 'Designation', - 'Marine': 'Marine Area', - 'Proclamation': 'Approved, Proclamation or Extent Boundary', -} - -STATE_MAP = { - 'AL': 'Alabama', 'AK': 'Alaska', 'AZ': 'Arizona', 'AR': 'Arkansas', - 'CA': 'California', 'CO': 'Colorado', 'CT': 'Connecticut', 'DE': 'Delaware', - 'DC': 'District of Columbia', 'FL': 'Florida', 'GA': 'Georgia', 'HI': 'Hawaii', - 'ID': 'Idaho', 'IL': 'Illinois', 'IN': 'Indiana', 'IA': 'Iowa', - 'KS': 'Kansas', 'KY': 'Kentucky', 'LA': 'Louisiana', 'ME': 'Maine', - 'MD': 'Maryland', 'MA': 'Massachusetts', 'MI': 'Michigan', 'MN': 'Minnesota', - 'MS': 'Mississippi', 'MO': 'Missouri', 'MT': 'Montana', 'NE': 'Nebraska', - 'NV': 'Nevada', 'NH': 'New Hampshire', 'NJ': 'New Jersey', 'NM': 'New Mexico', - 'NY': 'New York', 'NC': 'North Carolina', 'ND': 'North Dakota', 'OH': 'Ohio', - 'OK': 'Oklahoma', 'OR': 'Oregon', 'PA': 'Pennsylvania', 'RI': 'Rhode Island', - 'SC': 'South Carolina', 'SD': 'South Dakota', 'TN': 'Tennessee', 'TX': 'Texas', - 'UT': 'Utah', 'VT': 'Vermont', 'VA': 'Virginia', 'WA': 'Washington', - 'WV': 'West Virginia', 'WI': 'Wisconsin', 'WY': 'Wyoming', -} - - -def _decode(code, label_map): - """Decode a PAD-US code using a label map. Returns decoded label or the raw code.""" - if not code: - return '' - code = str(code).strip() - return label_map.get(code, code) - - -def _get_pool(): - """Lazy-init the connection pool. Returns None if Postgres is unreachable.""" - global _pool, _pool_failed - if _pool is not None: - return _pool - if _pool_failed: - return None - - try: - _pool = psycopg2.pool.SimpleConnectionPool( - minconn=1, - maxconn=3, - host=os.environ.get('PADUS_DB_HOST', 'localhost'), - port=int(os.environ.get('PADUS_DB_PORT', '5432')), - dbname=os.environ.get('PADUS_DB_NAME', 'padus'), - user=os.environ.get('PADUS_DB_USER', 'overture'), - password=os.environ.get('PADUS_DB_PASSWORD', ''), - connect_timeout=5, - ) - logger.info("PAD-US PostgreSQL connection pool initialized") - return _pool - except Exception as e: - _pool_failed = True - logger.warning(f"PAD-US PostgreSQL unavailable, land classification disabled: {e}") - return None - - -def _query_all(sql, params): - """Execute a query and return all rows as a list of dicts, or empty list.""" - pool = _get_pool() - if pool is None: - return [] - - conn = None - try: - conn = pool.getconn() - with conn.cursor() as cur: - cur.execute(sql, params) - rows = cur.fetchall() - if not rows: - return [] - cols = [desc[0] for desc in cur.description] - return [dict(zip(cols, row)) for row in rows] - except Exception as e: - logger.warning(f"PAD-US query error: {e}") - if conn: - try: - conn.rollback() - except Exception: - pass - return [] - finally: - if conn: - try: - pool.putconn(conn) - except Exception: - pass - - -def lookup_landclass(lat, lon): - """ - Look up PAD-US land classifications for a point. - - Returns a list of classification dicts, ordered by area ascending - (smallest/most specific first). Empty list on error or no results. - """ - rows = _query_all( - """SELECT unit_nm, mang_name, mang_type, own_name, own_type, - des_tp, gap_sts, pub_access, category, gis_acres, state_nm - FROM pad_units - WHERE ST_Intersects(geom, ST_SetSRID(ST_MakePoint(%s, %s), 4326)) - -- exclude antimeridian-wrapping polygons: 47 BOEM marine artifacts - -- span ~360 deg longitude and false-match non-US points at their lat band - AND (ST_XMax(geom) - ST_XMin(geom)) < 60 - ORDER BY gis_acres ASC - LIMIT 10""", - (lon, lat) - ) - - results = [] - for row in rows: - pa_code = str(row.get('pub_access', '')).strip() - - results.append({ - 'unit_name': (row.get('unit_nm') or '').strip(), - 'manager_name': _decode(row.get('mang_name'), AGENCY_NAME_MAP), - 'manager_type': _decode(row.get('mang_type'), AGENCY_TYPE_MAP), - 'owner_type': _decode(row.get('own_type'), AGENCY_TYPE_MAP), - 'designation_type': _decode(row.get('des_tp'), DESIGNATION_TYPE_MAP), - 'gap_status': str(row.get('gap_sts', '')).strip(), - 'public_access': _decode(pa_code, PUBLIC_ACCESS_MAP), - 'public_access_code': pa_code, - 'category': _decode(row.get('category'), CATEGORY_MAP), - 'acres': row.get('gis_acres'), - 'state': _decode(row.get('state_nm'), STATE_MAP), - }) - - return results - - -def format_summary(classifications): - """ - Format a human-readable summary from classification results. - - Returns the most specific unit name, or None if no results. - """ - if not classifications: - return None - # First result is smallest/most specific (ordered by acres ASC) - return classifications[0].get('unit_name') or None diff --git a/lib/landclass_test.py b/lib/landclass_test.py deleted file mode 100644 index cba8ca7..0000000 --- a/lib/landclass_test.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python3 -"""Tests for lib.landclass PAD-US lookups. - -Live-PostgreSQL regression test using the skip-if-not-available pattern -(matching test_real_timezone_db in reverse_bundle_test.py). Plain asserts + -a __main__ runner, matching the rest of lib/*_test.py. - -Note: lookup_landclass swallows DB errors and returns [] (it never raises), -so PG availability is probed via a known US point (Boise) rather than by -catching an exception. -""" - -import os -import sys - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from lib import landclass - - -def test_landclass_no_antimeridian_false_match(): - # Yosemite doubles as the liveness probe: a point on real US public land. - # (lookup_landclass returns [] when PG is unreachable AND when the point is - # off public land, so the probe must be a known-public-land point — e.g. - # downtown Boise is private and would yield [] even with PG up.) - yosemite = landclass.lookup_landclass(37.85, -119.55) - if not yosemite: - print(" SKIP: live PG not available (Yosemite returned no rows)") - return - # Filter must NOT drop legitimate (non-wrapping) US units. - assert len(yosemite) >= 1, f"Yosemite should match >=1 PAD-US unit, got {len(yosemite)}" - - # London (51.5074 N) previously false-matched the antimeridian-wrapping - # 'Rat Islands' record (ogc_fid 3974, ~360 deg lon span). The < 60 deg - # filter must now drop it -> empty result. - london = landclass.lookup_landclass(51.5074, -0.1278) - assert london == [], f"London should match no PAD-US unit, got {[r.get('unit_name') for r in london]}" - print(" PASS: antimeridian filter drops London false-match, keeps Yosemite coverage") - - -if __name__ == '__main__': - print("Running landclass tests...") - test_landclass_no_antimeridian_false_match() - print("All tests passed.") From 86c902f7b5666890cf8024d2281780e572efebf2 Mon Sep 17 00:00:00 2001 From: malice Date: Sat, 23 May 2026 04:25:54 -0600 Subject: [PATCH 33/41] cleanup: remove /api/offroute + /api/mvum handlers + lib/offroute/ package (extraction #8 shadow) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /api/offroute (POST) and /api/mvum (GET) are edge-shadowed since extraction #8 — navi-offroute :8428 serves both via nginx. Cleanup #4 removed the last in-process consumer of lib/offroute/dem.py (netsyms_api._reverse_elevation + the module-level _DEM = DEMReader()), so the entire 9-file lib/offroute/ package is now orphaned and goes with this PR. - api.py: drop both handlers (api_offroute, api_mvum) + their section comments. Both used in-function lazy imports of offroute, so no top-of-file import survives. - DELETE lib/offroute/ wholesale (__init__, router, mvum, cost, barriers, dem, friction, trails, prototype). prototype.py was already dead at runtime. Closes the recon->navi navi-shadow cleanup loop: recon now serves zero navi-* shadow routes. Co-authored-by: Claude Opus 4.7 (1M context) --- lib/api.py | 211 ----- lib/offroute/__init__.py | 1 - lib/offroute/barriers.py | 440 ---------- lib/offroute/cost.py | 494 ----------- lib/offroute/dem.py | 211 ----- lib/offroute/friction.py | 137 --- lib/offroute/mvum.py | 623 -------------- lib/offroute/prototype.py | 414 --------- lib/offroute/router.py | 1682 ------------------------------------- lib/offroute/trails.py | 174 ---- 10 files changed, 4387 deletions(-) delete mode 100644 lib/offroute/__init__.py delete mode 100644 lib/offroute/barriers.py delete mode 100644 lib/offroute/cost.py delete mode 100644 lib/offroute/dem.py delete mode 100644 lib/offroute/friction.py delete mode 100644 lib/offroute/mvum.py delete mode 100755 lib/offroute/prototype.py delete mode 100644 lib/offroute/router.py delete mode 100644 lib/offroute/trails.py diff --git a/lib/api.py b/lib/api.py index 9d57de1..3a7e5ca 100644 --- a/lib/api.py +++ b/lib/api.py @@ -2553,214 +2553,3 @@ def api_auth_whoami(): 'authenticated': False, 'username': None, }) - - -# ── OFFROUTE API ── - -@app.route("/api/offroute", methods=["POST"]) -def api_offroute(): - """ - Off-network routing from wilderness to destination. - - Request body: - { - "start": [lat, lon], - "end": [lat, lon], - "mode": "foot" | "mtb" | "atv", (default: "foot") - "boundary_mode": "strict" | "pragmatic" | "emergency" (default: "pragmatic") - } - - Response: - { - "status": "ok", - "route": { GeoJSON FeatureCollection with wilderness + network segments }, - "summary": { total_distance_km, total_effort_minutes, ... } - } - """ - try: - data = request.get_json() - if not data: - return jsonify({"status": "error", "message": "No JSON body provided"}), 400 - - # Parse coordinates - start = data.get("start") - end = data.get("end") - - if not start or not end: - return jsonify({"status": "error", "message": "Missing start or end coordinates"}), 400 - - if not isinstance(start, (list, tuple)) or len(start) != 2: - return jsonify({"status": "error", "message": "start must be [lat, lon]"}), 400 - if not isinstance(end, (list, tuple)) or len(end) != 2: - return jsonify({"status": "error", "message": "end must be [lat, lon]"}), 400 - - start_lat, start_lon = float(start[0]), float(start[1]) - end_lat, end_lon = float(end[0]), float(end[1]) - - # Parse options - mode = data.get("mode", "foot") - if mode not in ("auto", "foot", "mtb", "atv", "vehicle"): - return jsonify({"status": "error", "message": "mode must be auto, foot, mtb, atv, or vehicle"}), 400 - - boundary_mode = data.get("boundary_mode", "pragmatic") - if boundary_mode not in ("strict", "pragmatic", "emergency"): - return jsonify({"status": "error", "message": "boundary_mode must be strict, pragmatic, or emergency"}), 400 - - # Import and run router - from .offroute.router import OffrouteRouter - - router = OffrouteRouter() - try: - result = router.route( - start_lat=start_lat, - start_lon=start_lon, - end_lat=end_lat, - end_lon=end_lon, - mode=mode, - boundary_mode=boundary_mode - ) - finally: - router.close() - - if result.get("status") == "error": - return jsonify(result), 400 - - return jsonify(result) - - except Exception as e: - logger.exception("Offroute error") - return jsonify({"status": "error", "message": str(e)}), 500 - - -# ── MVUM Places Panel API ── - -@app.route("/api/mvum", methods=["GET"]) -def api_mvum(): - """ - Query MVUM (Motor Vehicle Use Map) features near a point. - - Used by the Navi frontend places panel when a user taps near a road/trail. - - Query params: - lat: Latitude - lon: Longitude - radius: Search radius in meters (default: 50) - - Response: - { - "status": "ok", - "feature": { - "id": "FR 123", - "name": "Some Forest Road", - "forest": "Sawtooth National Forest", - "district": "Ketchum Ranger District", - "surface": "NAT", - "maintenance_level": 2, - "seasonal": "Seasonal", - "symbol": 2, - "access": { - "passenger_vehicle": { "status": "Open", "dates": "06/15-10/15" }, - "high_clearance": { "status": "Open", "dates": "06/15-10/15" }, - "atv": { "status": "Open", "dates": "06/15-10/15" }, - ... - }, - "geometry": { GeoJSON LineString } - } - } - - If no MVUM feature within radius: - { "status": "ok", "feature": null } - """ - try: - lat = request.args.get("lat", type=float) - lon = request.args.get("lon", type=float) - radius = request.args.get("radius", 50, type=float) - - if lat is None or lon is None: - return jsonify({"status": "error", "message": "lat and lon required"}), 400 - - from .offroute.mvum import MVUMReader - - reader = MVUMReader() - try: - # Try roads first, then trails - feature = reader.query_nearest(lat, lon, radius, "mvum_roads") - if feature is None: - feature = reader.query_nearest(lat, lon, radius, "mvum_trails") - - if feature is None: - return jsonify({"status": "ok", "feature": None}) - - # Format access info - access = { - "passenger_vehicle": { - "status": feature.get("passengervehicle"), - "dates": feature.get("passengervehicle_datesopen") - }, - "high_clearance": { - "status": feature.get("highclearancevehicle"), - "dates": feature.get("highclearancevehicle_datesopen") - }, - "atv": { - "status": feature.get("atv"), - "dates": feature.get("atv_datesopen") - }, - "motorcycle": { - "status": feature.get("motorcycle"), - "dates": feature.get("motorcycle_datesopen") - }, - "4wd_gt50": { - "status": feature.get("fourwd_gt50inches"), - "dates": feature.get("fourwd_gt50_datesopen") - }, - "2wd_gt50": { - "status": feature.get("twowd_gt50inches"), - "dates": feature.get("twowd_gt50_datesopen") - }, - "e_bike_class1": { - "status": feature.get("e_bike_class1"), - "dates": feature.get("e_bike_class1_dur") - }, - "e_bike_class2": { - "status": feature.get("e_bike_class2"), - "dates": feature.get("e_bike_class2_dur") - }, - "e_bike_class3": { - "status": feature.get("e_bike_class3"), - "dates": feature.get("e_bike_class3_dur") - }, - } - - # Parse maintenance level - maint_level = feature.get("operationalmaintlevel", "") - maint_num = None - if maint_level: - # Extract first digit: "2 - HIGH CLEARANCE VEHICLES" -> 2 - import re - match = re.match(r"(\d+)", maint_level) - if match: - maint_num = int(match.group(1)) - - result = { - "id": feature.get("id"), - "name": feature.get("name"), - "forest": feature.get("forestname"), - "district": feature.get("districtname"), - "surface": feature.get("surfacetype"), - "maintenance_level": maint_num, - "seasonal": feature.get("seasonal"), - "symbol": feature.get("symbol"), - "trail_class": feature.get("trailclass"), - "trail_system": feature.get("trailsystem"), - "access": access, - "geometry": feature.get("geojson") - } - - return jsonify({"status": "ok", "feature": result}) - - finally: - reader.close() - - except Exception as e: - logger.exception("MVUM query error") - return jsonify({"status": "error", "message": str(e)}), 500 diff --git a/lib/offroute/__init__.py b/lib/offroute/__init__.py deleted file mode 100644 index b0536cd..0000000 --- a/lib/offroute/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""OFFROUTE: Off-network effort-based routing module.""" diff --git a/lib/offroute/barriers.py b/lib/offroute/barriers.py deleted file mode 100644 index f68e892..0000000 --- a/lib/offroute/barriers.py +++ /dev/null @@ -1,440 +0,0 @@ -""" -PAD-US barrier and wilderness layers for OFFROUTE. - -Provides access to: -1. Barrier raster (Pub_Access = 'XA' - closed/restricted areas) -2. Wilderness raster (Des_Tp = 'WA' - designated wilderness areas) - -Build functions rasterize PAD-US geodatabase to aligned GeoTIFFs. -Runtime functions read the rasters and resample to match elevation grids. -""" -import numpy as np -from pathlib import Path -from typing import Tuple, Optional -import subprocess -import tempfile -import os - -try: - import rasterio - from rasterio.windows import from_bounds - from rasterio.enums import Resampling -except ImportError: - raise ImportError("rasterio is required for barriers layer support") - -# Paths -DEFAULT_BARRIERS_PATH = Path("/mnt/nav/worldcover/padus_barriers.tif") -DEFAULT_WILDERNESS_PATH = Path("/mnt/nav/worldcover/wilderness.tif") -PADUS_GDB_PATH = Path("/mnt/nav/padus/PADUS4_0_Geodatabase.gdb") -PADUS_LAYER = "PADUS4_0Combined_Proclamation_Marine_Fee_Designation_Easement" - -# CONUS bounding box in WGS84 -CONUS_BOUNDS = { - "west": -125.0, - "east": -66.0, - "south": 24.0, - "north": 50.0, -} - -# Resolution in degrees (~30m at mid-latitudes) -PIXEL_SIZE = 0.0003 # ~33m - - -class BarrierReader: - """Reader for PAD-US barrier raster (closed/restricted areas).""" - - def __init__(self, barrier_path: Path = DEFAULT_BARRIERS_PATH): - self.barrier_path = barrier_path - self._dataset = None - - def _open(self): - """Lazy open the dataset.""" - if self._dataset is None: - if not self.barrier_path.exists(): - raise FileNotFoundError( - f"Barrier raster not found at {self.barrier_path}. " - f"Run build_barriers_raster() first." - ) - self._dataset = rasterio.open(self.barrier_path) - return self._dataset - - def get_barrier_grid( - self, - south: float, - north: float, - west: float, - east: float, - target_shape: Tuple[int, int] - ) -> np.ndarray: - """ - Get barrier values for a bounding box, resampled to target shape. - - Args: - south, north, west, east: Bounding box coordinates (WGS84) - target_shape: (rows, cols) to resample to (matches elevation grid) - - Returns: - np.ndarray of uint8 barrier values: - 255 = closed/restricted (impassable when respect_boundaries=True) - 0 = public/accessible - """ - ds = self._open() - window = from_bounds(west, south, east, north, ds.transform) - barriers = ds.read( - 1, - window=window, - out_shape=target_shape, - resampling=Resampling.nearest - ) - return barriers - - def sample_point(self, lat: float, lon: float) -> int: - """Sample barrier value at a single point.""" - ds = self._open() - row, col = ds.index(lon, lat) - if row < 0 or row >= ds.height or col < 0 or col >= ds.width: - return 0 - window = rasterio.windows.Window(col, row, 1, 1) - value = ds.read(1, window=window) - return int(value[0, 0]) - - def close(self): - """Close the dataset.""" - if self._dataset is not None: - self._dataset.close() - self._dataset = None - - -class WildernessReader: - """Reader for PAD-US wilderness raster (designated wilderness areas).""" - - def __init__(self, wilderness_path: Path = DEFAULT_WILDERNESS_PATH): - self.wilderness_path = wilderness_path - self._dataset = None - - def _open(self): - """Lazy open the dataset.""" - if self._dataset is None: - if not self.wilderness_path.exists(): - raise FileNotFoundError( - f"Wilderness raster not found at {self.wilderness_path}. " - f"Run build_wilderness_raster() first." - ) - self._dataset = rasterio.open(self.wilderness_path) - return self._dataset - - def get_wilderness_grid( - self, - south: float, - north: float, - west: float, - east: float, - target_shape: Tuple[int, int] - ) -> np.ndarray: - """ - Get wilderness values for a bounding box, resampled to target shape. - - Args: - south, north, west, east: Bounding box coordinates (WGS84) - target_shape: (rows, cols) to resample to (matches elevation grid) - - Returns: - np.ndarray of uint8 wilderness values: - 255 = designated wilderness area - 0 = not wilderness - """ - ds = self._open() - window = from_bounds(west, south, east, north, ds.transform) - wilderness = ds.read( - 1, - window=window, - out_shape=target_shape, - resampling=Resampling.nearest - ) - return wilderness - - def sample_point(self, lat: float, lon: float) -> int: - """Sample wilderness value at a single point.""" - ds = self._open() - row, col = ds.index(lon, lat) - if row < 0 or row >= ds.height or col < 0 or col >= ds.width: - return 0 - window = rasterio.windows.Window(col, row, 1, 1) - value = ds.read(1, window=window) - return int(value[0, 0]) - - def close(self): - """Close the dataset.""" - if self._dataset is not None: - self._dataset.close() - self._dataset = None - - -def build_barriers_raster( - output_path: Path = DEFAULT_BARRIERS_PATH, - gdb_path: Path = PADUS_GDB_PATH, - pixel_size: float = PIXEL_SIZE, - bounds: dict = CONUS_BOUNDS, -) -> Path: - """ - Build the PAD-US barriers raster from the source geodatabase. - - Extracts polygons where Pub_Access = 'XA' (Closed) and rasterizes them. - """ - import shutil - - if not gdb_path.exists(): - raise FileNotFoundError(f"PAD-US geodatabase not found at {gdb_path}") - - if not shutil.which('ogr2ogr'): - raise RuntimeError("ogr2ogr not found. Install GDAL.") - if not shutil.which('gdal_rasterize'): - raise RuntimeError("gdal_rasterize not found. Install GDAL.") - - output_path.parent.mkdir(parents=True, exist_ok=True) - - print(f"Building PAD-US barriers raster...") - print(f" Source: {gdb_path}") - print(f" Output: {output_path}") - print(f" Pixel size: {pixel_size} degrees (~{pixel_size * 111000:.0f}m)") - print(f" Bounds: {bounds}") - - with tempfile.TemporaryDirectory() as tmpdir: - closed_gpkg = Path(tmpdir) / "closed_areas.gpkg" - - print(f"\n[1/3] Extracting closed areas (Pub_Access = 'XA')...") - - ogr_cmd = [ - "ogr2ogr", - "-f", "GPKG", - str(closed_gpkg), - str(gdb_path), - PADUS_LAYER, - "-where", "Pub_Access = 'XA'", - "-t_srs", "EPSG:4326", - "-nlt", "MULTIPOLYGON", - "-nln", "closed_areas", - ] - - result = subprocess.run(ogr_cmd, capture_output=True, text=True) - if result.returncode != 0: - print(f"STDERR: {result.stderr}") - raise RuntimeError(f"ogr2ogr failed: {result.stderr}") - - info_cmd = ["ogrinfo", "-so", str(closed_gpkg), "closed_areas"] - info_result = subprocess.run(info_cmd, capture_output=True, text=True) - print(f" Extraction result:\n{info_result.stdout}") - - print(f"\n[2/3] Creating raster grid...") - - width = int((bounds['east'] - bounds['west']) / pixel_size) - height = int((bounds['north'] - bounds['south']) / pixel_size) - print(f" Grid size: {width} x {height} pixels") - - print(f"\n[3/3] Rasterizing closed areas...") - - rasterize_cmd = [ - "gdal_rasterize", - "-burn", "255", - "-init", "0", - "-a_nodata", "0", - "-te", str(bounds['west']), str(bounds['south']), - str(bounds['east']), str(bounds['north']), - "-tr", str(pixel_size), str(pixel_size), - "-ot", "Byte", - "-co", "COMPRESS=LZW", - "-co", "TILED=YES", - "-l", "closed_areas", - str(closed_gpkg), - str(output_path), - ] - - result = subprocess.run(rasterize_cmd, capture_output=True, text=True) - if result.returncode != 0: - print(f"STDERR: {result.stderr}") - raise RuntimeError(f"gdal_rasterize failed: {result.stderr}") - - print(f"\n[Done] Verifying output...") - with rasterio.open(output_path) as ds: - print(f" Size: {ds.width} x {ds.height}") - print(f" CRS: {ds.crs}") - sample = ds.read(1, window=rasterio.windows.Window(0, 0, 1000, 1000)) - closed_count = np.sum(sample == 255) - print(f" Sample (1000x1000): {closed_count} closed cells") - - file_size = output_path.stat().st_size / (1024**2) - print(f" File size: {file_size:.1f} MB") - - return output_path - - -def build_wilderness_raster( - output_path: Path = DEFAULT_WILDERNESS_PATH, - gdb_path: Path = PADUS_GDB_PATH, - pixel_size: float = PIXEL_SIZE, - bounds: dict = CONUS_BOUNDS, -) -> Path: - """ - Build the PAD-US wilderness raster from the source geodatabase. - - Extracts polygons where Des_Tp = 'WA' (Wilderness Area) and rasterizes them. - """ - import shutil - - if not gdb_path.exists(): - raise FileNotFoundError(f"PAD-US geodatabase not found at {gdb_path}") - - if not shutil.which('ogr2ogr'): - raise RuntimeError("ogr2ogr not found. Install GDAL.") - if not shutil.which('gdal_rasterize'): - raise RuntimeError("gdal_rasterize not found. Install GDAL.") - - output_path.parent.mkdir(parents=True, exist_ok=True) - - print(f"Building PAD-US wilderness raster...") - print(f" Source: {gdb_path}") - print(f" Output: {output_path}") - print(f" Pixel size: {pixel_size} degrees (~{pixel_size * 111000:.0f}m)") - print(f" Bounds: {bounds}") - - with tempfile.TemporaryDirectory() as tmpdir: - wilderness_gpkg = Path(tmpdir) / "wilderness_areas.gpkg" - - print(f"\n[1/3] Extracting wilderness areas (Des_Tp = 'WA')...") - - ogr_cmd = [ - "ogr2ogr", - "-f", "GPKG", - str(wilderness_gpkg), - str(gdb_path), - PADUS_LAYER, - "-where", "Des_Tp = 'WA'", - "-t_srs", "EPSG:4326", - "-nlt", "MULTIPOLYGON", - "-nln", "wilderness_areas", - ] - - result = subprocess.run(ogr_cmd, capture_output=True, text=True) - if result.returncode != 0: - print(f"STDERR: {result.stderr}") - raise RuntimeError(f"ogr2ogr failed: {result.stderr}") - - info_cmd = ["ogrinfo", "-so", str(wilderness_gpkg), "wilderness_areas"] - info_result = subprocess.run(info_cmd, capture_output=True, text=True) - print(f" Extraction result:\n{info_result.stdout}") - - print(f"\n[2/3] Creating raster grid...") - - width = int((bounds['east'] - bounds['west']) / pixel_size) - height = int((bounds['north'] - bounds['south']) / pixel_size) - print(f" Grid size: {width} x {height} pixels") - - print(f"\n[3/3] Rasterizing wilderness areas...") - - rasterize_cmd = [ - "gdal_rasterize", - "-burn", "255", - "-init", "0", - "-a_nodata", "0", - "-te", str(bounds['west']), str(bounds['south']), - str(bounds['east']), str(bounds['north']), - "-tr", str(pixel_size), str(pixel_size), - "-ot", "Byte", - "-co", "COMPRESS=LZW", - "-co", "TILED=YES", - "-l", "wilderness_areas", - str(wilderness_gpkg), - str(output_path), - ] - - result = subprocess.run(rasterize_cmd, capture_output=True, text=True) - if result.returncode != 0: - print(f"STDERR: {result.stderr}") - raise RuntimeError(f"gdal_rasterize failed: {result.stderr}") - - print(f"\n[Done] Verifying output...") - with rasterio.open(output_path) as ds: - print(f" Size: {ds.width} x {ds.height}") - print(f" CRS: {ds.crs}") - sample = ds.read(1, window=rasterio.windows.Window(0, 0, 1000, 1000)) - wilderness_count = np.sum(sample == 255) - print(f" Sample (1000x1000): {wilderness_count} wilderness cells") - - file_size = output_path.stat().st_size / (1024**2) - print(f" File size: {file_size:.1f} MB") - - return output_path - - -if __name__ == "__main__": - import sys - - if len(sys.argv) > 1: - cmd = sys.argv[1] - - if cmd == "build": - print("=" * 60) - print("PAD-US Barriers Raster Build") - print("=" * 60) - build_barriers_raster() - - elif cmd == "build-wilderness": - print("=" * 60) - print("PAD-US Wilderness Raster Build") - print("=" * 60) - build_wilderness_raster() - - elif cmd == "build-all": - print("=" * 60) - print("Building all PAD-US rasters") - print("=" * 60) - build_barriers_raster() - print("\n") - build_wilderness_raster() - - else: - print(f"Unknown command: {cmd}") - print("Usage:") - print(" python barriers.py build # Build barriers raster") - print(" python barriers.py build-wilderness # Build wilderness raster") - print(" python barriers.py build-all # Build both rasters") - sys.exit(1) - - else: - # Test readers - print("Testing BarrierReader...") - - if not DEFAULT_BARRIERS_PATH.exists(): - print(f"Barrier raster not found at {DEFAULT_BARRIERS_PATH}") - print(f"Run: python barriers.py build") - sys.exit(1) - - reader = BarrierReader() - barriers = reader.get_barrier_grid( - south=42.2, north=42.6, west=-114.8, east=-113.8, - target_shape=(400, 1000) - ) - print(f"\nBarrier grid shape: {barriers.shape}") - print(f"Unique values: {np.unique(barriers)}") - closed_cells = np.sum(barriers == 255) - print(f"Closed cells: {closed_cells} ({100*closed_cells/barriers.size:.2f}%)") - reader.close() - - print("\nTesting WildernessReader...") - - if not DEFAULT_WILDERNESS_PATH.exists(): - print(f"Wilderness raster not found at {DEFAULT_WILDERNESS_PATH}") - print(f"Run: python barriers.py build-wilderness") - else: - wilderness_reader = WildernessReader() - wilderness = wilderness_reader.get_wilderness_grid( - south=42.2, north=42.6, west=-114.8, east=-113.8, - target_shape=(400, 1000) - ) - print(f"Wilderness grid shape: {wilderness.shape}") - print(f"Unique values: {np.unique(wilderness)}") - wilderness_cells = np.sum(wilderness == 255) - print(f"Wilderness cells: {wilderness_cells} ({100*wilderness_cells/wilderness.size:.2f}%)") - wilderness_reader.close() - - print("\nDone.") diff --git a/lib/offroute/cost.py b/lib/offroute/cost.py deleted file mode 100644 index 16b8514..0000000 --- a/lib/offroute/cost.py +++ /dev/null @@ -1,494 +0,0 @@ -""" -Multi-mode travel cost functions for OFFROUTE. - -Supports four travel modes: foot, mtb, atv, vehicle. -Each mode has its own speed function, max slope, trail access rules, -and terrain friction overrides. - -Mode profiles are data-driven — adding a new mode means adding a profile entry. -""" -import math -import numpy as np -from dataclasses import dataclass, field -from typing import Optional, Literal, Dict, Callable - -# ═══════════════════════════════════════════════════════════════════════════════ -# SPEED FUNCTIONS -# ═══════════════════════════════════════════════════════════════════════════════ - -def tobler_off_path_speed(grade: np.ndarray, base_speed: float = 6.0) -> np.ndarray: - """ - Tobler off-path hiking function. - - W = 0.6 * base_speed * exp(-3.5 * |S + 0.05|) - - Peak ~3.6 km/h at grade = -0.05 (slight downhill). - The 0.6 multiplier is the off-trail penalty. - """ - return 0.6 * base_speed * np.exp(-3.5 * np.abs(grade + 0.05)) - - -def herzog_wheeled_speed(grade: np.ndarray, base_speed: float = 12.0) -> np.ndarray: - """ - Herzog wheeled-transport polynomial. - - Relative speed factor: - 1 / (1337.8·S^6 + 278.19·S^5 − 517.39·S^4 − 78.199·S^3 + 93.419·S^2 + 19.825·|S| + 1.64) - - Multiply by base_speed to get km/h. - """ - S = grade - S_abs = np.abs(S) - - # Herzog polynomial (returns relative speed factor 0-1) - denom = (1337.8 * S**6 + 278.19 * S**5 - 517.39 * S**4 - - 78.199 * S**3 + 93.419 * S**2 + 19.825 * S_abs + 1.64) - - # Avoid division by zero and negative speeds - denom = np.maximum(denom, 0.1) - rel_speed = 1.0 / denom - - # Clamp relative speed to reasonable bounds (0.05 to 1.5) - rel_speed = np.clip(rel_speed, 0.05, 1.5) - - return base_speed * rel_speed - - -def linear_degrade_speed(grade: np.ndarray, base_speed: float = 40.0, max_grade: float = 0.364) -> np.ndarray: - """ - Linear speed degradation with slope. - - speed = base_speed * max(0, 1 - |grade| / max_grade) - - max_grade = tan(20°) ≈ 0.364 for 20° max slope. - """ - speed = base_speed * np.maximum(0, 1.0 - np.abs(grade) / max_grade) - return np.maximum(speed, 0.1) # Minimum crawl speed - - -# ═══════════════════════════════════════════════════════════════════════════════ -# MODE PROFILES (Data-driven configuration) -# ═══════════════════════════════════════════════════════════════════════════════ - -@dataclass -class ModeProfile: - """Configuration for a travel mode.""" - - name: str - description: str - - # Speed function parameters - speed_function: str # "tobler", "herzog", "linear" - base_speed_kmh: float - max_slope_deg: float - - # Trail access: trail_value -> friction multiplier (None = impassable) - # Trail values: 5=road, 15=track, 25=foot trail - trail_friction: Dict[int, Optional[float]] = field(default_factory=dict) - - # Off-trail terrain friction overrides (by WorldCover class) - # These MULTIPLY the base WorldCover friction - # None = use default, np.inf = impassable - # WorldCover values: 10=tree, 20=shrub, 30=grass, 40=crop, 50=urban, - # 60=bare, 80=water, 90=wetland, 95=mangrove, 100=moss - terrain_friction_override: Dict[int, Optional[float]] = field(default_factory=dict) - - # Should wilderness areas be impassable? - wilderness_impassable: bool = False - - # For vehicle mode: can traverse off-trail flat terrain? - off_trail_flat_threshold_deg: float = 0.0 # 0 = no off-trail allowed - off_trail_flat_friction: float = np.inf # friction if allowed - - -# Define all mode profiles -MODE_PROFILES: Dict[str, ModeProfile] = { - "foot": ModeProfile( - name="foot", - description="Hiking on foot (Tobler off-path model)", - speed_function="tobler", - base_speed_kmh=6.0, - max_slope_deg=40.0, - trail_friction={ - 5: 0.1, # road - 15: 0.3, # track - 25: 0.5, # foot trail - }, - terrain_friction_override={ - # Use default WorldCover friction for foot mode - }, - wilderness_impassable=False, - ), - - "mtb": ModeProfile( - name="mtb", - description="Mountain bike / dirt bike (Herzog wheeled model)", - speed_function="herzog", - base_speed_kmh=12.0, - max_slope_deg=25.0, - trail_friction={ - 5: 0.1, # road - 15: 0.2, # track - 25: 0.5, # foot trail (rideable but slow) - }, - terrain_friction_override={ - 30: 2.0, # Grassland: rideable but slow - 20: 4.0, # Shrubland: barely rideable - 10: 8.0, # Tree cover/forest: effectively impassable - 60: 3.0, # Bare/rocky - 90: np.inf, # Wetland: impassable - 95: np.inf, # Mangrove: impassable - 80: np.inf, # Water: impassable - }, - wilderness_impassable=True, - ), - - "atv": ModeProfile( - name="atv", - description="ATV / side-by-side (Herzog wheeled model, higher base speed)", - speed_function="herzog", - base_speed_kmh=25.0, - max_slope_deg=30.0, - trail_friction={ - 5: 0.1, # road - 15: 0.3, # track - 25: None, # foot trail: impassable (too narrow) - }, - terrain_friction_override={ - 30: 1.5, # Grassland: passable - 20: 3.0, # Shrubland: rough - 10: np.inf, # Forest: impassable - 60: 2.0, # Bare/rocky - 90: np.inf, # Wetland: impassable - 95: np.inf, # Mangrove: impassable - 80: np.inf, # Water: impassable - }, - wilderness_impassable=True, - ), - - "vehicle": ModeProfile( - name="vehicle", - description="4x4 truck / jeep (linear speed degradation)", - speed_function="linear", - base_speed_kmh=40.0, - max_slope_deg=20.0, - trail_friction={ - 5: 0.1, # road - 15: 0.5, # track (rough but passable) - 25: None, # foot trail: impassable - }, - terrain_friction_override={ - # All off-trail terrain is impassable by default - 10: np.inf, # Forest - 20: np.inf, # Shrubland - 30: np.inf, # Grassland (except flat - see below) - 40: np.inf, # Cropland (except flat - see below) - 60: np.inf, # Bare - 90: np.inf, # Wetland - 95: np.inf, # Mangrove - 80: np.inf, # Water - }, - wilderness_impassable=True, - off_trail_flat_threshold_deg=5.0, # Can drive on flat fields - off_trail_flat_friction=5.0, # But very slow - ), -} - - -# Pragmatic mode friction multiplier for private land -PRAGMATIC_BARRIER_MULTIPLIER = 5.0 - - -# ═══════════════════════════════════════════════════════════════════════════════ -# COST GRID COMPUTATION -# ═══════════════════════════════════════════════════════════════════════════════ - -def compute_cost_grid( - elevation: np.ndarray, - cell_size_m: float, - cell_size_lat_m: float = None, - cell_size_lon_m: float = None, - friction: Optional[np.ndarray] = None, - friction_raw: Optional[np.ndarray] = None, - trails: Optional[np.ndarray] = None, - barriers: Optional[np.ndarray] = None, - wilderness: Optional[np.ndarray] = None, - mvum: Optional[np.ndarray] = None, - boundary_mode: Literal["strict", "pragmatic", "emergency"] = "pragmatic", - mode: Literal["foot", "mtb", "atv", "vehicle"] = "foot" -) -> np.ndarray: - """ - Compute isotropic travel cost grid from elevation data. - - Args: - elevation: 2D array of elevation values in meters - cell_size_m: Average cell size in meters - cell_size_lat_m: Cell size in latitude direction (optional) - cell_size_lon_m: Cell size in longitude direction (optional) - friction: Optional 2D array of friction multipliers (WorldCover). - Values should be float (1.0 = baseline, 2.0 = 2x slower). - np.inf marks impassable cells. - friction_raw: Optional 2D array of raw WorldCover class values (uint8). - Used for mode-specific terrain overrides. - Values: 10=tree, 20=shrub, 30=grass, etc. - trails: Optional 2D array of trail values (uint8). - 0 = no trail, 5 = road, 15 = track, 25 = foot trail - barriers: Optional 2D array of barrier values (uint8). - 255 = closed/restricted area (PAD-US Pub_Access = XA). - wilderness: Optional[np.ndarray] of wilderness values (uint8). - 255 = designated wilderness area. - mvum: Optional[np.ndarray] of MVUM access values (uint8). - 0 = no MVUM data, 1 = open, 255 = closed to this mode. - MVUM closures respond to boundary_mode (strict/pragmatic/emergency). - Foot mode should pass None (MVUM is motor-vehicle specific). - boundary_mode: How to handle barriers ("strict", "pragmatic", "emergency") - mode: Travel mode ("foot", "mtb", "atv", "vehicle") - - Returns: - 2D array of travel cost in seconds per cell. - np.inf for impassable cells. - """ - if boundary_mode not in ("strict", "pragmatic", "emergency"): - raise ValueError(f"boundary_mode must be 'strict', 'pragmatic', or 'emergency'") - - if mode not in MODE_PROFILES: - raise ValueError(f"mode must be one of {list(MODE_PROFILES.keys())}") - - profile = MODE_PROFILES[mode] - - if cell_size_lat_m is None: - cell_size_lat_m = cell_size_m - if cell_size_lon_m is None: - cell_size_lon_m = cell_size_m - - rows, cols = elevation.shape - - # ─── Compute gradients (in-place where possible) ───────────────────────── - # Use float32 to reduce memory footprint - grade = np.zeros(elevation.shape, dtype=np.float32) - - # Compute dy contribution to grade squared - dy_contrib = np.zeros(elevation.shape, dtype=np.float32) - dy_contrib[1:-1, :] = ((elevation[:-2, :] - elevation[2:, :]) / (2 * cell_size_lat_m)) ** 2 - dy_contrib[0, :] = ((elevation[0, :] - elevation[1, :]) / cell_size_lat_m) ** 2 - dy_contrib[-1, :] = ((elevation[-2, :] - elevation[-1, :]) / cell_size_lat_m) ** 2 - - # Compute dx contribution and add to dy_contrib in-place - dy_contrib[:, 1:-1] += ((elevation[:, 2:] - elevation[:, :-2]) / (2 * cell_size_lon_m)) ** 2 - dy_contrib[:, 0] += ((elevation[:, 1] - elevation[:, 0]) / cell_size_lon_m) ** 2 - dy_contrib[:, -1] += ((elevation[:, -1] - elevation[:, -2]) / cell_size_lon_m) ** 2 - - # grade = sqrt(dx^2 + dy^2) - np.sqrt(dy_contrib, out=grade) - del dy_contrib # Free memory immediately - - # ─── Compute speed based on mode ───────────────────────────────────────── - max_grade_val = np.tan(np.radians(profile.max_slope_deg)) - - if profile.speed_function == "tobler": - speed_kmh = tobler_off_path_speed(grade, profile.base_speed_kmh) - elif profile.speed_function == "herzog": - speed_kmh = herzog_wheeled_speed(grade, profile.base_speed_kmh) - elif profile.speed_function == "linear": - speed_kmh = linear_degrade_speed(grade, profile.base_speed_kmh, max_grade_val) - else: - raise ValueError(f"Unknown speed function: {profile.speed_function}") - - # ─── Base cost (seconds per cell) ───────────────────────────────────────── - avg_cell_size = (cell_size_lat_m + cell_size_lon_m) / 2 - cost = (avg_cell_size * 3.6) / speed_kmh - del speed_kmh - - # ─── Max slope limit ────────────────────────────────────────────────────── - cost[grade > max_grade_val] = np.inf - - # ─── NaN elevations ────────────────────────────────────────────────────── - cost[np.isnan(elevation)] = np.inf - - # ─── Apply friction in-place ───────────────────────────────────────────── - # Instead of creating effective_friction copy, apply directly to cost - - # Start with base friction - if friction is not None: - if friction.shape != elevation.shape: - raise ValueError(f"Friction shape mismatch") - np.multiply(cost, friction, out=cost) - - # ─── Mode-specific terrain friction overrides (memory-efficient) ───────── - if friction_raw is not None and profile.terrain_friction_override: - if friction_raw.shape != elevation.shape: - raise ValueError(f"Friction_raw shape mismatch") - - # Process all overrides without creating large intermediate masks - for wc_class, override in profile.terrain_friction_override.items(): - if override is not None: - if override == np.inf: - # Use np.where for in-place-like behavior - np.putmask(cost, friction_raw == wc_class, np.inf) - else: - # Multiply cost where friction_raw matches - # Using a loop with putmask is more memory efficient - mask = friction_raw == wc_class - cost[mask] *= override - del mask - - # ─── Vehicle mode: allow flat grassland/cropland ───────────────────────── - if mode == "vehicle" and profile.off_trail_flat_threshold_deg > 0: - if friction_raw is not None: - # Compute slope in degrees for flat terrain check - slope_deg = np.degrees(np.arctan(grade)) - # Flat grassland or cropland - recompute cost for these cells - flat_field_mask = ( - (slope_deg <= profile.off_trail_flat_threshold_deg) & - ((friction_raw == 30) | (friction_raw == 40)) - ) - del slope_deg - # Recalculate cost for these cells with flat field friction - if np.any(flat_field_mask): - base_time = avg_cell_size * 3.6 / linear_degrade_speed( - grade[flat_field_mask], profile.base_speed_kmh, max_grade_val - ) - cost[flat_field_mask] = base_time * profile.off_trail_flat_friction - del base_time - del flat_field_mask - - # ─── Trail friction (mode-specific) ────────────────────────────────────── - if trails is not None: - if trails.shape != elevation.shape: - raise ValueError(f"Trails shape mismatch") - - for trail_value, trail_friction in profile.trail_friction.items(): - if trail_friction is None: - # Impassable for this mode - np.putmask(cost, trails == trail_value, np.inf) - else: - # Trail friction REPLACES terrain friction - # Recalculate cost = base_time * trail_friction - trail_mask = trails == trail_value - if np.any(trail_mask): - # Get base travel time (without friction) - if profile.speed_function == "tobler": - trail_speed = tobler_off_path_speed(grade[trail_mask], profile.base_speed_kmh) - elif profile.speed_function == "herzog": - trail_speed = herzog_wheeled_speed(grade[trail_mask], profile.base_speed_kmh) - else: - trail_speed = linear_degrade_speed( - grade[trail_mask], profile.base_speed_kmh, max_grade_val - ) - cost[trail_mask] = (avg_cell_size * 3.6 / trail_speed) * trail_friction - del trail_speed - del trail_mask - - # ─── Wilderness areas (mode-specific) ──────────────────────────────────── - if wilderness is not None and profile.wilderness_impassable: - if wilderness.shape != elevation.shape: - raise ValueError(f"Wilderness shape mismatch") - np.putmask(cost, wilderness == 255, np.inf) - - # ─── Barriers (private land) ───────────────────────────────────────────── - if barriers is not None and boundary_mode != "emergency": - if barriers.shape != elevation.shape: - raise ValueError(f"Barriers shape mismatch") - - if boundary_mode == "strict": - np.putmask(cost, barriers == 255, np.inf) - elif boundary_mode == "pragmatic": - barrier_mask = barriers == 255 - cost[barrier_mask] *= PRAGMATIC_BARRIER_MULTIPLIER - del barrier_mask - - # ─── MVUM closures (motor vehicle restrictions) ────────────────────────── - # MVUM only applies to motorized modes, not foot. Foot mode should pass mvum=None. - # MVUM closures respond to the same boundary_mode as PAD-US barriers: - # "strict" = MVUM-closed road/trail is impassable - # "pragmatic" = MVUM-closed road/trail gets 5× friction penalty - # "emergency" = MVUM closures ignored entirely - if mvum is not None and mode != "foot" and boundary_mode != "emergency": - if mvum.shape != elevation.shape: - raise ValueError(f"MVUM shape mismatch") - - # Value 255 = road/trail exists but is closed to this mode - mvum_closed_mask = mvum == 255 - - if boundary_mode == "strict": - np.putmask(cost, mvum_closed_mask, np.inf) - elif boundary_mode == "pragmatic": - cost[mvum_closed_mask] *= PRAGMATIC_BARRIER_MULTIPLIER - - del mvum_closed_mask - - return cost - - -# ═══════════════════════════════════════════════════════════════════════════════ -# LEGACY API (backward compatibility) -# ═══════════════════════════════════════════════════════════════════════════════ - -def tobler_speed(grade: float) -> float: - """Legacy single-value Tobler speed function.""" - return 0.6 * 6.0 * math.exp(-3.5 * abs(grade + 0.05)) - - -# ═══════════════════════════════════════════════════════════════════════════════ -# TESTING -# ═══════════════════════════════════════════════════════════════════════════════ - -if __name__ == "__main__": - print("=" * 70) - print("OFFROUTE Multi-Mode Cost Function Tests") - print("=" * 70) - - print("\n[1] Speed functions at various grades:") - print(f"{'Grade':<10} {'Foot':<12} {'MTB':<12} {'ATV':<12} {'Vehicle':<12}") - print("-" * 60) - - for grade_val in [-0.3, -0.1, 0.0, 0.1, 0.2, 0.3]: - grade_arr = np.array([grade_val]) - foot = tobler_off_path_speed(grade_arr, 6.0)[0] - mtb = herzog_wheeled_speed(grade_arr, 12.0)[0] - atv = herzog_wheeled_speed(grade_arr, 25.0)[0] - veh = linear_degrade_speed(grade_arr, 40.0, np.tan(np.radians(20)))[0] - print(f"{grade_val:+.2f} {foot:>6.2f} km/h {mtb:>6.2f} km/h {atv:>6.2f} km/h {veh:>6.2f} km/h") - - print("\n[2] Mode profiles:") - for name, profile in MODE_PROFILES.items(): - print(f"\n {name.upper()}: {profile.description}") - print(f" Max slope: {profile.max_slope_deg}°") - print(f" Trail access: {profile.trail_friction}") - print(f" Wilderness blocked: {profile.wilderness_impassable}") - - print("\n[3] Cost grid test (flat terrain, forest):") - elev = np.ones((10, 10), dtype=np.float32) * 1000 - friction = np.ones((10, 10), dtype=np.float32) * 2.0 # Forest friction - friction_raw = np.ones((10, 10), dtype=np.uint8) * 10 # Tree cover class - - trails = np.zeros((10, 10), dtype=np.uint8) - trails[5, :] = 5 # Road across middle - - for mode_name in ["foot", "mtb", "atv", "vehicle"]: - cost = compute_cost_grid( - elev, cell_size_m=30.0, - friction=friction, - friction_raw=friction_raw, - trails=trails, - mode=mode_name - ) - off_trail_cost = cost[0, 0] - road_cost = cost[5, 0] - impassable = np.sum(np.isinf(cost)) - print(f" {mode_name:8s}: off-trail={off_trail_cost:>8.1f}s, road={road_cost:>6.1f}s, impassable={impassable}") - - print("\n[4] Wilderness blocking test:") - wilderness = np.zeros((10, 10), dtype=np.uint8) - wilderness[3:7, 3:7] = 255 - - for mode_name in ["foot", "mtb", "atv", "vehicle"]: - cost = compute_cost_grid( - elev, cell_size_m=30.0, - wilderness=wilderness, - mode=mode_name - ) - wilderness_impassable = np.sum(np.isinf(cost[3:7, 3:7])) - print(f" {mode_name:8s}: wilderness cells impassable = {wilderness_impassable}/16") - - print("\nDone.") diff --git a/lib/offroute/dem.py b/lib/offroute/dem.py deleted file mode 100644 index 06cfcea..0000000 --- a/lib/offroute/dem.py +++ /dev/null @@ -1,211 +0,0 @@ -""" -DEM tile reader for OFFROUTE. - -Reads elevation tiles from planet-dem.pmtiles (Terrarium-encoded WebP), -decodes them into numpy arrays, and provides a stitched elevation grid -for a given bounding box. -""" -import math -from functools import lru_cache -from io import BytesIO -from pathlib import Path -from typing import Tuple, Optional - -import numpy as np -from PIL import Image -from pmtiles.reader import MmapSource, Reader as PMTilesReader - -# Default path to the planet DEM PMTiles file -DEFAULT_DEM_PATH = Path("/mnt/nas/nav/planet-dem.pmtiles") - -# Tile size in pixels (z12 tiles are 512x512 in this tileset) -TILE_SIZE = 512 - -# Zoom level to use for elevation data -ZOOM_LEVEL = 12 - - -def terrarium_decode(rgb_array: np.ndarray) -> np.ndarray: - """ - Decode Terrarium-encoded RGB values to elevation in meters. - - Formula: elevation = (R * 256 + G + B/256) - 32768 - """ - r = rgb_array[:, :, 0].astype(np.float32) - g = rgb_array[:, :, 1].astype(np.float32) - b = rgb_array[:, :, 2].astype(np.float32) - - elevation = (r * 256.0 + g + b / 256.0) - 32768.0 - return elevation - - -def lat_lon_to_tile(lat: float, lon: float, zoom: int) -> Tuple[int, int]: - """Convert lat/lon to tile coordinates at given zoom level.""" - n = 2 ** zoom - x = int((lon + 180.0) / 360.0 * n) - lat_rad = math.radians(lat) - y = int((1.0 - math.asinh(math.tan(lat_rad)) / math.pi) / 2.0 * n) - return x, y - - -def tile_to_lat_lon(x: int, y: int, zoom: int) -> Tuple[float, float, float, float]: - """Convert tile coordinates to bounding box (north, south, west, east).""" - n = 2 ** zoom - lon_west = x / n * 360.0 - 180.0 - lon_east = (x + 1) / n * 360.0 - 180.0 - lat_north = math.degrees(math.atan(math.sinh(math.pi * (1 - 2 * y / n)))) - lat_south = math.degrees(math.atan(math.sinh(math.pi * (1 - 2 * (y + 1) / n)))) - return lat_north, lat_south, lon_west, lon_east - - -class DEMReader: - """Reader for Terrarium-encoded DEM tiles from PMTiles.""" - - def __init__(self, pmtiles_path: Path = DEFAULT_DEM_PATH, tile_cache_size: int = 128): - self.pmtiles_path = pmtiles_path - self._source = MmapSource(open(pmtiles_path, "rb")) - self._reader = PMTilesReader(self._source) - self._header = self._reader.header() - self._decode_tile = lru_cache(maxsize=tile_cache_size)(self._decode_tile_impl) - - def _decode_tile_impl(self, z: int, x: int, y: int) -> Optional[np.ndarray]: - """Fetch and decode a single tile.""" - tile_data = self._reader.get(z, x, y) - if tile_data is None: - return None - - img = Image.open(BytesIO(tile_data)) - rgb_array = np.array(img) - - if rgb_array.shape[2] == 4: - rgb_array = rgb_array[:, :, :3] - - elevation = terrarium_decode(rgb_array) - return elevation - - def get_elevation_grid( - self, - south: float, - north: float, - west: float, - east: float, - zoom: int = ZOOM_LEVEL - ) -> Tuple[np.ndarray, dict]: - """Get a stitched elevation grid for the given bounding box.""" - x_min, y_max = lat_lon_to_tile(south, west, zoom) - x_max, y_min = lat_lon_to_tile(north, east, zoom) - - n = 2 ** zoom - x_min = max(0, x_min) - x_max = min(n - 1, x_max) - y_min = max(0, y_min) - y_max = min(n - 1, y_max) - - n_tiles_x = x_max - x_min + 1 - n_tiles_y = y_max - y_min + 1 - out_height = n_tiles_y * TILE_SIZE - out_width = n_tiles_x * TILE_SIZE - - elevation = np.full((out_height, out_width), np.nan, dtype=np.float32) - - for ty in range(y_min, y_max + 1): - for tx in range(x_min, x_max + 1): - tile_elev = self._decode_tile(zoom, tx, ty) - if tile_elev is not None: - out_y = (ty - y_min) * TILE_SIZE - out_x = (tx - x_min) * TILE_SIZE - elevation[out_y:out_y + TILE_SIZE, out_x:out_x + TILE_SIZE] = tile_elev - - grid_north, _, grid_west, _ = tile_to_lat_lon(x_min, y_min, zoom) - _, grid_south, _, grid_east = tile_to_lat_lon(x_max, y_max, zoom) - - pixel_size_lat = (grid_north - grid_south) / out_height - pixel_size_lon = (grid_east - grid_west) / out_width - - origin_lat = grid_north - pixel_size_lat / 2 - origin_lon = grid_west + pixel_size_lon / 2 - - center_lat = (south + north) / 2 - lat_m = 111320.0 - lon_m = 111320.0 * math.cos(math.radians(center_lat)) - cell_size_lat_m = abs(pixel_size_lat) * lat_m - cell_size_lon_m = abs(pixel_size_lon) * lon_m - cell_size_m = (cell_size_lat_m + cell_size_lon_m) / 2 - - row_start = int((grid_north - north) / abs(pixel_size_lat)) - row_end = int((grid_north - south) / abs(pixel_size_lat)) - col_start = int((west - grid_west) / pixel_size_lon) - col_end = int((east - grid_west) / pixel_size_lon) - - row_start = max(0, row_start) - row_end = min(out_height, row_end) - col_start = max(0, col_start) - col_end = min(out_width, col_end) - - elevation = elevation[row_start:row_end, col_start:col_end] - - origin_lat = grid_north - (row_start + 0.5) * abs(pixel_size_lat) - origin_lon = grid_west + (col_start + 0.5) * pixel_size_lon - - metadata = { - "bounds": (south, north, west, east), - "pixel_size_lat": -abs(pixel_size_lat), - "pixel_size_lon": pixel_size_lon, - "origin_lat": origin_lat, - "origin_lon": origin_lon, - "cell_size_m": cell_size_m, - "shape": elevation.shape, - } - - return elevation, metadata - - def sample_point(self, lat: float, lon: float) -> Optional[float]: - """Return elevation in meters at a single point, or None if untiled. - - Reads one z12 Terrarium tile (LRU-cached) and indexes the matching - pixel. Sub-ms warm, ~15 ms cold per tile via NFS. Returns None when the - tile is absent (e.g. true ocean nodata) or lat is outside the - Web-Mercator pole cap (~+/-85.05 deg). - """ - if not -85.05112878 <= lat <= 85.05112878: - return None - n = 2 ** ZOOM_LEVEL - fx = (lon + 180.0) / 360.0 * n - fy = (1.0 - math.asinh(math.tan(math.radians(lat))) / math.pi) / 2.0 * n - tx, ty = int(fx), int(fy) - tile = self._decode_tile(ZOOM_LEVEL, tx, ty) - if tile is None: - return None - row = min(TILE_SIZE - 1, int((fy - ty) * TILE_SIZE)) - col = min(TILE_SIZE - 1, int((fx - tx) * TILE_SIZE)) - return float(tile[row, col]) - - def pixel_to_latlon(self, row: int, col: int, metadata: dict) -> Tuple[float, float]: - """Convert pixel coordinates to lat/lon.""" - lat = metadata["origin_lat"] + row * metadata["pixel_size_lat"] - lon = metadata["origin_lon"] + col * metadata["pixel_size_lon"] - return lat, lon - - def latlon_to_pixel(self, lat: float, lon: float, metadata: dict) -> Tuple[int, int]: - """Convert lat/lon to pixel coordinates.""" - row = int((metadata["origin_lat"] - lat) / abs(metadata["pixel_size_lat"])) - col = int((lon - metadata["origin_lon"]) / metadata["pixel_size_lon"]) - return row, col - - def close(self): - """Close the PMTiles file.""" - pass # MmapSource handles cleanup - - -if __name__ == "__main__": - reader = DEMReader() - elevation, meta = reader.get_elevation_grid( - south=42.4, north=42.6, west=-114.5, east=-114.3 - ) - print(f"Elevation grid shape: {elevation.shape}") - print(f"Cell size: {meta['cell_size_m']:.1f} m") - print(f"Elevation range: {np.nanmin(elevation):.1f} - {np.nanmax(elevation):.1f} m") - center_row, center_col = elevation.shape[0] // 2, elevation.shape[1] // 2 - lat, lon = reader.pixel_to_latlon(center_row, center_col, meta) - print(f"Center pixel lat/lon: {lat:.4f}, {lon:.4f}") - reader.close() diff --git a/lib/offroute/friction.py b/lib/offroute/friction.py deleted file mode 100644 index 32df0c0..0000000 --- a/lib/offroute/friction.py +++ /dev/null @@ -1,137 +0,0 @@ -""" -Friction layer reader for OFFROUTE. - -Reads friction values from the WorldCover friction VRT and resamples -to match the elevation grid dimensions. -""" -import numpy as np -from pathlib import Path -from typing import Tuple, Optional - -try: - import rasterio - from rasterio.windows import from_bounds - from rasterio.enums import Resampling -except ImportError: - raise ImportError("rasterio is required for friction layer support") - -# Default path to the friction VRT -DEFAULT_FRICTION_PATH = Path("/mnt/nav/worldcover/friction/friction_conus.vrt") - - -class FrictionReader: - """Reader for WorldCover friction raster.""" - - def __init__(self, friction_path: Path = DEFAULT_FRICTION_PATH): - self.friction_path = friction_path - self._dataset = None - - def _open(self): - """Lazy open the dataset.""" - if self._dataset is None: - self._dataset = rasterio.open(self.friction_path) - return self._dataset - - def get_friction_grid( - self, - south: float, - north: float, - west: float, - east: float, - target_shape: Tuple[int, int] - ) -> np.ndarray: - """ - Get friction values for a bounding box, resampled to target shape. - - Args: - south, north, west, east: Bounding box coordinates - target_shape: (rows, cols) to resample to (matches elevation grid) - - Returns: - np.ndarray of uint8 friction values, same shape as target_shape. - Values: 10-40 = friction multiplier (divide by 10) - 255 = impassable - 0 = nodata (treat as impassable) - """ - ds = self._open() - - # Create a window from the bounding box - window = from_bounds(west, south, east, north, ds.transform) - - # Read with resampling to target shape - # Use nearest neighbor for categorical data - friction = ds.read( - 1, - window=window, - out_shape=target_shape, - resampling=Resampling.nearest - ) - - return friction - - def sample_point(self, lat: float, lon: float) -> int: - """Sample friction value at a single point.""" - ds = self._open() - - # Get pixel coordinates - row, col = ds.index(lon, lat) - - # Check bounds - if row < 0 or row >= ds.height or col < 0 or col >= ds.width: - return 0 # Out of bounds = nodata - - # Read single pixel - window = rasterio.windows.Window(col, row, 1, 1) - value = ds.read(1, window=window) - return int(value[0, 0]) - - def close(self): - """Close the dataset.""" - if self._dataset is not None: - self._dataset.close() - self._dataset = None - - -def friction_to_multiplier(friction: np.ndarray) -> np.ndarray: - """ - Convert friction values to cost multipliers. - - Args: - friction: uint8 array of friction values - - Returns: - float32 array of multipliers. - Values 10-40 become 1.0-4.0 (divide by 10). - Values 0 or 255 become np.inf (impassable). - """ - multiplier = friction.astype(np.float32) / 10.0 - - # Mark impassable cells - multiplier[friction == 0] = np.inf # nodata - multiplier[friction == 255] = np.inf # water/impassable - - return multiplier - - -if __name__ == "__main__": - print("Testing FrictionReader...") - - reader = FrictionReader() - - # Test point sampling - Murtaugh Lake (should be water = 255) - lake_lat, lake_lon = 42.47, -114.15 - lake_friction = reader.sample_point(lake_lat, lake_lon) - print(f"Murtaugh Lake ({lake_lat}, {lake_lon}): friction = {lake_friction}") - print(f" Expected: 255 (water/impassable)") - - # Test grid read for small bbox - friction = reader.get_friction_grid( - south=42.4, north=42.5, west=-114.2, east=-114.1, - target_shape=(100, 100) - ) - print(f"\nGrid test shape: {friction.shape}") - print(f"Unique values: {np.unique(friction)}") - print(f"Water cells (255): {np.sum(friction == 255)}") - - reader.close() - print("\nFrictionReader test complete.") diff --git a/lib/offroute/mvum.py b/lib/offroute/mvum.py deleted file mode 100644 index 31e503d..0000000 --- a/lib/offroute/mvum.py +++ /dev/null @@ -1,623 +0,0 @@ -""" -MVUM (Motor Vehicle Use Map) legal access layer for OFFROUTE. - -Queries USFS MVUM data from navi.db and provides rasterized access grids -indicating which roads/trails are open or closed to specific vehicle modes. - -MVUM is motor-vehicle specific — foot mode should skip this layer entirely. -""" -import re -import sqlite3 -import warnings -from datetime import datetime -from pathlib import Path -from typing import Dict, List, Optional, Tuple, Literal - -import numpy as np - -# Path to navi.db -NAVI_DB_PATH = Path("/mnt/nav/navi.db") - - -def parse_date_range(date_str: str) -> List[Tuple[int, int, int, int]]: - """ - Parse MVUM date range strings like "05/01-11/30" or "06/15-10/15,12/01-03/31". - - Returns list of (start_month, start_day, end_month, end_day) tuples. - Returns empty list if unparseable. - """ - if not date_str or date_str.strip() == "": - return [] - - ranges = [] - # Split by comma for multi-period strings - for part in date_str.split(","): - part = part.strip() - # Match MM/DD-MM/DD pattern - match = re.match(r"(\d{1,2})/(\d{1,2})-(\d{1,2})/(\d{1,2})", part) - if match: - try: - sm, sd, em, ed = int(match.group(1)), int(match.group(2)), int(match.group(3)), int(match.group(4)) - if 1 <= sm <= 12 and 1 <= sd <= 31 and 1 <= em <= 12 and 1 <= ed <= 31: - ranges.append((sm, sd, em, ed)) - except ValueError: - pass - - return ranges - - -def is_date_in_range(month: int, day: int, ranges: List[Tuple[int, int, int, int]]) -> bool: - """ - Check if a given month/day falls within any of the date ranges. - Handles ranges that wrap around year end (e.g., 12/01-03/31). - """ - if not ranges: - return True # No ranges = assume open - - date_num = month * 100 + day # Simple numeric comparison - - for sm, sd, em, ed in ranges: - start_num = sm * 100 + sd - end_num = em * 100 + ed - - if start_num <= end_num: - # Normal range (e.g., 05/01-11/30) - if start_num <= date_num <= end_num: - return True - else: - # Wrapping range (e.g., 12/01-03/31) - if date_num >= start_num or date_num <= end_num: - return True - - return False - - -def check_access( - status_field: Optional[str], - dates_field: Optional[str], - seasonal: Optional[str], - check_date: Optional[Tuple[int, int]] = None -) -> Optional[bool]: - """ - Determine if a road/trail is open to a vehicle type. - - Args: - status_field: Value of vehicle-class field (e.g., "open", null) - dates_field: Value of *_DATESOPEN field (e.g., "05/01-11/30") - seasonal: Value of SEASONAL field ("yearlong", "seasonal") - check_date: Optional (month, day) tuple to check against date ranges - - Returns: - True = open - False = closed - None = no data (field not populated, defer to SYMBOL) - """ - if status_field is None or status_field.strip() == "": - return None # No data - - status = status_field.strip().lower() - - if status != "open": - return False # Explicitly closed or restricted - - # Status is "open" - check seasonal restrictions - if check_date is not None: - month, day = check_date - - # Parse date ranges - if dates_field: - ranges = parse_date_range(dates_field) - if ranges: - return is_date_in_range(month, day, ranges) - - # No date field but seasonal = "yearlong" means always open - if seasonal and seasonal.strip().lower() == "yearlong": - return True - - # Seasonal with no dates - assume open (data quality issue) - if seasonal and seasonal.strip().lower() == "seasonal": - warnings.warn(f"Seasonal road/trail with no DATESOPEN, assuming open") - return True - - return True # Open with no date check - - -def get_mode_field(mode: str) -> Tuple[str, str]: - """ - Get the MVUM field names for a given travel mode. - - Returns (status_field, dates_field) tuple. - """ - mode_mapping = { - "atv": ("atv", "atv_datesopen"), - "motorcycle": ("motorcycle", "motorcycle_datesopen"), - "mtb": ("e_bike_class1", "e_bike_class1_dur"), # Closest analog for e-bikes - "vehicle": ("highclearancevehicle", "highclearancevehicle_datesopen"), - "passenger": ("passengervehicle", "passengervehicle_datesopen"), - } - - return mode_mapping.get(mode, ("highclearancevehicle", "highclearancevehicle_datesopen")) - - -def symbol_to_access(symbol: str, mode: str, maint_level: Optional[str] = None) -> Optional[bool]: - """ - Fallback: interpret SYMBOL field when per-vehicle-class fields are null. - - MVUM SYMBOL meanings (roads): - 1 = Open to all vehicles - 2 = Open to highway legal vehicles only - 3 = Road closed to motorized - 4 = Road open seasonally - 11 = Administrative use only - 12 = Decommissioned - - For trails, similar logic applies based on TRAILCLASS. - """ - if symbol is None: - return None - - sym = str(symbol).strip() - - # Symbol 1: Open to all - if sym == "1": - return True - - # Symbol 2: Highway legal only - if sym == "2": - # ATVs/motorcycles typically not highway legal - if mode in ("atv", "motorcycle"): - return False - return True - - # Symbol 3: Closed to motorized - if sym == "3": - return False - - # Symbol 4: Seasonally open (assume open if no date check) - if sym == "4": - return True - - # Symbol 11/12: Administrative/decommissioned = closed - if sym in ("11", "12"): - return False - - # Unknown symbol - defer - return None - - -class MVUMReader: - """ - Reader for MVUM data from navi.db. - - Queries roads and trails by bounding box and returns access grids. - """ - - def __init__(self, db_path: Path = NAVI_DB_PATH): - self.db_path = db_path - self._conn = None - - def _get_conn(self) -> sqlite3.Connection: - if self._conn is None: - if not self.db_path.exists(): - raise FileNotFoundError(f"navi.db not found at {self.db_path}") - self._conn = sqlite3.connect(str(self.db_path)) - self._conn.row_factory = sqlite3.Row - # Load Spatialite extension if available - try: - self._conn.enable_load_extension(True) - self._conn.load_extension("mod_spatialite") - except Exception: - pass # Spatialite not available, will use manual bbox queries - return self._conn - - def table_exists(self, table_name: str) -> bool: - """Check if an MVUM table exists.""" - conn = self._get_conn() - cur = conn.execute( - "SELECT name FROM sqlite_master WHERE type='table' AND name=?", - (table_name,) - ) - return cur.fetchone() is not None - - def query_roads_bbox( - self, - south: float, north: float, west: float, east: float, - mode: str = "atv", - check_date: Optional[Tuple[int, int]] = None - ) -> List[Dict]: - """ - Query MVUM roads within a bounding box. - - Returns list of dicts with access info for the given mode. - """ - if not self.table_exists("mvum_roads"): - return [] - - conn = self._get_conn() - - # Query using bbox on geometry - # Since we don't have spatialite, we'll query all and filter in Python - # For production, consider pre-computing bbox columns - cur = conn.execute(""" - SELECT ogc_fid, id, name, symbol, operationalmaintlevel, seasonal, - atv, atv_datesopen, motorcycle, motorcycle_datesopen, - highclearancevehicle, highclearancevehicle_datesopen, - passengervehicle, passengervehicle_datesopen, - e_bike_class1, e_bike_class1_dur, - shape - FROM mvum_roads - """) - - status_field, dates_field = get_mode_field(mode) - results = [] - - for row in cur: - # Parse geometry to check bbox intersection - # The shape is stored as WKB blob - shape = row["shape"] - if shape is None: - continue - - # Quick bbox check using geometry extent - # Since we don't have Spatialite functions, we'll include all - # and let the rasterization handle it - - access = check_access( - row[status_field] if status_field in row.keys() else None, - row[dates_field] if dates_field in row.keys() else None, - row["seasonal"], - check_date - ) - - # Fallback to SYMBOL if no per-vehicle data - if access is None: - access = symbol_to_access(row["symbol"], mode, row["operationalmaintlevel"]) - - if access is not None: - results.append({ - "id": row["id"], - "name": row["name"], - "access": access, - "symbol": row["symbol"], - "maint_level": row["operationalmaintlevel"], - "shape": shape, - }) - - return results - - def query_trails_bbox( - self, - south: float, north: float, west: float, east: float, - mode: str = "atv", - check_date: Optional[Tuple[int, int]] = None - ) -> List[Dict]: - """ - Query MVUM trails within a bounding box. - """ - if not self.table_exists("mvum_trails"): - return [] - - conn = self._get_conn() - - cur = conn.execute(""" - SELECT ogc_fid, id, name, symbol, seasonal, trailclass, - atv, atv_datesopen, motorcycle, motorcycle_datesopen, - highclearancevehicle, highclearancevehicle_datesopen, - passengervehicle, passengervehicle_datesopen, - e_bike_class1, e_bike_class1_dur, - shape - FROM mvum_trails - """) - - status_field, dates_field = get_mode_field(mode) - results = [] - - for row in cur: - shape = row["shape"] - if shape is None: - continue - - access = check_access( - row[status_field] if status_field in row.keys() else None, - row[dates_field] if dates_field in row.keys() else None, - row["seasonal"], - check_date - ) - - if access is None: - access = symbol_to_access(row["symbol"], mode) - - if access is not None: - results.append({ - "id": row["id"], - "name": row["name"], - "access": access, - "symbol": row["symbol"], - "trail_class": row["trailclass"], - "shape": shape, - }) - - return results - - def query_nearest( - self, - lat: float, lon: float, - radius_m: float = 50, - table: str = "mvum_roads" - ) -> Optional[Dict]: - """ - Query the nearest MVUM feature to a point. - - Used for the places panel API. - """ - if not self.table_exists(table): - return None - - conn = self._get_conn() - - # Convert radius to degrees (approximate) - radius_deg = radius_m / 111000 - - # Query features in bbox around point - if table == "mvum_roads": - cur = conn.execute(""" - SELECT ogc_fid, id, name, forestname, districtname, symbol, - operationalmaintlevel, surfacetype, seasonal, jurisdiction, - passengervehicle, passengervehicle_datesopen, - highclearancevehicle, highclearancevehicle_datesopen, - atv, atv_datesopen, motorcycle, motorcycle_datesopen, - fourwd_gt50inches, fourwd_gt50_datesopen, - twowd_gt50inches, twowd_gt50_datesopen, - e_bike_class1, e_bike_class1_dur, - e_bike_class2, e_bike_class2_dur, - e_bike_class3, e_bike_class3_dur, - shape - FROM mvum_roads - LIMIT 1000 - """) - else: - cur = conn.execute(""" - SELECT ogc_fid, id, name, forestname, districtname, symbol, - seasonal, jurisdiction, trailclass, trailsystem, - passengervehicle, passengervehicle_datesopen, - highclearancevehicle, highclearancevehicle_datesopen, - atv, atv_datesopen, motorcycle, motorcycle_datesopen, - fourwd_gt50inches, fourwd_gt50_datesopen, - twowd_gt50inches, twowd_gt50_datesopen, - e_bike_class1, e_bike_class1_dur, - e_bike_class2, e_bike_class2_dur, - e_bike_class3, e_bike_class3_dur, - shape - FROM mvum_trails - LIMIT 1000 - """) - - # Find nearest feature - # This is a simplified approach - for production, use spatial index - try: - from shapely import wkb - from shapely.geometry import Point - - query_point = Point(lon, lat) - nearest = None - min_dist = float('inf') - - for row in cur: - try: - geom = wkb.loads(row["shape"]) - dist = query_point.distance(geom) - if dist < min_dist and dist < radius_deg: - min_dist = dist - nearest = dict(row) - nearest["geometry"] = geom - except Exception: - continue - - if nearest: - # Convert geometry to GeoJSON - nearest["geojson"] = nearest["geometry"].__geo_interface__ - del nearest["geometry"] - del nearest["shape"] - return nearest - - except ImportError: - warnings.warn("shapely not available for nearest query") - - return None - - def close(self): - if self._conn: - self._conn.close() - self._conn = None - - -def get_mvum_access_grid( - south: float, north: float, west: float, east: float, - target_shape: Tuple[int, int], - mode: Literal["foot", "mtb", "atv", "vehicle"] = "atv", - check_date: Optional[str] = None, - db_path: Path = NAVI_DB_PATH -) -> np.ndarray: - """ - Get MVUM access grid for pathfinding. - - Args: - south, north, west, east: Bounding box (WGS84) - target_shape: (rows, cols) to match elevation grid - mode: Travel mode (foot skips MVUM entirely) - check_date: Optional "MM/DD" string for seasonal checking - db_path: Path to navi.db - - Returns: - np.ndarray of uint8: - 0 = no MVUM data (defer to existing trail/friction logic) - 1 = road/trail is OPEN to this vehicle mode - 255 = road/trail EXISTS but is CLOSED to this mode - """ - # Foot mode bypasses MVUM entirely - if mode == "foot": - return np.zeros(target_shape, dtype=np.uint8) - - # Parse check_date if provided - parsed_date = None - if check_date: - match = re.match(r"(\d{1,2})/(\d{1,2})", check_date) - if match: - parsed_date = (int(match.group(1)), int(match.group(2))) - - # Initialize output grid - grid = np.zeros(target_shape, dtype=np.uint8) - rows, cols = target_shape - - # Pixel size - pixel_lat = (north - south) / rows - pixel_lon = (east - west) / cols - - reader = MVUMReader(db_path) - - try: - # Query roads and trails - roads = reader.query_roads_bbox(south, north, west, east, mode, parsed_date) - trails = reader.query_trails_bbox(south, north, west, east, mode, parsed_date) - - # Rasterize features - try: - from shapely import wkb - - for features in [roads, trails]: - for feat in features: - try: - geom = wkb.loads(feat["shape"]) - - # Get geometry bounds - minx, miny, maxx, maxy = geom.bounds - - # Check if intersects our bbox - if maxx < west or minx > east or maxy < south or miny > north: - continue - - # Rasterize line - value = 1 if feat["access"] else 255 - - # Simple line rasterization - if geom.geom_type in ("LineString", "MultiLineString"): - if geom.geom_type == "MultiLineString": - coords_list = [list(line.coords) for line in geom.geoms] - else: - coords_list = [list(geom.coords)] - - for coords in coords_list: - for i in range(len(coords) - 1): - x1, y1 = coords[i] - x2, y2 = coords[i + 1] - - # Convert to pixel coordinates - col1 = int((x1 - west) / pixel_lon) - row1 = int((north - y1) / pixel_lat) - col2 = int((x2 - west) / pixel_lon) - row2 = int((north - y2) / pixel_lat) - - # Bresenham's line algorithm - _draw_line(grid, row1, col1, row2, col2, value) - - except Exception as e: - continue - - except ImportError: - warnings.warn("shapely not available, MVUM rasterization skipped") - - finally: - reader.close() - - return grid - - -def _draw_line(grid: np.ndarray, r1: int, c1: int, r2: int, c2: int, value: int): - """Draw a line on the grid using Bresenham's algorithm.""" - rows, cols = grid.shape - - dr = abs(r2 - r1) - dc = abs(c2 - c1) - sr = 1 if r1 < r2 else -1 - sc = 1 if c1 < c2 else -1 - err = dr - dc - - r, c = r1, c1 - - while True: - if 0 <= r < rows and 0 <= c < cols: - # Only overwrite if current value is 0 (no data) or we're marking closed - if grid[r, c] == 0 or value == 255: - grid[r, c] = value - - if r == r2 and c == c2: - break - - e2 = 2 * err - if e2 > -dc: - err -= dc - r += sr - if e2 < dr: - err += dr - c += sc - - -if __name__ == "__main__": - import sys - - print("=" * 60) - print("MVUM Reader Test") - print("=" * 60) - - reader = MVUMReader() - - if not reader.table_exists("mvum_roads"): - print("ERROR: mvum_roads table not found in navi.db") - sys.exit(1) - - # Test bbox query (Sawtooth NF area) - print("\n[1] Testing bbox query (Sawtooth NF area)...") - roads = reader.query_roads_bbox( - south=43.5, north=44.0, west=-115.0, east=-114.0, - mode="atv" - ) - print(f" Found {len(roads)} roads") - - open_count = sum(1 for r in roads if r["access"]) - closed_count = sum(1 for r in roads if not r["access"]) - print(f" Open to ATV: {open_count}") - print(f" Closed to ATV: {closed_count}") - - # Test with seasonal date - print("\n[2] Testing with date check (July 15)...") - roads_summer = reader.query_roads_bbox( - south=43.5, north=44.0, west=-115.0, east=-114.0, - mode="atv", - check_date=(7, 15) - ) - open_summer = sum(1 for r in roads_summer if r["access"]) - print(f" Open to ATV on 07/15: {open_summer}") - - print("\n[3] Testing with date check (January 15)...") - roads_winter = reader.query_roads_bbox( - south=43.5, north=44.0, west=-115.0, east=-114.0, - mode="atv", - check_date=(1, 15) - ) - open_winter = sum(1 for r in roads_winter if r["access"]) - print(f" Open to ATV on 01/15: {open_winter}") - - # Test grid generation - print("\n[4] Testing grid generation...") - grid = get_mvum_access_grid( - south=43.5, north=44.0, west=-115.0, east=-114.0, - target_shape=(500, 1000), - mode="atv" - ) - print(f" Grid shape: {grid.shape}") - print(f" No data (0): {np.sum(grid == 0)}") - print(f" Open (1): {np.sum(grid == 1)}") - print(f" Closed (255): {np.sum(grid == 255)}") - - reader.close() - print("\nDone.") diff --git a/lib/offroute/prototype.py b/lib/offroute/prototype.py deleted file mode 100755 index c9b78f0..0000000 --- a/lib/offroute/prototype.py +++ /dev/null @@ -1,414 +0,0 @@ -#!/usr/bin/env python3 -""" -OFFROUTE Phase O3a Prototype - -Validates trail burn-in integration with the MCP pathfinder. -The path should actively seek out trails and roads when nearby. - -Compares paths with and without trail burn-in to show the benefit -of trail-seeking behavior. -""" -import json -import time -import sys -from pathlib import Path - -import numpy as np -from skimage.graph import MCP_Geometric - -# Add parent to path for imports -sys.path.insert(0, str(Path(__file__).parent.parent.parent)) - -from lib.offroute.dem import DEMReader -from lib.offroute.cost import compute_cost_grid -from lib.offroute.friction import FrictionReader, friction_to_multiplier -from lib.offroute.barriers import BarrierReader, DEFAULT_BARRIERS_PATH -from lib.offroute.trails import TrailReader, DEFAULT_TRAILS_PATH - -# Test bounding box - Idaho area -BBOX = { - "south": 42.21, - "north": 42.60, - "west": -114.76, - "east": -113.79, -} - -# Start point: wilderness area away from roads -START_LAT = 42.35 -START_LON = -114.60 - -# End point: near Twin Falls (has roads/trails) -END_LAT = 42.55 -END_LON = -114.20 - -# Output files -OUTPUT_PATH_WITH_TRAILS = Path("/opt/recon/data/offroute-test-trails.geojson") -OUTPUT_PATH_NO_TRAILS = Path("/opt/recon/data/offroute-test-no-trails.geojson") - -# Memory limit in GB -MEMORY_LIMIT_GB = 12 - - -def check_memory_usage(): - """Check current memory usage and abort if over limit.""" - try: - import psutil - process = psutil.Process() - mem_gb = process.memory_info().rss / (1024**3) - if mem_gb > MEMORY_LIMIT_GB: - print(f"ERROR: Memory usage {mem_gb:.1f}GB exceeds {MEMORY_LIMIT_GB}GB limit") - sys.exit(1) - return mem_gb - except ImportError: - return 0 - - -def run_pathfinder( - elevation: np.ndarray, - meta: dict, - friction_mult: np.ndarray, - trails: np.ndarray, - barriers: np.ndarray, - use_trails: bool, - start_row: int, - start_col: int, - end_row: int, - end_col: int, - dem_reader: DEMReader, -) -> dict: - """Run the MCP pathfinder with given parameters.""" - # Compute cost grid - cost = compute_cost_grid( - elevation, - cell_size_m=meta["cell_size_m"], - friction=friction_mult, - trails=trails if use_trails else None, - barriers=barriers, - boundary_mode="pragmatic", - ) - - # Run MCP - mcp = MCP_Geometric(cost, fully_connected=True) - cumulative_costs, traceback = mcp.find_costs([(start_row, start_col)]) - - end_cost = cumulative_costs[end_row, end_col] - - if np.isinf(end_cost): - return { - "success": False, - "reason": "No path found (blocked by impassable terrain)", - } - - # Traceback path - path_indices = mcp.traceback((end_row, end_col)) - - # Convert to coordinates and collect stats - coordinates = [] - elevations = [] - trail_values = [] - - for row, col in path_indices: - lat, lon = dem_reader.pixel_to_latlon(row, col, meta) - elev = elevation[row, col] - trail_val = trails[row, col] if trails is not None else 0 - coordinates.append([lon, lat]) - elevations.append(elev) - trail_values.append(trail_val) - - # Compute distance - total_distance_m = 0 - for i in range(1, len(coordinates)): - lon1, lat1 = coordinates[i-1] - lon2, lat2 = coordinates[i] - R = 6371000 - dlat = np.radians(lat2 - lat1) - dlon = np.radians(lon2 - lon1) - a = np.sin(dlat/2)**2 + np.cos(np.radians(lat1)) * np.cos(np.radians(lat2)) * np.sin(dlon/2)**2 - c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1-a)) - total_distance_m += R * c - - # Elevation stats - elev_arr = np.array(elevations) - elev_diff = np.diff(elev_arr) - elev_gain = np.sum(elev_diff[elev_diff > 0]) - elev_loss = np.sum(np.abs(elev_diff[elev_diff < 0])) - - # Trail stats - trail_arr = np.array(trail_values) - road_cells = np.sum(trail_arr == 5) - track_cells = np.sum(trail_arr == 15) - trail_cells = np.sum(trail_arr == 25) - off_trail_cells = np.sum(trail_arr == 0) - on_trail_cells = road_cells + track_cells + trail_cells - total_cells = len(trail_arr) - - return { - "success": True, - "coordinates": coordinates, - "total_time_seconds": float(end_cost), - "total_time_minutes": float(end_cost / 60), - "total_distance_m": float(total_distance_m), - "total_distance_km": float(total_distance_m / 1000), - "elevation_gain_m": float(elev_gain), - "elevation_loss_m": float(elev_loss), - "min_elevation_m": float(np.min(elev_arr)), - "max_elevation_m": float(np.max(elev_arr)), - "cell_count": total_cells, - "road_cells": int(road_cells), - "track_cells": int(track_cells), - "trail_cells": int(trail_cells), - "off_trail_cells": int(off_trail_cells), - "on_trail_pct": float(100 * on_trail_cells / total_cells) if total_cells > 0 else 0, - } - - -def main(): - print("=" * 80) - print("OFFROUTE Phase O3a Prototype (Trail Burn-In)") - print("=" * 80) - - t0 = time.time() - - # Check for required rasters - if not DEFAULT_BARRIERS_PATH.exists(): - print(f"\nERROR: Barrier raster not found at {DEFAULT_BARRIERS_PATH}") - sys.exit(1) - if not DEFAULT_TRAILS_PATH.exists(): - print(f"\nERROR: Trails raster not found at {DEFAULT_TRAILS_PATH}") - sys.exit(1) - - # Step 1: Load elevation data - print(f"\n[1] Loading DEM for bbox: {BBOX}") - dem_reader = DEMReader() - - elevation, meta = dem_reader.get_elevation_grid( - south=BBOX["south"], - north=BBOX["north"], - west=BBOX["west"], - east=BBOX["east"], - ) - - print(f" Elevation grid shape: {elevation.shape}") - print(f" Cell count: {elevation.size:,}") - print(f" Cell size: {meta['cell_size_m']:.1f} m") - - mem = check_memory_usage() - if mem > 0: - print(f" Memory usage: {mem:.1f} GB") - - # Step 2: Load friction data - print(f"\n[2] Loading WorldCover friction layer...") - friction_reader = FrictionReader() - - friction_raw = friction_reader.get_friction_grid( - south=BBOX["south"], - north=BBOX["north"], - west=BBOX["west"], - east=BBOX["east"], - target_shape=elevation.shape - ) - friction_mult = friction_to_multiplier(friction_raw) - - print(f" Friction grid shape: {friction_raw.shape}") - print(f" Water/impassable cells: {np.sum(np.isinf(friction_mult)):,}") - - # Step 3: Load barrier data - print(f"\n[3] Loading PAD-US barrier layer...") - barrier_reader = BarrierReader() - - barriers = barrier_reader.get_barrier_grid( - south=BBOX["south"], - north=BBOX["north"], - west=BBOX["west"], - east=BBOX["east"], - target_shape=elevation.shape - ) - - closed_cells = np.sum(barriers == 255) - print(f" Barrier grid shape: {barriers.shape}") - print(f" Closed/restricted cells: {closed_cells:,}") - - # Step 4: Load trails data - print(f"\n[4] Loading OSM trails layer...") - trail_reader = TrailReader() - - trails = trail_reader.get_trails_grid( - south=BBOX["south"], - north=BBOX["north"], - west=BBOX["west"], - east=BBOX["east"], - target_shape=elevation.shape - ) - - road_cells = np.sum(trails == 5) - track_cells = np.sum(trails == 15) - trail_cells = np.sum(trails == 25) - print(f" Trails grid shape: {trails.shape}") - print(f" Road cells: {road_cells:,}") - print(f" Track cells: {track_cells:,}") - print(f" Trail cells: {trail_cells:,}") - print(f" Total trail coverage: {100*(road_cells+track_cells+trail_cells)/trails.size:.2f}%") - - mem = check_memory_usage() - if mem > 0: - print(f" Memory usage: {mem:.1f} GB") - - # Step 5: Convert start/end to pixel coordinates - print(f"\n[5] Converting coordinates...") - start_row, start_col = dem_reader.latlon_to_pixel(START_LAT, START_LON, meta) - end_row, end_col = dem_reader.latlon_to_pixel(END_LAT, END_LON, meta) - - print(f" Start: ({START_LAT}, {START_LON}) -> pixel ({start_row}, {start_col})") - print(f" End: ({END_LAT}, {END_LON}) -> pixel ({end_row}, {end_col})") - - # Validate coordinates - rows, cols = elevation.shape - if not (0 <= start_row < rows and 0 <= start_col < cols): - print(f"ERROR: Start point outside grid bounds") - sys.exit(1) - if not (0 <= end_row < rows and 0 <= end_col < cols): - print(f"ERROR: End point outside grid bounds") - sys.exit(1) - - # Step 6: Run pathfinder WITH trails - print(f"\n[6] Running pathfinder WITH trail burn-in...") - t6a = time.time() - result_trails = run_pathfinder( - elevation, meta, friction_mult, trails, barriers, - use_trails=True, - start_row=start_row, start_col=start_col, - end_row=end_row, end_col=end_col, - dem_reader=dem_reader, - ) - t6b = time.time() - print(f" Completed in {t6b - t6a:.1f}s") - - # Step 7: Run pathfinder WITHOUT trails - print(f"\n[7] Running pathfinder WITHOUT trail burn-in...") - t7a = time.time() - result_no_trails = run_pathfinder( - elevation, meta, friction_mult, trails, barriers, - use_trails=False, - start_row=start_row, start_col=start_col, - end_row=end_row, end_col=end_col, - dem_reader=dem_reader, - ) - t7b = time.time() - print(f" Completed in {t7b - t7a:.1f}s") - - # Step 8: Save GeoJSON outputs - print(f"\n[8] Saving GeoJSON outputs...") - - OUTPUT_PATH_WITH_TRAILS.parent.mkdir(parents=True, exist_ok=True) - - if result_trails["success"]: - geojson = { - "type": "Feature", - "properties": { - "type": "offroute_with_trails", - "phase": "O3a", - "trail_burn_in": True, - "start": {"lat": START_LAT, "lon": START_LON}, - "end": {"lat": END_LAT, "lon": END_LON}, - **{k: v for k, v in result_trails.items() if k not in ["success", "coordinates"]}, - }, - "geometry": { - "type": "LineString", - "coordinates": result_trails["coordinates"], - } - } - with open(OUTPUT_PATH_WITH_TRAILS, "w") as f: - json.dump(geojson, f, indent=2) - print(f" Saved: {OUTPUT_PATH_WITH_TRAILS}") - - if result_no_trails["success"]: - geojson = { - "type": "Feature", - "properties": { - "type": "offroute_no_trails", - "phase": "O3a", - "trail_burn_in": False, - "start": {"lat": START_LAT, "lon": START_LON}, - "end": {"lat": END_LAT, "lon": END_LON}, - **{k: v for k, v in result_no_trails.items() if k not in ["success", "coordinates"]}, - }, - "geometry": { - "type": "LineString", - "coordinates": result_no_trails["coordinates"], - } - } - with open(OUTPUT_PATH_NO_TRAILS, "w") as f: - json.dump(geojson, f, indent=2) - print(f" Saved: {OUTPUT_PATH_NO_TRAILS}") - - t_total = time.time() - - # Final report - print(f"\n" + "=" * 80) - print("SIDE-BY-SIDE COMPARISON: Trail Burn-In Effect") - print("=" * 80) - - if result_trails["success"] and result_no_trails["success"]: - print(f"{'Metric':<25} {'WITH TRAILS':<20} {'WITHOUT TRAILS':<20} {'Delta':<15}") - print("-" * 80) - - metrics = [ - ("Distance (km)", "total_distance_km", ".2f"), - ("Effort time (min)", "total_time_minutes", ".1f"), - ("Cell count", "cell_count", "d"), - ("Elevation gain (m)", "elevation_gain_m", ".0f"), - ("On-trail %", "on_trail_pct", ".1f"), - ("Road cells", "road_cells", "d"), - ("Track cells", "track_cells", "d"), - ("Trail cells", "trail_cells", "d"), - ] - - for label, key, fmt in metrics: - val_with = result_trails[key] - val_without = result_no_trails[key] - if isinstance(val_with, int): - delta = val_with - val_without - delta_str = f"{delta:+d}" - else: - delta = val_with - val_without - delta_str = f"{delta:+.2f}" - print(f"{label:<25} {val_with:<20{fmt}} {val_without:<20{fmt}} {delta_str:<15}") - - # Analysis - print(f"\n" + "-" * 80) - print("ANALYSIS") - print("-" * 80) - - time_saved = result_no_trails["total_time_minutes"] - result_trails["total_time_minutes"] - if time_saved > 0: - print(f"Trail burn-in saves {time_saved:.1f} minutes ({100*time_saved/result_no_trails['total_time_minutes']:.1f}% faster)") - elif time_saved < 0: - print(f"Trail burn-in adds {-time_saved:.1f} minutes (path seeks trails even if longer)") - - on_trail_with = result_trails["on_trail_pct"] - on_trail_without = result_no_trails["on_trail_pct"] - if on_trail_with > on_trail_without: - print(f"Trail burn-in increases on-trail travel: {on_trail_without:.1f}% → {on_trail_with:.1f}%") - else: - print(f"Both paths have similar on-trail percentage") - - else: - if not result_trails["success"]: - print(f"WITH TRAILS: FAILED - {result_trails.get('reason', 'unknown')}") - if not result_no_trails["success"]: - print(f"WITHOUT TRAILS: FAILED - {result_no_trails.get('reason', 'unknown')}") - - print(f"\n" + "-" * 80) - print(f"Total wall time: {t_total - t0:.1f}s") - - # Cleanup - dem_reader.close() - friction_reader.close() - barrier_reader.close() - trail_reader.close() - - print("\nPrototype completed.") - - -if __name__ == "__main__": - main() diff --git a/lib/offroute/router.py b/lib/offroute/router.py deleted file mode 100644 index bd3d379..0000000 --- a/lib/offroute/router.py +++ /dev/null @@ -1,1682 +0,0 @@ -""" -OFFROUTE Router — Bidirectional wilderness-to-network path orchestration. - -Supports four routing scenarios: - A: off-network start → on-network end (wilderness then Valhalla) - B: off-network start → off-network end (wilderness, Valhalla, wilderness) - C: on-network start → off-network end (Valhalla then wilderness) - D: on-network start → on-network end (pure Valhalla passthrough) - -Off-network detection: Valhalla /locate snap distance > 500m = off-network. - -IMPORTANT: The wilderness segment ALWAYS uses foot mode for pathfinding. -The user's selected mode affects: - 1. Which entry points are valid (foot=any, mtb=tracks+roads, vehicle=roads only) - 2. The Valhalla costing profile for the network segment -""" -import gc -import json -import math -import subprocess -import tempfile -import time -from pathlib import Path -from typing import Dict, List, Optional, Tuple, Literal, Set - -import numpy as np -import requests -import psycopg2 -import psycopg2.extras -from shapely.geometry import LineString -from skimage.graph import MCP_Geometric - -from .dem import DEMReader -from .cost import compute_cost_grid -from .friction import FrictionReader, friction_to_multiplier -from .barriers import BarrierReader, WildernessReader, DEFAULT_WILDERNESS_PATH -from .trails import TrailReader -from .mvum import get_mvum_access_grid -from ..deployment_config import get_deployment_config - -# Load configuration -_deploy_config = get_deployment_config() -_offroute_config = _deploy_config.get("offroute", {}) - -# Paths (configurable via home.yaml) -OSM_PBF_PATH = Path(_offroute_config.get("osm_pbf_path", "/mnt/nav/sources/idaho-latest.osm.pbf")) -DENSIFY_INTERVAL_M = _offroute_config.get("densify_interval_m", 100) -POSTGIS_DSN = _offroute_config.get("postgis_dsn", "dbname=padus user=postgres") - -# Legacy SQLite path (still used by MVUM) -NAVI_DB_PATH = Path("/mnt/nav/navi.db") - -# Valhalla endpoint -VALHALLA_URL = "http://localhost:8002" - -# Search radius for entry points (km) -DEFAULT_SEARCH_RADIUS_KM = 50 -EXPANDED_SEARCH_RADIUS_KM = 100 - -# Memory limit -MEMORY_LIMIT_GB = 12 - -# Off-network detection threshold (meters) -OFF_NETWORK_THRESHOLD_M = 10 - -# Mode to Valhalla costing mapping -MODE_TO_COSTING = { - "auto": "auto", - "foot": "pedestrian", - "mtb": "bicycle", - "atv": "auto", - "vehicle": "auto", -} - -# Mode to valid entry point highway classes -# foot = any trail/track/road, mtb = tracks and roads, vehicle = roads only -MODE_TO_VALID_HIGHWAYS = { - "auto": {"primary", "secondary", "tertiary", "unclassified", "residential", - "service"}, - "foot": {"primary", "secondary", "tertiary", "unclassified", "residential", - "service", "track", "path", "footway", "bridleway"}, - "mtb": {"primary", "secondary", "tertiary", "unclassified", "residential", - "service", "track"}, - "atv": {"primary", "secondary", "tertiary", "unclassified", "residential", - "service", "track"}, - "vehicle": {"primary", "secondary", "tertiary", "unclassified", "residential", - "service"}, -} - - -def haversine_distance(lat1: float, lon1: float, lat2: float, lon2: float) -> float: - """Calculate distance between two points in meters.""" - R = 6371000 - dlat = math.radians(lat2 - lat1) - dlon = math.radians(lon2 - lon1) - a = math.sin(dlat/2)**2 + math.cos(math.radians(lat1)) * math.cos(math.radians(lat2)) * math.sin(dlon/2)**2 - c = 2 * math.atan2(math.sqrt(a), math.sqrt(1-a)) - return R * c - - -def check_memory_usage() -> float: - """Check current memory usage in GB.""" - try: - import psutil - process = psutil.Process() - return process.memory_info().rss / (1024**3) - except ImportError: - return 0 - - -class EntryPointIndex: - """ - PostGIS-backed spatial index of road/trail entry points. - Uses ST_DWithin for fast radius queries with meter-accurate distances. - Densifies highway LineStrings at 100m intervals for better coverage. - """ - - def __init__(self, dsn: str = None): - self.dsn = dsn or POSTGIS_DSN - self._conn: Optional[psycopg2.extensions.connection] = None - - def _get_conn(self) -> psycopg2.extensions.connection: - if self._conn is None or self._conn.closed: - self._conn = psycopg2.connect(self.dsn) - return self._conn - - def table_exists(self) -> bool: - """Check if entry_points table exists.""" - conn = self._get_conn() - with conn.cursor() as cur: - cur.execute(""" - SELECT EXISTS ( - SELECT FROM information_schema.tables - WHERE table_name = 'entry_points' - ) - """) - return cur.fetchone()[0] - - def get_entry_point_count(self) -> int: - """Return the number of entry points in the index.""" - if not self.table_exists(): - return 0 - conn = self._get_conn() - with conn.cursor() as cur: - cur.execute("SELECT COUNT(*) FROM entry_points") - return cur.fetchone()[0] - - def query_bbox( - self, - south: float, - north: float, - west: float, - east: float, - valid_highways: Optional[Set[str]] = None - ) -> List[Dict]: - """Find entry points within a bounding box.""" - if not self.table_exists(): - return [] - - conn = self._get_conn() - - highway_filter = "" - params = [west, south, east, north] - if valid_highways: - placeholders = ','.join(['%s'] * len(valid_highways)) - highway_filter = f"AND highway_class IN ({placeholders})" - params.extend(list(valid_highways)) - - query = f""" - SELECT - id, - ST_Y(geom) as lat, - ST_X(geom) as lon, - highway_class, - name, - land_status - FROM entry_points - WHERE geom && ST_MakeEnvelope(%s, %s, %s, %s, 4326) - {highway_filter} - """ - - with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: - cur.execute(query, params) - return [dict(row) for row in cur.fetchall()] - - def query_radius( - self, - lat: float, - lon: float, - radius_km: float, - valid_highways: Optional[Set[str]] = None, - limit: int = 50 - ) -> List[Dict]: - """ - Find entry points within radius_km of (lat, lon). - Uses PostGIS ST_DWithin with geography cast for meter-accurate distance. - """ - if not self.table_exists(): - return [] - - conn = self._get_conn() - radius_m = radius_km * 1000 - - # Build query with optional highway filter - highway_filter = "" - params = [lon, lat, lon, lat, radius_m] - if valid_highways: - placeholders = ','.join(['%s'] * len(valid_highways)) - highway_filter = f"AND highway_class IN ({placeholders})" - params.extend(list(valid_highways)) - params.append(limit) - - query = f""" - SELECT - id, - ST_Y(geom) as lat, - ST_X(geom) as lon, - highway_class, - name, - land_status, - ST_Distance( - geom::geography, - ST_SetSRID(ST_Point(%s, %s), 4326)::geography - ) as distance_m - FROM entry_points - WHERE ST_DWithin( - geom::geography, - ST_SetSRID(ST_Point(%s, %s), 4326)::geography, - %s - ) - {highway_filter} - ORDER BY distance_m - LIMIT %s - """ - - with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: - cur.execute(query, params) - return [dict(row) for row in cur.fetchall()] - - def build_index(self, osm_pbf_path: Path = None) -> Dict: - """ - Build the entry point index from OSM PBF. - Densifies LineStrings to sample points every 100m. - Tags points with land_status from PAD-US. - """ - if osm_pbf_path is None: - osm_pbf_path = OSM_PBF_PATH - - if not osm_pbf_path.exists(): - raise FileNotFoundError(f"OSM PBF not found: {osm_pbf_path}") - - print(f"Building entry point index from {osm_pbf_path}...") - start_time = time.time() - - highway_types = [ - "primary", "secondary", "tertiary", "unclassified", - "residential", "service", "track", "path", "footway", "bridleway" - ] - - stats = {"total": 0, "by_class": {}, "lines_processed": 0} - - with tempfile.TemporaryDirectory() as tmpdir: - geojson_path = Path(tmpdir) / "highways.geojson" - - # Extract highways with osmium - print(" Extracting highways with osmium...") - cmd = ["osmium", "tags-filter", str(osm_pbf_path)] - for ht in highway_types: - cmd.append(f"w/highway={ht}") - cmd.extend(["-o", str(Path(tmpdir) / "filtered.osm.pbf"), "--overwrite"]) - subprocess.run(cmd, check=True, capture_output=True) - - # Convert to GeoJSON - print(" Converting to GeoJSON with ogr2ogr...") - cmd = [ - "ogr2ogr", "-f", "GeoJSON", - str(geojson_path), - str(Path(tmpdir) / "filtered.osm.pbf"), - "lines", "-t_srs", "EPSG:4326" - ] - subprocess.run(cmd, check=True, capture_output=True) - - # Load GeoJSON - print(" Loading GeoJSON...") - with open(geojson_path) as f: - data = json.load(f) - - # Process features and densify - print(f" Densifying LineStrings at {DENSIFY_INTERVAL_M}m intervals...") - points_to_insert = [] - seen_keys = set() - - features = data.get("features", []) - total_features = len(features) - - for idx, feature in enumerate(features): - if idx > 0 and idx % 100000 == 0: - print(f" Processed {idx}/{total_features} features...") - - props = feature.get("properties", {}) - geom = feature.get("geometry", {}) - - if geom.get("type") != "LineString": - continue - - coords = geom.get("coordinates", []) - if len(coords) < 2: - continue - - highway_class = props.get("highway", "unknown") - name = props.get("name", "") - stats["lines_processed"] += 1 - - # Densify this LineString - densified = self._densify_line(coords, DENSIFY_INTERVAL_M) - - for lon, lat in densified: - # Deduplicate by rounding to 5 decimal places (~1m precision) - key = (round(lat, 5), round(lon, 5)) - if key in seen_keys: - continue - seen_keys.add(key) - - points_to_insert.append((lon, lat, highway_class, name)) - - # Insert into PostGIS - print(f" Inserting {len(points_to_insert)} entry points into PostGIS...") - conn = self._get_conn() - - with conn.cursor() as cur: - # Truncate existing data - cur.execute("TRUNCATE entry_points RESTART IDENTITY") - - # Batch insert with execute_values for speed - batch_size = 50000 - for i in range(0, len(points_to_insert), batch_size): - batch = points_to_insert[i:i+batch_size] - psycopg2.extras.execute_values( - cur, - """ - INSERT INTO entry_points (geom, highway_class, name) - VALUES %s - """, - batch, - template="(ST_SetSRID(ST_Point(%s, %s), 4326), %s, %s)", - page_size=10000 - ) - if i > 0 and i % 500000 == 0: - print(f" Inserted {i}/{len(points_to_insert)} points...") - - conn.commit() - - # Tag land_status from PAD-US - print(" Tagging land_status from PAD-US subdivided polygons...") - with conn.cursor() as cur: - cur.execute(""" - UPDATE entry_points e - SET land_status = 'public' - FROM padus_sub p - WHERE ST_Intersects(e.geom, p.geom) - """) - public_count = cur.rowcount - print(f" Tagged {public_count} points as public land") - - conn.commit() - - # Gather stats - elapsed = time.time() - start_time - stats["total"] = len(points_to_insert) - stats["build_time_sec"] = round(elapsed, 1) - - for lon, lat, hc, name in points_to_insert: - stats["by_class"][hc] = stats["by_class"].get(hc, 0) + 1 - - print(f" Done in {elapsed:.1f}s. Total: {stats['total']} entry points from {stats['lines_processed']} lines") - for hc, count in sorted(stats["by_class"].items(), key=lambda x: -x[1]): - print(f" {hc}: {count}") - - return stats - - def _densify_line(self, coords: List[List[float]], interval_m: float) -> List[tuple]: - """ - Sample points along a LineString at regular intervals. - coords: [[lon, lat], ...] in GeoJSON order - Returns: [(lon, lat), ...] sampled points including first and last - """ - if len(coords) < 2: - return [(coords[0][0], coords[0][1])] if coords else [] - - # Calculate line length in meters using haversine on segments - total_m = 0 - for i in range(len(coords) - 1): - lon1, lat1 = coords[i] - lon2, lat2 = coords[i + 1] - total_m += haversine_distance(lat1, lon1, lat2, lon2) - - if total_m == 0: - return [(coords[0][0], coords[0][1])] - - # Create Shapely LineString - line = LineString(coords) - - # Calculate number of points needed - n_points = max(2, int(total_m / interval_m) + 1) - - # Sample using normalized interpolation - result = [] - for i in range(n_points): - fraction = min(i / (n_points - 1), 1.0) if n_points > 1 else 0 - point = line.interpolate(fraction, normalized=True) - result.append((point.x, point.y)) # (lon, lat) - - # Always ensure first and last original coordinates are included - first_coord = (coords[0][0], coords[0][1]) - last_coord = (coords[-1][0], coords[-1][1]) - - if result[0] != first_coord: - result[0] = first_coord - if result[-1] != last_coord: - result[-1] = last_coord - - return result - - def _highway_priority(self, highway_class: str) -> int: - """Lower number = better priority for entry points.""" - priority = { - "primary": 1, "secondary": 2, "tertiary": 3, - "unclassified": 4, "residential": 5, "service": 6, - "track": 7, "path": 8, "footway": 9, "bridleway": 10 - } - return priority.get(highway_class, 99) - - def close(self): - if self._conn and not self._conn.closed: - self._conn.close() - self._conn = None - - -class OffrouteRouter: - """ - OFFROUTE Router — orchestrates wilderness pathfinding and Valhalla stitching. - - Supports four scenarios: - A: off-network start → on-network end - B: off-network start → off-network end - C: on-network start → off-network end - D: on-network start → on-network end (pure Valhalla) - - IMPORTANT: Wilderness segment ALWAYS uses foot mode for pathfinding. - User's mode affects entry point selection and Valhalla costing only. - """ - - def __init__(self): - self.dem_reader = None - self.friction_reader = None - self.barrier_reader = None - self.wilderness_reader = None - self.trail_reader = None - self.entry_index = EntryPointIndex() - - def _init_readers(self): - """Lazy init readers.""" - if self.dem_reader is None: - self.dem_reader = DEMReader() - if self.friction_reader is None: - self.friction_reader = FrictionReader() - if self.barrier_reader is None: - self.barrier_reader = BarrierReader() - if self.wilderness_reader is None and DEFAULT_WILDERNESS_PATH.exists(): - self.wilderness_reader = WildernessReader() - if self.trail_reader is None: - self.trail_reader = TrailReader() - - def _locate_on_network(self, lat: float, lon: float, mode: str) -> Dict: - """ - Check if a point is on the routable network using Valhalla's /locate. - - Returns: - { - "on_network": bool, - "snap_distance_m": float, - "snapped_lat": float, - "snapped_lon": float - } - """ - costing = MODE_TO_COSTING.get(mode, "pedestrian") - try: - resp = requests.post( - f"{VALHALLA_URL}/locate", - json={"locations": [{"lat": lat, "lon": lon}], "costing": costing}, - timeout=10 - ) - - if resp.status_code == 200: - data = resp.json() - if data and len(data) > 0 and data[0].get("edges"): - edge = data[0]["edges"][0] - snap_lat = edge.get("correlated_lat", lat) - snap_lon = edge.get("correlated_lon", lon) - snap_dist = haversine_distance(lat, lon, snap_lat, snap_lon) - return { - "on_network": snap_dist <= OFF_NETWORK_THRESHOLD_M, - "snap_distance_m": snap_dist, - "snapped_lat": snap_lat, - "snapped_lon": snap_lon - } - except Exception: - pass - - return { - "on_network": False, - "snap_distance_m": float('inf'), - "snapped_lat": lat, - "snapped_lon": lon - } - - def route( - self, - start_lat: float, - start_lon: float, - end_lat: float, - end_lon: float, - mode: Literal["foot", "mtb", "atv", "vehicle"] = "foot", - boundary_mode: Literal["strict", "pragmatic", "emergency"] = "pragmatic" - ) -> Dict: - """ - Route between two points, handling all four scenarios. - - Scenarios: - A: off-network start → on-network end (wilderness then network) - B: off-network start → off-network end (wilderness, network, wilderness) - C: on-network start → off-network end (network then wilderness) - D: on-network start → on-network end (pure network) - - Args: - start_lat, start_lon: Starting coordinates - end_lat, end_lon: Destination coordinates - mode: Travel mode (foot, mtb, atv, vehicle) - boundary_mode: How to handle private land (strict, pragmatic, emergency) - - Returns a GeoJSON FeatureCollection with route segments. - """ - if mode not in MODE_TO_COSTING: - return {"status": "error", "message": f"Unknown mode: {mode}"} - - # Detect network status for both endpoints - start_status = self._locate_on_network(start_lat, start_lon, mode) - end_status = self._locate_on_network(end_lat, end_lon, mode) - - start_off_network = not start_status["on_network"] - end_off_network = not end_status["on_network"] - - # Dispatch to appropriate handler - if not start_off_network and not end_off_network: - # Scenario D: on-network → on-network (pure Valhalla) - return self._route_D_network_only( - start_lat, start_lon, end_lat, end_lon, mode - ) - elif not start_off_network and end_off_network: - # Scenario C: on-network → off-network - return self._route_C_network_to_wilderness( - start_lat, start_lon, end_lat, end_lon, mode, boundary_mode - ) - elif start_off_network and not end_off_network: - # Scenario A: off-network → on-network - return self._route_A_wilderness_to_network( - start_lat, start_lon, end_lat, end_lon, mode, boundary_mode - ) - else: - # Scenario B: off-network → off-network - return self._route_B_wilderness_both( - start_lat, start_lon, end_lat, end_lon, mode, boundary_mode - ) - - def _route_D_network_only( - self, - start_lat: float, start_lon: float, - end_lat: float, end_lon: float, - mode: str - ) -> Dict: - """ - Scenario D: Both endpoints on-network. Pure Valhalla routing. - """ - t0 = time.time() - costing = MODE_TO_COSTING.get(mode, "pedestrian") - - valhalla_request = { - "locations": [ - {"lat": start_lat, "lon": start_lon}, - {"lat": end_lat, "lon": end_lon} - ], - "costing": costing, - "directions_options": {"units": "kilometers"} - } - - try: - resp = requests.post(f"{VALHALLA_URL}/route", json=valhalla_request, timeout=30) - - if resp.status_code != 200: - return { - "status": "error", - "message": f"Network routing failed: {resp.text[:200]}" - } - - valhalla_data = resp.json() - trip = valhalla_data.get("trip", {}) - legs = trip.get("legs", []) - - if not legs: - return {"status": "error", "message": "No route found"} - - leg = legs[0] - shape = leg.get("shape", "") - network_coords = self._decode_polyline(shape) - - maneuvers = [] - for m in leg.get("maneuvers", []): - maneuvers.append({ - "instruction": m.get("instruction", ""), - "type": m.get("type", 0), - "distance_km": m.get("length", 0), - "time_seconds": m.get("time", 0), - "street_names": m.get("street_names", []), - }) - - summary = trip.get("summary", {}) - distance_km = summary.get("length", 0) - duration_min = summary.get("time", 0) / 60 - - # Build response in same format as wilderness routes - network_feature = { - "type": "Feature", - "properties": { - "segment_type": "network", - "distance_km": distance_km, - "duration_minutes": duration_min, - "maneuvers": maneuvers, - "network_mode": mode, - }, - "geometry": {"type": "LineString", "coordinates": network_coords} - } - - combined_feature = { - "type": "Feature", - "properties": { - "segment_type": "combined", - "network_mode": mode, - }, - "geometry": {"type": "LineString", "coordinates": network_coords} - } - - geojson = {"type": "FeatureCollection", "features": [network_feature, combined_feature]} - - result = { - "status": "ok", - "route": geojson, - "summary": { - "total_distance_km": float(distance_km), - "total_effort_minutes": float(duration_min), - "wilderness_distance_km": 0.0, - "wilderness_effort_minutes": 0.0, - "network_distance_km": float(distance_km), - "network_duration_minutes": float(duration_min), - "on_trail_pct": 100.0, - "barrier_crossings": 0, - "network_mode": mode, - "scenario": "D", - "computation_time_s": time.time() - t0, - } - } - return result - - except Exception as e: - return {"status": "error", "message": f"Network routing failed: {e}"} - - def _route_A_wilderness_to_network( - self, - start_lat: float, start_lon: float, - end_lat: float, end_lon: float, - mode: str, boundary_mode: str - ) -> Dict: - """ - Scenario A: Off-network start → on-network end. - Wilderness pathfinding from start to entry point, then Valhalla to end. - """ - t0 = time.time() - - # Ensure entry point index exists - if not self.entry_index.table_exists() or self.entry_index.get_entry_point_count() == 0: - return { - "status": "error", - "message": "Trail entry point index not built. Run build_entry_index() first." - } - - # Get valid highway classes for this mode - valid_highways = MODE_TO_VALID_HIGHWAYS.get(mode) - - # Find entry points near start, filtered by mode - MAX_ENTRY_POINTS = 10 - entry_points = self.entry_index.query_radius( - start_lat, start_lon, DEFAULT_SEARCH_RADIUS_KM, valid_highways - ) - - if not entry_points: - entry_points = self.entry_index.query_radius( - start_lat, start_lon, EXPANDED_SEARCH_RADIUS_KM, valid_highways - ) - if not entry_points: - if mode == "vehicle": - msg = f"No roads found within {EXPANDED_SEARCH_RADIUS_KM}km. Try a different mode." - elif mode in ("mtb", "atv"): - msg = f"No tracks or roads found within {EXPANDED_SEARCH_RADIUS_KM}km. Try foot mode." - else: - msg = f"No trail entry points found within {EXPANDED_SEARCH_RADIUS_KM}km of start." - return {"status": "error", "message": msg} - - entry_points = entry_points[:MAX_ENTRY_POINTS] - - # Run wilderness pathfinding - wilderness_result = self._pathfind_wilderness( - start_lat, start_lon, end_lat, end_lon, - entry_points, boundary_mode, "start" - ) - - if wilderness_result.get("status") == "error": - return wilderness_result - - # Extract results - wilderness_coords = wilderness_result["coords"] - wilderness_stats = wilderness_result["stats"] - wilderness_elevations = wilderness_result.get("elevations", []) - best_entry = wilderness_result["entry_point"] - - entry_lat = best_entry["lat"] - entry_lon = best_entry["lon"] - - # Call Valhalla from entry point to destination - network_result = self._valhalla_route(entry_lat, entry_lon, end_lat, end_lon, mode) - - # Build response - return self._build_response( - wilderness_start=wilderness_coords, - wilderness_start_stats=wilderness_stats, - wilderness_start_elevations=wilderness_elevations, - network_segment=network_result.get("segment"), - wilderness_end=None, - wilderness_end_stats=None, - wilderness_end_elevations=None, - mode=mode, - boundary_mode=boundary_mode, - entry_start=best_entry, - entry_end=None, - scenario="A", - t0=t0, - valhalla_error=network_result.get("error") - ) - - def _route_C_network_to_wilderness( - self, - start_lat: float, start_lon: float, - end_lat: float, end_lon: float, - mode: str, boundary_mode: str - ) -> Dict: - """ - Scenario C: On-network start → off-network end. - Valhalla from start to entry point, then wilderness pathfinding to end. - """ - t0 = time.time() - - if not self.entry_index.table_exists() or self.entry_index.get_entry_point_count() == 0: - return { - "status": "error", - "message": "Trail entry point index not built. Run build_entry_index() first." - } - - valid_highways = MODE_TO_VALID_HIGHWAYS.get(mode) - - # Find entry points near END (destination) - MAX_ENTRY_POINTS = 10 - entry_points = self.entry_index.query_radius( - end_lat, end_lon, DEFAULT_SEARCH_RADIUS_KM, valid_highways - ) - - if not entry_points: - entry_points = self.entry_index.query_radius( - end_lat, end_lon, EXPANDED_SEARCH_RADIUS_KM, valid_highways - ) - if not entry_points: - if mode == "vehicle": - msg = f"No roads found within {EXPANDED_SEARCH_RADIUS_KM}km of destination. Try a different mode." - elif mode in ("mtb", "atv"): - msg = f"No tracks or roads found within {EXPANDED_SEARCH_RADIUS_KM}km of destination. Try foot mode." - else: - msg = f"No trail entry points found within {EXPANDED_SEARCH_RADIUS_KM}km of destination." - return {"status": "error", "message": msg} - - entry_points = entry_points[:MAX_ENTRY_POINTS] - - # Run wilderness pathfinding FROM END toward entry points - wilderness_result = self._pathfind_wilderness( - end_lat, end_lon, start_lat, start_lon, - entry_points, boundary_mode, "end" - ) - - if wilderness_result.get("status") == "error": - return wilderness_result - - # The path is from end→entry, reverse it for display (entry→end) - wilderness_coords = list(reversed(wilderness_result["coords"])) - wilderness_stats = wilderness_result["stats"] - wilderness_elevations = list(reversed(wilderness_result.get("elevations", []))) - best_entry = wilderness_result["entry_point"] - - entry_lat = best_entry["lat"] - entry_lon = best_entry["lon"] - - # Call Valhalla from start to entry point - network_result = self._valhalla_route(start_lat, start_lon, entry_lat, entry_lon, mode) - - # Build response (network first, then wilderness) - return self._build_response( - wilderness_start=None, - wilderness_start_stats=None, - wilderness_start_elevations=None, - network_segment=network_result.get("segment"), - wilderness_end=wilderness_coords, - wilderness_end_stats=wilderness_stats, - wilderness_end_elevations=wilderness_elevations, - mode=mode, - boundary_mode=boundary_mode, - entry_start=None, - entry_end=best_entry, - scenario="C", - t0=t0, - valhalla_error=network_result.get("error") - ) - - def _route_B_wilderness_both( - self, - start_lat: float, start_lon: float, - end_lat: float, end_lon: float, - mode: str, boundary_mode: str - ) -> Dict: - """ - Scenario B: Off-network start → off-network end. - Wilderness from start to entry_A, Valhalla entry_A to entry_B, wilderness from entry_B to end. - """ - t0 = time.time() - - if not self.entry_index.table_exists() or self.entry_index.get_entry_point_count() == 0: - return { - "status": "error", - "message": "Trail entry point index not built. Run build_entry_index() first." - } - - valid_highways = MODE_TO_VALID_HIGHWAYS.get(mode) - MAX_ENTRY_POINTS = 10 - - # Find entry points near START - entry_points_start = self.entry_index.query_radius( - start_lat, start_lon, DEFAULT_SEARCH_RADIUS_KM, valid_highways - ) - if not entry_points_start: - entry_points_start = self.entry_index.query_radius( - start_lat, start_lon, EXPANDED_SEARCH_RADIUS_KM, valid_highways - ) - if not entry_points_start: - return {"status": "error", "message": f"No entry points found near start within {EXPANDED_SEARCH_RADIUS_KM}km."} - entry_points_start = entry_points_start[:MAX_ENTRY_POINTS] - - # Find entry points near END - entry_points_end = self.entry_index.query_radius( - end_lat, end_lon, DEFAULT_SEARCH_RADIUS_KM, valid_highways - ) - if not entry_points_end: - entry_points_end = self.entry_index.query_radius( - end_lat, end_lon, EXPANDED_SEARCH_RADIUS_KM, valid_highways - ) - if not entry_points_end: - return {"status": "error", "message": f"No entry points found near destination within {EXPANDED_SEARCH_RADIUS_KM}km."} - entry_points_end = entry_points_end[:MAX_ENTRY_POINTS] - - # Phase 1: Wilderness pathfinding from START - wilderness_start_result = self._pathfind_wilderness( - start_lat, start_lon, end_lat, end_lon, - entry_points_start, boundary_mode, "start" - ) - - if wilderness_start_result.get("status") == "error": - return wilderness_start_result - - wilderness_start_coords = wilderness_start_result["coords"] - wilderness_start_stats = wilderness_start_result["stats"] - wilderness_start_elevations = wilderness_start_result.get("elevations", []) - entry_A = wilderness_start_result["entry_point"] - - # Phase 2: Wilderness pathfinding from END (run after freeing phase 1 memory) - wilderness_end_result = self._pathfind_wilderness( - end_lat, end_lon, start_lat, start_lon, - entry_points_end, boundary_mode, "end" - ) - - if wilderness_end_result.get("status") == "error": - return wilderness_end_result - - # Reverse the end wilderness path (it's end→entry, we want entry→end for display) - wilderness_end_coords = list(reversed(wilderness_end_result["coords"])) - wilderness_end_stats = wilderness_end_result["stats"] - wilderness_end_elevations = list(reversed(wilderness_end_result.get("elevations", []))) - entry_B = wilderness_end_result["entry_point"] - - # Phase 3: Valhalla from entry_A to entry_B - network_result = self._valhalla_route( - entry_A["lat"], entry_A["lon"], - entry_B["lat"], entry_B["lon"], - mode - ) - - # Build response - return self._build_response( - wilderness_start=wilderness_start_coords, - wilderness_start_stats=wilderness_start_stats, - wilderness_start_elevations=wilderness_start_elevations, - network_segment=network_result.get("segment"), - wilderness_end=wilderness_end_coords, - wilderness_end_stats=wilderness_end_stats, - wilderness_end_elevations=wilderness_end_elevations, - mode=mode, - boundary_mode=boundary_mode, - entry_start=entry_A, - entry_end=entry_B, - scenario="B", - t0=t0, - valhalla_error=network_result.get("error") - ) - - def _pathfind_wilderness( - self, - origin_lat: float, origin_lon: float, - dest_lat: float, dest_lon: float, - entry_points: List[Dict], - boundary_mode: str, - label: str - ) -> Dict: - """ - Run MCP wilderness pathfinding from origin toward entry points. - - Args: - origin_lat, origin_lon: Starting point for pathfinding - dest_lat, dest_lon: Ultimate destination (for bbox calculation) - entry_points: List of candidate entry points - boundary_mode: How to handle barriers - label: "start" or "end" for error messages - - Returns: - {"status": "ok", "coords": [...], "stats": {...}, "entry_point": {...}} - or {"status": "error", "message": "..."} - """ - # Build bbox - only include origin and entry points, NOT distant destination - # The destination is handled by Valhalla, wilderness only needs to reach entry points - MAX_BBOX_DEGREES = 2.0 - all_lats = [origin_lat] + [p["lat"] for p in entry_points] - all_lons = [origin_lon] + [p["lon"] for p in entry_points] - - padding = 0.05 - bbox = { - "south": min(all_lats) - padding, - "north": max(all_lats) + padding, - "west": min(all_lons) - padding, - "east": max(all_lons) + padding, - } - - # Clamp bbox size, centering on origin - lat_span = bbox["north"] - bbox["south"] - lon_span = bbox["east"] - bbox["west"] - if lat_span > MAX_BBOX_DEGREES or lon_span > MAX_BBOX_DEGREES: - half_span = MAX_BBOX_DEGREES / 2 - bbox = { - "south": origin_lat - half_span, - "north": origin_lat + half_span, - "west": origin_lon - half_span, - "east": origin_lon + half_span, - } - - # Initialize readers - self._init_readers() - - # Load elevation - try: - elevation, meta = self.dem_reader.get_elevation_grid( - south=bbox["south"], north=bbox["north"], - west=bbox["west"], east=bbox["east"], - ) - except Exception as e: - return {"status": "error", "message": f"Failed to load elevation for {label}: {e}"} - - # Check memory - mem = check_memory_usage() - if mem > MEMORY_LIMIT_GB: - return {"status": "error", "message": f"Memory limit exceeded: {mem:.1f}GB > {MEMORY_LIMIT_GB}GB"} - - # Load friction - friction_raw = self.friction_reader.get_friction_grid( - south=bbox["south"], north=bbox["north"], - west=bbox["west"], east=bbox["east"], - target_shape=elevation.shape - ) - friction_mult = friction_to_multiplier(friction_raw) - - # Load barriers - barriers = self.barrier_reader.get_barrier_grid( - south=bbox["south"], north=bbox["north"], - west=bbox["west"], east=bbox["east"], - target_shape=elevation.shape - ) - - # Load trails - trails = self.trail_reader.get_trails_grid( - south=bbox["south"], north=bbox["north"], - west=bbox["west"], east=bbox["east"], - target_shape=elevation.shape - ) - - # Compute cost grid (ALWAYS foot mode for wilderness) - cost = compute_cost_grid( - elevation, - cell_size_m=meta["cell_size_m"], - friction=friction_mult, - friction_raw=friction_raw, - trails=trails, - barriers=barriers, - wilderness=None, - mvum=None, - boundary_mode=boundary_mode, - mode="foot", - ) - - # Free intermediate arrays - del friction_mult, friction_raw - gc.collect() - - # Convert origin to pixel coordinates - origin_row, origin_col = self.dem_reader.latlon_to_pixel(origin_lat, origin_lon, meta) - - rows, cols = elevation.shape - if not (0 <= origin_row < rows and 0 <= origin_col < cols): - return {"status": "error", "message": f"{label.capitalize()} point outside grid bounds"} - - # Map entry points to pixels - entry_pixels = [] - for ep in entry_points: - row, col = self.dem_reader.latlon_to_pixel(ep["lat"], ep["lon"], meta) - if 0 <= row < rows and 0 <= col < cols: - entry_pixels.append({"row": row, "col": col, "entry_point": ep}) - - if not entry_pixels: - return {"status": "error", "message": f"No entry points map to grid bounds for {label}"} - - # Run MCP - mcp = MCP_Geometric(cost, fully_connected=True) - cumulative_costs, traceback = mcp.find_costs([(origin_row, origin_col)]) - - # Find nearest reachable entry point - best_entry = None - best_cost = np.inf - - for ep in entry_pixels: - ep_cost = cumulative_costs[ep["row"], ep["col"]] - if ep_cost < best_cost: - best_cost = ep_cost - best_entry = ep - - if best_entry is None or np.isinf(best_cost): - return { - "status": "error", - "message": f"No path found from {label} to any entry point (blocked by impassable terrain)" - } - - # Traceback path - path_indices = mcp.traceback((best_entry["row"], best_entry["col"])) - - # Convert to coordinates and collect stats - coords = [] - elevations = [] - trail_values = [] - barrier_crossings = 0 - - for row, col in path_indices: - lat, lon = self.dem_reader.pixel_to_latlon(row, col, meta) - coords.append([lon, lat]) - elevations.append(elevation[row, col]) - trail_values.append(trails[row, col]) - if barriers[row, col] == 255: - barrier_crossings += 1 - - # Calculate distance - distance_m = 0 - for i in range(1, len(coords)): - lon1, lat1 = coords[i-1] - lon2, lat2 = coords[i] - distance_m += haversine_distance(lat1, lon1, lat2, lon2) - - # Elevation stats - elev_arr = np.array(elevations) - elev_diff = np.diff(elev_arr) - elev_gain = float(np.sum(elev_diff[elev_diff > 0])) - elev_loss = float(np.sum(np.abs(elev_diff[elev_diff < 0]))) - - # Trail stats - trail_arr = np.array(trail_values) - on_trail_cells = np.sum(trail_arr > 0) - total_cells = len(trail_arr) - on_trail_pct = float(100 * on_trail_cells / total_cells) if total_cells > 0 else 0 - - # Free memory - del mcp, cumulative_costs, traceback, cost, trails, barriers, elevation - gc.collect() - - return { - "status": "ok", - "coords": coords, - "elevations": elevations, # Raw elevation values for maneuver generation - "stats": { - "distance_km": distance_m / 1000, - "effort_minutes": best_cost / 60, - "elevation_gain_m": elev_gain, - "elevation_loss_m": elev_loss, - "on_trail_pct": on_trail_pct, - "barrier_crossings": barrier_crossings, - "cell_count": total_cells, - }, - "entry_point": best_entry["entry_point"] - } - - def _valhalla_route( - self, - start_lat: float, start_lon: float, - end_lat: float, end_lon: float, - mode: str - ) -> Dict: - """ - Call Valhalla for network routing. - - Returns: - {"segment": {...}, "error": None} on success - {"segment": None, "error": "..."} on failure - """ - costing = MODE_TO_COSTING.get(mode, "pedestrian") - - valhalla_request = { - "locations": [ - {"lat": start_lat, "lon": start_lon}, - {"lat": end_lat, "lon": end_lon} - ], - "costing": costing, - "directions_options": {"units": "kilometers"} - } - - try: - resp = requests.post(f"{VALHALLA_URL}/route", json=valhalla_request, timeout=30) - - if resp.status_code == 200: - valhalla_data = resp.json() - trip = valhalla_data.get("trip", {}) - legs = trip.get("legs", []) - - if legs: - leg = legs[0] - shape = leg.get("shape", "") - coords = self._decode_polyline(shape) - - maneuvers = [] - for m in leg.get("maneuvers", []): - maneuvers.append({ - "instruction": m.get("instruction", ""), - "type": m.get("type", 0), - "distance_km": m.get("length", 0), - "time_seconds": m.get("time", 0), - "street_names": m.get("street_names", []), - }) - - summary = trip.get("summary", {}) - return { - "segment": { - "coordinates": coords, - "distance_km": summary.get("length", 0), - "duration_minutes": summary.get("time", 0) / 60, - "maneuvers": maneuvers, - }, - "error": None - } - - return {"segment": None, "error": f"Valhalla returned {resp.status_code}: {resp.text[:200]}"} - - except Exception as e: - return {"segment": None, "error": f"Valhalla request failed: {e}"} - - def _generate_wilderness_maneuvers( - self, - coords: List[List[float]], - elevations: List[float], - position: str = "start" - ) -> List[Dict]: - """ - Generate turn-by-turn maneuvers for a wilderness segment. - - Segment breaks occur when: - - Bearing changes more than 30° from segment start - - Grade category changes (flat→steep etc) - - Distance exceeds 0.5 miles without a break - - Args: - coords: [[lon, lat], ...] coordinate list - elevations: Elevation values (meters) for each coord - position: "start" or "end" for labeling - - Returns: - List of maneuver dicts with instruction, distance, elevation, grade, bearing - """ - if not coords or len(coords) < 2: - return [] - - # Constants - COMPASS = ["N", "NNE", "NE", "ENE", "E", "ESE", "SE", "SSE", - "S", "SSW", "SW", "WSW", "W", "WNW", "NW", "NNW"] - MAX_SEGMENT_M = 804.672 # 0.5 miles in meters - BEARING_THRESHOLD = 30 # degrees - M_TO_FT = 3.28084 - M_TO_MI = 0.000621371 - - def get_bearing(lat1, lon1, lat2, lon2): - """Calculate bearing between two points (degrees 0-360).""" - dlon = math.radians(lon2 - lon1) - lat1_r, lat2_r = math.radians(lat1), math.radians(lat2) - x = math.sin(dlon) * math.cos(lat2_r) - y = math.cos(lat1_r) * math.sin(lat2_r) - math.sin(lat1_r) * math.cos(lat2_r) * math.cos(dlon) - return (math.degrees(math.atan2(x, y)) + 360) % 360 - - def bearing_to_cardinal(bearing): - """Convert bearing to 16-point compass direction.""" - return COMPASS[round(bearing / 22.5) % 16] - - def get_grade_category(grade_deg): - """Categorize grade angle: flat (0-2°), gentle (2-5°), moderate (5-10°), steep (10-15°), very steep (15°+).""" - grade_abs = abs(grade_deg) - if grade_abs < 2: - return "flat" - elif grade_abs < 5: - return "gentle" - elif grade_abs < 10: - return "moderate" - elif grade_abs < 15: - return "steep" - else: - return "very steep" - - def format_distance(meters): - """Format distance: feet with commas if under 1 mile, miles with one decimal if over.""" - miles = meters * M_TO_MI - if miles < 1.0: - feet = round(meters * M_TO_FT) - return f"{feet:,} ft" - else: - return f"{miles:.1f} mi" - - def build_instruction(cardinal, gain_ft, loss_ft, grade_cat, distance_m): - """Build instruction string per spec.""" - dist_str = format_distance(distance_m) - if grade_cat == "flat": - return f"Head {cardinal} on level ground — {dist_str}" - elif gain_ft > loss_ft: - return f"Head {cardinal}, gaining {gain_ft:,} ft ({grade_cat} uphill) — {dist_str}" - else: - return f"Head {cardinal}, descending {loss_ft:,} ft ({grade_cat} downhill) — {dist_str}" - - maneuvers = [] - i = 0 - - while i < len(coords) - 1: - seg_start_idx = i - seg_start_lon, seg_start_lat = coords[i] - seg_start_elev = elevations[i] if i < len(elevations) else 0 - - # Initial bearing for this segment - next_lon, next_lat = coords[i + 1] - seg_bearing = get_bearing(seg_start_lat, seg_start_lon, next_lat, next_lon) - - # Accumulate elevation changes within segment - seg_distance_m = 0 - seg_elev_gain = 0 - seg_elev_loss = 0 - prev_elev = seg_start_elev - - # Calculate initial grade category - step_dist = haversine_distance(seg_start_lat, seg_start_lon, next_lat, next_lon) - step_elev_change = (elevations[i + 1] if i + 1 < len(elevations) else seg_start_elev) - seg_start_elev - initial_grade = math.degrees(math.atan(step_elev_change / step_dist)) if step_dist > 0 else 0 - seg_grade_cat = get_grade_category(initial_grade) - - j = i - while j < len(coords) - 1: - lon1, lat1 = coords[j] - lon2, lat2 = coords[j + 1] - elev1 = elevations[j] if j < len(elevations) else prev_elev - elev2 = elevations[j + 1] if j + 1 < len(elevations) else elev1 - - step_dist = haversine_distance(lat1, lon1, lat2, lon2) - step_bearing = get_bearing(lat1, lon1, lat2, lon2) - step_elev_change = elev2 - elev1 - step_grade = math.degrees(math.atan(step_elev_change / step_dist)) if step_dist > 0 else 0 - step_grade_cat = get_grade_category(step_grade) - - # Check break conditions - bearing_diff = abs(step_bearing - seg_bearing) - if bearing_diff > 180: - bearing_diff = 360 - bearing_diff - - # Break if: bearing changed >30°, grade category changed, or distance >0.5mi - if seg_distance_m > 0: # Don't break on first step - if bearing_diff > BEARING_THRESHOLD: - break - if step_grade_cat != seg_grade_cat: - break - if seg_distance_m >= MAX_SEGMENT_M: - break - - # Accumulate - seg_distance_m += step_dist - if step_elev_change > 0: - seg_elev_gain += step_elev_change - else: - seg_elev_loss += abs(step_elev_change) - prev_elev = elev2 - j += 1 - - # Compute segment stats - seg_end_idx = j - gain_ft = round(seg_elev_gain * M_TO_FT) - loss_ft = round(seg_elev_loss * M_TO_FT) - - # Net elevation change for grade calculation - net_elev_change = seg_elev_gain - seg_elev_loss - grade_deg = math.degrees(math.atan(net_elev_change / seg_distance_m)) if seg_distance_m > 0 else 0 - grade_cat = get_grade_category(grade_deg) - - cardinal = bearing_to_cardinal(seg_bearing) - instruction = build_instruction(cardinal, gain_ft, loss_ft, grade_cat, seg_distance_m) - - maneuvers.append({ - "instruction": instruction, - "type": "wilderness", - "distance_m": round(seg_distance_m, 1), - "elevation_gain_ft": gain_ft, - "elevation_loss_ft": loss_ft, - "grade_degrees": round(grade_deg, 1), - "grade_category": grade_cat, - "bearing": round(seg_bearing, 1), - "cardinal": cardinal, - }) - - i = seg_end_idx - - # Add arrival maneuver - arrival_text = "Arrive at trail/road" if position == "start" else "Arrive at destination" - last_bearing = maneuvers[-1]["bearing"] if maneuvers else 0 - last_cardinal = maneuvers[-1]["cardinal"] if maneuvers else "N" - - maneuvers.append({ - "instruction": arrival_text, - "type": "arrival", - "distance_m": 0, - "elevation_gain_ft": 0, - "elevation_loss_ft": 0, - "grade_degrees": 0, - "grade_category": "flat", - "bearing": last_bearing, - "cardinal": last_cardinal, - }) - - return maneuvers - - def _build_response( - self, - wilderness_start: Optional[List], - wilderness_start_stats: Optional[Dict], - wilderness_start_elevations: Optional[List], - network_segment: Optional[Dict], - wilderness_end: Optional[List], - wilderness_end_stats: Optional[Dict], - wilderness_end_elevations: Optional[List], - mode: str, - boundary_mode: str, - entry_start: Optional[Dict], - entry_end: Optional[Dict], - scenario: str, - t0: float, - valhalla_error: Optional[str] - ) -> Dict: - """Build the final GeoJSON response.""" - features = [] - - # Wilderness start segment - if wilderness_start and wilderness_start_stats: - wild_start_maneuvers = [] - if wilderness_start_elevations: - wild_start_maneuvers = self._generate_wilderness_maneuvers( - wilderness_start, wilderness_start_elevations, position="start" - ) - features.append({ - "type": "Feature", - "properties": { - "segment_type": "wilderness", - "segment_position": "start", - "effort_minutes": float(wilderness_start_stats["effort_minutes"]), - "distance_km": float(wilderness_start_stats["distance_km"]), - "elevation_gain_m": wilderness_start_stats["elevation_gain_m"], - "elevation_loss_m": wilderness_start_stats["elevation_loss_m"], - "boundary_mode": boundary_mode, - "on_trail_pct": wilderness_start_stats["on_trail_pct"], - "barrier_crossings": wilderness_start_stats["barrier_crossings"], - "wilderness_mode": "foot", - "maneuvers": wild_start_maneuvers, - }, - "geometry": {"type": "LineString", "coordinates": wilderness_start} - }) - - # Network segment - if network_segment: - features.append({ - "type": "Feature", - "properties": { - "segment_type": "network", - "distance_km": network_segment["distance_km"], - "duration_minutes": network_segment["duration_minutes"], - "maneuvers": network_segment["maneuvers"], - "network_mode": mode, - }, - "geometry": {"type": "LineString", "coordinates": network_segment["coordinates"]} - }) - - # Wilderness end segment - if wilderness_end and wilderness_end_stats: - wild_end_maneuvers = [] - if wilderness_end_elevations: - wild_end_maneuvers = self._generate_wilderness_maneuvers( - wilderness_end, wilderness_end_elevations, position="end" - ) - features.append({ - "type": "Feature", - "properties": { - "segment_type": "wilderness", - "segment_position": "end", - "effort_minutes": float(wilderness_end_stats["effort_minutes"]), - "distance_km": float(wilderness_end_stats["distance_km"]), - "elevation_gain_m": wilderness_end_stats["elevation_gain_m"], - "elevation_loss_m": wilderness_end_stats["elevation_loss_m"], - "boundary_mode": boundary_mode, - "on_trail_pct": wilderness_end_stats["on_trail_pct"], - "barrier_crossings": wilderness_end_stats["barrier_crossings"], - "wilderness_mode": "foot", - "maneuvers": wild_end_maneuvers, - }, - "geometry": {"type": "LineString", "coordinates": wilderness_end} - }) - - # Combined path - combined_coords = [] - if wilderness_start: - combined_coords.extend(wilderness_start) - if network_segment: - # Skip first coord if we already have wilderness_start (avoid duplicate) - start_idx = 1 if wilderness_start else 0 - combined_coords.extend(network_segment["coordinates"][start_idx:]) - if wilderness_end: - # Skip first coord (avoid duplicate with network end) - start_idx = 1 if (wilderness_start or network_segment) else 0 - combined_coords.extend(wilderness_end[start_idx:]) - - if combined_coords: - features.append({ - "type": "Feature", - "properties": { - "segment_type": "combined", - "wilderness_mode": "foot", - "network_mode": mode, - "boundary_mode": boundary_mode, - "scenario": scenario, - }, - "geometry": {"type": "LineString", "coordinates": combined_coords} - }) - - geojson = {"type": "FeatureCollection", "features": features} - - # Calculate totals - total_distance_km = 0.0 - total_effort_minutes = 0.0 - wilderness_distance_km = 0.0 - wilderness_effort_minutes = 0.0 - network_distance_km = 0.0 - network_duration_minutes = 0.0 - barrier_crossings = 0 - on_trail_pct = 0.0 - - if wilderness_start_stats: - wilderness_distance_km += wilderness_start_stats["distance_km"] - wilderness_effort_minutes += wilderness_start_stats["effort_minutes"] - barrier_crossings += wilderness_start_stats["barrier_crossings"] - on_trail_pct = wilderness_start_stats["on_trail_pct"] - - if wilderness_end_stats: - wilderness_distance_km += wilderness_end_stats["distance_km"] - wilderness_effort_minutes += wilderness_end_stats["effort_minutes"] - barrier_crossings += wilderness_end_stats["barrier_crossings"] - # Average on-trail percentage if we have both - if wilderness_start_stats: - on_trail_pct = (on_trail_pct + wilderness_end_stats["on_trail_pct"]) / 2 - else: - on_trail_pct = wilderness_end_stats["on_trail_pct"] - - if network_segment: - network_distance_km = network_segment["distance_km"] - network_duration_minutes = network_segment["duration_minutes"] - - total_distance_km = wilderness_distance_km + network_distance_km - total_effort_minutes = wilderness_effort_minutes + network_duration_minutes - - summary = { - "total_distance_km": float(total_distance_km), - "total_effort_minutes": float(total_effort_minutes), - "wilderness_distance_km": float(wilderness_distance_km), - "wilderness_effort_minutes": float(wilderness_effort_minutes), - "network_distance_km": float(network_distance_km), - "network_duration_minutes": float(network_duration_minutes), - "on_trail_pct": float(on_trail_pct), - "barrier_crossings": barrier_crossings, - "boundary_mode": boundary_mode, - "wilderness_mode": "foot", - "network_mode": mode, - "scenario": scenario, - "computation_time_s": time.time() - t0, - } - - if entry_start: - summary["entry_point_start"] = { - "lat": entry_start["lat"], - "lon": entry_start["lon"], - "highway_class": entry_start["highway_class"], - "name": entry_start.get("name", ""), - } - - if entry_end: - summary["entry_point_end"] = { - "lat": entry_end["lat"], - "lon": entry_end["lon"], - "highway_class": entry_end["highway_class"], - "name": entry_end.get("name", ""), - } - - result = {"status": "ok", "route": geojson, "summary": summary} - - if valhalla_error: - result["warning"] = f"Network segment incomplete: {valhalla_error}" - - return result - - def _decode_polyline(self, encoded: str, precision: int = 6) -> List[List[float]]: - """Decode a polyline string into coordinates [lon, lat].""" - coords = [] - index = 0 - lat = 0 - lon = 0 - - while index < len(encoded): - shift = 0 - result = 0 - while True: - b = ord(encoded[index]) - 63 - index += 1 - result |= (b & 0x1f) << shift - shift += 5 - if b < 0x20: - break - dlat = ~(result >> 1) if result & 1 else result >> 1 - lat += dlat - - shift = 0 - result = 0 - while True: - b = ord(encoded[index]) - 63 - index += 1 - result |= (b & 0x1f) << shift - shift += 5 - if b < 0x20: - break - dlon = ~(result >> 1) if result & 1 else result >> 1 - lon += dlon - - coords.append([lon / (10 ** precision), lat / (10 ** precision)]) - - return coords - - def close(self): - """Close all readers.""" - if self.dem_reader: - self.dem_reader.close() - if self.friction_reader: - self.friction_reader.close() - if self.barrier_reader: - self.barrier_reader.close() - if self.wilderness_reader: - self.wilderness_reader.close() - if self.trail_reader: - self.trail_reader.close() - self.entry_index.close() - - -def build_entry_index(): - """Build the trail entry point index.""" - index = EntryPointIndex() - stats = index.build_index() - index.close() - return stats - - -if __name__ == "__main__": - import sys - - if len(sys.argv) > 1 and sys.argv[1] == "build": - print("Building trail entry point index...") - stats = build_entry_index() - print(f"\nDone. Total entry points: {stats['total']}") - - elif len(sys.argv) > 1 and sys.argv[1] == "test": - print("Testing router (all scenarios)...") - print("=" * 60) - - router = OffrouteRouter() - - # Test points - wilderness_start = (44.0543, -115.4237) # Off-network - wilderness_end = (45.2, -115.5) # Deep wilderness (Frank Church) - road_start = (43.6150, -116.2023) # Boise downtown (on-network) - road_end = (43.5867, -116.5625) # Nampa (on-network) - - tests = [ - ("A: wilderness→road", wilderness_start, (44.0814, -115.5021)), - ("B: wilderness→wilderness", wilderness_start, wilderness_end), - ("C: road→wilderness", road_start, wilderness_start), - ("D: road→road", road_start, road_end), - ] - - for label, (slat, slon), (elat, elon) in tests: - print(f"\n{label}") - print("-" * 40) - - result = router.route( - start_lat=slat, start_lon=slon, - end_lat=elat, end_lon=elon, - mode="foot", boundary_mode="pragmatic" - ) - - if result["status"] == "ok": - s = result["summary"] - print(f" Scenario: {s.get('scenario', '?')}") - print(f" Total: {s['total_distance_km']:.2f} km, {s['total_effort_minutes']:.1f} min") - print(f" Wilderness: {s['wilderness_distance_km']:.2f} km") - print(f" Network: {s['network_distance_km']:.2f} km") - if s.get('entry_point_start'): - ep = s['entry_point_start'] - print(f" Entry (start): {ep['highway_class']} at {ep['lat']:.4f}, {ep['lon']:.4f}") - if s.get('entry_point_end'): - ep = s['entry_point_end'] - print(f" Entry (end): {ep['highway_class']} at {ep['lat']:.4f}, {ep['lon']:.4f}") - else: - print(f" ERROR: {result['message']}") - - router.close() - - else: - print("Usage:") - print(" python router.py build # Build entry point index") - print(" python router.py test # Test all scenarios") diff --git a/lib/offroute/trails.py b/lib/offroute/trails.py deleted file mode 100644 index 9d9185e..0000000 --- a/lib/offroute/trails.py +++ /dev/null @@ -1,174 +0,0 @@ -""" -Trail corridor reader for OFFROUTE. - -Provides access to the OSM-derived trail raster for pathfinding. -Trail values replace WorldCover friction where trails exist. - -Raster values: - 0 = no trail (use WorldCover friction) - 5 = road (0.1× friction) - 15 = track (0.3× friction) - 25 = foot trail (0.5× friction) -""" -import numpy as np -from pathlib import Path -from typing import Tuple, Optional - -try: - import rasterio - from rasterio.windows import from_bounds - from rasterio.enums import Resampling -except ImportError: - raise ImportError("rasterio is required for trails layer support") - -# Default path to the trails raster -DEFAULT_TRAILS_PATH = Path("/mnt/nav/worldcover/trails.tif") - -# Trail value to friction multiplier mapping -TRAIL_FRICTION_MAP = { - 5: 0.1, # road - 15: 0.3, # track - 25: 0.5, # foot trail -} - - -class TrailReader: - """Reader for OSM-derived trail corridor raster.""" - - def __init__(self, trails_path: Path = DEFAULT_TRAILS_PATH): - self.trails_path = trails_path - self._dataset = None - - def _open(self): - """Lazy open the dataset.""" - if self._dataset is None: - if not self.trails_path.exists(): - raise FileNotFoundError( - f"Trails raster not found at {self.trails_path}. " - f"Run the Phase B rasterization script first." - ) - self._dataset = rasterio.open(self.trails_path) - return self._dataset - - def get_trails_grid( - self, - south: float, - north: float, - west: float, - east: float, - target_shape: Tuple[int, int] - ) -> np.ndarray: - """ - Get trail values for a bounding box, resampled to target shape. - - Args: - south, north, west, east: Bounding box coordinates (WGS84) - target_shape: (rows, cols) to resample to (matches elevation grid) - - Returns: - np.ndarray of uint8 trail values: - 0 = no trail - 5 = road (0.1× friction) - 15 = track (0.3× friction) - 25 = foot trail (0.5× friction) - """ - ds = self._open() - - # Create a window from the bounding box - window = from_bounds(west, south, east, north, ds.transform) - - # Read with resampling to target shape - # Use nearest neighbor to preserve discrete values - trails = ds.read( - 1, - window=window, - out_shape=target_shape, - resampling=Resampling.nearest - ) - - return trails - - def sample_point(self, lat: float, lon: float) -> int: - """Sample trail value at a single point.""" - ds = self._open() - - # Get pixel coordinates - row, col = ds.index(lon, lat) - - # Check bounds - if row < 0 or row >= ds.height or col < 0 or col >= ds.width: - return 0 # Out of bounds = no trail - - # Read single pixel - window = rasterio.windows.Window(col, row, 1, 1) - value = ds.read(1, window=window) - return int(value[0, 0]) - - def close(self): - """Close the dataset.""" - if self._dataset is not None: - self._dataset.close() - self._dataset = None - - -def trails_to_friction(trails: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: - """ - Convert trail values to friction multipliers. - - Args: - trails: uint8 array of trail values (0, 5, 15, or 25) - - Returns: - Tuple of: - - friction: float32 array of friction multipliers - - has_trail: bool array indicating where trails exist - """ - friction = np.ones_like(trails, dtype=np.float32) - has_trail = trails > 0 - - # Apply friction values where trails exist - friction[trails == 5] = 0.1 # road - friction[trails == 15] = 0.3 # track - friction[trails == 25] = 0.5 # foot trail - - return friction, has_trail - - -if __name__ == "__main__": - print("Testing TrailReader...") - - if not DEFAULT_TRAILS_PATH.exists(): - print(f"Trails raster not found at {DEFAULT_TRAILS_PATH}") - print("Run Phase B rasterization first.") - exit(1) - - reader = TrailReader() - - # Test point sampling - Twin Falls downtown (should have roads) - test_lat, test_lon = 42.563, -114.461 - trail_value = reader.sample_point(test_lat, test_lon) - print(f"\nTwin Falls ({test_lat}, {test_lon}): trail value = {trail_value}") - label = {0: "no trail", 5: "road", 15: "track", 25: "trail"}.get(trail_value, "unknown") - print(f" Type: {label}") - - # Test grid read for test bbox - trails = reader.get_trails_grid( - south=42.21, north=42.60, west=-114.76, east=-113.79, - target_shape=(400, 1000) - ) - print(f"\nGrid test shape: {trails.shape}") - - unique, counts = np.unique(trails, return_counts=True) - print("Value distribution:") - for v, c in zip(unique, counts): - pct = 100 * c / trails.size - label = {0: "no trail", 5: "road", 15: "track", 25: "trail"}.get(v, f"unknown({v})") - print(f" {label}: {c:,} pixels ({pct:.2f}%)") - - # Test conversion to friction - friction, has_trail = trails_to_friction(trails) - print(f"\nTrail coverage: {100 * np.sum(has_trail) / trails.size:.2f}%") - print(f"Friction range (on trails): {friction[has_trail].min():.1f} - {friction[has_trail].max():.1f}") - - reader.close() - print("\nTrailReader test complete.") From adee6d5a69ff23b96ee247563c71807762bc7481 Mon Sep 17 00:00:00 2001 From: malice Date: Sat, 23 May 2026 10:21:31 -0600 Subject: [PATCH 34/41] cleanup: remove dead place_detail orchestrator cluster + lib/google_places.py (post-PR-11 dead code) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #11 (cleanup #2) deleted the /api/place* HTTP handlers but left their orchestrator functions in lib/place_detail.py as dead code. Pre-flight for the original Task #27 (delete google_places.py) surfaced that _enrich_with_google is NOT a no-caller leaf — it's called by the unreachable get_place_detail. A full caller-graph trace showed ~90% of place_detail.py is dead orchestration. Scope expanded (Matt confirmed in chat) to remove the whole dead cluster: - lib/google_places.py (entire file) - place_detail.py: get_place_detail, get_place_by_wikidata, _enrich_with_google, _apply_google_data, _enrich_with_overture, _enrich_with_wiki_index, _enrich_wiki_links, _parse_nominatim, _parse_nominatim_address, _parse_overpass, _build_overpass_query, cache_get, cache_put, _get_db + their now-unused imports/constants (json, time, requests, osm_categories, NOMINATIM_URL, etc.) KEEP only lookup_wiki_index + _get_wiki_index_db (the wiki_enrich_api survivor path) — preserved byte-exact. Module docstring refreshed. Flagged separately (not touched): overture.py + osm_categories.py are now orphaned (only consumers were the deleted cluster); stale docstrings; the deployment_config.py:9 catalog comment. Co-authored-by: Claude Opus 4.7 (1M context) --- lib/google_places.py | 397 -------------------- lib/place_detail.py | 877 +------------------------------------------ 2 files changed, 4 insertions(+), 1270 deletions(-) delete mode 100644 lib/google_places.py diff --git a/lib/google_places.py b/lib/google_places.py deleted file mode 100644 index 8272b81..0000000 --- a/lib/google_places.py +++ /dev/null @@ -1,397 +0,0 @@ -""" -Google Places (New) API client for tertiary enrichment. - -Searches for business POIs and fetches details (opening hours, phone, website) -when OSM + Overture data is incomplete. Uses field masks to minimize cost. - -API docs: https://developers.google.com/maps/documentation/places/web-service -""" -import json -import os -import sqlite3 -import time -from datetime import date, timezone, datetime - -import requests - -from .utils import setup_logging - -logger = setup_logging('recon.google_places') - -API_BASE = 'https://places.googleapis.com/v1' -DEFAULT_DAILY_CAP = 500 -REQUEST_TIMEOUT = 3 # seconds - -# Google day index → OSM abbreviation -_DAY_ABBR = ['Su', 'Mo', 'Tu', 'We', 'Th', 'Fr', 'Sa'] - -_db_conn = None - - -def _get_db(): - """Return a module-level SQLite connection (lazy init).""" - global _db_conn - if _db_conn is not None: - return _db_conn - - db_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'data') - db_path = os.path.join(db_dir, 'place_cache.db') - _db_conn = sqlite3.connect(db_path, check_same_thread=False) - _db_conn.execute("PRAGMA journal_mode=WAL") - _db_conn.execute("PRAGMA synchronous=NORMAL") - # Ensure google_api_calls table exists - _db_conn.execute(""" - CREATE TABLE IF NOT EXISTS google_api_calls ( - call_date TEXT PRIMARY KEY, - call_count INTEGER NOT NULL DEFAULT 0 - ) - """) - _db_conn.commit() - return _db_conn - - -def _get_api_key(): - """Return the Google Places API key from environment.""" - key = os.environ.get('GOOGLE_PLACES_API_KEY') - if not key: - logger.error("GOOGLE_PLACES_API_KEY not set in environment") - return key - - -def _get_daily_cap(): - """Return the daily API call cap (configurable via deployment config).""" - try: - from .deployment_config import get_deployment_config - config = get_deployment_config() - return config.get('google_places', {}).get('daily_cap', DEFAULT_DAILY_CAP) - except Exception: - return DEFAULT_DAILY_CAP - - -# ── Daily call counter ────────────────────────────────────────────────── - -def check_daily_cap(): - """Return True if under daily cap, False if limit reached.""" - db = _get_db() - today = date.today().isoformat() - row = db.execute( - "SELECT call_count FROM google_api_calls WHERE call_date = ?", (today,) - ).fetchone() - current = row[0] if row else 0 - cap = _get_daily_cap() - if current >= cap: - logger.info(f"google_places: daily_cap_reached count={current} cap={cap}") - return False - return True - - -def get_daily_count(): - """Return today's API call count.""" - db = _get_db() - today = date.today().isoformat() - row = db.execute( - "SELECT call_count FROM google_api_calls WHERE call_date = ?", (today,) - ).fetchone() - return row[0] if row else 0 - - -def increment_call_counter(): - """Atomically increment today's API call counter.""" - db = _get_db() - today = date.today().isoformat() - db.execute(""" - INSERT INTO google_api_calls (call_date, call_count) VALUES (?, 1) - ON CONFLICT(call_date) DO UPDATE SET call_count = call_count + 1 - """, (today,)) - db.commit() - - -def _set_daily_count_to_cap(): - """Set today's counter to the cap value (soft-stop on quota error).""" - db = _get_db() - today = date.today().isoformat() - cap = _get_daily_cap() - db.execute(""" - INSERT INTO google_api_calls (call_date, call_count) VALUES (?, ?) - ON CONFLICT(call_date) DO UPDATE SET call_count = ? - """, (today, cap, cap)) - db.commit() - - -# ── Google Places cache (on place_cache table) ───────────────────────── - -def cache_get_google(osm_type, osm_id): - """Return (google_place_id, google_data_dict) or (None, None).""" - db = _get_db() - row = db.execute( - "SELECT google_place_id, google_data FROM place_cache WHERE osm_type=? AND osm_id=?", - (osm_type, osm_id) - ).fetchone() - if row and row[0]: - data = None - if row[1]: - try: - data = json.loads(row[1]) - except (json.JSONDecodeError, TypeError): - pass - return row[0], data - return None, None - - -def cache_put_google(osm_type, osm_id, place_id, data): - """Store Google Places data for a cache entry (UPSERT on google columns).""" - db = _get_db() - now = int(time.time()) - db.execute(""" - INSERT INTO place_cache (osm_type, osm_id, data, source, cached_at, google_place_id, google_data, google_fetched_at) - VALUES (?, ?, '', 'pending', 0, ?, ?, ?) - ON CONFLICT(osm_type, osm_id) DO UPDATE SET - google_place_id = excluded.google_place_id, - google_data = excluded.google_data, - google_fetched_at = excluded.google_fetched_at - """, (osm_type, osm_id, place_id, json.dumps(data) if data else None, now)) - db.commit() - - -# ── API calls ─────────────────────────────────────────────────────────── - -def search_place(name, lat, lon, radius_m=200): - """ - Search Google Places (New) for a business by name + location. - Returns the Google Place ID of the best match, or None. - """ - key = _get_api_key() - if not key: - return None - - if not check_daily_cap(): - return None - - try: - resp = requests.post( - f'{API_BASE}/places:searchText', - headers={ - 'Content-Type': 'application/json', - 'X-Goog-Api-Key': key, - 'X-Goog-FieldMask': 'places.id,places.displayName,places.location', - }, - json={ - 'textQuery': name, - 'locationBias': { - 'circle': { - 'center': {'latitude': lat, 'longitude': lon}, - 'radius': float(radius_m), - } - }, - 'maxResultCount': 1, - }, - timeout=REQUEST_TIMEOUT, - ) - - increment_call_counter() - - if resp.status_code == 429: - logger.warning("google_places: action=search place=%s result=rate_limited", name) - _set_daily_count_to_cap() - return None - - if resp.status_code == 403: - logger.error("google_places: action=search place=%s result=forbidden (invalid key?)", name) - return None - - if resp.status_code != 200: - logger.warning("google_places: action=search place=%s result=error status=%d", name, resp.status_code) - return None - - data = resp.json() - places = data.get('places', []) - if not places: - logger.info("google_places: action=search place=%s result=miss", name) - return None - - place_id = places[0].get('id') - display = places[0].get('displayName', {}).get('text', '?') - logger.info("google_places: action=search place=%s result=hit google_name=%s id=%s", name, display, place_id) - return place_id - - except requests.exceptions.Timeout: - logger.warning("google_places: action=search place=%s result=timeout", name) - return None - except Exception as e: - logger.error("google_places: action=search place=%s result=error err=%s", name, e) - return None - - -def get_place_details(place_id): - """ - Fetch details for a Google Place ID. - Returns dict with {opening_hours, phone_number, website} or None. - """ - key = _get_api_key() - if not key: - return None - - if not check_daily_cap(): - return None - - try: - resp = requests.get( - f'{API_BASE}/places/{place_id}', - headers={ - 'X-Goog-Api-Key': key, - 'X-Goog-FieldMask': 'regularOpeningHours,internationalPhoneNumber,websiteUri', - }, - timeout=REQUEST_TIMEOUT, - ) - - increment_call_counter() - - if resp.status_code == 429: - logger.warning("google_places: action=details id=%s result=rate_limited", place_id) - _set_daily_count_to_cap() - return None - - if resp.status_code != 200: - logger.warning("google_places: action=details id=%s result=error status=%d", place_id, resp.status_code) - return None - - data = resp.json() - result = { - 'opening_hours': None, - 'opening_hours_raw': None, - 'phone_number': None, - 'website': None, - } - - # Phone - phone = data.get('internationalPhoneNumber') - if phone: - result['phone_number'] = phone.replace(' ', '').replace('-', '') - - # Website - result['website'] = data.get('websiteUri') - - # Opening hours - hours = data.get('regularOpeningHours') - if hours: - # Try OSM-compatible format from periods - periods = hours.get('periods', []) - if periods: - osm_str = _periods_to_osm(periods) - if osm_str: - result['opening_hours'] = osm_str - - # Fallback: weekday descriptions (human-readable) - if not result['opening_hours']: - descriptions = hours.get('weekdayDescriptions') - if descriptions: - result['opening_hours_raw'] = descriptions - - logger.info("google_places: action=details id=%s result=hit hours=%s phone=%s website=%s", - place_id, - 'yes' if result['opening_hours'] or result['opening_hours_raw'] else 'no', - 'yes' if result['phone_number'] else 'no', - 'yes' if result['website'] else 'no') - return result - - except requests.exceptions.Timeout: - logger.warning("google_places: action=details id=%s result=timeout", place_id) - return None - except Exception as e: - logger.error("google_places: action=details id=%s result=error err=%s", place_id, e) - return None - - -# ── Opening hours conversion ──────────────────────────────────────────── - -def _periods_to_osm(periods): - """ - Convert Google Places periods array to OSM opening_hours string. - - Google periods: [{"open": {"day": 0-6, "hour": H, "minute": M}, - "close": {"day": 0-6, "hour": H, "minute": M}}, ...] - Where day 0 = Sunday. - - OSM format: "Mo-Fr 06:00-23:00; Sa-Su 07:00-23:00" - """ - if not periods: - return None - - # Check for 24/7: single period with no close, or open 00:00 close 00:00 next day - if len(periods) == 1: - p = periods[0] - o = p.get('open', {}) - c = p.get('close') - if c is None and o.get('hour', 0) == 0 and o.get('minute', 0) == 0: - return '24/7' - - # Build a map: day_index → "HH:MM-HH:MM" - day_hours = {} # day_index → time_range string - for p in periods: - o = p.get('open', {}) - c = p.get('close', {}) - day = o.get('day', 0) - open_time = f"{o.get('hour', 0):02d}:{o.get('minute', 0):02d}" - - if c: - close_time = f"{c.get('hour', 0):02d}:{c.get('minute', 0):02d}" - # Handle midnight closing (00:00 means end of day) - if close_time == '00:00': - close_time = '24:00' - else: - close_time = '24:00' - - time_range = f"{open_time}-{close_time}" - - # A day can have multiple periods (e.g., lunch break) - if day in day_hours: - day_hours[day] = day_hours[day] + ',' + time_range - else: - day_hours[day] = time_range - - if not day_hours: - return None - - # Check if all 7 days have same hours - unique_ranges = set(day_hours.values()) - if len(day_hours) == 7 and len(unique_ranges) == 1: - hours = unique_ranges.pop() - if hours == '00:00-24:00': - return '24/7' - return hours # implicit "every day" - - # Group consecutive days with same hours - # Reorder to OSM convention: Mo(1) Tu(2) We(3) Th(4) Fr(5) Sa(6) Su(0) - osm_day_order = [1, 2, 3, 4, 5, 6, 0] - groups = [] - current_days = [] - current_hours = None - - for day_idx in osm_day_order: - hours = day_hours.get(day_idx) - if hours == current_hours: - current_days.append(day_idx) - else: - if current_days and current_hours: - groups.append((current_days, current_hours)) - current_days = [day_idx] - current_hours = hours - - if current_days and current_hours: - groups.append((current_days, current_hours)) - - if not groups: - return None - - # Format each group - parts = [] - for days, hours in groups: - if len(days) == 1: - day_str = _DAY_ABBR[days[0]] - elif len(days) == 2: - day_str = f"{_DAY_ABBR[days[0]]},{_DAY_ABBR[days[1]]}" - else: - day_str = f"{_DAY_ABBR[days[0]]}-{_DAY_ABBR[days[-1]]}" - parts.append(f"{day_str} {hours}") - - return '; '.join(parts) diff --git a/lib/place_detail.py b/lib/place_detail.py index 46aa8b0..6f6f1ba 100644 --- a/lib/place_detail.py +++ b/lib/place_detail.py @@ -1,291 +1,17 @@ """ -Place detail proxy — local Nominatim first, Overpass API fallback, SQLite cache. -Overture Maps enrichment layer fills sparse extratags (phone, website, brand). +Wiki-index lookup for place enrichment. -Provides get_place_detail(osm_type, osm_id) which returns a cleaned dict -matching the response shape for /api/place//. +Provides lookup_wiki_index(wikidata_id, name, country_code) — a pure read of the +local wiki_index.db, used by the /api/wiki-enrich endpoint (navi-places +HTTP-fetches wiki enrichment instead of reading the 2.1 GB DB directly). """ -import json import os import sqlite3 -import time -import requests as http_requests - -from .osm_categories import humanize_category from .utils import setup_logging logger = setup_logging('recon.place_detail') -NOMINATIM_URL = "http://localhost:8010/details.php" -OVERPASS_URL = "https://overpass-api.de/api/interpreter" -OVERPASS_UA = "Navi/1.0 (forge.echo6.co/matt/recon)" -VALID_OSM_TYPES = {"N", "W", "R"} - -_db_conn = None - - -# ── SQLite cache ──────────────────────────────────────────────────────── - -def _get_db(): - """Return a module-level SQLite connection (lazy init).""" - global _db_conn - if _db_conn is not None: - return _db_conn - - db_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'data') - os.makedirs(db_dir, exist_ok=True) - db_path = os.path.join(db_dir, 'place_cache.db') - - _db_conn = sqlite3.connect(db_path, check_same_thread=False) - _db_conn.execute("PRAGMA journal_mode=WAL") - _db_conn.execute("PRAGMA synchronous=NORMAL") - _db_conn.execute(""" - CREATE TABLE IF NOT EXISTS place_cache ( - osm_type TEXT NOT NULL, - osm_id INTEGER NOT NULL, - data TEXT NOT NULL, - source TEXT NOT NULL, - cached_at INTEGER NOT NULL, - PRIMARY KEY (osm_type, osm_id) - ) - """) - _db_conn.commit() - logger.info(f"Place cache DB ready at {db_path}") - return _db_conn - - -def cache_get(osm_type, osm_id): - """Return cached place dict or None.""" - db = _get_db() - row = db.execute( - "SELECT data FROM place_cache WHERE osm_type=? AND osm_id=?", - (osm_type, osm_id) - ).fetchone() - if row: - try: - result = json.loads(row[0]) - result['source'] = 'cache' - return result - except (json.JSONDecodeError, TypeError): - pass - return None - - -def cache_put(osm_type, osm_id, data, source): - """Store a place detail result in the cache (preserves google columns).""" - db = _get_db() - now = int(time.time()) - db.execute(""" - INSERT INTO place_cache (osm_type, osm_id, data, source, cached_at) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT(osm_type, osm_id) DO UPDATE SET - data = excluded.data, - source = excluded.source, - cached_at = excluded.cached_at - """, (osm_type, osm_id, json.dumps(data), source, now)) - db.commit() - - -# ── Overture enrichment ───────────────────────────────────────────────── - -def _enrich_with_overture(result, osm_type, osm_id): - """ - Attempt to enrich a place result with Overture Maps data. - Fills sparse extratags (phone, website, brand) without overwriting existing values. - Returns the (possibly enriched) result dict. - """ - try: - from .deployment_config import get_deployment_config - deploy_config = get_deployment_config() - features = deploy_config.get('features', {}) - if not features.get('has_overture_enrichment', False): - return result - except Exception: - return result - - try: - from .overture import find_by_osm_id, find_by_coords_and_name - except ImportError: - logger.debug("Overture module not available") - return result - - enrichment = None - match_method = None - - # Strategy 1: OSM cross-reference (exact) - enrichment = find_by_osm_id(osm_type, osm_id) - if enrichment: - match_method = 'osm_xref' - - # Strategy 2: Coordinate + name fuzzy (fallback) - if not enrichment and result.get('centroid') and result.get('name'): - centroid = result['centroid'] - if centroid.get('lat') and centroid.get('lon'): - enrichment = find_by_coords_and_name( - centroid['lat'], centroid['lon'], result['name'] - ) - if enrichment: - match_method = 'coord_name_fuzzy' - - if not enrichment: - return result - - # Fill sparse extratags (never overwrite existing non-null values) - extratags = result.get('extratags', {}) - fill_map = [ - ('phone', 'phone'), - ('website', 'website'), - ('brand', 'brand_name'), - ('brand:wikidata', 'brand_wikidata'), - ] - for osm_key, overture_key in fill_map: - if not extratags.get(osm_key) and enrichment.get(overture_key): - extratags[osm_key] = enrichment[overture_key] - result['extratags'] = extratags - - # Add source metadata - result['sources'] = { - 'primary': result.get('source', 'unknown'), - 'enrichment': 'overture', - 'overture_match_method': match_method, - 'overture_gers_id': enrichment.get('gers_id'), - 'overture_confidence': enrichment.get('confidence'), - 'overture_basic_category': enrichment.get('basic_category'), - } - - logger.debug(f"Overture enrichment for {osm_type}/{osm_id}: {match_method}") - return result - - - -# ── Google Places enrichment (tertiary, gap-fill only) ────────────── - -# Business POI classes eligible for Google enrichment -_BUSINESS_CLASSES = {'amenity', 'shop', 'tourism', 'leisure', 'office', 'craft'} - -# Fields Google can fill -_GOOGLE_GAP_FIELDS = ('opening_hours', 'phone', 'website') - - -def _enrich_with_google(result, osm_type, osm_id): - """ - Tertiary enrichment via Google Places (New) API. - Only fires for business-type POIs when opening_hours, phone, or website - are still missing after OSM + Overture enrichment. - Fills only empty fields — never overwrites existing values. - """ - # Check feature flag - try: - from .deployment_config import get_deployment_config - deploy_config = get_deployment_config() - features = deploy_config.get('features', {}) - if not features.get('has_google_places_enrichment', False): - return result - except Exception: - return result - - # Only enrich business-type POIs - poi_class = result.get('class', '') - if poi_class not in _BUSINESS_CLASSES: - return result - - # Check if any gap fields are missing - extratags = result.get('extratags', {}) - gaps = [f for f in _GOOGLE_GAP_FIELDS if not extratags.get(f)] - if not gaps: - logger.debug(f"google_places: skip {osm_type}/{osm_id} — no gaps") - return result - - try: - from . import google_places - except ImportError: - logger.debug("google_places module not available") - return result - - # Check Google cache first - cached_pid, cached_data = google_places.cache_get_google(osm_type, osm_id) - if cached_pid and cached_data: - _apply_google_data(result, cached_data, gaps) - result.setdefault('sources', {})['google_places'] = { - 'place_id': cached_pid, - 'source': 'cache', - } - logger.debug(f"google_places: cache hit for {osm_type}/{osm_id}") - return result - - # Skip if already looked up and found nothing (cached_pid is None) - if cached_pid is not None: - return result - - # Skip new Google API calls for guest users (cached data already returned above) - from .auth import get_user_id - if not get_user_id(): - logger.debug(f"google_places: skip API call for {osm_type}/{osm_id} — guest user") - return result - - # Daily cap check - if not google_places.check_daily_cap(): - return result - - # Search for the place - name = result.get('name', '') - centroid = result.get('centroid', {}) - lat = centroid.get('lat') - lon = centroid.get('lon') - if not name or not lat or not lon: - return result - - place_id = google_places.search_place(name, lat, lon) - if not place_id: - # Cache the miss to avoid repeated lookups - google_places.cache_put_google(osm_type, osm_id, '__miss__', None) - return result - - # Get details - details = google_places.get_place_details(place_id) - if not details: - google_places.cache_put_google(osm_type, osm_id, place_id, None) - return result - - # Cache the result - google_places.cache_put_google(osm_type, osm_id, place_id, details) - - # Apply to result - _apply_google_data(result, details, gaps) - result.setdefault('sources', {})['google_places'] = { - 'place_id': place_id, - 'source': 'api', - 'daily_count': google_places.get_daily_count(), - } - - return result - - -def _apply_google_data(result, google_data, gaps): - """Apply Google Places data to fill gap fields only.""" - extratags = result.get('extratags', {}) - if 'opening_hours' in gaps: - osm_hours = google_data.get('opening_hours') - if osm_hours: - extratags['opening_hours'] = osm_hours - elif google_data.get('opening_hours_raw'): - extratags['opening_hours_raw'] = google_data['opening_hours_raw'] - if 'phone' in gaps and google_data.get('phone_number'): - extratags['phone'] = google_data['phone_number'] - if 'website' in gaps and google_data.get('website'): - extratags['website'] = google_data['website'] - result['extratags'] = extratags - - - - -# ── Wiki link rewriting ───────────────────────────────────────────────── - -# Extratag keys that may contain wiki references -_WIKI_TAGS = ('wikipedia', 'wikidata', 'wikivoyage', 'appropedia') - - # ── Wiki Index enrichment ─────────────────────────────────────────────── @@ -369,598 +95,3 @@ def lookup_wiki_index(wikidata_id=None, name=None, country_code=None): except Exception as e: logger.debug(f"wiki_index lookup error: {e}") return None - - -def _enrich_with_wiki_index(result): - try: - from .deployment_config import get_deployment_config - deploy_config = get_deployment_config() - features = deploy_config.get("features", {}) - if not features.get("has_kiwix_wiki", False): - return result - except Exception: - return result - - db = _get_wiki_index_db() - if not db: - return result - - try: - cur = db.cursor() - row = None - - extratags = result.get("extratags", {}) - wikidata_id = result.get("wikidata_id") or extratags.get("wikidata") - if wikidata_id: - if isinstance(wikidata_id, str) and wikidata_id.startswith("http"): - wikidata_id = wikidata_id.split("/")[-1] - cur.execute( - "SELECT summary, wiki_population, wikipedia_title, wikivoyage_title FROM wiki_places WHERE wikidata_id = ?", - (wikidata_id,) - ) - row = cur.fetchone() - - if not row: - name = result.get("name") - address = result.get("address") or {} - country_code = address.get("country_code") or result.get("country_code") - if name and country_code: - cur.execute( - "SELECT summary, wiki_population, wikipedia_title, wikivoyage_title FROM wiki_places WHERE place_name = ? AND country_code = ? LIMIT 1", - (name, country_code.lower()) - ) - row = cur.fetchone() - - if row: - if row["summary"]: - result["wiki_summary"] = row["summary"] - if row["wiki_population"]: - try: - result["wiki_population"] = int(row["wiki_population"]) - except (ValueError, TypeError): - result["wiki_population"] = row["wiki_population"] - if row["wikipedia_title"]: - title = row["wikipedia_title"].replace(" ", "_") - result["wiki_url"] = f"https://en.wikipedia.org/wiki/{title}" - if row["wikivoyage_title"]: - title = row["wikivoyage_title"].replace(" ", "_") - result["wikivoyage_url"] = f"https://en.wikivoyage.org/wiki/{title}" - logger.debug(f"Wiki index enrichment hit for {result.get(name)}") - - except Exception as e: - logger.debug(f"Wiki index enrichment error: {e}") - - return result - -def _enrich_wiki_links(result): - """ - Rewrite wiki-related extratags to local Kiwix URLs where available. - Falls back to public URLs. Only runs when has_wiki_rewriting is enabled. - Returns the (possibly enriched) result dict. - """ - try: - from .deployment_config import get_deployment_config - deploy_config = get_deployment_config() - features = deploy_config.get('features', {}) - if not features.get('has_wiki_rewriting', False): - return result - except Exception: - return result - - try: - from .wiki_rewrite import rewrite_wiki_link - except ImportError: - logger.debug("wiki_rewrite module not available") - return result - - extratags = result.get('extratags', {}) - if not extratags: - return result - - rewrites = {} - for tag in _WIKI_TAGS: - value = extratags.get(tag) - if not value: - continue - url, status = rewrite_wiki_link(tag, value) - if status != 'original': - extratags[tag] = url - rewrites[tag] = status - - if rewrites: - result['extratags'] = extratags - result.setdefault('sources', {})['wiki_rewrites'] = rewrites - logger.debug(f"Wiki rewrites for {result.get('osm_type')}/{result.get('osm_id')}: {rewrites}") - - return result - -# ── Nominatim parsing ─────────────────────────────────────────────────── - -# Nominatim address array uses rank_address to indicate what each entry is. -# We map rank ranges to our flat address fields. -RANK_TO_FIELD = { - 4: 'country', - 5: 'postcode', - 6: 'state', # rank 6 = county in US, but we try name matching - 8: 'state', - 12: 'county', - 16: 'city', - 20: 'neighbourhood', - 22: 'neighbourhood', - 26: 'road', - 28: 'house_number', -} - - -def _parse_nominatim_address(address_array, country_code=None): - """Parse Nominatim's ranked address array into a flat address dict.""" - addr = { - 'house_number': None, - 'road': None, - 'neighbourhood': None, - 'city': None, - 'county': None, - 'state': None, - 'postcode': None, - 'country': None, - 'country_code': country_code, - } - - if not address_array: - return addr - - for entry in address_array: - if not entry.get('isaddress', False): - continue - - name = entry.get('localname', '') - rank = entry.get('rank_address', 0) - etype = entry.get('type', '') - eclass = entry.get('class', '') - - # Explicit type-based assignments (more reliable than rank alone) - if etype == 'country' and eclass == 'place': - addr['country'] = name - elif etype == 'state' or (eclass == 'boundary' and etype == 'administrative' and rank == 8): - if not addr['state']: - addr['state'] = name - elif etype == 'county' or (eclass == 'boundary' and etype == 'administrative' and rank in (10, 12)): - if not addr['county']: - addr['county'] = name - elif etype in ('city', 'town', 'village', 'hamlet') and eclass == 'place': - if not addr['city']: - addr['city'] = name - elif eclass == 'boundary' and etype == 'administrative' and rank == 16: - # City-level admin boundary (common in US) - if not addr['city']: - addr['city'] = name - elif etype == 'postcode': - addr['postcode'] = name - elif eclass == 'highway' or rank == 26: - if not addr['road']: - addr['road'] = name - elif etype == 'house_number' or rank == 28: - addr['house_number'] = name - elif rank in (20, 22) and not addr['neighbourhood']: - addr['neighbourhood'] = name - - # Remove county from output (not in spec) - addr.pop('county', None) - - return addr - - -def _parse_nominatim(data): - """Parse a Nominatim /details response into our canonical shape.""" - osm_type = data.get('osm_type', '') - osm_id = data.get('osm_id', 0) - osm_class = data.get('category', '') - osm_type_tag = data.get('type', '') - - # Centroid - centroid_geom = data.get('centroid', {}) - coords = centroid_geom.get('coordinates', [0, 0]) - centroid = {'lat': coords[1], 'lon': coords[0]} if len(coords) >= 2 else {'lat': 0, 'lon': 0} - - # Names - names = data.get('names', {}) - display_name = data.get('localname', '') or names.get('name', '') - - # Address - address = _parse_nominatim_address( - data.get('address', []), - country_code=data.get('country_code') - ) - - # Use calculated_postcode if address parse didn't find one - if not address.get('postcode') and data.get('calculated_postcode'): - address['postcode'] = data['calculated_postcode'] - - # Extratags - raw_extra = data.get('extratags', {}) - extratags = { - 'opening_hours': raw_extra.get('opening_hours'), - 'phone': raw_extra.get('phone') or raw_extra.get('contact:phone'), - 'website': raw_extra.get('website') or raw_extra.get('contact:website') or raw_extra.get('url'), - 'email': raw_extra.get('email') or raw_extra.get('contact:email'), - 'wikipedia': raw_extra.get('wikipedia'), - 'wikidata': raw_extra.get('wikidata'), - 'cuisine': raw_extra.get('cuisine'), - 'operator': raw_extra.get('operator'), - 'wheelchair': raw_extra.get('wheelchair'), - 'fee': raw_extra.get('fee'), - 'takeaway': raw_extra.get('takeaway'), - } - - # Category: use extratags.place for boundaries (e.g. "city"), else class/type - effective_class = osm_class - effective_type = osm_type_tag - if osm_class == 'boundary' and osm_type_tag == 'administrative': - place_tag = raw_extra.get('place') or raw_extra.get('linked_place') - if place_tag: - effective_class = 'place' - effective_type = place_tag - - category = humanize_category(effective_class, effective_type) - - # Filter names: only include extra name tags, not the bare "name" - extra_names = {k: v for k, v in names.items() if k != 'name'} if names else {} - - # Boundary geometry (polygon/multipolygon from Nominatim) - boundary = None - geom = data.get('geometry') - if geom and geom.get('type') in ('Polygon', 'MultiPolygon'): - boundary = geom - - return { - 'osm_type': osm_type, - 'osm_id': osm_id, - 'name': display_name, - 'category': category, - 'class': osm_class, - 'type': osm_type_tag, - 'address': address, - 'centroid': centroid, - 'extratags': extratags, - 'names': extra_names if extra_names else None, - 'source': 'nominatim_local', - 'boundary': boundary, - } - - -# ── Overpass parsing ──────────────────────────────────────────────────── - -OVERPASS_TYPE_MAP = {'N': 'node', 'W': 'way', 'R': 'relation'} - - -def _build_overpass_query(osm_type, osm_id): - """Build an Overpass QL query for a single element.""" - elem = OVERPASS_TYPE_MAP.get(osm_type) - if not elem: - return None - return f"[out:json][timeout:10];{elem}({osm_id});out tags center;" - - -def _parse_overpass(data, osm_type, osm_id): - """Parse an Overpass API response into our canonical shape.""" - elements = data.get('elements', []) - if not elements: - return None - - elem = elements[0] - tags = elem.get('tags', {}) - - # Centroid: Overpass returns lat/lon for nodes, center for ways/relations - lat = elem.get('lat') or (elem.get('center', {}).get('lat')) - lon = elem.get('lon') or (elem.get('center', {}).get('lon')) - centroid = {'lat': lat, 'lon': lon} if lat and lon else {'lat': 0, 'lon': 0} - - # Determine class/type from tags — Overpass doesn't have a canonical class field - # Use the first recognized class tag - osm_class = '' - osm_type_tag = '' - for cls in ('amenity', 'shop', 'leisure', 'tourism', 'natural', 'highway', - 'boundary', 'place', 'building', 'waterway', 'landuse', 'historic'): - if cls in tags: - osm_class = cls - osm_type_tag = tags[cls] - break - - category = humanize_category(osm_class, osm_type_tag) - - # Address from addr:* tags - address = { - 'house_number': tags.get('addr:housenumber'), - 'road': tags.get('addr:street'), - 'neighbourhood': tags.get('addr:suburb') or tags.get('addr:neighbourhood'), - 'city': tags.get('addr:city'), - 'state': tags.get('addr:state'), - 'postcode': tags.get('addr:postcode'), - 'country': tags.get('addr:country'), - 'country_code': tags.get('addr:country_code', - tags.get('addr:country', '')).lower()[:2] or None, - } - - # Extratags - extratags = { - 'opening_hours': tags.get('opening_hours'), - 'phone': tags.get('phone') or tags.get('contact:phone'), - 'website': tags.get('website') or tags.get('contact:website') or tags.get('url'), - 'email': tags.get('email') or tags.get('contact:email'), - 'wikipedia': tags.get('wikipedia'), - 'wikidata': tags.get('wikidata'), - 'cuisine': tags.get('cuisine'), - 'operator': tags.get('operator'), - 'wheelchair': tags.get('wheelchair'), - 'fee': tags.get('fee'), - 'takeaway': tags.get('takeaway'), - } - - # Names - name = tags.get('name', '') - extra_names = {} - for k, v in tags.items(): - if k.startswith('name:') or k in ('alt_name', 'old_name', 'short_name', 'official_name'): - extra_names[k] = v - - return { - 'osm_type': osm_type, - 'osm_id': osm_id, - 'name': name, - 'category': category, - 'class': osm_class, - 'type': osm_type_tag, - 'address': address, - 'centroid': centroid, - 'extratags': extratags, - 'names': extra_names if extra_names else None, - 'source': 'overpass', - } - - -# ── Public API ────────────────────────────────────────────────────────── - -def get_place_detail(osm_type, osm_id): - """ - Fetch place details for an OSM element. - - Returns (dict, status_code): - - (data, 200) on success - - (error_dict, 404) if not found in any source - - (error_dict, 502) if both sources error - """ - osm_type = osm_type.upper() - if osm_type not in VALID_OSM_TYPES: - return {'error': f'Invalid osm_type: {osm_type}. Must be N, W, or R.'}, 400 - - if osm_id <= 0: - return {'error': 'osm_id must be a positive integer'}, 400 - - # 1. Check cache - cached = cache_get(osm_type, osm_id) - if cached: - logger.debug(f"Cache hit: {osm_type}/{osm_id}") - return cached, 200 - - # 2. Try local Nominatim first - nominatim_result = None - nominatim_error = None - try: - resp = http_requests.get(NOMINATIM_URL, params={ - 'osmtype': osm_type, - 'osmid': osm_id, - 'format': 'json', - 'addressdetails': 1, - 'hierarchy': 0, - 'keywords': 0, - 'polygon_geojson': 1, - }, timeout=5) - - if resp.status_code == 200: - data = resp.json() - # Nominatim returns a result even for IDs not in its DB, - # but they'll have empty/minimal data. Check for osm_id match. - if data.get('osm_id') == osm_id: - nominatim_result = _parse_nominatim(data) - logger.debug(f"Nominatim hit: {osm_type}/{osm_id}") - except Exception as e: - nominatim_error = str(e) - logger.warning(f"Nominatim error for {osm_type}/{osm_id}: {e}") - - if nominatim_result: - nominatim_result = _enrich_with_overture(nominatim_result, osm_type, osm_id) - nominatim_result = _enrich_with_google(nominatim_result, osm_type, osm_id) - nominatim_result = _enrich_wiki_links(nominatim_result) - nominatim_result = _enrich_with_wiki_index(nominatim_result) - cache_put(osm_type, osm_id, nominatim_result, 'nominatim_local') - return nominatim_result, 200 - - # 3. Fallback to Overpass - overpass_result = None - overpass_error = None - try: - query = _build_overpass_query(osm_type, osm_id) - if query: - resp = http_requests.post( - OVERPASS_URL, - data={'data': query}, - headers={'User-Agent': OVERPASS_UA}, - timeout=10, - ) - if resp.status_code == 200: - data = resp.json() - overpass_result = _parse_overpass(data, osm_type, osm_id) - if overpass_result: - logger.debug(f"Overpass hit: {osm_type}/{osm_id}") - elif resp.status_code == 429: - overpass_error = "Overpass rate limited" - logger.warning(f"Overpass 429 for {osm_type}/{osm_id}") - else: - overpass_error = f"Overpass HTTP {resp.status_code}" - except Exception as e: - overpass_error = str(e) - logger.warning(f"Overpass error for {osm_type}/{osm_id}: {e}") - - if overpass_result: - overpass_result = _enrich_with_overture(overpass_result, osm_type, osm_id) - overpass_result = _enrich_with_google(overpass_result, osm_type, osm_id) - overpass_result = _enrich_wiki_links(overpass_result) - overpass_result = _enrich_with_wiki_index(overpass_result) - cache_put(osm_type, osm_id, overpass_result, 'overpass') - return overpass_result, 200 - - # 4. Both failed - if nominatim_error and overpass_error: - logger.error(f"Both sources failed for {osm_type}/{osm_id}: " - f"Nominatim={nominatim_error}, Overpass={overpass_error}") - return {'error': 'Both data sources unavailable'}, 502 - - # Not found in either source (no errors, just empty results) - return {'error': f'{osm_type}/{osm_id} not found'}, 404 - - -# ── Wikidata lookup ───────────────────────────────────────────────────── - -WIKIDATA_API_URL = "https://www.wikidata.org/w/api.php" - -def get_place_by_wikidata(wikidata_id): - """ - Fetch place details from Wikidata entity. - - Returns (dict, status_code): - - (data, 200) on success - - (error_dict, 404) if entity not found - - (error_dict, 400) if invalid ID format - - (error_dict, 502) on API error - """ - # Validate wikidata ID format (Q followed by digits) - wikidata_id = wikidata_id.upper().strip() - if not wikidata_id.startswith("Q") or not wikidata_id[1:].isdigit(): - return {"error": f"Invalid wikidata ID: {wikidata_id}. Must be Q followed by digits."}, 400 - - try: - resp = http_requests.get(WIKIDATA_API_URL, params={ - "action": "wbgetentities", - "ids": wikidata_id, - "format": "json", - "languages": "en", - "props": "labels|descriptions|claims|sitelinks", - }, timeout=10, headers={"User-Agent": "Navi/1.0 (forge.echo6.co/matt/recon)"}) - - if resp.status_code != 200: - logger.warning(f"Wikidata API error for {wikidata_id}: HTTP {resp.status_code}") - return {"error": "Wikidata API error"}, 502 - - data = resp.json() - entities = data.get("entities", {}) - entity = entities.get(wikidata_id) - - if not entity or entity.get("missing"): - return {"error": f"Wikidata entity {wikidata_id} not found"}, 404 - - # Extract basic info - labels = entity.get("labels", {}) - descriptions = entity.get("descriptions", {}) - claims = entity.get("claims", {}) - - name = labels.get("en", {}).get("value", wikidata_id) - description = descriptions.get("en", {}).get("value", "") - - # Extract coordinates from P625 (coordinate location) - lat, lon = None, None - if "P625" in claims: - coord_claim = claims["P625"] - if coord_claim and coord_claim[0].get("mainsnak", {}).get("datavalue"): - coord_val = coord_claim[0]["mainsnak"]["datavalue"]["value"] - lat = coord_val.get("latitude") - lon = coord_val.get("longitude") - - # Extract population from P1082 - population = None - if "P1082" in claims: - pop_claims = claims["P1082"] - if pop_claims: - # Get the most recent population value - for claim in pop_claims: - if claim.get("mainsnak", {}).get("datavalue"): - try: - population = int(claim["mainsnak"]["datavalue"]["value"]["amount"].lstrip("+")) - break - except (KeyError, ValueError): - pass - - # Extract country from P17 - country = None - if "P17" in claims: - country_claims = claims["P17"] - if country_claims and country_claims[0].get("mainsnak", {}).get("datavalue"): - country_id = country_claims[0]["mainsnak"]["datavalue"]["value"]["id"] - # Could resolve this to a name, but for now just store the ID - - # Extract instance of (P31) for type classification - instance_of = [] - if "P31" in claims: - for claim in claims["P31"]: - if claim.get("mainsnak", {}).get("datavalue"): - instance_of.append(claim["mainsnak"]["datavalue"]["value"]["id"]) - - # Extract OSM relation ID if available (P402) - osm_relation_id = None - if "P402" in claims: - osm_claims = claims["P402"] - if osm_claims and osm_claims[0].get("mainsnak", {}).get("datavalue"): - osm_relation_id = osm_claims[0]["mainsnak"]["datavalue"]["value"] - - # Extract Wikipedia sitelink - sitelinks = entity.get("sitelinks", {}) - wikipedia = None - if "enwiki" in sitelinks: - wiki_title = sitelinks["enwiki"].get("title", "") - if wiki_title: - wikipedia = f"en:{wiki_title}" - - result = { - "wikidata_id": wikidata_id, - "name": name, - "description": description, - "centroid": {"lat": lat, "lon": lon} if lat and lon else None, - "population": population, - "instance_of": instance_of, - "osm_relation_id": osm_relation_id, - "source": "wikidata", - "extratags": { - "wikidata": wikidata_id, - }, - } - - if wikipedia: - result["extratags"]["wikipedia"] = wikipedia - - # Fetch boundary polygon from Nominatim if we have an OSM relation ID - boundary = None - if osm_relation_id: - try: - nom_resp = http_requests.get(NOMINATIM_URL, params={ - 'osmtype': 'R', - 'osmid': osm_relation_id, - 'format': 'json', - 'polygon_geojson': 1, - }, timeout=5) - if nom_resp.status_code == 200: - nom_data = nom_resp.json() - geom = nom_data.get('geometry') - if geom and geom.get('type') in ('Polygon', 'MultiPolygon'): - boundary = geom - logger.debug(f"Wikidata boundary hit for {wikidata_id}") - except Exception as e: - logger.debug(f"Wikidata boundary fetch failed: {e}") - - result["boundary"] = boundary - - result = _enrich_with_wiki_index(result) - logger.debug(f"Wikidata hit: {wikidata_id} -> {name}") - return result, 200 - - except Exception as e: - logger.warning(f"Wikidata error for {wikidata_id}: {e}") - return {"error": "Wikidata lookup failed"}, 502 From 79d7b2b343478a6bac025e9ffd63051bc43116e2 Mon Sep 17 00:00:00 2001 From: malice Date: Sat, 23 May 2026 10:29:39 -0600 Subject: [PATCH 35/41] cleanup: remove orphaned lib/address_book.py (post-cleanup-4 dead code) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After cleanup #4 deleted lib/geocode.py, the only remaining address_book references in recon were lib/address_book_test.py (test of the dying SUT) and a dead `from . import address_book` import at the top of lib/netsyms_api.py (never referenced in the body). This PR removes all three. - DELETE lib/address_book.py + lib/address_book_test.py - netsyms_api.py: drop the dead `from . import address_book` import config/address_book.yaml stays — vendored data, navi-contacts (:8423) consumes its own copy via NAVI_ADDRESS_BOOK_YAML. Co-authored-by: Claude Opus 4.7 (1M context) --- lib/address_book.py | 160 --------------------------------------- lib/address_book_test.py | 91 ---------------------- lib/netsyms_api.py | 1 - 3 files changed, 252 deletions(-) delete mode 100644 lib/address_book.py delete mode 100644 lib/address_book_test.py diff --git a/lib/address_book.py b/lib/address_book.py deleted file mode 100644 index f9827f6..0000000 --- a/lib/address_book.py +++ /dev/null @@ -1,160 +0,0 @@ -""" -RECON Address Book — YAML-backed saved-location lookup. - -Provides named locations (home, work, etc.) that short-circuit Photon -geocoding when an exact alias match is found. - -Config: /opt/recon/config/address_book.yaml -""" - -import os -import re -import threading - -import yaml - -from .utils import setup_logging - -logger = setup_logging('recon.address_book') - -_CONFIG_PATH = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - 'config', 'address_book.yaml', -) - -_lock = threading.Lock() -_entries: list[dict] = [] -_mtime: float = 0.0 - - -def _reload_if_changed(): - """Reload the YAML file if its mtime has changed.""" - global _entries, _mtime - try: - st = os.stat(_CONFIG_PATH) - except FileNotFoundError: - logger.warning("Address book not found: %s", _CONFIG_PATH) - _entries = [] - _mtime = 0.0 - return - - if st.st_mtime == _mtime: - return - - with _lock: - # Double-check after acquiring lock - try: - st = os.stat(_CONFIG_PATH) - except FileNotFoundError: - _entries = [] - _mtime = 0.0 - return - if st.st_mtime == _mtime: - return - - with open(_CONFIG_PATH, 'r') as f: - data = yaml.safe_load(f) or {} - - raw = data.get('entries', []) - loaded = [] - for entry in raw: - # Normalise aliases to lowercase for matching - aliases = [a.lower() for a in entry.get('aliases', [])] - loaded.append({ - 'id': entry.get('id', ''), - 'name': entry.get('name', ''), - 'aliases': aliases, - 'address': entry.get('address', ''), - 'lat': entry.get('lat'), - 'lon': entry.get('lon'), - 'tags': entry.get('tags', []), - }) - _entries = loaded - _mtime = st.st_mtime - logger.info("Address book loaded: %d entries from %s", len(_entries), _CONFIG_PATH) - - -def load(): - """Ensure the address book is loaded (and refreshed if the file changed).""" - _reload_if_changed() - return _entries - - -def _normalize(text: str) -> str: - """Lowercase, strip, remove commas, collapse whitespace.""" - t = text.strip().lower() - t = t.replace(',', ' ') - return ' '.join(t.split()) - - -def lookup(query: str): - """ - Look up a query against name and aliases. - - Returns dict with the matching entry plus a 'confidence' field: - - "exact": full name/alias match, OR query starts with alias + word boundary - - "partial": alias starts with query + word boundary, or alias appears - as a contiguous token sequence inside the query - - None if no match - - Matching order (first exact wins, else first partial): - 1. normalized(query) == normalized(name or alias) → exact - 2. normalized(query) starts with normalized(alias) + " " → exact - 3. normalized(alias) starts with normalized(query) + " " → partial - 4. normalized(alias) is a contiguous token sub-sequence → partial - """ - _reload_if_changed() - q = _normalize(query) - if not q: - return None - - first_exact = None - first_partial = None - - for entry in _entries: - norm_name = _normalize(entry['name']) - check_aliases = [_normalize(a) for a in entry.get('aliases', [])] - all_forms = [norm_name] + check_aliases - - for form in all_forms: - if not form: - continue - - # Rule 1: exact match - if q == form: - return {**entry, 'confidence': 'exact'} - - # Rule 2: query starts with alias + word boundary - if q.startswith(form + ' '): - if first_exact is None: - first_exact = entry - continue - - # Rule 3: alias starts with query (user still typing) - if form.startswith(q) and len(q) < len(form): - if first_partial is None: - first_partial = entry - continue - - # Rule 4: alias is contiguous token sub-sequence in query - # Build regex: token1\s+token2\s+...tokenN - tokens = form.split() - if len(tokens) >= 1: - pattern = r'(?:^|\s)' + r'\s+'.join(re.escape(t) for t in tokens) + r'(?:\s|$)' - if re.search(pattern, q): - if first_partial is None: - first_partial = entry - - if first_exact is not None: - return {**first_exact, 'confidence': 'exact'} - - if first_partial is not None: - return {**first_partial, 'confidence': 'partial'} - - return None - - -def list_all(): - """Return all address book entries.""" - _reload_if_changed() - return list(_entries) diff --git a/lib/address_book_test.py b/lib/address_book_test.py deleted file mode 100644 index 75905f0..0000000 --- a/lib/address_book_test.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python3 -"""Tests for RECON address book module.""" -import sys -import os - -# Add project root to path -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from lib import address_book - -TESTS = [ - # ── Existing tests ── - ("lookup('home') → exact", - lambda: address_book.lookup("home"), - lambda r: r is not None and r['confidence'] == 'exact' and r['id'] == 'home'), - - ("lookup('Home') → exact (case-insensitive)", - lambda: address_book.lookup("Home"), - lambda r: r is not None and r['confidence'] == 'exact' and r['id'] == 'home'), - - ("lookup('214 north st') → exact via alias", - lambda: address_book.lookup("214 north st"), - lambda r: r is not None and r['confidence'] == 'exact' and r['id'] == 'home'), - - ("lookup('214 North Street') → exact via alias", - lambda: address_book.lookup("214 North Street"), - lambda r: r is not None and r['confidence'] == 'exact' and r['id'] == 'home'), - - ("lookup('nonexistent place') → None", - lambda: address_book.lookup("nonexistent place"), - lambda r: r is None), - - ("list_all() → 1 entry", - lambda: address_book.list_all(), - lambda r: isinstance(r, list) and len(r) == 1 and r[0]['id'] == 'home'), - - # ── New prefix+boundary tests ── - ("lookup('214 north st filer') → exact (query starts with alias)", - lambda: address_book.lookup("214 north st filer"), - lambda r: r is not None and r['confidence'] == 'exact' and r['id'] == 'home'), - - ("lookup('214 North St Filer ID') → exact (case + trailing state)", - lambda: address_book.lookup("214 North St Filer ID"), - lambda r: r is not None and r['confidence'] == 'exact' and r['id'] == 'home'), - - ("lookup('214 north st, filer, id') → exact (commas stripped)", - lambda: address_book.lookup("214 north st, filer, id"), - lambda r: r is not None and r['confidence'] == 'exact' and r['id'] == 'home'), - - ("lookup('home today') → exact (short alias + trailing text)", - lambda: address_book.lookup("home today"), - lambda r: r is not None and r['confidence'] == 'exact' and r['id'] == 'home'), - - ("lookup('214') → partial (query is prefix of alias)", - lambda: address_book.lookup("214"), - lambda r: r is not None and r['confidence'] == 'partial'), - - ("lookup('214 n') → partial (partial prefix of alias)", - lambda: address_book.lookup("214 n"), - lambda r: r is not None and r['confidence'] == 'partial'), - - ("lookup('completely unrelated query') → None", - lambda: address_book.lookup("completely unrelated query"), - lambda r: r is None), - - ("lookup('214 north streets of filer') → None (no word boundary after st)", - lambda: address_book.lookup("214 north streets of filer"), - lambda r: r is None), -] - -passed = 0 -failed = 0 -for name, fn, check in TESTS: - try: - result = fn() - ok = check(result) - except Exception as e: - ok = False - result = f"EXCEPTION: {e}" - - status = "PASS" if ok else "FAIL" - if ok: - passed += 1 - else: - failed += 1 - print(f" [{status}] {name}") - if not ok: - print(f" got: {result}") - -print(f"\n{passed} passed, {failed} failed") -sys.exit(0 if failed == 0 else 1) diff --git a/lib/netsyms_api.py b/lib/netsyms_api.py index 2caf47c..dbae24e 100644 --- a/lib/netsyms_api.py +++ b/lib/netsyms_api.py @@ -8,7 +8,6 @@ GET /api/netsyms/health from flask import Blueprint, request, jsonify from . import netsyms -from . import address_book from .utils import setup_logging logger = setup_logging('recon.netsyms_api') From aa6e972260d89a7c03c65aa6fd291013123f96dd Mon Sep 17 00:00:00 2001 From: malice Date: Sat, 23 May 2026 10:35:15 -0600 Subject: [PATCH 36/41] cleanup: remove orphaned lib/overture.py + lib/osm_categories.py (post-#27 dead code) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both modules were flagged in cleanup #27 (PR #16) as fully orphaned once the place_detail orchestrator cluster was deleted; Matt confirmed scope in chat. - lib/overture.py (170L): only consumer was place_detail._enrich_with_overture (deleted in #27). - lib/osm_categories.py (143L): humanize_category's only callers were place_detail._parse_nominatim / _parse_overpass (both deleted in #27). Re-probed against master 79d7b2b: zero import/usage references anywhere outside the modules themselves, zero template/JS refs, no test files. compileall lib/ passes. Note: scripts/overture_import.py (the Overture-Maps→PostGIS ETL script) is independent — imports nothing from lib/ — and is left untouched. After this PR the `overture` PostGIS DB it populates has no remaining recon reader; that's a data-ops follow-up, not code touched here. Co-authored-by: Claude Opus 4.7 (1M context) --- lib/osm_categories.py | 143 ----------------------------------- lib/overture.py | 170 ------------------------------------------ 2 files changed, 313 deletions(-) delete mode 100644 lib/osm_categories.py delete mode 100644 lib/overture.py diff --git a/lib/osm_categories.py b/lib/osm_categories.py deleted file mode 100644 index dd5217c..0000000 --- a/lib/osm_categories.py +++ /dev/null @@ -1,143 +0,0 @@ -""" -Human-readable category names for OSM class/type pairs. - -Used by the place detail proxy to turn ("amenity", "cafe") into "Coffee shop". -Covers ~50 common categories; unmapped pairs fall back to title-cased class:type. -""" - -# Exact (class, type) → label -CATEGORY_MAP = { - # Amenity - ("amenity", "cafe"): "Coffee shop", - ("amenity", "restaurant"): "Restaurant", - ("amenity", "fast_food"): "Fast food restaurant", - ("amenity", "bar"): "Bar", - ("amenity", "pub"): "Pub", - ("amenity", "biergarten"): "Beer garden", - ("amenity", "ice_cream"): "Ice cream shop", - ("amenity", "fuel"): "Gas station", - ("amenity", "charging_station"): "EV charging station", - ("amenity", "parking"): "Parking", - ("amenity", "bank"): "Bank", - ("amenity", "atm"): "ATM", - ("amenity", "pharmacy"): "Pharmacy", - ("amenity", "hospital"): "Hospital", - ("amenity", "clinic"): "Clinic", - ("amenity", "dentist"): "Dentist", - ("amenity", "doctors"): "Doctor's office", - ("amenity", "veterinary"): "Veterinarian", - ("amenity", "school"): "School", - ("amenity", "university"): "University", - ("amenity", "college"): "College", - ("amenity", "library"): "Library", - ("amenity", "post_office"): "Post office", - ("amenity", "fire_station"): "Fire station", - ("amenity", "police"): "Police station", - ("amenity", "townhall"): "Town hall", - ("amenity", "place_of_worship"): "Place of worship", - ("amenity", "theatre"): "Theatre", - ("amenity", "cinema"): "Cinema", - ("amenity", "community_centre"): "Community center", - ("amenity", "toilets"): "Restrooms", - ("amenity", "drinking_water"): "Drinking water", - ("amenity", "shelter"): "Shelter", - ("amenity", "camping"): "Campground", - # Shop - ("shop", "supermarket"): "Supermarket", - ("shop", "convenience"): "Convenience store", - ("shop", "hardware"): "Hardware store", - ("shop", "clothes"): "Clothing store", - ("shop", "car_repair"): "Auto repair", - ("shop", "car"): "Car dealership", - ("shop", "bakery"): "Bakery", - ("shop", "butcher"): "Butcher", - # Leisure - ("leisure", "park"): "Park", - ("leisure", "playground"): "Playground", - ("leisure", "sports_centre"): "Sports center", - ("leisure", "swimming_pool"): "Swimming pool", - ("leisure", "golf_course"): "Golf course", - ("leisure", "nature_reserve"): "Nature reserve", - ("leisure", "campsite"): "Campsite", - # Tourism - ("tourism", "hotel"): "Hotel", - ("tourism", "motel"): "Motel", - ("tourism", "guest_house"): "Guest house", - ("tourism", "hostel"): "Hostel", - ("tourism", "camp_site"): "Campsite", - ("tourism", "viewpoint"): "Viewpoint", - ("tourism", "museum"): "Museum", - ("tourism", "information"): "Information", - ("tourism", "attraction"): "Tourist attraction", - ("tourism", "picnic_site"): "Picnic site", - # Natural - ("natural", "peak"): "Peak", - ("natural", "spring"): "Spring", - ("natural", "hot_spring"): "Hot spring", - ("natural", "lake"): "Lake", - ("natural", "water"): "Water body", - ("natural", "cliff"): "Cliff", - ("natural", "cave_entrance"): "Cave", - # Highway - ("highway", "bus_stop"): "Bus stop", - ("highway", "rest_area"): "Rest area", - # Boundary - ("boundary", "administrative"): "Administrative boundary", - ("boundary", "protected_area"): "Protected area", - ("boundary", "national_park"): "National park", - # Place - ("place", "city"): "City", - ("place", "town"): "Town", - ("place", "village"): "Village", - ("place", "hamlet"): "Hamlet", - ("place", "suburb"): "Suburb", - ("place", "neighbourhood"): "Neighborhood", - # Building - ("building", "yes"): "Building", - # Waterway - ("waterway", "river"): "River", - ("waterway", "stream"): "Stream", - ("waterway", "waterfall"): "Waterfall", - # Landuse - ("landuse", "cemetery"): "Cemetery", - ("landuse", "forest"): "Forest", - # Historic - ("historic", "monument"): "Monument", - ("historic", "memorial"): "Memorial", - ("historic", "ruins"): "Ruins", -} - -# Class-level wildcard fallbacks (when exact type isn't mapped) -CLASS_FALLBACKS = { - "shop": "Shop", - "amenity": "Amenity", - "leisure": "Leisure", - "tourism": "Tourism", - "natural": "Natural feature", - "historic": "Historic site", -} - - -def humanize_category(osm_class, osm_type): - """Return a human-readable category string for an OSM class/type pair.""" - if not osm_class or not osm_type: - return "Place" - - osm_class = osm_class.lower() - osm_type = osm_type.lower() - - # Exact match - label = CATEGORY_MAP.get((osm_class, osm_type)) - if label: - return label - - # Class-level wildcard with formatted type - prefix = CLASS_FALLBACKS.get(osm_class) - if prefix: - nice_type = osm_type.replace("_", " ").title() - return f"{prefix}: {nice_type}" if prefix != nice_type else prefix - - # Generic fallback - nice_class = osm_class.replace("_", " ").title() - nice_type = osm_type.replace("_", " ").title() - return f"{nice_class}: {nice_type}" diff --git a/lib/overture.py b/lib/overture.py deleted file mode 100644 index fcbdd18..0000000 --- a/lib/overture.py +++ /dev/null @@ -1,170 +0,0 @@ -""" -Overture Maps enrichment layer. - -Provides lookup functions against the local PostgreSQL Overture Places database. -Two strategies: - 1. find_by_osm_id — exact match via OSM cross-reference index - 2. find_by_coords_and_name — spatial + fuzzy name fallback - -Connection pool is lazy-initialized on first call. If PostgreSQL is unreachable, -functions return None gracefully (feature degrades, doesn't crash). -""" -import json -import os - -import psycopg2 -import psycopg2.pool - -from .utils import setup_logging - -logger = setup_logging('recon.overture') - -_pool = None -_pool_failed = False - -# Map full OSM type names to single-letter codes used in Overture sources -OSM_TYPE_MAP = { - 'N': 'n', 'W': 'w', 'R': 'r', - 'node': 'n', 'way': 'w', 'relation': 'r', - 'n': 'n', 'w': 'w', 'r': 'r', -} - - -def _get_pool(): - """Lazy-init the connection pool. Returns None if Postgres is unreachable.""" - global _pool, _pool_failed - if _pool is not None: - return _pool - if _pool_failed: - return None - - try: - _pool = psycopg2.pool.SimpleConnectionPool( - minconn=1, - maxconn=3, - host=os.environ.get('OVERTURE_DB_HOST', 'localhost'), - port=int(os.environ.get('OVERTURE_DB_PORT', '5432')), - dbname=os.environ.get('OVERTURE_DB_NAME', 'overture'), - user=os.environ.get('OVERTURE_DB_USER', 'overture'), - password=os.environ.get('OVERTURE_DB_PASSWORD', ''), - connect_timeout=5, - ) - logger.info("Overture PostgreSQL connection pool initialized") - return _pool - except Exception as e: - _pool_failed = True - logger.warning(f"Overture PostgreSQL unavailable, enrichment disabled: {e}") - return None - - -def _query(sql, params): - """Execute a query and return the first row as a dict, or None.""" - pool = _get_pool() - if pool is None: - return None - - conn = None - try: - conn = pool.getconn() - with conn.cursor() as cur: - cur.execute(sql, params) - row = cur.fetchone() - if row is None: - return None - cols = [desc[0] for desc in cur.description] - return dict(zip(cols, row)) - except Exception as e: - logger.warning(f"Overture query error: {e}") - if conn: - try: - conn.rollback() - except Exception: - pass - return None - finally: - if conn: - try: - pool.putconn(conn) - except Exception: - pass - - -def _format_result(row, match_method): - """Convert a database row dict to the enrichment result shape.""" - if not row: - return None - - socials = row.get('socials') - if isinstance(socials, str): - try: - socials = json.loads(socials) - except (json.JSONDecodeError, TypeError): - socials = None - - return { - 'phone': row.get('phone'), - 'website': row.get('website'), - 'socials': socials, - 'brand_name': row.get('brand_name'), - 'brand_wikidata': row.get('brand_wikidata'), - 'basic_category': row.get('basic_category'), - 'confidence': row.get('confidence'), - 'gers_id': row.get('id'), - 'match_method': match_method, - } - - -def find_by_osm_id(osm_type, osm_id): - """ - Look up an Overture place by its OSM cross-reference. - - Args: - osm_type: OSM type — 'N', 'W', 'R', 'node', 'way', 'relation', or single letter - osm_id: OSM numeric ID - - Returns: - Enrichment dict or None - """ - type_letter = OSM_TYPE_MAP.get(osm_type) - if not type_letter: - return None - - row = _query( - """SELECT id, name, basic_category, confidence, - phone, website, socials, brand_name, brand_wikidata - FROM places - WHERE osm_type = %s AND osm_id = %s - LIMIT 1""", - (type_letter, int(osm_id)) - ) - return _format_result(row, 'osm_xref') - - -def find_by_coords_and_name(lat, lon, name, radius_m=100): - """ - Look up an Overture place by spatial proximity + fuzzy name match. - - Args: - lat: Latitude - lon: Longitude - name: Place name to fuzzy-match - radius_m: Search radius in meters (default 100) - - Returns: - Enrichment dict or None - """ - if not name or not lat or not lon: - return None - - row = _query( - """SELECT id, name, basic_category, confidence, - phone, website, socials, brand_name, brand_wikidata, - similarity(name, %s) AS sim - FROM places - WHERE ST_DWithin(geometry::geography, ST_MakePoint(%s, %s)::geography, %s) - AND similarity(name, %s) > 0.4 - ORDER BY sim DESC, ST_Distance(geometry::geography, ST_MakePoint(%s, %s)::geography) ASC - LIMIT 1""", - (name, lon, lat, radius_m, name, lon, lat) - ) - return _format_result(row, 'coord_name_fuzzy') From 879df84b7a80296d62a95a519adbfa689b6b356d Mon Sep 17 00:00:00 2001 From: malice Date: Sat, 23 May 2026 13:34:06 -0600 Subject: [PATCH 37/41] decouple: remove /api/auth/whoami handler (migrated to navi-admin) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR-B of the 2-PR whoami migration. The route is now served by navi-admin :8427 via nginx (`^~ /api/auth/whoami` cutover verified live — edge responses carry navi-admin's X-Cache-Status: BYPASS), so recon's handler is edge-unreachable and safe to remove. - lib/api.py: delete the @app.route('/api/auth/whoami') api_auth_whoami handler + its dedicated section comment. It was the file tail (post-cleanup-#6), so api.py now ends on the metrics-history handler. Sequenced after PR-A (navi-backend, merged + deployed) and the nginx edge cutover, so the route never 404s. recon serves zero navi-facing auth-state endpoints now. Co-authored-by: Claude Opus 4.7 (1M context) --- lib/api.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/lib/api.py b/lib/api.py index 3a7e5ca..e83a98f 100644 --- a/lib/api.py +++ b/lib/api.py @@ -2535,21 +2535,3 @@ def api_metrics_history(): return jsonify({'type': metric_type, 'hours': hours, 'points': points}) except Exception as e: return jsonify({'type': metric_type, 'hours': hours, 'points': [], 'error': str(e)}) - - -# ── Auth state endpoint ───────────────────────────────────────────────────── -# Returns current auth state for frontend consumption. -# This endpoint must be behind Caddy forward_auth to receive X-Authentik-* headers. -@app.route('/api/auth/whoami') -def api_auth_whoami(): - """Return auth state for frontend. Behind forward_auth, so headers are present when authenticated.""" - username = request.headers.get('X-Authentik-Username') - if username: - return jsonify({ - 'authenticated': True, - 'username': username, - }) - return jsonify({ - 'authenticated': False, - 'username': None, - }) From 21c0f11eff17d3de8f44dc771ff2bd55bb05c71d Mon Sep 17 00:00:00 2001 From: malice Date: Sat, 23 May 2026 13:57:14 -0600 Subject: [PATCH 38/41] decouple: remove scripts/overture_import.py (migrated to navi-backend) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR-B of the Overture-import relocation. The ETL now lives in zvx-echo6/navi-backend at the same scripts/ path (PR-A, navi-backend 475739d: script ported verbatim + duckdb dep + docs; verified live — imports cleanly, overture PG reachable with ~20.9M rows). recon no longer produces overture data it doesn't consume. - DELETE scripts/overture_import.py. Context: cleanup #29 removed lib/overture.py (recon's only overture *reader*), leaving this ETL as recon's last orphan overture code path. PR-A moved the writer to the navi side; this removes recon's now-orphan copy. The `overture` PG database is unchanged — only the writer moved. OVERTURE_DB_* vars in /opt/recon/.env are now dead in recon (zero overture code paths remain) — flagged for out-of-band post-merge prune, same pattern as PADUS_DB_* (cleanup #5). Co-authored-by: Claude Opus 4.7 (1M context) --- scripts/overture_import.py | 350 ------------------------------------- 1 file changed, 350 deletions(-) delete mode 100644 scripts/overture_import.py diff --git a/scripts/overture_import.py b/scripts/overture_import.py deleted file mode 100644 index 0b6ba67..0000000 --- a/scripts/overture_import.py +++ /dev/null @@ -1,350 +0,0 @@ -#!/usr/bin/env python3 -"""Overture Maps Places → PostgreSQL import script (v2). - -Downloads Overture Places Parquet from S3 via DuckDB (public bucket, no credentials), -filters to North America bounding box, and inserts into local PostgreSQL with PostGIS. - -Usage: - cd /opt/recon && venv/bin/python scripts/overture_import.py - -Re-runnable (idempotent via UPSERT). -""" - -import json -import logging -import os -import re -import sys -import time - -import duckdb -import psycopg2 -import psycopg2.extras - -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s %(levelname)s %(message)s', - datefmt='%H:%M:%S' -) -log = logging.getLogger('overture_import') - -# --- Config --- -OVERTURE_RELEASE = '2026-04-15.0' -S3_PATH = f's3://overturemaps-us-west-2/release/{OVERTURE_RELEASE}/theme=places/type=place/*' - -# North America bounding box (generous — includes Hawaii, Puerto Rico, Canada) -BBOX = { - 'xmin': -170.0, - 'xmax': -50.0, - 'ymin': 15.0, - 'ymax': 85.0, -} - -BATCH_SIZE = 50_000 -OSM_RECORD_RE = re.compile(r'^([nwr])(\d+)@\d+$') - -DB_CONFIG = { - 'host': os.environ.get('OVERTURE_DB_HOST', 'localhost'), - 'port': int(os.environ.get('OVERTURE_DB_PORT', '5432')), - 'dbname': os.environ.get('OVERTURE_DB_NAME', 'overture'), - 'user': os.environ.get('OVERTURE_DB_USER', 'overture'), - 'password': os.environ.get('OVERTURE_DB_PASSWORD', ''), -} - - -def create_table(conn): - """Create places table and indexes if they don't exist.""" - with conn.cursor() as cur: - cur.execute(""" - CREATE TABLE IF NOT EXISTS places ( - id TEXT PRIMARY KEY, - geometry GEOMETRY(Point, 4326), - name TEXT, - basic_category TEXT, - confidence REAL, - phone TEXT, - website TEXT, - socials JSONB, - brand_name TEXT, - brand_wikidata TEXT, - osm_type CHAR(1), - osm_id BIGINT, - source_record_id TEXT, - raw_sources JSONB - ); - """) - cur.execute(""" - CREATE INDEX IF NOT EXISTS idx_places_osm - ON places(osm_type, osm_id) WHERE osm_type IS NOT NULL; - """) - cur.execute(""" - CREATE INDEX IF NOT EXISTS idx_places_geom - ON places USING GIST(geometry); - """) - cur.execute(""" - CREATE INDEX IF NOT EXISTS idx_places_name_trgm - ON places USING GIN(name gin_trgm_ops); - """) - conn.commit() - log.info('Table and indexes ready') - - -def parse_osm_ref(sources): - """Extract OSM type letter and ID from Overture sources array.""" - if not sources: - return None, None, None - for src in sources: - record_id = None - if isinstance(src, dict): - record_id = src.get('record_id', '') - elif hasattr(src, '__getitem__'): - # DuckDB struct — try attribute access - try: - record_id = src['record_id'] - except (KeyError, TypeError, IndexError): - pass - if not record_id: - continue - m = OSM_RECORD_RE.match(str(record_id)) - if m: - return m.group(1), int(m.group(2)), str(record_id) - return None, None, None - - -def run_import(): - """Main import: DuckDB reads S3 Parquet → PostgreSQL via chunked OFFSET/LIMIT.""" - log.info(f'Overture release: {OVERTURE_RELEASE}') - log.info(f'S3 path: {S3_PATH}') - log.info(f'Bounding box: {BBOX}') - - # Connect to PostgreSQL - conn = psycopg2.connect(**DB_CONFIG) - conn.autocommit = False - create_table(conn) - - # Set up DuckDB with httpfs and spatial for S3 access - duck = duckdb.connect() - duck.execute("INSTALL httpfs; LOAD httpfs;") - duck.execute("INSTALL spatial; LOAD spatial;") - duck.execute("SET s3_region='us-west-2';") - - # Use a materialized approach: DuckDB query → Arrow → iterate in Python - query = f""" - SELECT - id, - ST_X(geometry) AS lon, - ST_Y(geometry) AS lat, - names.primary AS name, - basic_category, - confidence, - phones, - websites, - socials, - brand, - sources - FROM read_parquet('{S3_PATH}', hive_partitioning=true) - WHERE bbox.xmin >= {BBOX['xmin']} - AND bbox.xmax <= {BBOX['xmax']} - AND bbox.ymin >= {BBOX['ymin']} - AND bbox.ymax <= {BBOX['ymax']} - """ - - log.info('Starting DuckDB query against S3 (this will take several minutes)...') - t_start = time.time() - - # Execute and fetch all as Arrow for efficient iteration - result_rel = duck.sql(query) - - upsert_sql = """ - INSERT INTO places (id, geometry, name, basic_category, confidence, - phone, website, socials, brand_name, brand_wikidata, - osm_type, osm_id, source_record_id, raw_sources) - VALUES %s - ON CONFLICT (id) DO UPDATE SET - geometry = EXCLUDED.geometry, - name = EXCLUDED.name, - basic_category = EXCLUDED.basic_category, - confidence = EXCLUDED.confidence, - phone = EXCLUDED.phone, - website = EXCLUDED.website, - socials = EXCLUDED.socials, - brand_name = EXCLUDED.brand_name, - brand_wikidata = EXCLUDED.brand_wikidata, - osm_type = EXCLUDED.osm_type, - osm_id = EXCLUDED.osm_id, - source_record_id = EXCLUDED.source_record_id, - raw_sources = EXCLUDED.raw_sources - """ - - template = """( - %(id)s, - ST_SetSRID(ST_MakePoint(%(lon)s, %(lat)s), 4326), - %(name)s, - %(basic_category)s, - %(confidence)s, - %(phone)s, - %(website)s, - %(socials)s::jsonb, - %(brand_name)s, - %(brand_wikidata)s, - %(osm_type)s, - %(osm_id)s, - %(source_record_id)s, - %(raw_sources)s::jsonb - )""" - - total = 0 - osm_refs = 0 - batch = [] - - log.info('DuckDB query executing, fetching results in chunks...') - - # Fetch in chunks using fetchmany on the relation - chunk_size = BATCH_SIZE - while True: - chunk = result_rel.fetchmany(chunk_size) - if not chunk: - break - - for row in chunk: - row_id = row[0] - lon = row[1] - lat = row[2] - name = row[3] - basic_cat = row[4] - conf = row[5] - phones = row[6] - websites = row[7] - socials_raw = row[8] - brand_raw = row[9] - sources_raw = row[10] - - if lon is None or lat is None: - continue - - # Phone: first element of VARCHAR[] - phone = None - if phones and len(phones) > 0: - phone = str(phones[0]) if phones[0] else None - - # Website: first element of VARCHAR[] - website = None - if websites and len(websites) > 0: - website = str(websites[0]) if websites[0] else None - - # Socials: VARCHAR[] → JSON array of strings - socials_json = None - if socials_raw and len(socials_raw) > 0: - socials_json = json.dumps([str(s) for s in socials_raw if s]) - - # Brand: struct with wikidata and names.primary - brand_name = None - brand_wikidata = None - if brand_raw: - try: - if isinstance(brand_raw, dict): - brand_wikidata = brand_raw.get('wikidata') - names_struct = brand_raw.get('names') - if names_struct and isinstance(names_struct, dict): - brand_name = names_struct.get('primary') - else: - # DuckDB struct — access by key - brand_wikidata = brand_raw['wikidata'] if 'wikidata' in dir(brand_raw) else None - try: - brand_wikidata = brand_raw[0] # wikidata is first field - names_struct = brand_raw[1] # names is second field - if names_struct: - brand_name = names_struct[0] # primary is first field - except (IndexError, TypeError): - pass - except Exception: - pass - - # Sources: parse OSM cross-reference - sources_list = None - if sources_raw: - if isinstance(sources_raw, (list, tuple)): - sources_list = [] - for s in sources_raw: - if isinstance(s, dict): - sources_list.append(s) - else: - # DuckDB struct tuple — convert - try: - sources_list.append({ - 'dataset': s[1] if len(s) > 1 else None, - 'record_id': s[3] if len(s) > 3 else None, - }) - except (TypeError, IndexError): - pass - - osm_type_letter, osm_id_val, source_record_id = parse_osm_ref(sources_list) - if osm_type_letter: - osm_refs += 1 - - raw_sources_json = json.dumps(sources_list) if sources_list else None - - batch.append({ - 'id': row_id, - 'lon': float(lon), - 'lat': float(lat), - 'name': name, - 'basic_category': basic_cat, - 'confidence': float(conf) if conf is not None else None, - 'phone': phone, - 'website': website, - 'socials': socials_json, - 'brand_name': brand_name, - 'brand_wikidata': brand_wikidata, - 'osm_type': osm_type_letter, - 'osm_id': osm_id_val, - 'source_record_id': source_record_id, - 'raw_sources': raw_sources_json, - }) - - if len(batch) >= BATCH_SIZE: - with conn.cursor() as cur: - psycopg2.extras.execute_values( - cur, upsert_sql, batch, - template=template, - page_size=BATCH_SIZE - ) - conn.commit() - total += len(batch) - elapsed = time.time() - t_start - rate = total / elapsed if elapsed > 0 else 0 - log.info(f'Inserted {total:,} rows ({osm_refs:,} OSM xrefs) ' - f'[{rate:.0f} rows/sec, {elapsed:.0f}s elapsed]') - batch = [] - - # Flush remaining - if batch: - with conn.cursor() as cur: - psycopg2.extras.execute_values( - cur, upsert_sql, batch, - template=template, - page_size=BATCH_SIZE - ) - conn.commit() - total += len(batch) - - duck.close() - - # Final stats - elapsed = time.time() - t_start - log.info(f'Import complete: {total:,} rows, {osm_refs:,} OSM cross-refs, ' - f'{elapsed:.0f}s total ({total/elapsed:.0f} rows/sec)') - - # Verify - with conn.cursor() as cur: - cur.execute("SELECT count(*) FROM places") - count = cur.fetchone()[0] - cur.execute("SELECT count(*) FROM places WHERE osm_type IS NOT NULL") - osm_count = cur.fetchone()[0] - log.info(f'Final table: {count:,} total rows, {osm_count:,} with OSM cross-references') - - conn.close() - - -if __name__ == '__main__': - run_import() From ac99723e514d9766d8c8d7994452cefd3d59c06a Mon Sep 17 00:00:00 2001 From: malice Date: Sat, 23 May 2026 19:25:53 -0600 Subject: [PATCH 39/41] decouple: remove /api/wiki-enrich + wiki_index read path (migrated to navi-places) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR-B of decouple #4-READ. navi-places now reads its own wiki_index.db directly (navi-backend a8f9520, deployed + verified: Horseshoe Falls enrichment served from /var/lib/navi-backend/wiki_index.db; admin-info dropped the recon-wiki-enrich dependency). recon's endpoint is edge-unreachable-unused, safe to remove. - DELETE lib/wiki_enrich_api.py (the /api/wiki-enrich blueprint). - DELETE lib/place_detail.py (97-line survivor: lookup_wiki_index + _get_wiki_index_db) — its only consumer was wiki_enrich_api.py (verified zero non-test code consumers). Fully orphaned. - DELETE lib/wiki_enrich_api_test.py (tests the deleted endpoint). - api.py: drop the wiki_enrich_bp import + register_blueprint. Untouched (separate decouple): /api/wiki-rewrite (wiki_rewrite_api.py + wiki_rewrite.py), still navi-consumed. /opt/recon/data/wiki_index.db left in place (data; now a harmless dead file). Internal localhost migration — no nginx. Flag (doc follow-up, not fixed): deployment_config.py:10 + wiki_rewrite_api.py:6 both have stale in-prose references to the deleted place_detail. Co-authored-by: Claude Opus 4.7 (1M context) --- lib/api.py | 4 -- lib/place_detail.py | 97 ------------------------------------- lib/wiki_enrich_api.py | 31 ------------ lib/wiki_enrich_api_test.py | 77 ----------------------------- 4 files changed, 209 deletions(-) delete mode 100644 lib/place_detail.py delete mode 100644 lib/wiki_enrich_api.py delete mode 100644 lib/wiki_enrich_api_test.py diff --git a/lib/api.py b/lib/api.py index e83a98f..63562f7 100644 --- a/lib/api.py +++ b/lib/api.py @@ -62,10 +62,6 @@ app.request_class = _LargeZimRequest from .netsyms_api import netsyms_bp app.register_blueprint(netsyms_bp) -# ── Wiki-enrich Blueprint (extraction #5 prep — HTTP wrapper over wiki_index) ── -from .wiki_enrich_api import wiki_enrich_bp -app.register_blueprint(wiki_enrich_bp) - # ── Wiki-rewrite Blueprint (extraction #5 prep — HTTP wrapper over rewrite_wiki_link) ── from .wiki_rewrite_api import wiki_rewrite_bp app.register_blueprint(wiki_rewrite_bp) diff --git a/lib/place_detail.py b/lib/place_detail.py deleted file mode 100644 index 6f6f1ba..0000000 --- a/lib/place_detail.py +++ /dev/null @@ -1,97 +0,0 @@ -""" -Wiki-index lookup for place enrichment. - -Provides lookup_wiki_index(wikidata_id, name, country_code) — a pure read of the -local wiki_index.db, used by the /api/wiki-enrich endpoint (navi-places -HTTP-fetches wiki enrichment instead of reading the 2.1 GB DB directly). -""" -import os -import sqlite3 - -from .utils import setup_logging - -logger = setup_logging('recon.place_detail') - - -# ── Wiki Index enrichment ─────────────────────────────────────────────── - -_wiki_index_conn = None - -def _get_wiki_index_db(): - global _wiki_index_conn - if _wiki_index_conn is not None: - return _wiki_index_conn - - db_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data", "wiki_index.db") - if not os.path.exists(db_path): - logger.debug(f"wiki_index.db not found at {db_path}") - return None - - _wiki_index_conn = sqlite3.connect(db_path, check_same_thread=False) - _wiki_index_conn.row_factory = sqlite3.Row - logger.info(f"Wiki index DB ready at {db_path}") - return _wiki_index_conn - - -def lookup_wiki_index(wikidata_id=None, name=None, country_code=None): - """Standalone wiki_index lookup, extracted for the /api/wiki-enrich endpoint - (extraction #5: navi-places HTTP-fetches wiki enrichment instead of reading - the 2.1 GB wiki_index.db directly). - - Mirrors the lookup that `_enrich_with_wiki_index` performs in-process: - by wikidata_id first, then a name + country_code fallback. Returns a dict of - wiki enrichment fields (only those present), or None if there is no match or - the wiki_index DB is unavailable. Pure DB read — no feature-flag gating - (callers decide whether to call) and never raises. - - NOTE: additive only — `_enrich_with_wiki_index` is intentionally left - untouched here; it can be DRY-refactored to delegate to this in a later PR. - """ - db = _get_wiki_index_db() - if not db: - return None - - try: - cur = db.cursor() - row = None - - if wikidata_id: - wid = wikidata_id - if isinstance(wid, str) and wid.startswith("http"): - wid = wid.split("/")[-1] - cur.execute( - "SELECT summary, wiki_population, wikipedia_title, wikivoyage_title FROM wiki_places WHERE wikidata_id = ?", - (wid,) - ) - row = cur.fetchone() - - if not row and name and country_code: - cur.execute( - "SELECT summary, wiki_population, wikipedia_title, wikivoyage_title FROM wiki_places WHERE place_name = ? AND country_code = ? LIMIT 1", - (name, country_code.lower()) - ) - row = cur.fetchone() - - if not row: - return None - - out = {} - if row["summary"]: - out["wiki_summary"] = row["summary"] - if row["wiki_population"]: - try: - out["wiki_population"] = int(row["wiki_population"]) - except (ValueError, TypeError): - out["wiki_population"] = row["wiki_population"] - if row["wikipedia_title"]: - title = row["wikipedia_title"].replace(" ", "_") - out["wiki_url"] = f"https://en.wikipedia.org/wiki/{title}" - if row["wikivoyage_title"]: - title = row["wikivoyage_title"].replace(" ", "_") - out["wikivoyage_url"] = f"https://en.wikivoyage.org/wiki/{title}" - - return out or None - - except Exception as e: - logger.debug(f"wiki_index lookup error: {e}") - return None diff --git a/lib/wiki_enrich_api.py b/lib/wiki_enrich_api.py deleted file mode 100644 index ff0f9c7..0000000 --- a/lib/wiki_enrich_api.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Wiki-enrich API — read-only HTTP wrapper over the wiki_index lookup. - -Extraction #5 prep: lets the (future) navi-places service fetch wiki enrichment -over HTTP instead of reading recon's 2.1 GB data/wiki_index.db directly. Additive -only — does not change place_detail's in-process `_enrich_with_wiki_index` path. - - GET /api/wiki-enrich?wikidata= (primary key) - GET /api/wiki-enrich?name=&country= (fallback key) - -Public (no auth), matching /api/place/*. 400 if no usable key; 404 on no match. -""" -from flask import Blueprint, request, jsonify - -from .place_detail import lookup_wiki_index - -wiki_enrich_bp = Blueprint('wiki_enrich', __name__) - - -@wiki_enrich_bp.route('/api/wiki-enrich') -def api_wiki_enrich(): - wikidata = (request.args.get('wikidata') or '').strip() or None - name = (request.args.get('name') or '').strip() or None - country = (request.args.get('country') or '').strip() or None - - if not wikidata and not (name and country): - return jsonify({'error': 'provide ?wikidata= or ?name=&country='}), 400 - - result = lookup_wiki_index(wikidata_id=wikidata, name=name, country_code=country) - if result is None: - return jsonify({'error': 'no wiki match'}), 404 - return jsonify(result) diff --git a/lib/wiki_enrich_api_test.py b/lib/wiki_enrich_api_test.py deleted file mode 100644 index 681e5cb..0000000 --- a/lib/wiki_enrich_api_test.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Tests for the /api/wiki-enrich endpoint (extraction #5 prep). - -Plain-assert style (matching the other lib *_test.py; recon's venv has no -pytest). Builds a minimal Flask app with only wiki_enrich_bp registered (avoids -importing the full recon app) and points place_detail's lazy wiki_index -connection at an in-memory fixture DB. Run with pytest, or directly: - python -m lib.wiki_enrich_api_test -""" -import sqlite3 - -from flask import Flask - -from lib import place_detail -from lib.wiki_enrich_api import wiki_enrich_bp - - -def _client(): - """Fresh in-memory wiki_index fixture + a minimal app with just the route.""" - conn = sqlite3.connect(":memory:", check_same_thread=False) - conn.row_factory = sqlite3.Row - conn.execute( - "CREATE TABLE wiki_places (wikidata_id TEXT, place_name TEXT, country_code TEXT, " - "summary TEXT, wiki_population TEXT, wikipedia_title TEXT, wikivoyage_title TEXT)" - ) - conn.execute( - "INSERT INTO wiki_places VALUES (?,?,?,?,?,?,?)", - ("Q830149", "Filer", "us", "A city in Idaho.", "2508", "Filer, Idaho", "Filer"), - ) - conn.commit() - # Point the lazy module-level connection at the fixture so - # _get_wiki_index_db()/lookup_wiki_index() use it (bypasses the file path). - place_detail._wiki_index_conn = conn - app = Flask(__name__) - app.register_blueprint(wiki_enrich_bp) - return app.test_client() - - -def test_wiki_enrich_hit_by_wikidata(): - resp = _client().get("/api/wiki-enrich?wikidata=Q830149") - assert resp.status_code == 200, resp.status_code - d = resp.get_json() - assert d["wiki_summary"] == "A city in Idaho." - assert d["wiki_population"] == 2508 # cast to int - assert d["wiki_url"] == "https://en.wikipedia.org/wiki/Filer,_Idaho" - assert d["wikivoyage_url"] == "https://en.wikivoyage.org/wiki/Filer" - - -def test_wiki_enrich_no_match_404(): - resp = _client().get("/api/wiki-enrich?wikidata=Q9999999") - assert resp.status_code == 404, resp.status_code - - -def test_wiki_enrich_name_country_fallback(): - resp = _client().get("/api/wiki-enrich?name=Filer&country=US") - assert resp.status_code == 200, resp.status_code - assert resp.get_json()["wiki_summary"] == "A city in Idaho." - - -def test_wiki_enrich_no_key_400(): - c = _client() - assert c.get("/api/wiki-enrich").status_code == 400 - # name without country is not a usable key - assert c.get("/api/wiki-enrich?name=Filer").status_code == 400 - - -if __name__ == "__main__": - failures = 0 - for _name, _fn in sorted(globals().items()): - if _name.startswith("test_") and callable(_fn): - try: - _fn() - print(f"PASS {_name}") - except Exception as exc: # noqa: BLE001 - failures += 1 - print(f"FAIL {_name}: {exc!r}") - print("OK" if failures == 0 else f"{failures} FAILED") - raise SystemExit(1 if failures else 0) From 6365fe67565ffabe67f5a0d3647f1aaa263bc983 Mon Sep 17 00:00:00 2001 From: malice Date: Sat, 23 May 2026 21:21:22 -0600 Subject: [PATCH 40/41] decouple: remove /api/wiki-rewrite (migrated to navi-places) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR-B of decouple #4-REWRITE — the LAST recon→navi decoupling step. navi-places now owns the Kiwix link-rewrite logic in-process (navi-backend PR-A 7103c27, deployed + verified: Twin Falls live route returns wiki_rewrites local/public from navi's own wiki_cache.db; zero outbound calls to recon /api/wiki-rewrite). - DELETE lib/wiki_rewrite.py (the Kiwix rewrite logic — ported to navi-places). - DELETE lib/wiki_rewrite_api.py (the /api/wiki-rewrite blueprint). - DELETE lib/wiki_rewrite_api_test.py (tests the deleted endpoint). - api.py: drop the wiki_rewrite_bp import + register_blueprint + section comment. Verified zero recon consumers: nothing in recon imports wiki_rewrite — it was purely an HTTP endpoint for navi-places. After this, recon services make and receive zero navi-ecosystem runtime calls; recon is a fully separate product. Out-of-band (post-deploy): DROP TABLE wiki_cache from /opt/recon/data/place_cache.db (table only — place_cache + google_api_calls stay). Co-authored-by: Claude Opus 4.7 (1M context) --- lib/api.py | 5 - lib/wiki_rewrite.py | 324 ----------------------------------- lib/wiki_rewrite_api.py | 34 ---- lib/wiki_rewrite_api_test.py | 73 -------- 4 files changed, 436 deletions(-) delete mode 100644 lib/wiki_rewrite.py delete mode 100644 lib/wiki_rewrite_api.py delete mode 100644 lib/wiki_rewrite_api_test.py diff --git a/lib/api.py b/lib/api.py index 63562f7..576f4ee 100644 --- a/lib/api.py +++ b/lib/api.py @@ -62,11 +62,6 @@ app.request_class = _LargeZimRequest from .netsyms_api import netsyms_bp app.register_blueprint(netsyms_bp) -# ── Wiki-rewrite Blueprint (extraction #5 prep — HTTP wrapper over rewrite_wiki_link) ── -from .wiki_rewrite_api import wiki_rewrite_bp -app.register_blueprint(wiki_rewrite_bp) - - # ── Navigation Constants ── diff --git a/lib/wiki_rewrite.py b/lib/wiki_rewrite.py deleted file mode 100644 index d884635..0000000 --- a/lib/wiki_rewrite.py +++ /dev/null @@ -1,324 +0,0 @@ -""" -Wiki link rewriter — rewrites OSM wikipedia/wikidata/wikivoyage/appropedia -links to local Kiwix URLs where the article exists in a loaded ZIM. - -Falls back silently to public URLs when article is unavailable locally. -Caches positive results only in place_cache.db. - -Kiwix catalog is parsed from the OPDS Atom feed at startup and refreshed -hourly to pick up newly loaded ZIMs without a restart. - -Operations note: - - After loading a new ZIM, either restart RECON (forces fresh catalog - fetch) or wait up to 1 hour for automatic refresh. - - To invalidate the wiki cache (e.g. after ZIM update): - sqlite3 /opt/recon/data/place_cache.db "DELETE FROM wiki_cache;" -""" -import os -import re -import sqlite3 -import time -import xml.etree.ElementTree as ET -from urllib.parse import unquote, quote - -import requests as http_requests - -from .utils import setup_logging - -logger = setup_logging('recon.wiki_rewrite') - -# ── Configuration ─────────────────────────────────────────────────────── - -KIWIX_BASE = "http://localhost:8430" -KIWIX_PUBLIC_BASE = "https://wiki.echo6.co" -KIWIX_CATALOG_URL = f"{KIWIX_BASE}/catalog/v2/entries" -HEAD_TIMEOUT = 1.5 # seconds -CATALOG_REFRESH_INTERVAL = 3600 # 1 hour - -# OPDS Atom namespace -_ATOM_NS = "http://www.w3.org/2005/Atom" - -# ── ZIM catalog map ───────────────────────────────────────────────────── - -_zim_map = {} # source_type → content_path e.g. 'wikipedia' → 'wikipedia_en_all_maxi_2026-02' -_zim_map_ts = 0.0 # last refresh timestamp - -# Prefix-to-source-type mapping (order matters: longest prefix first) -_ZIM_PREFIX_MAP = [ - ('wikipedia_en_all', 'wikipedia'), - ('appropedia_en_all', 'appropedia'), - ('wikivoyage_en', 'wikivoyage'), - ('wikidata_en', 'wikidata'), -] - - -def _discover_zims(): - """Parse Kiwix OPDS Atom catalog to map source types to content paths.""" - global _zim_map, _zim_map_ts - - try: - resp = http_requests.get(KIWIX_CATALOG_URL, timeout=5) - if resp.status_code != 200: - logger.warning(f"Kiwix catalog returned HTTP {resp.status_code}") - return - - root = ET.fromstring(resp.content) - new_map = {} - - for entry in root.findall(f"{{{_ATOM_NS}}}entry"): - name_el = entry.find(f"{{{_ATOM_NS}}}name") - if name_el is None: - continue - book_name = name_el.text or "" - - # - content_path = None - for link in entry.findall(f"{{{_ATOM_NS}}}link"): - if link.get("type") == "text/html": - href = link.get("href", "") - if href.startswith("/content/"): - content_path = href[len("/content/"):] - break - - if not content_path: - continue - - # Match book name against known prefixes - for prefix, source_type in _ZIM_PREFIX_MAP: - if book_name.startswith(prefix): - new_map[source_type] = content_path - break - - _zim_map = new_map - _zim_map_ts = time.time() - logger.info(f"ZIM catalog refreshed: {new_map}") - - except Exception as e: - logger.warning(f"Failed to discover ZIMs from Kiwix catalog: {e}") - - -def _ensure_zim_map(): - """Lazy-load and refresh ZIM map if stale.""" - if not _zim_map or (time.time() - _zim_map_ts) > CATALOG_REFRESH_INTERVAL: - _discover_zims() - - -# ── Database (wiki_cache in place_cache.db) ───────────────────────────── - -_db_conn = None - - -def _get_db(): - """Return a module-level SQLite connection to place_cache.db (lazy init).""" - global _db_conn - if _db_conn is not None: - return _db_conn - - db_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'data') - os.makedirs(db_dir, exist_ok=True) - db_path = os.path.join(db_dir, 'place_cache.db') - - _db_conn = sqlite3.connect(db_path, check_same_thread=False) - _db_conn.execute("PRAGMA journal_mode=WAL") - _db_conn.execute("PRAGMA synchronous=NORMAL") - _db_conn.execute(""" - CREATE TABLE IF NOT EXISTS wiki_cache ( - source_type TEXT NOT NULL, - article_id TEXT NOT NULL, - kiwix_url TEXT NOT NULL, - cached_at INTEGER NOT NULL, - PRIMARY KEY (source_type, article_id) - ) - """) - _db_conn.commit() - logger.info(f"Wiki cache table ready in {db_path}") - return _db_conn - - -# ── URL classification ────────────────────────────────────────────────── - -# Patterns for OSM wikipedia/wikidata tag values -_WIKI_TAG_RE = re.compile(r'^(?:en:)?(.+)$') # "en:Title" or just "Title" -_WIKI_URL_RE = re.compile(r'https?://en\.wikipedia\.org/wiki/(.+)') -_WIKIDATA_TAG_RE = re.compile(r'^(Q\d+)$') -_WIKIDATA_URL_RE = re.compile(r'https?://(?:www\.)?wikidata\.org/wiki/(Q\d+)') -_WIKIVOYAGE_URL_RE = re.compile(r'https?://en\.wikivoyage\.org/wiki/(.+)') -_APPROPEDIA_URL_RE = re.compile(r'https?://(?:www\.)?appropedia\.org/(?:wiki/)?(.+)') - - -def _normalize_article_id(article_id): - """Normalize article ID to MediaWiki/Kiwix convention: spaces → underscores.""" - return article_id.replace(' ', '_') - - -def classify_wiki_link(tag_name, value): - """ - Classify an OSM extratag value into (source_type, article_id) or None. - - tag_name: the extratags key ('wikipedia', 'wikidata', etc.) - value: the raw tag value from OSM - - Article IDs are normalized to MediaWiki convention (spaces → underscores). - """ - if not value or not isinstance(value, str): - return None - - value = value.strip() - - if tag_name == 'wikidata': - m = _WIKIDATA_TAG_RE.match(value) - if m: - return ('wikidata', m.group(1)) - m = _WIKIDATA_URL_RE.match(value) - if m: - return ('wikidata', m.group(1)) - return None - - if tag_name == 'wikipedia': - # URL form: https://en.wikipedia.org/wiki/Title - m = _WIKI_URL_RE.match(value) - if m: - return ('wikipedia', _normalize_article_id(unquote(m.group(1)))) - # Tag form: "en:Title" or "Title" - m = _WIKI_TAG_RE.match(value) - if m: - return ('wikipedia', _normalize_article_id(m.group(1))) - return None - - if tag_name == 'wikivoyage': - m = _WIKIVOYAGE_URL_RE.match(value) - if m: - return ('wikivoyage', _normalize_article_id(unquote(m.group(1)))) - # Plain tag: "en:Title" or "Title" - m = _WIKI_TAG_RE.match(value) - if m: - return ('wikivoyage', _normalize_article_id(m.group(1))) - return None - - if tag_name == 'appropedia': - m = _APPROPEDIA_URL_RE.match(value) - if m: - return ('appropedia', _normalize_article_id(unquote(m.group(1)))) - return ('appropedia', _normalize_article_id(value)) - - return None - - -# ── URL builders ──────────────────────────────────────────────────────── - -def build_kiwix_url(source_type, article_id): - """Build a public Kiwix URL. Returns None if source_type not in ZIM map.""" - _ensure_zim_map() - content_path = _zim_map.get(source_type) - if not content_path: - return None - return f"{KIWIX_PUBLIC_BASE}/content/{content_path}/{quote(article_id, safe='/:@!$&\'()*+,;=')}" - - -_PUBLIC_URL_TEMPLATES = { - 'wikipedia': "https://en.wikipedia.org/wiki/{id}", - 'wikidata': "https://www.wikidata.org/wiki/{id}", - 'wikivoyage': "https://en.wikivoyage.org/wiki/{id}", - 'appropedia': "https://www.appropedia.org/wiki/{id}", -} - - -def build_public_url(source_type, article_id): - """Build the canonical public URL for a wiki article.""" - tmpl = _PUBLIC_URL_TEMPLATES.get(source_type) - if not tmpl: - return None - return tmpl.format(id=quote(article_id, safe='/:@!$&\'()*+,;=')) - - -# ── Kiwix availability check ─────────────────────────────────────────── - -def check_kiwix_has_article(source_type, article_id): - """ - Check if an article exists in local Kiwix. - - Returns (bool, url): - - (True, kiwix_public_url) if article exists locally - - (False, None) if not found or Kiwix unavailable - - Only positive results are cached. - """ - # Check cache first - db = _get_db() - row = db.execute( - "SELECT kiwix_url FROM wiki_cache WHERE source_type=? AND article_id=?", - (source_type, article_id) - ).fetchone() - if row: - return (True, row[0]) - - # Build local HEAD URL - _ensure_zim_map() - content_path = _zim_map.get(source_type) - if not content_path: - return (False, None) - - head_url = f"{KIWIX_BASE}/content/{content_path}/{quote(article_id, safe='/:@!$&\'()*+,;=')}" - - try: - resp = http_requests.head(head_url, timeout=HEAD_TIMEOUT, allow_redirects=True) - if resp.status_code == 200: - kiwix_url = build_kiwix_url(source_type, article_id) - # Cache positive result - now = int(time.time()) - db.execute(""" - INSERT OR REPLACE INTO wiki_cache (source_type, article_id, kiwix_url, cached_at) - VALUES (?, ?, ?, ?) - """, (source_type, article_id, kiwix_url, now)) - db.commit() - return (True, kiwix_url) - else: - return (False, None) - except Exception as e: - logger.debug(f"Kiwix HEAD failed for {source_type}/{article_id}: {e}") - return (False, None) - - -# ── Primary entry point ──────────────────────────────────────────────── - -def rewrite_wiki_link(tag_name, value): - """ - Rewrite an OSM wiki tag value to a local Kiwix URL if available. - - Returns (url, 'local'|'public') or (None, None) if unrecognized. - """ - classified = classify_wiki_link(tag_name, value) - if not classified: - return (value, 'original') - - source_type, article_id = classified - - # Try local Kiwix - found, kiwix_url = check_kiwix_has_article(source_type, article_id) - if found and kiwix_url: - return (kiwix_url, 'local') - - # Fall back to public URL - public_url = build_public_url(source_type, article_id) - if public_url: - return (public_url, 'public') - - return (value, 'original') - - -# ── Discovery stubs (disabled, for future activation) ─────────────────── - -def discover_wikivoyage_article(name, category, lat, lon): - """ - Discover a related Wikivoyage article for a place. - Enabled by has_wiki_discovery. Currently returns None. - """ - return None - - -def discover_appropedia_article(name, category): - """ - Discover a related Appropedia article for a place. - Enabled by has_wiki_discovery. Currently returns None. - """ - return None diff --git a/lib/wiki_rewrite_api.py b/lib/wiki_rewrite_api.py deleted file mode 100644 index ae1d52e..0000000 --- a/lib/wiki_rewrite_api.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Wiki-rewrite API — read-only HTTP wrapper over wiki_rewrite.rewrite_wiki_link. - -Extraction #5 prep: lets the (future) navi-places service rewrite OSM wiki tags -to local Kiwix URLs over HTTP instead of importing recon's wiki_rewrite module -(which talks to Kiwix and the wiki_cache table in /opt/recon/data/place_cache.db). -Additive only — does not change place_detail's in-process `_enrich_wiki_links`. - - GET /api/wiki-rewrite?tag=&value= - -Public (no auth), matching /api/place/* and /api/wiki-enrich. 400 on missing -value or unknown tag. No 404 — an unclassifiable value returns the original -value with status "original" (mirrors rewrite_wiki_link). -""" -from flask import Blueprint, request, jsonify - -from .wiki_rewrite import rewrite_wiki_link - -wiki_rewrite_bp = Blueprint('wiki_rewrite', __name__) - -_KNOWN_TAGS = {'wikipedia', 'wikidata', 'wikivoyage', 'appropedia'} - - -@wiki_rewrite_bp.route('/api/wiki-rewrite') -def api_wiki_rewrite(): - tag = (request.args.get('tag') or '').strip().lower() - value = (request.args.get('value') or '').strip() - - if not value: - return jsonify({'error': 'value is required'}), 400 - if tag not in _KNOWN_TAGS: - return jsonify({'error': f"tag must be one of {sorted(_KNOWN_TAGS)}"}), 400 - - url, status = rewrite_wiki_link(tag, value) - return jsonify({'url': url, 'status': status}) diff --git a/lib/wiki_rewrite_api_test.py b/lib/wiki_rewrite_api_test.py deleted file mode 100644 index 2bc50f4..0000000 --- a/lib/wiki_rewrite_api_test.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Tests for the /api/wiki-rewrite endpoint (extraction #5 prep). - -Plain-assert style (recon's venv has no pytest). Builds a minimal Flask app -with only wiki_rewrite_bp registered. Mocks `wiki_rewrite.check_kiwix_has_article` -to control the local-Kiwix-hit vs. fallback paths without touching Kiwix or the -wiki_cache DB. classify_wiki_link (pure regex) runs for real. Run with pytest, -or directly: python -m lib.wiki_rewrite_api_test -""" -from flask import Flask - -from lib import wiki_rewrite -from lib.wiki_rewrite_api import wiki_rewrite_bp - - -def _client(kiwix_hit): - """kiwix_hit: (found_bool, url) returned by a stubbed check_kiwix_has_article.""" - wiki_rewrite.check_kiwix_has_article = lambda source_type, article_id: kiwix_hit - app = Flask(__name__) - app.register_blueprint(wiki_rewrite_bp) - return app.test_client() - - -def test_local_kiwix_hit(): - url = "https://wiki.echo6.co/content/wikipedia/Filer,_Idaho" - c = _client((True, url)) - resp = c.get("/api/wiki-rewrite?tag=wikipedia&value=Filer, Idaho") - assert resp.status_code == 200, resp.status_code - d = resp.get_json() - assert d["status"] == "local" - assert d["url"] == url - - -def test_public_fallback_when_not_in_kiwix(): - c = _client((False, None)) # not in Kiwix -> canonical public URL - resp = c.get("/api/wiki-rewrite?tag=wikipedia&value=Filer") - assert resp.status_code == 200, resp.status_code - d = resp.get_json() - assert d["status"] == "public" - assert d["url"] == "https://en.wikipedia.org/wiki/Filer" - - -def test_unclassifiable_returns_original(): - # 'wikidata' requires a Q-id; a non-matching value -> classify None -> original. - c = _client((False, None)) - resp = c.get("/api/wiki-rewrite?tag=wikidata&value=not-a-qid") - assert resp.status_code == 200, resp.status_code - d = resp.get_json() - assert d["status"] == "original" - assert d["url"] == "not-a-qid" - - -def test_missing_value_400(): - c = _client((False, None)) - assert c.get("/api/wiki-rewrite?tag=wikipedia").status_code == 400 - - -def test_unknown_tag_400(): - c = _client((False, None)) - assert c.get("/api/wiki-rewrite?tag=facebook&value=x").status_code == 400 - - -if __name__ == "__main__": - failures = 0 - for _name, _fn in sorted(globals().items()): - if _name.startswith("test_") and callable(_fn): - try: - _fn() - print(f"PASS {_name}") - except Exception as exc: # noqa: BLE001 - failures += 1 - print(f"FAIL {_name}: {exc!r}") - print("OK" if failures == 0 else f"{failures} FAILED") - raise SystemExit(1 if failures else 0) From e840a119dd8a2f68ba994ba7fdc099c504800384 Mon Sep 17 00:00:00 2001 From: malice Date: Sat, 23 May 2026 23:09:49 -0600 Subject: [PATCH 41/41] cleanup: drop dead deployment_config references + orphaned deleted_contacts template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tidies stale references left behind by the navi extraction + decoupling work. - lib/deployment_config.py: the consumer-catalog docstring listed four in-process consumers that were all extracted/removed across cleanups #4/#5/#6/#27 (/api/landclass gate, google_places.py, place_detail.py, offroute/router.py). Replaced the stale 4-bullet list with an accurate note: recon has no remaining caller of get_deployment_config() today; the module is retained per cleanup #1. - lib/api.py: removed the now-dead `from .deployment_config import get_deployment_config` import (its only caller was the /api/landclass handler removed in #5 — zero call sites remain). - templates/knowledge/deleted_contacts.html: deleted — orphaned since cleanup #3 removed the contacts/dashboard routes; zero callers in recon. No functional change (the removed import was unused; the template unrendered). Co-authored-by: Claude Opus 4.7 (1M context) --- lib/api.py | 1 - lib/deployment_config.py | 12 ++--- templates/knowledge/deleted_contacts.html | 56 ----------------------- 3 files changed, 6 insertions(+), 63 deletions(-) delete mode 100644 templates/knowledge/deleted_contacts.html diff --git a/lib/api.py b/lib/api.py index 576f4ee..a0697bf 100644 --- a/lib/api.py +++ b/lib/api.py @@ -24,7 +24,6 @@ from werkzeug.utils import secure_filename from .utils import get_config, content_hash, clean_filename_to_title, derive_source_and_category, generate_download_url, setup_logging from .status import StatusDB -from .deployment_config import get_deployment_config logger = setup_logging('recon.api') diff --git a/lib/deployment_config.py b/lib/deployment_config.py index 83cc864..ab6aa17 100644 --- a/lib/deployment_config.py +++ b/lib/deployment_config.py @@ -4,12 +4,12 @@ Deployment profile loader. Reads RECON_PROFILE env var (default: "home"), loads the matching YAML from config/profiles/.yaml, and caches the parsed dict in memory. -Provides get_deployment_config() for in-process consumers of the profile: - - lib/api.py:api_landclass — the has_landclass feature-flag gate - - lib/google_places.py — Google Places enrichment config - - lib/place_detail.py — place-detail enrichment config (×4 call sites) - - lib/offroute/router.py — profile.offroute.* (osm_pbf_path / postgis_dsn / - densify_interval_m) +Exposes get_deployment_config() as the in-process accessor for the profile. + +Note: its former consumers (the /api/landclass gate, google_places, +place_detail, offroute/router) were all extracted to navi-* services or removed +across cleanups #4–#6/#27 — recon has no remaining caller of +get_deployment_config() today; the module is retained per cleanup #1. (The former /api/config HTTP endpoint that served this dict to the frontend was removed once navi-config (:8422) took over that route.) """ diff --git a/templates/knowledge/deleted_contacts.html b/templates/knowledge/deleted_contacts.html deleted file mode 100644 index 58a9ff5..0000000 --- a/templates/knowledge/deleted_contacts.html +++ /dev/null @@ -1,56 +0,0 @@ -{% extends "base.html" %} -{% block content %} -

    Deleted Contacts

    -{% if not contacts %} -

    No deleted contacts.

    -{% else %} - - - {% for c in contacts %} - - - - - - - - - {% endfor %} -
    LabelNameCategoryPhoneDeleted AtActions
    {{ c.label }}{{ c.name or '' }}{{ c.category or '' }}{{ c.phone or '' }}{{ c.deleted_at or '' }} - - -
    -{% endif %} -{% endblock %} -{% block scripts %} - -{% endblock %}