mirror of
https://github.com/zvx-echo6/recon.git
synced 2026-05-20 14:44:54 +02:00
Add Overture Maps POI enrichment layer for place details
Ingests 20.9M North America places from Overture Maps Foundation (release 2026-04-15.0) into PostgreSQL. Enriches /api/place responses with phone, website, and brand data via spatial + fuzzy name matching when OSM extratags are sparse. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
2121ee4936
commit
65693d15aa
6 changed files with 597 additions and 0 deletions
170
lib/overture.py
Normal file
170
lib/overture.py
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
"""
|
||||
Overture Maps enrichment layer.
|
||||
|
||||
Provides lookup functions against the local PostgreSQL Overture Places database.
|
||||
Two strategies:
|
||||
1. find_by_osm_id — exact match via OSM cross-reference index
|
||||
2. find_by_coords_and_name — spatial + fuzzy name fallback
|
||||
|
||||
Connection pool is lazy-initialized on first call. If PostgreSQL is unreachable,
|
||||
functions return None gracefully (feature degrades, doesn't crash).
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
|
||||
import psycopg2
|
||||
import psycopg2.pool
|
||||
|
||||
from .utils import setup_logging
|
||||
|
||||
logger = setup_logging('recon.overture')
|
||||
|
||||
_pool = None
|
||||
_pool_failed = False
|
||||
|
||||
# Map full OSM type names to single-letter codes used in Overture sources
|
||||
OSM_TYPE_MAP = {
|
||||
'N': 'n', 'W': 'w', 'R': 'r',
|
||||
'node': 'n', 'way': 'w', 'relation': 'r',
|
||||
'n': 'n', 'w': 'w', 'r': 'r',
|
||||
}
|
||||
|
||||
|
||||
def _get_pool():
|
||||
"""Lazy-init the connection pool. Returns None if Postgres is unreachable."""
|
||||
global _pool, _pool_failed
|
||||
if _pool is not None:
|
||||
return _pool
|
||||
if _pool_failed:
|
||||
return None
|
||||
|
||||
try:
|
||||
_pool = psycopg2.pool.SimpleConnectionPool(
|
||||
minconn=1,
|
||||
maxconn=3,
|
||||
host=os.environ.get('OVERTURE_DB_HOST', 'localhost'),
|
||||
port=int(os.environ.get('OVERTURE_DB_PORT', '5432')),
|
||||
dbname=os.environ.get('OVERTURE_DB_NAME', 'overture'),
|
||||
user=os.environ.get('OVERTURE_DB_USER', 'overture'),
|
||||
password=os.environ.get('OVERTURE_DB_PASSWORD', ''),
|
||||
connect_timeout=5,
|
||||
)
|
||||
logger.info("Overture PostgreSQL connection pool initialized")
|
||||
return _pool
|
||||
except Exception as e:
|
||||
_pool_failed = True
|
||||
logger.warning(f"Overture PostgreSQL unavailable, enrichment disabled: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def _query(sql, params):
|
||||
"""Execute a query and return the first row as a dict, or None."""
|
||||
pool = _get_pool()
|
||||
if pool is None:
|
||||
return None
|
||||
|
||||
conn = None
|
||||
try:
|
||||
conn = pool.getconn()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(sql, params)
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
cols = [desc[0] for desc in cur.description]
|
||||
return dict(zip(cols, row))
|
||||
except Exception as e:
|
||||
logger.warning(f"Overture query error: {e}")
|
||||
if conn:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
finally:
|
||||
if conn:
|
||||
try:
|
||||
pool.putconn(conn)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _format_result(row, match_method):
|
||||
"""Convert a database row dict to the enrichment result shape."""
|
||||
if not row:
|
||||
return None
|
||||
|
||||
socials = row.get('socials')
|
||||
if isinstance(socials, str):
|
||||
try:
|
||||
socials = json.loads(socials)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
socials = None
|
||||
|
||||
return {
|
||||
'phone': row.get('phone'),
|
||||
'website': row.get('website'),
|
||||
'socials': socials,
|
||||
'brand_name': row.get('brand_name'),
|
||||
'brand_wikidata': row.get('brand_wikidata'),
|
||||
'basic_category': row.get('basic_category'),
|
||||
'confidence': row.get('confidence'),
|
||||
'gers_id': row.get('id'),
|
||||
'match_method': match_method,
|
||||
}
|
||||
|
||||
|
||||
def find_by_osm_id(osm_type, osm_id):
|
||||
"""
|
||||
Look up an Overture place by its OSM cross-reference.
|
||||
|
||||
Args:
|
||||
osm_type: OSM type — 'N', 'W', 'R', 'node', 'way', 'relation', or single letter
|
||||
osm_id: OSM numeric ID
|
||||
|
||||
Returns:
|
||||
Enrichment dict or None
|
||||
"""
|
||||
type_letter = OSM_TYPE_MAP.get(osm_type)
|
||||
if not type_letter:
|
||||
return None
|
||||
|
||||
row = _query(
|
||||
"""SELECT id, name, basic_category, confidence,
|
||||
phone, website, socials, brand_name, brand_wikidata
|
||||
FROM places
|
||||
WHERE osm_type = %s AND osm_id = %s
|
||||
LIMIT 1""",
|
||||
(type_letter, int(osm_id))
|
||||
)
|
||||
return _format_result(row, 'osm_xref')
|
||||
|
||||
|
||||
def find_by_coords_and_name(lat, lon, name, radius_m=100):
|
||||
"""
|
||||
Look up an Overture place by spatial proximity + fuzzy name match.
|
||||
|
||||
Args:
|
||||
lat: Latitude
|
||||
lon: Longitude
|
||||
name: Place name to fuzzy-match
|
||||
radius_m: Search radius in meters (default 100)
|
||||
|
||||
Returns:
|
||||
Enrichment dict or None
|
||||
"""
|
||||
if not name or not lat or not lon:
|
||||
return None
|
||||
|
||||
row = _query(
|
||||
"""SELECT id, name, basic_category, confidence,
|
||||
phone, website, socials, brand_name, brand_wikidata,
|
||||
similarity(name, %s) AS sim
|
||||
FROM places
|
||||
WHERE ST_DWithin(geometry::geography, ST_MakePoint(%s, %s)::geography, %s)
|
||||
AND similarity(name, %s) > 0.4
|
||||
ORDER BY sim DESC, ST_Distance(geometry::geography, ST_MakePoint(%s, %s)::geography) ASC
|
||||
LIMIT 1""",
|
||||
(name, lon, lat, radius_m, name, lon, lat)
|
||||
)
|
||||
return _format_result(row, 'coord_name_fuzzy')
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
"""
|
||||
Place detail proxy — local Nominatim first, Overpass API fallback, SQLite cache.
|
||||
Overture Maps enrichment layer fills sparse extratags (phone, website, brand).
|
||||
|
||||
Provides get_place_detail(osm_type, osm_id) which returns a cleaned dict
|
||||
matching the response shape for /api/place/<osm_type>/<osm_id>.
|
||||
|
|
@ -82,6 +83,77 @@ def cache_put(osm_type, osm_id, data, source):
|
|||
db.commit()
|
||||
|
||||
|
||||
# ── Overture enrichment ─────────────────────────────────────────────────
|
||||
|
||||
def _enrich_with_overture(result, osm_type, osm_id):
|
||||
"""
|
||||
Attempt to enrich a place result with Overture Maps data.
|
||||
Fills sparse extratags (phone, website, brand) without overwriting existing values.
|
||||
Returns the (possibly enriched) result dict.
|
||||
"""
|
||||
try:
|
||||
from .deployment_config import get_deployment_config
|
||||
deploy_config = get_deployment_config()
|
||||
features = deploy_config.get('features', {})
|
||||
if not features.get('has_overture_enrichment', False):
|
||||
return result
|
||||
except Exception:
|
||||
return result
|
||||
|
||||
try:
|
||||
from .overture import find_by_osm_id, find_by_coords_and_name
|
||||
except ImportError:
|
||||
logger.debug("Overture module not available")
|
||||
return result
|
||||
|
||||
enrichment = None
|
||||
match_method = None
|
||||
|
||||
# Strategy 1: OSM cross-reference (exact)
|
||||
enrichment = find_by_osm_id(osm_type, osm_id)
|
||||
if enrichment:
|
||||
match_method = 'osm_xref'
|
||||
|
||||
# Strategy 2: Coordinate + name fuzzy (fallback)
|
||||
if not enrichment and result.get('centroid') and result.get('name'):
|
||||
centroid = result['centroid']
|
||||
if centroid.get('lat') and centroid.get('lon'):
|
||||
enrichment = find_by_coords_and_name(
|
||||
centroid['lat'], centroid['lon'], result['name']
|
||||
)
|
||||
if enrichment:
|
||||
match_method = 'coord_name_fuzzy'
|
||||
|
||||
if not enrichment:
|
||||
return result
|
||||
|
||||
# Fill sparse extratags (never overwrite existing non-null values)
|
||||
extratags = result.get('extratags', {})
|
||||
fill_map = [
|
||||
('phone', 'phone'),
|
||||
('website', 'website'),
|
||||
('brand', 'brand_name'),
|
||||
('brand:wikidata', 'brand_wikidata'),
|
||||
]
|
||||
for osm_key, overture_key in fill_map:
|
||||
if not extratags.get(osm_key) and enrichment.get(overture_key):
|
||||
extratags[osm_key] = enrichment[overture_key]
|
||||
result['extratags'] = extratags
|
||||
|
||||
# Add source metadata
|
||||
result['sources'] = {
|
||||
'primary': result.get('source', 'unknown'),
|
||||
'enrichment': 'overture',
|
||||
'overture_match_method': match_method,
|
||||
'overture_gers_id': enrichment.get('gers_id'),
|
||||
'overture_confidence': enrichment.get('confidence'),
|
||||
'overture_basic_category': enrichment.get('basic_category'),
|
||||
}
|
||||
|
||||
logger.debug(f"Overture enrichment for {osm_type}/{osm_id}: {match_method}")
|
||||
return result
|
||||
|
||||
|
||||
# ── Nominatim parsing ───────────────────────────────────────────────────
|
||||
|
||||
# Nominatim address array uses rank_address to indicate what each entry is.
|
||||
|
|
@ -368,6 +440,7 @@ def get_place_detail(osm_type, osm_id):
|
|||
logger.warning(f"Nominatim error for {osm_type}/{osm_id}: {e}")
|
||||
|
||||
if nominatim_result:
|
||||
nominatim_result = _enrich_with_overture(nominatim_result, osm_type, osm_id)
|
||||
cache_put(osm_type, osm_id, nominatim_result, 'nominatim_local')
|
||||
return nominatim_result, 200
|
||||
|
||||
|
|
@ -398,6 +471,7 @@ def get_place_detail(osm_type, osm_id):
|
|||
logger.warning(f"Overpass error for {osm_type}/{osm_id}: {e}")
|
||||
|
||||
if overpass_result:
|
||||
overpass_result = _enrich_with_overture(overpass_result, osm_type, osm_id)
|
||||
cache_put(osm_type, osm_id, overpass_result, 'overpass')
|
||||
return overpass_result, 200
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue