Add Overture Maps POI enrichment layer for place details

Ingests 20.9M North America places from Overture Maps Foundation
(release 2026-04-15.0) into PostgreSQL. Enriches /api/place responses
with phone, website, and brand data via spatial + fuzzy name matching
when OSM extratags are sparse.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matt 2026-04-21 16:51:25 +00:00
commit 65693d15aa
6 changed files with 597 additions and 0 deletions

170
lib/overture.py Normal file
View file

@ -0,0 +1,170 @@
"""
Overture Maps enrichment layer.
Provides lookup functions against the local PostgreSQL Overture Places database.
Two strategies:
1. find_by_osm_id exact match via OSM cross-reference index
2. find_by_coords_and_name spatial + fuzzy name fallback
Connection pool is lazy-initialized on first call. If PostgreSQL is unreachable,
functions return None gracefully (feature degrades, doesn't crash).
"""
import json
import os
import psycopg2
import psycopg2.pool
from .utils import setup_logging
logger = setup_logging('recon.overture')
_pool = None
_pool_failed = False
# Map full OSM type names to single-letter codes used in Overture sources
OSM_TYPE_MAP = {
'N': 'n', 'W': 'w', 'R': 'r',
'node': 'n', 'way': 'w', 'relation': 'r',
'n': 'n', 'w': 'w', 'r': 'r',
}
def _get_pool():
"""Lazy-init the connection pool. Returns None if Postgres is unreachable."""
global _pool, _pool_failed
if _pool is not None:
return _pool
if _pool_failed:
return None
try:
_pool = psycopg2.pool.SimpleConnectionPool(
minconn=1,
maxconn=3,
host=os.environ.get('OVERTURE_DB_HOST', 'localhost'),
port=int(os.environ.get('OVERTURE_DB_PORT', '5432')),
dbname=os.environ.get('OVERTURE_DB_NAME', 'overture'),
user=os.environ.get('OVERTURE_DB_USER', 'overture'),
password=os.environ.get('OVERTURE_DB_PASSWORD', ''),
connect_timeout=5,
)
logger.info("Overture PostgreSQL connection pool initialized")
return _pool
except Exception as e:
_pool_failed = True
logger.warning(f"Overture PostgreSQL unavailable, enrichment disabled: {e}")
return None
def _query(sql, params):
"""Execute a query and return the first row as a dict, or None."""
pool = _get_pool()
if pool is None:
return None
conn = None
try:
conn = pool.getconn()
with conn.cursor() as cur:
cur.execute(sql, params)
row = cur.fetchone()
if row is None:
return None
cols = [desc[0] for desc in cur.description]
return dict(zip(cols, row))
except Exception as e:
logger.warning(f"Overture query error: {e}")
if conn:
try:
conn.rollback()
except Exception:
pass
return None
finally:
if conn:
try:
pool.putconn(conn)
except Exception:
pass
def _format_result(row, match_method):
"""Convert a database row dict to the enrichment result shape."""
if not row:
return None
socials = row.get('socials')
if isinstance(socials, str):
try:
socials = json.loads(socials)
except (json.JSONDecodeError, TypeError):
socials = None
return {
'phone': row.get('phone'),
'website': row.get('website'),
'socials': socials,
'brand_name': row.get('brand_name'),
'brand_wikidata': row.get('brand_wikidata'),
'basic_category': row.get('basic_category'),
'confidence': row.get('confidence'),
'gers_id': row.get('id'),
'match_method': match_method,
}
def find_by_osm_id(osm_type, osm_id):
"""
Look up an Overture place by its OSM cross-reference.
Args:
osm_type: OSM type 'N', 'W', 'R', 'node', 'way', 'relation', or single letter
osm_id: OSM numeric ID
Returns:
Enrichment dict or None
"""
type_letter = OSM_TYPE_MAP.get(osm_type)
if not type_letter:
return None
row = _query(
"""SELECT id, name, basic_category, confidence,
phone, website, socials, brand_name, brand_wikidata
FROM places
WHERE osm_type = %s AND osm_id = %s
LIMIT 1""",
(type_letter, int(osm_id))
)
return _format_result(row, 'osm_xref')
def find_by_coords_and_name(lat, lon, name, radius_m=100):
"""
Look up an Overture place by spatial proximity + fuzzy name match.
Args:
lat: Latitude
lon: Longitude
name: Place name to fuzzy-match
radius_m: Search radius in meters (default 100)
Returns:
Enrichment dict or None
"""
if not name or not lat or not lon:
return None
row = _query(
"""SELECT id, name, basic_category, confidence,
phone, website, socials, brand_name, brand_wikidata,
similarity(name, %s) AS sim
FROM places
WHERE ST_DWithin(geometry::geography, ST_MakePoint(%s, %s)::geography, %s)
AND similarity(name, %s) > 0.4
ORDER BY sim DESC, ST_Distance(geometry::geography, ST_MakePoint(%s, %s)::geography) ASC
LIMIT 1""",
(name, lon, lat, radius_m, name, lon, lat)
)
return _format_result(row, 'coord_name_fuzzy')

View file

@ -1,5 +1,6 @@
"""
Place detail proxy local Nominatim first, Overpass API fallback, SQLite cache.
Overture Maps enrichment layer fills sparse extratags (phone, website, brand).
Provides get_place_detail(osm_type, osm_id) which returns a cleaned dict
matching the response shape for /api/place/<osm_type>/<osm_id>.
@ -82,6 +83,77 @@ def cache_put(osm_type, osm_id, data, source):
db.commit()
# ── Overture enrichment ─────────────────────────────────────────────────
def _enrich_with_overture(result, osm_type, osm_id):
"""
Attempt to enrich a place result with Overture Maps data.
Fills sparse extratags (phone, website, brand) without overwriting existing values.
Returns the (possibly enriched) result dict.
"""
try:
from .deployment_config import get_deployment_config
deploy_config = get_deployment_config()
features = deploy_config.get('features', {})
if not features.get('has_overture_enrichment', False):
return result
except Exception:
return result
try:
from .overture import find_by_osm_id, find_by_coords_and_name
except ImportError:
logger.debug("Overture module not available")
return result
enrichment = None
match_method = None
# Strategy 1: OSM cross-reference (exact)
enrichment = find_by_osm_id(osm_type, osm_id)
if enrichment:
match_method = 'osm_xref'
# Strategy 2: Coordinate + name fuzzy (fallback)
if not enrichment and result.get('centroid') and result.get('name'):
centroid = result['centroid']
if centroid.get('lat') and centroid.get('lon'):
enrichment = find_by_coords_and_name(
centroid['lat'], centroid['lon'], result['name']
)
if enrichment:
match_method = 'coord_name_fuzzy'
if not enrichment:
return result
# Fill sparse extratags (never overwrite existing non-null values)
extratags = result.get('extratags', {})
fill_map = [
('phone', 'phone'),
('website', 'website'),
('brand', 'brand_name'),
('brand:wikidata', 'brand_wikidata'),
]
for osm_key, overture_key in fill_map:
if not extratags.get(osm_key) and enrichment.get(overture_key):
extratags[osm_key] = enrichment[overture_key]
result['extratags'] = extratags
# Add source metadata
result['sources'] = {
'primary': result.get('source', 'unknown'),
'enrichment': 'overture',
'overture_match_method': match_method,
'overture_gers_id': enrichment.get('gers_id'),
'overture_confidence': enrichment.get('confidence'),
'overture_basic_category': enrichment.get('basic_category'),
}
logger.debug(f"Overture enrichment for {osm_type}/{osm_id}: {match_method}")
return result
# ── Nominatim parsing ───────────────────────────────────────────────────
# Nominatim address array uses rank_address to indicate what each entry is.
@ -368,6 +440,7 @@ def get_place_detail(osm_type, osm_id):
logger.warning(f"Nominatim error for {osm_type}/{osm_id}: {e}")
if nominatim_result:
nominatim_result = _enrich_with_overture(nominatim_result, osm_type, osm_id)
cache_put(osm_type, osm_id, nominatim_result, 'nominatim_local')
return nominatim_result, 200
@ -398,6 +471,7 @@ def get_place_detail(osm_type, osm_id):
logger.warning(f"Overpass error for {osm_type}/{osm_id}: {e}")
if overpass_result:
overpass_result = _enrich_with_overture(overpass_result, osm_type, osm_id)
cache_put(osm_type, osm_id, overpass_result, 'overpass')
return overpass_result, 200