diff --git a/backend/services/navi_places/admin.py b/backend/services/navi_places/admin.py index 31d8592..64e4fd0 100644 --- a/backend/services/navi_places/admin.py +++ b/backend/services/navi_places/admin.py @@ -13,6 +13,7 @@ from shared.admin_info import build_info_response, mask_key from . import overture from . import place_cache +from . import wiki_index bp = Blueprint('places_admin', __name__) @@ -56,6 +57,7 @@ def _recon_probe(name, path): def navi_places_info(): metrics = current_app.config['METRICS'] cache_path = place_cache.db_path() + wiki_path = wiki_index.db_path() # Two real secrets -> mask_key. The rest are non-secret paths/URLs/params. info = build_info_response( service='navi-places', @@ -71,11 +73,11 @@ def navi_places_info(): {'name': 'GOOGLE_PLACES_API_KEY', 'value': mask_key(os.environ.get('GOOGLE_PLACES_API_KEY'))}, {'name': 'RECON_BASE_URL', 'value': _recon_base()}, {'name': 'NAVI_PLACE_CACHE_DB', 'value': cache_path}, + {'name': 'NAVI_WIKI_INDEX_DB', 'value': wiki_path}, {'name': 'NAVI_PROFILES_DIR', 'value': os.environ.get('NAVI_PROFILES_DIR', '(default vendored)')}, ], dependencies=[ _overture_dependency(), - _recon_probe('recon-wiki-enrich', '/api/wiki-enrich'), _recon_probe('recon-wiki-rewrite', '/api/wiki-rewrite'), ], filesystem=[{ @@ -83,6 +85,10 @@ def navi_places_info(): 'exists': os.path.exists(cache_path), 'readable': os.access(cache_path, os.R_OK), 'writable': os.access(cache_path, os.W_OK), + }, { + 'path': wiki_path, + 'exists': os.path.exists(wiki_path), + 'readable': os.access(wiki_path, os.R_OK), }], runtime={ 'uptime_s': round(time.time() - metrics['start_time'], 1), diff --git a/backend/services/navi_places/app.py b/backend/services/navi_places/app.py index 74e50f4..56b8d19 100644 --- a/backend/services/navi_places/app.py +++ b/backend/services/navi_places/app.py @@ -12,6 +12,7 @@ from shared.git_sha import git_short_sha from . import place_route, admin from . import overture from . import place_cache +from . import wiki_index from . import config as places_config @@ -29,6 +30,7 @@ def create_app(): # gunicorn worker (and each test) picks up the current env. overture.reset_pool() place_cache.reset_cache() + wiki_index.reset() places_config.reset_config() @app.before_request diff --git a/backend/services/navi_places/place_detail.py b/backend/services/navi_places/place_detail.py index 28e880a..254f9b5 100644 --- a/backend/services/navi_places/place_detail.py +++ b/backend/services/navi_places/place_detail.py @@ -1,9 +1,10 @@ """Place detail orchestrator — port of recon's lib/place_detail.py. Local Nominatim first, Overpass fallback, SQLite cache, then enrichment: -Overture (PostGIS) + Google Places + wiki. The two wiki paths are now HTTP to -recon (the 2.1 GB wiki_index.db and Kiwix/wiki_cache stay in recon): - - wiki_index summary/links -> wiki_client.enrich_via_recon (/api/wiki-enrich, PR #8) +Overture (PostGIS) + Google Places + wiki. wiki_index enrichment is a direct +local read of navi-places' own wiki_index.db (NAVI_WIKI_INDEX_DB); the Kiwix +offline-wiki rewrite is still HTTP to recon (separate decouple): + - wiki_index summary/links -> wiki_index.lookup (local wiki_index.db) - Kiwix offline-wiki rewrite -> wiki_rewrite_client.rewrite_via_recon (/api/wiki-rewrite, PR #9) Feature-flag gates (has_overture_enrichment / has_google_places_enrichment / has_kiwix_wiki / has_wiki_rewriting) read from the vendored profile via config.py. @@ -18,7 +19,7 @@ from shared.auth import get_user_id from . import config from . import overture from . import google_places -from . import wiki_client +from . import wiki_index from . import wiki_rewrite_client from .osm_categories import humanize_category from .place_cache import cache_get, cache_put @@ -161,11 +162,11 @@ def _apply_google_data(result, google_data, gaps): result['extratags'] = extratags -# ── Wiki enrichment via HTTP to recon (replaces in-process wiki_index/Kiwix) ── +# ── Wiki enrichment: wiki_index (local DB) + Kiwix link rewrite (HTTP to recon) ── -def _enrich_with_wiki_via_http(result): - """Merge wiki_index fields (summary/population/urls) via recon /api/wiki-enrich. - Replaces recon's in-process _enrich_with_wiki_index. Gated on has_kiwix_wiki.""" +def _enrich_with_wiki_index(result): + """Merge wiki_index fields (summary/population/urls) via a direct local read + of wiki_index.db. Port of recon's in-process lookup. Gated on has_kiwix_wiki.""" if not config.has_feature('has_kiwix_wiki'): return result @@ -177,7 +178,7 @@ def _enrich_with_wiki_via_http(result): name = result.get('name') country_code = address.get('country_code') or result.get('country_code') - fields = wiki_client.enrich_via_recon( + fields = wiki_index.lookup( wikidata_id=wikidata_id, name=name, country_code=country_code ) if fields: @@ -401,7 +402,7 @@ def _enrich_all(result, osm_type, osm_id): result = _enrich_with_overture(result, osm_type, osm_id) result = _enrich_with_google(result, osm_type, osm_id) result = _enrich_wiki_links_via_http(result) - result = _enrich_with_wiki_via_http(result) + result = _enrich_with_wiki_index(result) return result @@ -562,7 +563,7 @@ def get_place_by_wikidata(wikidata_id): logger.debug(f"Wikidata boundary fetch failed: {e}") result["boundary"] = boundary - result = _enrich_with_wiki_via_http(result) + result = _enrich_with_wiki_index(result) logger.debug(f"Wikidata hit: {wikidata_id} -> {name}") return result, 200 diff --git a/backend/services/navi_places/tests/test_place.py b/backend/services/navi_places/tests/test_place.py index d07d62f..2edd9ea 100644 --- a/backend/services/navi_places/tests/test_place.py +++ b/backend/services/navi_places/tests/test_place.py @@ -2,13 +2,17 @@ All upstreams are mocked: Nominatim/Overpass/Wikidata via a fake http_requests on place_detail; Overture via monkeypatched overture functions; Google via the -gate; wiki via monkeypatched wiki_client / wiki_rewrite_client. Feature flags via -a stubbed config.has_feature. The cache uses a real tmp SQLite (auto-created). +gate; wiki_index via a real tmp SQLite DB; wiki-rewrite via monkeypatched +wiki_rewrite_client. Feature flags via a stubbed config.has_feature. The cache +uses a real tmp SQLite (auto-created). """ +import sqlite3 + import pytest import services.navi_places.place_detail as pd import services.navi_places.place_cache as place_cache +import services.navi_places.wiki_index as wiki_index from services.navi_places.app import create_app @@ -177,15 +181,27 @@ def test_wiki_rewrite_original_passes_through(tmp_path, monkeypatch): assert 'wikipedia' not in d.get('sources', {}).get('wiki_rewrites', {}) -# ── wiki index summary via HTTP ── +# ── wiki index summary via local wiki_index.db ── -def test_wiki_enrich_via_http_merges_fields(tmp_path, monkeypatch): +def test_wiki_enrich_via_local_db_merges_fields(tmp_path, monkeypatch): monkeypatch.setenv('NAVI_PLACE_CACHE_DB', str(tmp_path / 'pc.db')) + # Hermetic wiki_index.db: one wiki_places row keyed by wikidata_id. + wi_path = tmp_path / 'wi.db' + conn = sqlite3.connect(str(wi_path)) + conn.execute( + "CREATE TABLE wiki_places (wikidata_id TEXT, place_name TEXT, " + "country_code TEXT, summary TEXT, wiki_population INTEGER, " + "wikipedia_title TEXT, wikivoyage_title TEXT)") + conn.execute( + "INSERT INTO wiki_places (wikidata_id, summary, wikipedia_title) VALUES (?,?,?)", + ('Q830149', 'A city.', 'Filer')) + conn.commit() + conn.close() + monkeypatch.setenv('NAVI_WIKI_INDEX_DB', str(wi_path)) + wiki_index.reset() _flags(monkeypatch, enabled=('has_kiwix_wiki',)) nom = {**NOMINATIM_CAFE, 'extratags': {'wikidata': 'Q830149'}} monkeypatch.setattr(pd, 'http_requests', FakeHTTP(get=lambda url, **kw: FakeResp(200, nom))) - monkeypatch.setattr(pd.wiki_client, 'enrich_via_recon', - lambda **kw: {'wiki_summary': 'A city.', 'wiki_url': 'https://en.wikipedia.org/wiki/Filer'}) client = create_app().test_client() d = client.get('/api/place/W/123').get_json() assert d['wiki_summary'] == 'A city.' diff --git a/backend/services/navi_places/wiki_client.py b/backend/services/navi_places/wiki_client.py deleted file mode 100644 index 8c57f43..0000000 --- a/backend/services/navi_places/wiki_client.py +++ /dev/null @@ -1,37 +0,0 @@ -"""HTTP client for recon's /api/wiki-enrich (PR #8). - -Replaces the in-process wiki_index.db read (the 2.1 GB DB stays in recon). -Returns the wiki enrichment fields dict, or None on no-match/error/timeout. -""" -import logging -import os - -import requests - -logger = logging.getLogger('navi_places.wiki_client') - - -def _base_url(): - return os.environ.get('RECON_BASE_URL', 'http://127.0.0.1:8420') - - -def enrich_via_recon(wikidata_id=None, name=None, country_code=None, timeout=3.0): - """GET ${RECON_BASE_URL}/api/wiki-enrich. Returns the fields dict on 200, - or None on 404 / 400 / any error / timeout (graceful — wiki enrichment is - optional).""" - params = {} - if wikidata_id: - params['wikidata'] = wikidata_id - if name and country_code: - params['name'] = name - params['country'] = country_code - if not params: - return None - try: - resp = requests.get(f"{_base_url()}/api/wiki-enrich", params=params, timeout=timeout) - if resp.status_code == 200: - return resp.json() - return None - except Exception as e: - logger.debug(f"wiki-enrich call failed: {e}") - return None diff --git a/backend/services/navi_places/wiki_index.py b/backend/services/navi_places/wiki_index.py new file mode 100644 index 0000000..0ae116c --- /dev/null +++ b/backend/services/navi_places/wiki_index.py @@ -0,0 +1,91 @@ +"""Direct read of the local wiki_index.db (wiki_places table) for navi-places. + +Ports recon's lib/place_detail.lookup_wiki_index (the /api/wiki-enrich read path) +to an in-process SQLite read, matching place_cache.py's path/conn pattern. The +2.1 GB wiki_index.db is now owned by navi-places (NAVI_WIKI_INDEX_DB). Pure +read; never raises (missing DB / errors → None so enrichment no-ops). +""" +import logging +import os +import sqlite3 + +logger = logging.getLogger('navi_places.wiki_index') + +DEFAULT_DB_PATH = '/var/lib/navi-backend/wiki_index.db' + +_db_conn = None + + +def db_path(): + return os.environ.get('NAVI_WIKI_INDEX_DB', DEFAULT_DB_PATH) + + +def _get_db(): + """Lazy module-level read-only connection. Returns None if the file is absent + (enrichment then silently no-ops). row_factory=Row for name access.""" + global _db_conn + if _db_conn is not None: + return _db_conn + path = db_path() + if not os.path.exists(path): + logger.debug(f"wiki_index.db not found at {path}") + return None + _db_conn = sqlite3.connect(path, check_same_thread=False) + _db_conn.row_factory = sqlite3.Row + logger.info(f"Wiki index DB ready at {path}") + return _db_conn + + +def reset(): + """Close + drop the cached connection (per-worker / per-test refresh).""" + global _db_conn + if _db_conn is not None: + try: + _db_conn.close() + except Exception: + pass + _db_conn = None + + +def lookup(wikidata_id=None, name=None, country_code=None): + """wikidata_id first, then name+country_code fallback — exact port of recon's + lookup_wiki_index. Returns {wiki_summary, wiki_population, wiki_url, + wikivoyage_url} (only keys present), or None on no match / no DB.""" + db = _get_db() + if not db: + return None + try: + cur = db.cursor() + row = None + if wikidata_id: + wid = wikidata_id + if isinstance(wid, str) and wid.startswith("http"): + wid = wid.split("/")[-1] + cur.execute( + "SELECT summary, wiki_population, wikipedia_title, wikivoyage_title " + "FROM wiki_places WHERE wikidata_id = ?", (wid,)) + row = cur.fetchone() + if not row and name and country_code: + cur.execute( + "SELECT summary, wiki_population, wikipedia_title, wikivoyage_title " + "FROM wiki_places WHERE place_name = ? AND country_code = ? LIMIT 1", + (name, country_code.lower())) + row = cur.fetchone() + if not row: + return None + out = {} + if row["summary"]: + out["wiki_summary"] = row["summary"] + if row["wiki_population"]: + try: + out["wiki_population"] = int(row["wiki_population"]) + except (ValueError, TypeError): + out["wiki_population"] = row["wiki_population"] + if row["wikipedia_title"]: + out["wiki_url"] = f"https://en.wikipedia.org/wiki/{row['wikipedia_title'].replace(' ', '_')}" + if row["wikivoyage_title"]: + out["wikivoyage_url"] = f"https://en.wikivoyage.org/wiki/{row['wikivoyage_title'].replace(' ', '_')}" + return out or None + except Exception as e: + logger.debug(f"wiki_index lookup error: {e}") + return None