decouple: read wiki_index.db directly in navi-places (drop /api/wiki-enrich HTTP)

PR-A of decouple #4-READ. navi-places now reads its own wiki_index.db directly
(NAVI_WIKI_INDEX_DB) instead of HTTP-calling recon's /api/wiki-enrich — same
pattern it already uses for place_cache.db. The 2.1GB DB was copied to
/var/lib/navi-backend/wiki_index.db out-of-band (5,061,763 rows verified).

- NEW services/navi_places/wiki_index.py: verbatim port of recon's
  lookup_wiki_index + _get_wiki_index_db, reading NAVI_WIKI_INDEX_DB, mirroring
  place_cache.py's db_path()/lazy-conn/reset() pattern. Returns the same
  {wiki_summary, wiki_population, wiki_url, wikivoyage_url} shape /api/wiki-enrich
  did, so it's a drop-in for the HTTP client.
- place_detail.py: _enrich_with_wiki_via_http -> _enrich_with_wiki_index; call
  wiki_index.lookup() instead of wiki_client.enrich_via_recon(); docstrings.
- app.py: wiki_index.reset() per worker/test (alongside place_cache.reset_cache()).
- admin.py: drop the recon-wiki-enrich dependency probe; add NAVI_WIKI_INDEX_DB
  env + a read-only filesystem entry. (recon-wiki-rewrite probe kept — separate
  decouple.)
- DELETE wiki_client.py (fully replaced).
- test_place.py: convert the wiki test from a monkeypatched HTTP client to a
  hermetic tmp wiki_index.db.

Internal localhost migration — no nginx/edge involvement. recon's /api/wiki-enrich
stays live until PR-B (deploy PR-A first so nothing calls the route after removal).

Co-authored-by: Matt Johnson <mj@k7zvx.com>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

backend/services/navi_places/admin.py

@@ -13,6 +13,7 @@ from shared.admin_info import build_info_response, mask_key
 
 from . import overture
 from . import place_cache
+from . import wiki_index
 
 bp = Blueprint('places_admin', __name__)
 
@@ -56,6 +57,7 @@ def _recon_probe(name, path):
 def navi_places_info():
     metrics = current_app.config['METRICS']
     cache_path = place_cache.db_path()
+    wiki_path = wiki_index.db_path()
     # Two real secrets -> mask_key. The rest are non-secret paths/URLs/params.
     info = build_info_response(
         service='navi-places',
@@ -71,11 +73,11 @@ def navi_places_info():
             {'name': 'GOOGLE_PLACES_API_KEY', 'value': mask_key(os.environ.get('GOOGLE_PLACES_API_KEY'))},
             {'name': 'RECON_BASE_URL', 'value': _recon_base()},
             {'name': 'NAVI_PLACE_CACHE_DB', 'value': cache_path},
+            {'name': 'NAVI_WIKI_INDEX_DB', 'value': wiki_path},
             {'name': 'NAVI_PROFILES_DIR', 'value': os.environ.get('NAVI_PROFILES_DIR', '(default vendored)')},
         ],
         dependencies=[
             _overture_dependency(),
-            _recon_probe('recon-wiki-enrich', '/api/wiki-enrich'),
             _recon_probe('recon-wiki-rewrite', '/api/wiki-rewrite'),
         ],
         filesystem=[{
@@ -83,6 +85,10 @@ def navi_places_info():
             'exists': os.path.exists(cache_path),
             'readable': os.access(cache_path, os.R_OK),
             'writable': os.access(cache_path, os.W_OK),
+        }, {
+            'path': wiki_path,
+            'exists': os.path.exists(wiki_path),
+            'readable': os.access(wiki_path, os.R_OK),
         }],
         runtime={
             'uptime_s': round(time.time() - metrics['start_time'], 1),

backend/services/navi_places/app.py

@@ -12,6 +12,7 @@ from shared.git_sha import git_short_sha
 from . import place_route, admin
 from . import overture
 from . import place_cache
+from . import wiki_index
 from . import config as places_config
 
 
@@ -29,6 +30,7 @@ def create_app():
     # gunicorn worker (and each test) picks up the current env.
     overture.reset_pool()
     place_cache.reset_cache()
+    wiki_index.reset()
     places_config.reset_config()
 
     @app.before_request

backend/services/navi_places/place_detail.py

@@ -1,9 +1,10 @@
 """Place detail orchestrator — port of recon's lib/place_detail.py.
 
 Local Nominatim first, Overpass fallback, SQLite cache, then enrichment:
-Overture (PostGIS) + Google Places + wiki. The two wiki paths are now HTTP to
-recon (the 2.1 GB wiki_index.db and Kiwix/wiki_cache stay in recon):
-  - wiki_index summary/links  -> wiki_client.enrich_via_recon  (/api/wiki-enrich, PR #8)
+Overture (PostGIS) + Google Places + wiki. wiki_index enrichment is a direct
+local read of navi-places' own wiki_index.db (NAVI_WIKI_INDEX_DB); the Kiwix
+offline-wiki rewrite is still HTTP to recon (separate decouple):
+  - wiki_index summary/links  -> wiki_index.lookup  (local wiki_index.db)
   - Kiwix offline-wiki rewrite -> wiki_rewrite_client.rewrite_via_recon (/api/wiki-rewrite, PR #9)
 Feature-flag gates (has_overture_enrichment / has_google_places_enrichment /
 has_kiwix_wiki / has_wiki_rewriting) read from the vendored profile via config.py.
@@ -18,7 +19,7 @@ from shared.auth import get_user_id
 from . import config
 from . import overture
 from . import google_places
-from . import wiki_client
+from . import wiki_index
 from . import wiki_rewrite_client
 from .osm_categories import humanize_category
 from .place_cache import cache_get, cache_put
@@ -161,11 +162,11 @@ def _apply_google_data(result, google_data, gaps):
     result['extratags'] = extratags
 
 
-# ── Wiki enrichment via HTTP to recon (replaces in-process wiki_index/Kiwix) ──
+# ── Wiki enrichment: wiki_index (local DB) + Kiwix link rewrite (HTTP to recon) ──
 
-def _enrich_with_wiki_via_http(result):
-    """Merge wiki_index fields (summary/population/urls) via recon /api/wiki-enrich.
-    Replaces recon's in-process _enrich_with_wiki_index. Gated on has_kiwix_wiki."""
+def _enrich_with_wiki_index(result):
+    """Merge wiki_index fields (summary/population/urls) via a direct local read
+    of wiki_index.db. Port of recon's in-process lookup. Gated on has_kiwix_wiki."""
     if not config.has_feature('has_kiwix_wiki'):
         return result
 
@@ -177,7 +178,7 @@ def _enrich_with_wiki_via_http(result):
     name = result.get('name')
     country_code = address.get('country_code') or result.get('country_code')
 
-    fields = wiki_client.enrich_via_recon(
+    fields = wiki_index.lookup(
         wikidata_id=wikidata_id, name=name, country_code=country_code
     )
     if fields:
@@ -401,7 +402,7 @@ def _enrich_all(result, osm_type, osm_id):
     result = _enrich_with_overture(result, osm_type, osm_id)
     result = _enrich_with_google(result, osm_type, osm_id)
     result = _enrich_wiki_links_via_http(result)
-    result = _enrich_with_wiki_via_http(result)
+    result = _enrich_with_wiki_index(result)
     return result
 
 
@@ -562,7 +563,7 @@ def get_place_by_wikidata(wikidata_id):
                 logger.debug(f"Wikidata boundary fetch failed: {e}")
         result["boundary"] = boundary
 
-        result = _enrich_with_wiki_via_http(result)
+        result = _enrich_with_wiki_index(result)
         logger.debug(f"Wikidata hit: {wikidata_id} -> {name}")
         return result, 200
 

backend/services/navi_places/tests/test_place.py

@@ -2,13 +2,17 @@
 
 All upstreams are mocked: Nominatim/Overpass/Wikidata via a fake http_requests
 on place_detail; Overture via monkeypatched overture functions; Google via the
-gate; wiki via monkeypatched wiki_client / wiki_rewrite_client. Feature flags via
-a stubbed config.has_feature. The cache uses a real tmp SQLite (auto-created).
+gate; wiki_index via a real tmp SQLite DB; wiki-rewrite via monkeypatched
+wiki_rewrite_client. Feature flags via a stubbed config.has_feature. The cache
+uses a real tmp SQLite (auto-created).
 """
+import sqlite3
+
 import pytest
 
 import services.navi_places.place_detail as pd
 import services.navi_places.place_cache as place_cache
+import services.navi_places.wiki_index as wiki_index
 from services.navi_places.app import create_app
 
 
@@ -177,15 +181,27 @@ def test_wiki_rewrite_original_passes_through(tmp_path, monkeypatch):
     assert 'wikipedia' not in d.get('sources', {}).get('wiki_rewrites', {})
 
 
-# ── wiki index summary via HTTP ──
+# ── wiki index summary via local wiki_index.db ──
 
-def test_wiki_enrich_via_http_merges_fields(tmp_path, monkeypatch):
+def test_wiki_enrich_via_local_db_merges_fields(tmp_path, monkeypatch):
     monkeypatch.setenv('NAVI_PLACE_CACHE_DB', str(tmp_path / 'pc.db'))
+    # Hermetic wiki_index.db: one wiki_places row keyed by wikidata_id.
+    wi_path = tmp_path / 'wi.db'
+    conn = sqlite3.connect(str(wi_path))
+    conn.execute(
+        "CREATE TABLE wiki_places (wikidata_id TEXT, place_name TEXT, "
+        "country_code TEXT, summary TEXT, wiki_population INTEGER, "
+        "wikipedia_title TEXT, wikivoyage_title TEXT)")
+    conn.execute(
+        "INSERT INTO wiki_places (wikidata_id, summary, wikipedia_title) VALUES (?,?,?)",
+        ('Q830149', 'A city.', 'Filer'))
+    conn.commit()
+    conn.close()
+    monkeypatch.setenv('NAVI_WIKI_INDEX_DB', str(wi_path))
+    wiki_index.reset()
     _flags(monkeypatch, enabled=('has_kiwix_wiki',))
     nom = {**NOMINATIM_CAFE, 'extratags': {'wikidata': 'Q830149'}}
     monkeypatch.setattr(pd, 'http_requests', FakeHTTP(get=lambda url, **kw: FakeResp(200, nom)))
-    monkeypatch.setattr(pd.wiki_client, 'enrich_via_recon',
-                        lambda **kw: {'wiki_summary': 'A city.', 'wiki_url': 'https://en.wikipedia.org/wiki/Filer'})
     client = create_app().test_client()
     d = client.get('/api/place/W/123').get_json()
     assert d['wiki_summary'] == 'A city.'

backend/services/navi_places/wiki_client.py

@@ -1,37 +0,0 @@
-"""HTTP client for recon's /api/wiki-enrich (PR #8).
-
-Replaces the in-process wiki_index.db read (the 2.1 GB DB stays in recon).
-Returns the wiki enrichment fields dict, or None on no-match/error/timeout.
-"""
-import logging
-import os
-
-import requests
-
-logger = logging.getLogger('navi_places.wiki_client')
-
-
-def _base_url():
-    return os.environ.get('RECON_BASE_URL', 'http://127.0.0.1:8420')
-
-
-def enrich_via_recon(wikidata_id=None, name=None, country_code=None, timeout=3.0):
-    """GET ${RECON_BASE_URL}/api/wiki-enrich. Returns the fields dict on 200,
-    or None on 404 / 400 / any error / timeout (graceful — wiki enrichment is
-    optional)."""
-    params = {}
-    if wikidata_id:
-        params['wikidata'] = wikidata_id
-    if name and country_code:
-        params['name'] = name
-        params['country'] = country_code
-    if not params:
-        return None
-    try:
-        resp = requests.get(f"{_base_url()}/api/wiki-enrich", params=params, timeout=timeout)
-        if resp.status_code == 200:
-            return resp.json()
-        return None
-    except Exception as e:
-        logger.debug(f"wiki-enrich call failed: {e}")
-        return None

backend/services/navi_places/wiki_index.py

@@ -0,0 +1,91 @@
+"""Direct read of the local wiki_index.db (wiki_places table) for navi-places.
+
+Ports recon's lib/place_detail.lookup_wiki_index (the /api/wiki-enrich read path)
+to an in-process SQLite read, matching place_cache.py's path/conn pattern. The
+2.1 GB wiki_index.db is now owned by navi-places (NAVI_WIKI_INDEX_DB). Pure
+read; never raises (missing DB / errors → None so enrichment no-ops).
+"""
+import logging
+import os
+import sqlite3
+
+logger = logging.getLogger('navi_places.wiki_index')
+
+DEFAULT_DB_PATH = '/var/lib/navi-backend/wiki_index.db'
+
+_db_conn = None
+
+
+def db_path():
+    return os.environ.get('NAVI_WIKI_INDEX_DB', DEFAULT_DB_PATH)
+
+
+def _get_db():
+    """Lazy module-level read-only connection. Returns None if the file is absent
+    (enrichment then silently no-ops). row_factory=Row for name access."""
+    global _db_conn
+    if _db_conn is not None:
+        return _db_conn
+    path = db_path()
+    if not os.path.exists(path):
+        logger.debug(f"wiki_index.db not found at {path}")
+        return None
+    _db_conn = sqlite3.connect(path, check_same_thread=False)
+    _db_conn.row_factory = sqlite3.Row
+    logger.info(f"Wiki index DB ready at {path}")
+    return _db_conn
+
+
+def reset():
+    """Close + drop the cached connection (per-worker / per-test refresh)."""
+    global _db_conn
+    if _db_conn is not None:
+        try:
+            _db_conn.close()
+        except Exception:
+            pass
+    _db_conn = None
+
+
+def lookup(wikidata_id=None, name=None, country_code=None):
+    """wikidata_id first, then name+country_code fallback — exact port of recon's
+    lookup_wiki_index. Returns {wiki_summary, wiki_population, wiki_url,
+    wikivoyage_url} (only keys present), or None on no match / no DB."""
+    db = _get_db()
+    if not db:
+        return None
+    try:
+        cur = db.cursor()
+        row = None
+        if wikidata_id:
+            wid = wikidata_id
+            if isinstance(wid, str) and wid.startswith("http"):
+                wid = wid.split("/")[-1]
+            cur.execute(
+                "SELECT summary, wiki_population, wikipedia_title, wikivoyage_title "
+                "FROM wiki_places WHERE wikidata_id = ?", (wid,))
+            row = cur.fetchone()
+        if not row and name and country_code:
+            cur.execute(
+                "SELECT summary, wiki_population, wikipedia_title, wikivoyage_title "
+                "FROM wiki_places WHERE place_name = ? AND country_code = ? LIMIT 1",
+                (name, country_code.lower()))
+            row = cur.fetchone()
+        if not row:
+            return None
+        out = {}
+        if row["summary"]:
+            out["wiki_summary"] = row["summary"]
+        if row["wiki_population"]:
+            try:
+                out["wiki_population"] = int(row["wiki_population"])
+            except (ValueError, TypeError):
+                out["wiki_population"] = row["wiki_population"]
+        if row["wikipedia_title"]:
+            out["wiki_url"] = f"https://en.wikipedia.org/wiki/{row['wikipedia_title'].replace(' ', '_')}"
+        if row["wikivoyage_title"]:
+            out["wikivoyage_url"] = f"https://en.wikivoyage.org/wiki/{row['wikivoyage_title'].replace(' ', '_')}"
+        return out or None
+    except Exception as e:
+        logger.debug(f"wiki_index lookup error: {e}")
+        return None