From 65911d320c7bf3c60dca9baa5b4607c1596b9c6c Mon Sep 17 00:00:00 2001 From: malice Date: Sat, 23 May 2026 21:12:27 -0600 Subject: [PATCH] decouple: move /api/wiki-rewrite logic from recon to navi-places MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR-A of decouple #4-REWRITE — the LAST navi→recon coupling. navi-places now decides "is this wiki article in the local Kiwix mirror?" in-process instead of HTTP-calling recon's /api/wiki-rewrite. Intra-process swap, no nginx changes. Mirrors decouple #4-READ (which moved wiki_index.db reads the same way). - NEW services/navi_places/wiki_rewrite.py: verbatim port of recon's lib/wiki_rewrite.py. Only adjustments: setup_logging -> stdlib logging; KIWIX_BASE -> NAVI_KIWIX_BASE_URL env; KIWIX_PUBLIC_BASE -> NAVI_KIWIX_PUBLIC_BASE env; cache DB -> NAVI_WIKI_CACHE_DB (default /var/lib/navi-backend/wiki_cache.db, auto-created); + a reset() to match the place_cache/wiki_index per-worker pattern. No logic changes — same classify, same lazy hourly catalog refresh, same HEAD timeout, same status values (local/public/original), same cache semantics. - place_detail.py: _enrich_wiki_links_via_http -> _enrich_wiki_links; calls wiki_rewrite.rewrite_wiki_link(tag,value) (TUPLE) and unpacks it, replacing the dict-returning HTTP client. Import + docstrings updated. - app.py: wiki_rewrite.reset() per worker/test (alongside place_cache/wiki_index). - DELETE services/navi_places/wiki_rewrite_client.py (HTTP shim dead). - tests: the 2 wiki-rewrite tests now monkeypatch the local wiki_rewrite.rewrite_wiki_link (tuple) instead of the deleted client. Recon's endpoint stays live but unused until PR-B (safe co-existence). Co-authored-by: Matt Johnson Co-authored-by: Claude Opus 4.7 (1M context) --- backend/services/navi_places/app.py | 2 + backend/services/navi_places/place_detail.py | 38 +- .../services/navi_places/tests/test_place.py | 16 +- backend/services/navi_places/wiki_rewrite.py | 343 ++++++++++++++++++ .../navi_places/wiki_rewrite_client.py | 37 -- 5 files changed, 373 insertions(+), 63 deletions(-) create mode 100644 backend/services/navi_places/wiki_rewrite.py delete mode 100644 backend/services/navi_places/wiki_rewrite_client.py diff --git a/backend/services/navi_places/app.py b/backend/services/navi_places/app.py index 56b8d19..193da60 100644 --- a/backend/services/navi_places/app.py +++ b/backend/services/navi_places/app.py @@ -13,6 +13,7 @@ from . import place_route, admin from . import overture from . import place_cache from . import wiki_index +from . import wiki_rewrite from . import config as places_config @@ -31,6 +32,7 @@ def create_app(): overture.reset_pool() place_cache.reset_cache() wiki_index.reset() + wiki_rewrite.reset() places_config.reset_config() @app.before_request diff --git a/backend/services/navi_places/place_detail.py b/backend/services/navi_places/place_detail.py index 254f9b5..dcf05ac 100644 --- a/backend/services/navi_places/place_detail.py +++ b/backend/services/navi_places/place_detail.py @@ -1,11 +1,12 @@ """Place detail orchestrator — port of recon's lib/place_detail.py. Local Nominatim first, Overpass fallback, SQLite cache, then enrichment: -Overture (PostGIS) + Google Places + wiki. wiki_index enrichment is a direct -local read of navi-places' own wiki_index.db (NAVI_WIKI_INDEX_DB); the Kiwix -offline-wiki rewrite is still HTTP to recon (separate decouple): - - wiki_index summary/links -> wiki_index.lookup (local wiki_index.db) - - Kiwix offline-wiki rewrite -> wiki_rewrite_client.rewrite_via_recon (/api/wiki-rewrite, PR #9) +Overture (PostGIS) + Google Places + wiki. Both wiki steps are now in-process +local reads (no recon HTTP) — wiki_index summary from navi-places' own +wiki_index.db, and the Kiwix offline-wiki rewrite from the local Kiwix catalog ++ wiki_cache.db: + - wiki_index summary/links -> wiki_index.lookup (local wiki_index.db, NAVI_WIKI_INDEX_DB) + - Kiwix offline-wiki rewrite -> wiki_rewrite.rewrite_wiki_link (local Kiwix catalog + wiki_cache.db, NAVI_WIKI_CACHE_DB) Feature-flag gates (has_overture_enrichment / has_google_places_enrichment / has_kiwix_wiki / has_wiki_rewriting) read from the vendored profile via config.py. """ @@ -20,7 +21,7 @@ from . import config from . import overture from . import google_places from . import wiki_index -from . import wiki_rewrite_client +from . import wiki_rewrite from .osm_categories import humanize_category from .place_cache import cache_get, cache_put @@ -162,7 +163,7 @@ def _apply_google_data(result, google_data, gaps): result['extratags'] = extratags -# ── Wiki enrichment: wiki_index (local DB) + Kiwix link rewrite (HTTP to recon) ── +# ── Wiki enrichment: wiki_index (local DB) + Kiwix link rewrite (local catalog + cache) ── def _enrich_with_wiki_index(result): """Merge wiki_index fields (summary/population/urls) via a direct local read @@ -188,11 +189,11 @@ def _enrich_with_wiki_index(result): return result -def _enrich_wiki_links_via_http(result): - """Per-tag HTTP rewrite via recon /api/wiki-rewrite. Mirrors recon's - in-process _enrich_wiki_links loop: for each of the 4 wiki tag keys present - in extratags, call /api/wiki-rewrite; for any status != 'original', set - extratags[tag] = url + record under sources.wiki_rewrites[tag] = status. +def _enrich_wiki_links(result): + """Per-tag local Kiwix rewrite. Mirrors recon's in-process _enrich_wiki_links + loop: for each of the 4 wiki tag keys present in extratags, rewrite to a + local Kiwix URL when the article is mirrored; for any status != 'original', + set extratags[tag] = url + record under sources.wiki_rewrites[tag] = status. Gated on has_wiki_rewriting.""" if not config.has_feature('has_wiki_rewriting'): return result @@ -203,10 +204,10 @@ def _enrich_wiki_links_via_http(result): value = extratags.get(tag) if not value: continue - out = wiki_rewrite_client.rewrite_via_recon(tag, value) - if out.get('status') and out['status'] != 'original': - extratags[tag] = out['url'] - sources_wr[tag] = out['status'] + url, status = wiki_rewrite.rewrite_wiki_link(tag, value) + if status and status != 'original': + extratags[tag] = url + sources_wr[tag] = status return result @@ -398,10 +399,11 @@ def _parse_overpass(data, osm_type, osm_id): def _enrich_all(result, osm_type, osm_id): """Run the enrichment chain in recon's order (overture, google, wiki rewrite, - wiki index) — the two wiki steps now go over HTTP to recon.""" + wiki index). Overture (PostGIS) and Google are external upstreams; both wiki + steps are in-process local reads (no recon HTTP).""" result = _enrich_with_overture(result, osm_type, osm_id) result = _enrich_with_google(result, osm_type, osm_id) - result = _enrich_wiki_links_via_http(result) + result = _enrich_wiki_links(result) result = _enrich_with_wiki_index(result) return result diff --git a/backend/services/navi_places/tests/test_place.py b/backend/services/navi_places/tests/test_place.py index 2edd9ea..a219eff 100644 --- a/backend/services/navi_places/tests/test_place.py +++ b/backend/services/navi_places/tests/test_place.py @@ -3,8 +3,8 @@ All upstreams are mocked: Nominatim/Overpass/Wikidata via a fake http_requests on place_detail; Overture via monkeypatched overture functions; Google via the gate; wiki_index via a real tmp SQLite DB; wiki-rewrite via monkeypatched -wiki_rewrite_client. Feature flags via a stubbed config.has_feature. The cache -uses a real tmp SQLite (auto-created). +wiki_rewrite.rewrite_wiki_link. Feature flags via a stubbed config.has_feature. +The cache uses a real tmp SQLite (auto-created). """ import sqlite3 @@ -153,15 +153,15 @@ def test_overture_gated_off_no_pg_call(client, monkeypatch): assert client.get('/api/place/W/123').status_code == 200 -# ── wiki rewrite via HTTP (the new coupling) ── +# ── wiki rewrite via local Kiwix ── -def test_wiki_rewrite_via_http_local_hit(tmp_path, monkeypatch): +def test_wiki_rewrite_local_hit(tmp_path, monkeypatch): monkeypatch.setenv('NAVI_PLACE_CACHE_DB', str(tmp_path / 'pc.db')) _flags(monkeypatch, enabled=('has_wiki_rewriting',)) nom = {**NOMINATIM_CAFE, 'extratags': {'wikipedia': 'en:Filer, Idaho'}} monkeypatch.setattr(pd, 'http_requests', FakeHTTP(get=lambda url, **kw: FakeResp(200, nom))) - monkeypatch.setattr(pd.wiki_rewrite_client, 'rewrite_via_recon', - lambda tag, value, **kw: {'url': 'https://wiki.echo6.co/content/z/Filer,_Idaho', 'status': 'local'}) + monkeypatch.setattr(pd.wiki_rewrite, 'rewrite_wiki_link', + lambda tag, value: ('https://wiki.echo6.co/content/z/Filer,_Idaho', 'local')) client = create_app().test_client() d = client.get('/api/place/W/123').get_json() assert d['extratags']['wikipedia'] == 'https://wiki.echo6.co/content/z/Filer,_Idaho' @@ -173,8 +173,8 @@ def test_wiki_rewrite_original_passes_through(tmp_path, monkeypatch): _flags(monkeypatch, enabled=('has_wiki_rewriting',)) nom = {**NOMINATIM_CAFE, 'extratags': {'wikipedia': 'en:Nowhere'}} monkeypatch.setattr(pd, 'http_requests', FakeHTTP(get=lambda url, **kw: FakeResp(200, nom))) - monkeypatch.setattr(pd.wiki_rewrite_client, 'rewrite_via_recon', - lambda tag, value, **kw: {'url': value, 'status': 'original'}) + monkeypatch.setattr(pd.wiki_rewrite, 'rewrite_wiki_link', + lambda tag, value: (value, 'original')) client = create_app().test_client() d = client.get('/api/place/W/123').get_json() assert d['extratags']['wikipedia'] == 'en:Nowhere' # unchanged diff --git a/backend/services/navi_places/wiki_rewrite.py b/backend/services/navi_places/wiki_rewrite.py new file mode 100644 index 0000000..d2afbe2 --- /dev/null +++ b/backend/services/navi_places/wiki_rewrite.py @@ -0,0 +1,343 @@ +""" +Wiki link rewriter — rewrites OSM wikipedia/wikidata/wikivoyage/appropedia +links to local Kiwix URLs where the article exists in a loaded ZIM. + +Falls back silently to public URLs when article is unavailable locally. +Caches positive results only in the wiki_cache table (NAVI_WIKI_CACHE_DB). + +Kiwix catalog is parsed from the OPDS Atom feed at startup and refreshed +hourly to pick up newly loaded ZIMs without a restart. + +Ported verbatim from recon's lib/wiki_rewrite.py (decouple #4-REWRITE) — the +Kiwix base + public base + cache DB are env-configurable; logic is unchanged. + +Operations note: + - After loading a new ZIM, either restart navi-places (forces fresh catalog + fetch) or wait up to 1 hour for automatic refresh. + - To invalidate the wiki cache (e.g. after ZIM update): + sqlite3 /var/lib/navi-backend/wiki_cache.db "DELETE FROM wiki_cache;" +""" +import logging +import os +import re +import sqlite3 +import time +import xml.etree.ElementTree as ET +from urllib.parse import unquote, quote + +import requests as http_requests + +logger = logging.getLogger('navi_places.wiki_rewrite') + +# ── Configuration ─────────────────────────────────────────────────────── + +KIWIX_BASE = os.environ.get('NAVI_KIWIX_BASE_URL', 'http://localhost:8430') +KIWIX_PUBLIC_BASE = os.environ.get('NAVI_KIWIX_PUBLIC_BASE', 'https://wiki.echo6.co') +KIWIX_CATALOG_URL = f"{KIWIX_BASE}/catalog/v2/entries" +HEAD_TIMEOUT = 1.5 # seconds +CATALOG_REFRESH_INTERVAL = 3600 # 1 hour + +# OPDS Atom namespace +_ATOM_NS = "http://www.w3.org/2005/Atom" + +# ── ZIM catalog map ───────────────────────────────────────────────────── + +_zim_map = {} # source_type → content_path e.g. 'wikipedia' → 'wikipedia_en_all_maxi_2026-02' +_zim_map_ts = 0.0 # last refresh timestamp + +# Prefix-to-source-type mapping (order matters: longest prefix first) +_ZIM_PREFIX_MAP = [ + ('wikipedia_en_all', 'wikipedia'), + ('appropedia_en_all', 'appropedia'), + ('wikivoyage_en', 'wikivoyage'), + ('wikidata_en', 'wikidata'), +] + + +def _discover_zims(): + """Parse Kiwix OPDS Atom catalog to map source types to content paths.""" + global _zim_map, _zim_map_ts + + try: + resp = http_requests.get(KIWIX_CATALOG_URL, timeout=5) + if resp.status_code != 200: + logger.warning(f"Kiwix catalog returned HTTP {resp.status_code}") + return + + root = ET.fromstring(resp.content) + new_map = {} + + for entry in root.findall(f"{{{_ATOM_NS}}}entry"): + name_el = entry.find(f"{{{_ATOM_NS}}}name") + if name_el is None: + continue + book_name = name_el.text or "" + + # + content_path = None + for link in entry.findall(f"{{{_ATOM_NS}}}link"): + if link.get("type") == "text/html": + href = link.get("href", "") + if href.startswith("/content/"): + content_path = href[len("/content/"):] + break + + if not content_path: + continue + + # Match book name against known prefixes + for prefix, source_type in _ZIM_PREFIX_MAP: + if book_name.startswith(prefix): + new_map[source_type] = content_path + break + + _zim_map = new_map + _zim_map_ts = time.time() + logger.info(f"ZIM catalog refreshed: {new_map}") + + except Exception as e: + logger.warning(f"Failed to discover ZIMs from Kiwix catalog: {e}") + + +def _ensure_zim_map(): + """Lazy-load and refresh ZIM map if stale.""" + if not _zim_map or (time.time() - _zim_map_ts) > CATALOG_REFRESH_INTERVAL: + _discover_zims() + + +# ── Database (wiki_cache in NAVI_WIKI_CACHE_DB) ───────────────────────── + +DEFAULT_WIKI_CACHE_DB = '/var/lib/navi-backend/wiki_cache.db' + +_db_conn = None + + +def db_path(): + return os.environ.get('NAVI_WIKI_CACHE_DB', DEFAULT_WIKI_CACHE_DB) + + +def reset(): + """Close + drop the cached connection (per-worker / per-test refresh).""" + global _db_conn + if _db_conn is not None: + try: + _db_conn.close() + except Exception: + pass + _db_conn = None + + +def _get_db(): + """Return a module-level SQLite connection to NAVI_WIKI_CACHE_DB (lazy init, + auto-creates the wiki_cache table).""" + global _db_conn + if _db_conn is not None: + return _db_conn + + path = db_path() + os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True) + + _db_conn = sqlite3.connect(path, check_same_thread=False) + _db_conn.execute("PRAGMA journal_mode=WAL") + _db_conn.execute("PRAGMA synchronous=NORMAL") + _db_conn.execute(""" + CREATE TABLE IF NOT EXISTS wiki_cache ( + source_type TEXT NOT NULL, + article_id TEXT NOT NULL, + kiwix_url TEXT NOT NULL, + cached_at INTEGER NOT NULL, + PRIMARY KEY (source_type, article_id) + ) + """) + _db_conn.commit() + logger.info(f"Wiki cache table ready in {path}") + return _db_conn + + +# ── URL classification ────────────────────────────────────────────────── + +# Patterns for OSM wikipedia/wikidata tag values +_WIKI_TAG_RE = re.compile(r'^(?:en:)?(.+)$') # "en:Title" or just "Title" +_WIKI_URL_RE = re.compile(r'https?://en\.wikipedia\.org/wiki/(.+)') +_WIKIDATA_TAG_RE = re.compile(r'^(Q\d+)$') +_WIKIDATA_URL_RE = re.compile(r'https?://(?:www\.)?wikidata\.org/wiki/(Q\d+)') +_WIKIVOYAGE_URL_RE = re.compile(r'https?://en\.wikivoyage\.org/wiki/(.+)') +_APPROPEDIA_URL_RE = re.compile(r'https?://(?:www\.)?appropedia\.org/(?:wiki/)?(.+)') + + +def _normalize_article_id(article_id): + """Normalize article ID to MediaWiki/Kiwix convention: spaces → underscores.""" + return article_id.replace(' ', '_') + + +def classify_wiki_link(tag_name, value): + """ + Classify an OSM extratag value into (source_type, article_id) or None. + + tag_name: the extratags key ('wikipedia', 'wikidata', etc.) + value: the raw tag value from OSM + + Article IDs are normalized to MediaWiki convention (spaces → underscores). + """ + if not value or not isinstance(value, str): + return None + + value = value.strip() + + if tag_name == 'wikidata': + m = _WIKIDATA_TAG_RE.match(value) + if m: + return ('wikidata', m.group(1)) + m = _WIKIDATA_URL_RE.match(value) + if m: + return ('wikidata', m.group(1)) + return None + + if tag_name == 'wikipedia': + # URL form: https://en.wikipedia.org/wiki/Title + m = _WIKI_URL_RE.match(value) + if m: + return ('wikipedia', _normalize_article_id(unquote(m.group(1)))) + # Tag form: "en:Title" or "Title" + m = _WIKI_TAG_RE.match(value) + if m: + return ('wikipedia', _normalize_article_id(m.group(1))) + return None + + if tag_name == 'wikivoyage': + m = _WIKIVOYAGE_URL_RE.match(value) + if m: + return ('wikivoyage', _normalize_article_id(unquote(m.group(1)))) + # Plain tag: "en:Title" or "Title" + m = _WIKI_TAG_RE.match(value) + if m: + return ('wikivoyage', _normalize_article_id(m.group(1))) + return None + + if tag_name == 'appropedia': + m = _APPROPEDIA_URL_RE.match(value) + if m: + return ('appropedia', _normalize_article_id(unquote(m.group(1)))) + return ('appropedia', _normalize_article_id(value)) + + return None + + +# ── URL builders ──────────────────────────────────────────────────────── + +def build_kiwix_url(source_type, article_id): + """Build a public Kiwix URL. Returns None if source_type not in ZIM map.""" + _ensure_zim_map() + content_path = _zim_map.get(source_type) + if not content_path: + return None + return f"{KIWIX_PUBLIC_BASE}/content/{content_path}/{quote(article_id, safe='/:@!$&\'()*+,;=')}" + + +_PUBLIC_URL_TEMPLATES = { + 'wikipedia': "https://en.wikipedia.org/wiki/{id}", + 'wikidata': "https://www.wikidata.org/wiki/{id}", + 'wikivoyage': "https://en.wikivoyage.org/wiki/{id}", + 'appropedia': "https://www.appropedia.org/wiki/{id}", +} + + +def build_public_url(source_type, article_id): + """Build the canonical public URL for a wiki article.""" + tmpl = _PUBLIC_URL_TEMPLATES.get(source_type) + if not tmpl: + return None + return tmpl.format(id=quote(article_id, safe='/:@!$&\'()*+,;=')) + + +# ── Kiwix availability check ─────────────────────────────────────────── + +def check_kiwix_has_article(source_type, article_id): + """ + Check if an article exists in local Kiwix. + + Returns (bool, url): + - (True, kiwix_public_url) if article exists locally + - (False, None) if not found or Kiwix unavailable + + Only positive results are cached. + """ + # Check cache first + db = _get_db() + row = db.execute( + "SELECT kiwix_url FROM wiki_cache WHERE source_type=? AND article_id=?", + (source_type, article_id) + ).fetchone() + if row: + return (True, row[0]) + + # Build local HEAD URL + _ensure_zim_map() + content_path = _zim_map.get(source_type) + if not content_path: + return (False, None) + + head_url = f"{KIWIX_BASE}/content/{content_path}/{quote(article_id, safe='/:@!$&\'()*+,;=')}" + + try: + resp = http_requests.head(head_url, timeout=HEAD_TIMEOUT, allow_redirects=True) + if resp.status_code == 200: + kiwix_url = build_kiwix_url(source_type, article_id) + # Cache positive result + now = int(time.time()) + db.execute(""" + INSERT OR REPLACE INTO wiki_cache (source_type, article_id, kiwix_url, cached_at) + VALUES (?, ?, ?, ?) + """, (source_type, article_id, kiwix_url, now)) + db.commit() + return (True, kiwix_url) + else: + return (False, None) + except Exception as e: + logger.debug(f"Kiwix HEAD failed for {source_type}/{article_id}: {e}") + return (False, None) + + +# ── Primary entry point ──────────────────────────────────────────────── + +def rewrite_wiki_link(tag_name, value): + """ + Rewrite an OSM wiki tag value to a local Kiwix URL if available. + + Returns (url, 'local'|'public') or (None, None) if unrecognized. + """ + classified = classify_wiki_link(tag_name, value) + if not classified: + return (value, 'original') + + source_type, article_id = classified + + # Try local Kiwix + found, kiwix_url = check_kiwix_has_article(source_type, article_id) + if found and kiwix_url: + return (kiwix_url, 'local') + + # Fall back to public URL + public_url = build_public_url(source_type, article_id) + if public_url: + return (public_url, 'public') + + return (value, 'original') + + +# ── Discovery stubs (disabled, for future activation) ─────────────────── + +def discover_wikivoyage_article(name, category, lat, lon): + """ + Discover a related Wikivoyage article for a place. + Enabled by has_wiki_discovery. Currently returns None. + """ + return None + + +def discover_appropedia_article(name, category): + """ + Discover a related Appropedia article for a place. + Enabled by has_wiki_discovery. Currently returns None. + """ + return None diff --git a/backend/services/navi_places/wiki_rewrite_client.py b/backend/services/navi_places/wiki_rewrite_client.py deleted file mode 100644 index 22cb06e..0000000 --- a/backend/services/navi_places/wiki_rewrite_client.py +++ /dev/null @@ -1,37 +0,0 @@ -"""HTTP client for recon's /api/wiki-rewrite (PR #9). - -Replaces the in-process wiki_rewrite/Kiwix call (Kiwix + the wiki_cache table -stay in recon). Rewrites a single OSM wiki tag value to a local Kiwix URL. -""" -import logging -import os - -import requests - -logger = logging.getLogger('navi_places.wiki_rewrite_client') - - -def _base_url(): - return os.environ.get('RECON_BASE_URL', 'http://127.0.0.1:8420') - - -def rewrite_via_recon(tag: str, value: str, timeout: float = 3.0) -> dict: - """HTTP-call recon's /api/wiki-rewrite. On any error/timeout, return - {'url': value, 'status': 'original'} so the orchestrator can safely keep - the original extratag unchanged (mirrors _enrich_wiki_links' ImportError - self-degrade path).""" - try: - resp = requests.get( - f"{_base_url()}/api/wiki-rewrite", - params={'tag': tag, 'value': value}, - timeout=timeout, - ) - if resp.status_code == 200: - data = resp.json() - # Defensive: ensure the expected shape - if isinstance(data, dict) and 'url' in data and 'status' in data: - return data - return {'url': value, 'status': 'original'} - except Exception as e: - logger.debug(f"wiki-rewrite call failed for {tag}: {e}") - return {'url': value, 'status': 'original'}