From 3a9bc624c8263abecb24649a7b4c316cdb4941ea Mon Sep 17 00:00:00 2001 From: malice Date: Tue, 26 May 2026 00:42:06 -0600 Subject: [PATCH] wiki_rewrite: paginate fully + extend rewriter to wikivoyage (#29) The Twin Falls Wikivoyage investigation surfaced two coordinated gaps that kept OSM wikivoyage tags from rewriting to local Kiwix URLs even though the wikivoyage ZIM is loaded and serving. Fix 1 (pagination) -- wiki_rewrite.py: append ?count=-1 to the Kiwix OPDS catalog fetch. kiwix-serve's /catalog/v2/entries defaults to the first 10 entries; the library has 17, so wikivoyage (and other page-2 ZIMs) were never seen by _discover_zims and never entered _zim_map, so their tags always fell back to public. Fix 2 (tag passthrough) -- place_detail.py: add wikivoyage to both nominatim extratags whitelists, mirroring the existing wikipedia/wikidata lines. The rewriter (classify_wiki_link / build_kiwix_url / rewrite_wiki_link) and the _enrich_wiki_links loop were already source_type-generic and covered wikivoyage; the only missing link was that the nominatim parser dropped the wikivoyage tag before enrichment ever saw it. No rewriter refactor was needed. Tests (test_place.py): - test_catalog_url_requests_full_library: the OPDS fetch URL contains count=-1. - test_wikivoyage_tag_rewrites_to_local: a wikivoyage OSM tag for a mirrored article rewrites to https://wiki.echo6.co/content/wikivoyage_en_all_maxi_/... with sources.wiki_rewrites.wikivoyage == "local" (Kiwix mocked). Full navi-places suite: 15 passed. Follow-up (separate ops step, not in this PR): prune 3 dangling library.xml entries on the Kiwix host (wikiloc.com_eng_2026-04_18, meshtastic.org_eng_2026-04_14, meshtastic.org_eng_2026-04_15) whose ZIM files are absent; kiwix-serve silently skips them. Note: Twin Falls (R/121355) itself has no OSM wikivoyage tag, so it still will not get a local Wikivoyage link from tag rewrite -- that needs the separate name-based discovery feature (discover_wikivoyage_article stub). Co-authored-by: Matt Co-authored-by: Claude Opus 4.7 (1M context) --- backend/services/navi_places/place_detail.py | 2 ++ .../services/navi_places/tests/test_place.py | 34 +++++++++++++++++++ backend/services/navi_places/wiki_rewrite.py | 5 ++- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/backend/services/navi_places/place_detail.py b/backend/services/navi_places/place_detail.py index dcf05ac..94b2ab7 100644 --- a/backend/services/navi_places/place_detail.py +++ b/backend/services/navi_places/place_detail.py @@ -291,6 +291,7 @@ def _parse_nominatim(data): 'email': raw_extra.get('email') or raw_extra.get('contact:email'), 'wikipedia': raw_extra.get('wikipedia'), 'wikidata': raw_extra.get('wikidata'), + 'wikivoyage': raw_extra.get('wikivoyage'), 'cuisine': raw_extra.get('cuisine'), 'operator': raw_extra.get('operator'), 'wheelchair': raw_extra.get('wheelchair'), @@ -376,6 +377,7 @@ def _parse_overpass(data, osm_type, osm_id): 'email': tags.get('email') or tags.get('contact:email'), 'wikipedia': tags.get('wikipedia'), 'wikidata': tags.get('wikidata'), + 'wikivoyage': tags.get('wikivoyage'), 'cuisine': tags.get('cuisine'), 'operator': tags.get('operator'), 'wheelchair': tags.get('wheelchair'), diff --git a/backend/services/navi_places/tests/test_place.py b/backend/services/navi_places/tests/test_place.py index a219eff..d99ab3c 100644 --- a/backend/services/navi_places/tests/test_place.py +++ b/backend/services/navi_places/tests/test_place.py @@ -181,6 +181,40 @@ def test_wiki_rewrite_original_passes_through(tmp_path, monkeypatch): assert 'wikipedia' not in d.get('sources', {}).get('wiki_rewrites', {}) +def test_catalog_url_requests_full_library(): + # Fix 1: the OPDS fetch must pull the whole library (kiwix-serve defaults to the + # first 10 entries, which hid wikivoyage and other page-2 ZIMs from discovery). + from services.navi_places import wiki_rewrite + assert 'count=-1' in wiki_rewrite.KIWIX_CATALOG_URL + + +def test_wikivoyage_tag_rewrites_to_local(tmp_path, monkeypatch): + # Fix 2: a wikivoyage OSM tag for a mirrored article rewrites to a local Kiwix + # URL via the same source_type-generic path as wikipedia. Kiwix is mocked: the + # ZIM map is seeded with the wikivoyage book and the HEAD probe returns 200. + monkeypatch.setenv('NAVI_PLACE_CACHE_DB', str(tmp_path / 'pc.db')) + monkeypatch.setenv('NAVI_WIKI_CACHE_DB', str(tmp_path / 'wiki_cache.db')) + _flags(monkeypatch, enabled=('has_wiki_rewriting',)) + + wr = pd.wiki_rewrite + wr.reset() + monkeypatch.setattr(wr, '_ensure_zim_map', lambda: None) + monkeypatch.setattr(wr, '_zim_map', {'wikivoyage': 'wikivoyage_en_all_maxi_2026-03'}) + + class FakeKiwixHTTP: + def head(self, url, **kw): + return FakeResp(200) + monkeypatch.setattr(wr, 'http_requests', FakeKiwixHTTP()) + + nom = {**NOMINATIM_CAFE, 'extratags': {'wikivoyage': 'en:Twin Falls'}} + monkeypatch.setattr(pd, 'http_requests', FakeHTTP(get=lambda url, **kw: FakeResp(200, nom))) + + d = create_app().test_client().get('/api/place/W/123').get_json() + assert d['extratags']['wikivoyage'] == \ + 'https://wiki.echo6.co/content/wikivoyage_en_all_maxi_2026-03/Twin_Falls' + assert d['sources']['wiki_rewrites']['wikivoyage'] == 'local' + + # ── wiki index summary via local wiki_index.db ── def test_wiki_enrich_via_local_db_merges_fields(tmp_path, monkeypatch): diff --git a/backend/services/navi_places/wiki_rewrite.py b/backend/services/navi_places/wiki_rewrite.py index d2afbe2..16bde27 100644 --- a/backend/services/navi_places/wiki_rewrite.py +++ b/backend/services/navi_places/wiki_rewrite.py @@ -33,7 +33,10 @@ logger = logging.getLogger('navi_places.wiki_rewrite') KIWIX_BASE = os.environ.get('NAVI_KIWIX_BASE_URL', 'http://localhost:8430') KIWIX_PUBLIC_BASE = os.environ.get('NAVI_KIWIX_PUBLIC_BASE', 'https://wiki.echo6.co') -KIWIX_CATALOG_URL = f"{KIWIX_BASE}/catalog/v2/entries" +# count=-1 returns the full library; kiwix-serve OPDS defaults to only the first +# 10 entries, which silently hid ZIMs past page 1 (e.g. wikivoyage) from +# _discover_zims so their tags never rewrote to local. +KIWIX_CATALOG_URL = f"{KIWIX_BASE}/catalog/v2/entries?count=-1" HEAD_TIMEOUT = 1.5 # seconds CATALOG_REFRESH_INTERVAL = 3600 # 1 hour