From f537a552669b059769fcddb9b564d7906223a68d Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 26 May 2026 06:23:24 +0000 Subject: [PATCH] wiki_rewrite: paginate fully + extend rewriter to wikivoyage The Twin Falls Wikivoyage investigation surfaced two coordinated gaps that kept OSM wikivoyage tags from rewriting to local Kiwix URLs even though the wikivoyage ZIM is loaded and serving. Fix 1 (pagination) -- wiki_rewrite.py: append ?count=-1 to the Kiwix OPDS catalog fetch. kiwix-serve's /catalog/v2/entries defaults to the first 10 entries; the library has 17, so wikivoyage (and other page-2 ZIMs) were never seen by _discover_zims and never entered _zim_map, so their tags always fell back to public. Fix 2 (tag passthrough) -- place_detail.py: add wikivoyage to both nominatim extratags whitelists, mirroring the existing wikipedia/wikidata lines. The rewriter (classify_wiki_link / build_kiwix_url / rewrite_wiki_link) and the _enrich_wiki_links loop were already source_type-generic and covered wikivoyage; the only missing link was that the nominatim parser dropped the wikivoyage tag before enrichment ever saw it. No rewriter refactor was needed. Tests (test_place.py): - test_catalog_url_requests_full_library: the OPDS fetch URL contains count=-1. - test_wikivoyage_tag_rewrites_to_local: a wikivoyage OSM tag for a mirrored article rewrites to https://wiki.echo6.co/content/wikivoyage_en_all_maxi_/... with sources.wiki_rewrites.wikivoyage == "local" (Kiwix mocked). Full navi-places suite: 15 passed. Follow-up (separate ops step, not in this PR): prune 3 dangling library.xml entries on the Kiwix host (wikiloc.com_eng_2026-04_18, meshtastic.org_eng_2026-04_14, meshtastic.org_eng_2026-04_15) whose ZIM files are absent; kiwix-serve silently skips them. Note: Twin Falls (R/121355) itself has no OSM wikivoyage tag, so it still will not get a local Wikivoyage link from tag rewrite -- that needs the separate name-based discovery feature (discover_wikivoyage_article stub). Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/services/navi_places/place_detail.py | 2 ++ .../services/navi_places/tests/test_place.py | 34 +++++++++++++++++++ backend/services/navi_places/wiki_rewrite.py | 5 ++- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/backend/services/navi_places/place_detail.py b/backend/services/navi_places/place_detail.py index dcf05ac..94b2ab7 100644 --- a/backend/services/navi_places/place_detail.py +++ b/backend/services/navi_places/place_detail.py @@ -291,6 +291,7 @@ def _parse_nominatim(data): 'email': raw_extra.get('email') or raw_extra.get('contact:email'), 'wikipedia': raw_extra.get('wikipedia'), 'wikidata': raw_extra.get('wikidata'), + 'wikivoyage': raw_extra.get('wikivoyage'), 'cuisine': raw_extra.get('cuisine'), 'operator': raw_extra.get('operator'), 'wheelchair': raw_extra.get('wheelchair'), @@ -376,6 +377,7 @@ def _parse_overpass(data, osm_type, osm_id): 'email': tags.get('email') or tags.get('contact:email'), 'wikipedia': tags.get('wikipedia'), 'wikidata': tags.get('wikidata'), + 'wikivoyage': tags.get('wikivoyage'), 'cuisine': tags.get('cuisine'), 'operator': tags.get('operator'), 'wheelchair': tags.get('wheelchair'), diff --git a/backend/services/navi_places/tests/test_place.py b/backend/services/navi_places/tests/test_place.py index a219eff..d99ab3c 100644 --- a/backend/services/navi_places/tests/test_place.py +++ b/backend/services/navi_places/tests/test_place.py @@ -181,6 +181,40 @@ def test_wiki_rewrite_original_passes_through(tmp_path, monkeypatch): assert 'wikipedia' not in d.get('sources', {}).get('wiki_rewrites', {}) +def test_catalog_url_requests_full_library(): + # Fix 1: the OPDS fetch must pull the whole library (kiwix-serve defaults to the + # first 10 entries, which hid wikivoyage and other page-2 ZIMs from discovery). + from services.navi_places import wiki_rewrite + assert 'count=-1' in wiki_rewrite.KIWIX_CATALOG_URL + + +def test_wikivoyage_tag_rewrites_to_local(tmp_path, monkeypatch): + # Fix 2: a wikivoyage OSM tag for a mirrored article rewrites to a local Kiwix + # URL via the same source_type-generic path as wikipedia. Kiwix is mocked: the + # ZIM map is seeded with the wikivoyage book and the HEAD probe returns 200. + monkeypatch.setenv('NAVI_PLACE_CACHE_DB', str(tmp_path / 'pc.db')) + monkeypatch.setenv('NAVI_WIKI_CACHE_DB', str(tmp_path / 'wiki_cache.db')) + _flags(monkeypatch, enabled=('has_wiki_rewriting',)) + + wr = pd.wiki_rewrite + wr.reset() + monkeypatch.setattr(wr, '_ensure_zim_map', lambda: None) + monkeypatch.setattr(wr, '_zim_map', {'wikivoyage': 'wikivoyage_en_all_maxi_2026-03'}) + + class FakeKiwixHTTP: + def head(self, url, **kw): + return FakeResp(200) + monkeypatch.setattr(wr, 'http_requests', FakeKiwixHTTP()) + + nom = {**NOMINATIM_CAFE, 'extratags': {'wikivoyage': 'en:Twin Falls'}} + monkeypatch.setattr(pd, 'http_requests', FakeHTTP(get=lambda url, **kw: FakeResp(200, nom))) + + d = create_app().test_client().get('/api/place/W/123').get_json() + assert d['extratags']['wikivoyage'] == \ + 'https://wiki.echo6.co/content/wikivoyage_en_all_maxi_2026-03/Twin_Falls' + assert d['sources']['wiki_rewrites']['wikivoyage'] == 'local' + + # ── wiki index summary via local wiki_index.db ── def test_wiki_enrich_via_local_db_merges_fields(tmp_path, monkeypatch): diff --git a/backend/services/navi_places/wiki_rewrite.py b/backend/services/navi_places/wiki_rewrite.py index d2afbe2..16bde27 100644 --- a/backend/services/navi_places/wiki_rewrite.py +++ b/backend/services/navi_places/wiki_rewrite.py @@ -33,7 +33,10 @@ logger = logging.getLogger('navi_places.wiki_rewrite') KIWIX_BASE = os.environ.get('NAVI_KIWIX_BASE_URL', 'http://localhost:8430') KIWIX_PUBLIC_BASE = os.environ.get('NAVI_KIWIX_PUBLIC_BASE', 'https://wiki.echo6.co') -KIWIX_CATALOG_URL = f"{KIWIX_BASE}/catalog/v2/entries" +# count=-1 returns the full library; kiwix-serve OPDS defaults to only the first +# 10 entries, which silently hid ZIMs past page 1 (e.g. wikivoyage) from +# _discover_zims so their tags never rewrote to local. +KIWIX_CATALOG_URL = f"{KIWIX_BASE}/catalog/v2/entries?count=-1" HEAD_TIMEOUT = 1.5 # seconds CATALOG_REFRESH_INTERVAL = 3600 # 1 hour