From d612633d5ada09b5e124085f8484d361ebf615f2 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 26 May 2026 15:46:45 +0000 Subject: [PATCH] navi-places: name-based wikivoyage article discovery Places without an OSM wikivoyage tag (the entire current dataset -- 0 of placex rows carry the tag, vs 32,862 wikipedia) never got a local Wikivoyage link even when the article exists in the mirror by name. This implements the long-standing discover_wikivoyage_article stub (a never-finished placeholder ported verbatim from recon -- not disabled for any flaw) so a place name can resolve to a local Kiwix article. - wiki_rewrite.py: discover_wikivoyage_article(name, ...) normalizes the name to a MediaWiki title and runs it through the existing check_kiwix_has_article ('wikivoyage', ...) path -- same catalog discovery, HEAD probe, and positive cache as tag rewriting. Returns (kiwix_url, "local") on a HEAD 200, else (None, None). No public fallback: without an OSM tag we can't confirm a public Wikivoyage article exists, so a name miss yields no link rather than a guessed (possibly dead) public URL. - place_detail._enrich_wiki_links: after the tag-rewrite loop, when extratags has no wikivoyage value, attempt name-based discovery on result["name"] and, on a local hit, set extratags["wikivoyage"] + sources.wiki_rewrites["wikivoyage"] = "local". Tag rewrite always wins when a tag is present (discovery only fills the gap). Gated by the existing has_wiki_rewriting flag (discovery lives inside that already-enabled, flag-gated function) -- no new flag / cross-repo profile edit; the docstring-only has_wiki_discovery flag was never defined in any profile. Tests (test_place.py): finds-local (HEAD 200 -> local URL + source), 404 -> no link / no source, and runs-only-when-tag-missing (tag present -> tag rewrite wins, discovery not consulted). Full navi-places suite: 18 passed. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/services/navi_places/place_detail.py | 9 +++ .../services/navi_places/tests/test_place.py | 61 +++++++++++++++++++ backend/services/navi_places/wiki_rewrite.py | 24 ++++++-- 3 files changed, 89 insertions(+), 5 deletions(-) diff --git a/backend/services/navi_places/place_detail.py b/backend/services/navi_places/place_detail.py index 94b2ab7..92b2358 100644 --- a/backend/services/navi_places/place_detail.py +++ b/backend/services/navi_places/place_detail.py @@ -208,6 +208,15 @@ def _enrich_wiki_links(result): if status and status != 'original': extratags[tag] = url sources_wr[tag] = status + + # Name-based Wikivoyage discovery: many travel destinations are in the local + # Wikivoyage mirror without carrying an OSM wikivoyage tag (e.g. Twin Falls). + # Only when the tag is absent, try the place name against the wikivoyage ZIM. + if not extratags.get('wikivoyage'): + url, status = wiki_rewrite.discover_wikivoyage_article(result.get('name')) + if status == 'local' and url: + extratags['wikivoyage'] = url + sources_wr['wikivoyage'] = status return result diff --git a/backend/services/navi_places/tests/test_place.py b/backend/services/navi_places/tests/test_place.py index d99ab3c..ddd7813 100644 --- a/backend/services/navi_places/tests/test_place.py +++ b/backend/services/navi_places/tests/test_place.py @@ -215,6 +215,67 @@ def test_wikivoyage_tag_rewrites_to_local(tmp_path, monkeypatch): assert d['sources']['wiki_rewrites']['wikivoyage'] == 'local' +# ── name-based wikivoyage discovery (place has no OSM wikivoyage tag) ── + +def _seed_wikivoyage_zim(monkeypatch, head_status): + """Seed the wikivoyage ZIM map and mock the Kiwix HEAD probe for discovery.""" + wr = pd.wiki_rewrite + wr.reset() + monkeypatch.setattr(wr, '_ensure_zim_map', lambda: None) + monkeypatch.setattr(wr, '_zim_map', {'wikivoyage': 'wikivoyage_en_all_maxi_2026-03'}) + + class FakeKiwixHTTP: + def head(self, url, **kw): + return FakeResp(head_status) + monkeypatch.setattr(wr, 'http_requests', FakeKiwixHTTP()) + + +def test_name_based_discovery_finds_local(tmp_path, monkeypatch): + monkeypatch.setenv('NAVI_PLACE_CACHE_DB', str(tmp_path / 'pc.db')) + monkeypatch.setenv('NAVI_WIKI_CACHE_DB', str(tmp_path / 'wc.db')) + _flags(monkeypatch, enabled=('has_wiki_rewriting',)) + _seed_wikivoyage_zim(monkeypatch, head_status=200) + # No OSM wikivoyage tag; the place name ("Twin Falls") drives discovery. + nom = {**NOMINATIM_CAFE, 'localname': 'Twin Falls', 'extratags': {}} + monkeypatch.setattr(pd, 'http_requests', FakeHTTP(get=lambda url, **kw: FakeResp(200, nom))) + d = create_app().test_client().get('/api/place/W/123').get_json() + assert d['extratags']['wikivoyage'] == \ + 'https://wiki.echo6.co/content/wikivoyage_en_all_maxi_2026-03/Twin_Falls' + assert d['sources']['wiki_rewrites']['wikivoyage'] == 'local' + + +def test_name_based_discovery_404_falls_back(tmp_path, monkeypatch): + monkeypatch.setenv('NAVI_PLACE_CACHE_DB', str(tmp_path / 'pc.db')) + monkeypatch.setenv('NAVI_WIKI_CACHE_DB', str(tmp_path / 'wc.db')) + _flags(monkeypatch, enabled=('has_wiki_rewriting',)) + _seed_wikivoyage_zim(monkeypatch, head_status=404) + nom = {**NOMINATIM_CAFE, 'localname': 'Nowhere Town', 'extratags': {}} + monkeypatch.setattr(pd, 'http_requests', FakeHTTP(get=lambda url, **kw: FakeResp(200, nom))) + d = create_app().test_client().get('/api/place/W/123').get_json() + # No local article -> no link emitted (no public guess), no source recorded. + assert d['extratags'].get('wikivoyage') is None + assert 'wikivoyage' not in d.get('sources', {}).get('wiki_rewrites', {}) + + +def test_name_based_discovery_runs_only_when_tag_missing(tmp_path, monkeypatch): + monkeypatch.setenv('NAVI_PLACE_CACHE_DB', str(tmp_path / 'pc.db')) + _flags(monkeypatch, enabled=('has_wiki_rewriting',)) + # Place DOES carry an OSM wikivoyage tag -> tag rewrite wins, discovery skipped. + nom = {**NOMINATIM_CAFE, 'localname': 'Twin Falls', + 'extratags': {'wikivoyage': 'en:Twin Falls'}} + monkeypatch.setattr(pd, 'http_requests', FakeHTTP(get=lambda url, **kw: FakeResp(200, nom))) + monkeypatch.setattr(pd.wiki_rewrite, 'rewrite_wiki_link', + lambda tag, value: ('https://wiki.echo6.co/content/TAG/Twin_Falls', 'local')) + + def _no_discovery(*a, **k): + raise AssertionError('discovery must not run when the wikivoyage tag is present') + monkeypatch.setattr(pd.wiki_rewrite, 'discover_wikivoyage_article', _no_discovery) + + d = create_app().test_client().get('/api/place/W/123').get_json() + assert d['extratags']['wikivoyage'] == 'https://wiki.echo6.co/content/TAG/Twin_Falls' + assert d['sources']['wiki_rewrites']['wikivoyage'] == 'local' + + # ── wiki index summary via local wiki_index.db ── def test_wiki_enrich_via_local_db_merges_fields(tmp_path, monkeypatch): diff --git a/backend/services/navi_places/wiki_rewrite.py b/backend/services/navi_places/wiki_rewrite.py index 16bde27..0c64c97 100644 --- a/backend/services/navi_places/wiki_rewrite.py +++ b/backend/services/navi_places/wiki_rewrite.py @@ -330,12 +330,26 @@ def rewrite_wiki_link(tag_name, value): # ── Discovery stubs (disabled, for future activation) ─────────────────── -def discover_wikivoyage_article(name, category, lat, lon): +def discover_wikivoyage_article(name, category=None, lat=None, lon=None): + """Find a local Wikivoyage article by place NAME, for places that lack an OSM + wikivoyage tag (e.g. Twin Falls). Returns (kiwix_url, "local") when an article + titled ``name`` exists in the wikivoyage ZIM — verified via the same catalog + + HEAD + positive-cache path as tag rewriting — else (None, None). + + No public fallback: without an OSM tag we cannot confirm a public Wikivoyage + article exists, so guessing a public URL from the name would risk dead links; + a miss simply yields no Wikivoyage link. category/lat/lon are reserved for + future disambiguation. """ - Discover a related Wikivoyage article for a place. - Enabled by has_wiki_discovery. Currently returns None. - """ - return None + if not name or not isinstance(name, str): + return (None, None) + article_id = _normalize_article_id(name.strip()) + if not article_id: + return (None, None) + found, kiwix_url = check_kiwix_has_article('wikivoyage', article_id) + if found and kiwix_url: + return (kiwix_url, 'local') + return (None, None) def discover_appropedia_article(name, category):