mirror of
https://github.com/zvx-echo6/navi.git
synced 2026-08-26 17:31:37 +00:00
decouple: move /api/wiki-rewrite logic from recon to navi-places
PR-A of decouple #4-REWRITE — the LAST navi→recon coupling. navi-places now decides "is this wiki article in the local Kiwix mirror?" in-process instead of HTTP-calling recon's /api/wiki-rewrite. Intra-process swap, no nginx changes. Mirrors decouple #4-READ (which moved wiki_index.db reads the same way). - NEW services/navi_places/wiki_rewrite.py: verbatim port of recon's lib/wiki_rewrite.py. Only adjustments: setup_logging -> stdlib logging; KIWIX_BASE -> NAVI_KIWIX_BASE_URL env; KIWIX_PUBLIC_BASE -> NAVI_KIWIX_PUBLIC_BASE env; cache DB -> NAVI_WIKI_CACHE_DB (default /var/lib/navi-backend/wiki_cache.db, auto-created); + a reset() to match the place_cache/wiki_index per-worker pattern. No logic changes — same classify, same lazy hourly catalog refresh, same HEAD timeout, same status values (local/public/original), same cache semantics. - place_detail.py: _enrich_wiki_links_via_http -> _enrich_wiki_links; calls wiki_rewrite.rewrite_wiki_link(tag,value) (TUPLE) and unpacks it, replacing the dict-returning HTTP client. Import + docstrings updated. - app.py: wiki_rewrite.reset() per worker/test (alongside place_cache/wiki_index). - DELETE services/navi_places/wiki_rewrite_client.py (HTTP shim dead). - tests: the 2 wiki-rewrite tests now monkeypatch the local wiki_rewrite.rewrite_wiki_link (tuple) instead of the deleted client. Recon's endpoint stays live but unused until PR-B (safe co-existence). Co-authored-by: Matt Johnson <mj@k7zvx.com> Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
GitHub
parent
8b59284158
commit
65911d320c
5 changed files with 373 additions and 63 deletions
|
|
@ -13,6 +13,7 @@ from . import place_route, admin
|
|||
from . import overture
|
||||
from . import place_cache
|
||||
from . import wiki_index
|
||||
from . import wiki_rewrite
|
||||
from . import config as places_config
|
||||
|
||||
|
||||
|
|
@ -31,6 +32,7 @@ def create_app():
|
|||
overture.reset_pool()
|
||||
place_cache.reset_cache()
|
||||
wiki_index.reset()
|
||||
wiki_rewrite.reset()
|
||||
places_config.reset_config()
|
||||
|
||||
@app.before_request
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
"""Place detail orchestrator — port of recon's lib/place_detail.py.
|
||||
|
||||
Local Nominatim first, Overpass fallback, SQLite cache, then enrichment:
|
||||
Overture (PostGIS) + Google Places + wiki. wiki_index enrichment is a direct
|
||||
local read of navi-places' own wiki_index.db (NAVI_WIKI_INDEX_DB); the Kiwix
|
||||
offline-wiki rewrite is still HTTP to recon (separate decouple):
|
||||
- wiki_index summary/links -> wiki_index.lookup (local wiki_index.db)
|
||||
- Kiwix offline-wiki rewrite -> wiki_rewrite_client.rewrite_via_recon (/api/wiki-rewrite, PR #9)
|
||||
Overture (PostGIS) + Google Places + wiki. Both wiki steps are now in-process
|
||||
local reads (no recon HTTP) — wiki_index summary from navi-places' own
|
||||
wiki_index.db, and the Kiwix offline-wiki rewrite from the local Kiwix catalog
|
||||
+ wiki_cache.db:
|
||||
- wiki_index summary/links -> wiki_index.lookup (local wiki_index.db, NAVI_WIKI_INDEX_DB)
|
||||
- Kiwix offline-wiki rewrite -> wiki_rewrite.rewrite_wiki_link (local Kiwix catalog + wiki_cache.db, NAVI_WIKI_CACHE_DB)
|
||||
Feature-flag gates (has_overture_enrichment / has_google_places_enrichment /
|
||||
has_kiwix_wiki / has_wiki_rewriting) read from the vendored profile via config.py.
|
||||
"""
|
||||
|
|
@ -20,7 +21,7 @@ from . import config
|
|||
from . import overture
|
||||
from . import google_places
|
||||
from . import wiki_index
|
||||
from . import wiki_rewrite_client
|
||||
from . import wiki_rewrite
|
||||
from .osm_categories import humanize_category
|
||||
from .place_cache import cache_get, cache_put
|
||||
|
||||
|
|
@ -162,7 +163,7 @@ def _apply_google_data(result, google_data, gaps):
|
|||
result['extratags'] = extratags
|
||||
|
||||
|
||||
# ── Wiki enrichment: wiki_index (local DB) + Kiwix link rewrite (HTTP to recon) ──
|
||||
# ── Wiki enrichment: wiki_index (local DB) + Kiwix link rewrite (local catalog + cache) ──
|
||||
|
||||
def _enrich_with_wiki_index(result):
|
||||
"""Merge wiki_index fields (summary/population/urls) via a direct local read
|
||||
|
|
@ -188,11 +189,11 @@ def _enrich_with_wiki_index(result):
|
|||
return result
|
||||
|
||||
|
||||
def _enrich_wiki_links_via_http(result):
|
||||
"""Per-tag HTTP rewrite via recon /api/wiki-rewrite. Mirrors recon's
|
||||
in-process _enrich_wiki_links loop: for each of the 4 wiki tag keys present
|
||||
in extratags, call /api/wiki-rewrite; for any status != 'original', set
|
||||
extratags[tag] = url + record under sources.wiki_rewrites[tag] = status.
|
||||
def _enrich_wiki_links(result):
|
||||
"""Per-tag local Kiwix rewrite. Mirrors recon's in-process _enrich_wiki_links
|
||||
loop: for each of the 4 wiki tag keys present in extratags, rewrite to a
|
||||
local Kiwix URL when the article is mirrored; for any status != 'original',
|
||||
set extratags[tag] = url + record under sources.wiki_rewrites[tag] = status.
|
||||
Gated on has_wiki_rewriting."""
|
||||
if not config.has_feature('has_wiki_rewriting'):
|
||||
return result
|
||||
|
|
@ -203,10 +204,10 @@ def _enrich_wiki_links_via_http(result):
|
|||
value = extratags.get(tag)
|
||||
if not value:
|
||||
continue
|
||||
out = wiki_rewrite_client.rewrite_via_recon(tag, value)
|
||||
if out.get('status') and out['status'] != 'original':
|
||||
extratags[tag] = out['url']
|
||||
sources_wr[tag] = out['status']
|
||||
url, status = wiki_rewrite.rewrite_wiki_link(tag, value)
|
||||
if status and status != 'original':
|
||||
extratags[tag] = url
|
||||
sources_wr[tag] = status
|
||||
return result
|
||||
|
||||
|
||||
|
|
@ -398,10 +399,11 @@ def _parse_overpass(data, osm_type, osm_id):
|
|||
|
||||
def _enrich_all(result, osm_type, osm_id):
|
||||
"""Run the enrichment chain in recon's order (overture, google, wiki rewrite,
|
||||
wiki index) — the two wiki steps now go over HTTP to recon."""
|
||||
wiki index). Overture (PostGIS) and Google are external upstreams; both wiki
|
||||
steps are in-process local reads (no recon HTTP)."""
|
||||
result = _enrich_with_overture(result, osm_type, osm_id)
|
||||
result = _enrich_with_google(result, osm_type, osm_id)
|
||||
result = _enrich_wiki_links_via_http(result)
|
||||
result = _enrich_wiki_links(result)
|
||||
result = _enrich_with_wiki_index(result)
|
||||
return result
|
||||
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@
|
|||
All upstreams are mocked: Nominatim/Overpass/Wikidata via a fake http_requests
|
||||
on place_detail; Overture via monkeypatched overture functions; Google via the
|
||||
gate; wiki_index via a real tmp SQLite DB; wiki-rewrite via monkeypatched
|
||||
wiki_rewrite_client. Feature flags via a stubbed config.has_feature. The cache
|
||||
uses a real tmp SQLite (auto-created).
|
||||
wiki_rewrite.rewrite_wiki_link. Feature flags via a stubbed config.has_feature.
|
||||
The cache uses a real tmp SQLite (auto-created).
|
||||
"""
|
||||
import sqlite3
|
||||
|
||||
|
|
@ -153,15 +153,15 @@ def test_overture_gated_off_no_pg_call(client, monkeypatch):
|
|||
assert client.get('/api/place/W/123').status_code == 200
|
||||
|
||||
|
||||
# ── wiki rewrite via HTTP (the new coupling) ──
|
||||
# ── wiki rewrite via local Kiwix ──
|
||||
|
||||
def test_wiki_rewrite_via_http_local_hit(tmp_path, monkeypatch):
|
||||
def test_wiki_rewrite_local_hit(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv('NAVI_PLACE_CACHE_DB', str(tmp_path / 'pc.db'))
|
||||
_flags(monkeypatch, enabled=('has_wiki_rewriting',))
|
||||
nom = {**NOMINATIM_CAFE, 'extratags': {'wikipedia': 'en:Filer, Idaho'}}
|
||||
monkeypatch.setattr(pd, 'http_requests', FakeHTTP(get=lambda url, **kw: FakeResp(200, nom)))
|
||||
monkeypatch.setattr(pd.wiki_rewrite_client, 'rewrite_via_recon',
|
||||
lambda tag, value, **kw: {'url': 'https://wiki.echo6.co/content/z/Filer,_Idaho', 'status': 'local'})
|
||||
monkeypatch.setattr(pd.wiki_rewrite, 'rewrite_wiki_link',
|
||||
lambda tag, value: ('https://wiki.echo6.co/content/z/Filer,_Idaho', 'local'))
|
||||
client = create_app().test_client()
|
||||
d = client.get('/api/place/W/123').get_json()
|
||||
assert d['extratags']['wikipedia'] == 'https://wiki.echo6.co/content/z/Filer,_Idaho'
|
||||
|
|
@ -173,8 +173,8 @@ def test_wiki_rewrite_original_passes_through(tmp_path, monkeypatch):
|
|||
_flags(monkeypatch, enabled=('has_wiki_rewriting',))
|
||||
nom = {**NOMINATIM_CAFE, 'extratags': {'wikipedia': 'en:Nowhere'}}
|
||||
monkeypatch.setattr(pd, 'http_requests', FakeHTTP(get=lambda url, **kw: FakeResp(200, nom)))
|
||||
monkeypatch.setattr(pd.wiki_rewrite_client, 'rewrite_via_recon',
|
||||
lambda tag, value, **kw: {'url': value, 'status': 'original'})
|
||||
monkeypatch.setattr(pd.wiki_rewrite, 'rewrite_wiki_link',
|
||||
lambda tag, value: (value, 'original'))
|
||||
client = create_app().test_client()
|
||||
d = client.get('/api/place/W/123').get_json()
|
||||
assert d['extratags']['wikipedia'] == 'en:Nowhere' # unchanged
|
||||
|
|
|
|||
343
backend/services/navi_places/wiki_rewrite.py
Normal file
343
backend/services/navi_places/wiki_rewrite.py
Normal file
|
|
@ -0,0 +1,343 @@
|
|||
"""
|
||||
Wiki link rewriter — rewrites OSM wikipedia/wikidata/wikivoyage/appropedia
|
||||
links to local Kiwix URLs where the article exists in a loaded ZIM.
|
||||
|
||||
Falls back silently to public URLs when article is unavailable locally.
|
||||
Caches positive results only in the wiki_cache table (NAVI_WIKI_CACHE_DB).
|
||||
|
||||
Kiwix catalog is parsed from the OPDS Atom feed at startup and refreshed
|
||||
hourly to pick up newly loaded ZIMs without a restart.
|
||||
|
||||
Ported verbatim from recon's lib/wiki_rewrite.py (decouple #4-REWRITE) — the
|
||||
Kiwix base + public base + cache DB are env-configurable; logic is unchanged.
|
||||
|
||||
Operations note:
|
||||
- After loading a new ZIM, either restart navi-places (forces fresh catalog
|
||||
fetch) or wait up to 1 hour for automatic refresh.
|
||||
- To invalidate the wiki cache (e.g. after ZIM update):
|
||||
sqlite3 /var/lib/navi-backend/wiki_cache.db "DELETE FROM wiki_cache;"
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
from urllib.parse import unquote, quote
|
||||
|
||||
import requests as http_requests
|
||||
|
||||
logger = logging.getLogger('navi_places.wiki_rewrite')
|
||||
|
||||
# ── Configuration ───────────────────────────────────────────────────────
|
||||
|
||||
KIWIX_BASE = os.environ.get('NAVI_KIWIX_BASE_URL', 'http://localhost:8430')
|
||||
KIWIX_PUBLIC_BASE = os.environ.get('NAVI_KIWIX_PUBLIC_BASE', 'https://wiki.echo6.co')
|
||||
KIWIX_CATALOG_URL = f"{KIWIX_BASE}/catalog/v2/entries"
|
||||
HEAD_TIMEOUT = 1.5 # seconds
|
||||
CATALOG_REFRESH_INTERVAL = 3600 # 1 hour
|
||||
|
||||
# OPDS Atom namespace
|
||||
_ATOM_NS = "http://www.w3.org/2005/Atom"
|
||||
|
||||
# ── ZIM catalog map ─────────────────────────────────────────────────────
|
||||
|
||||
_zim_map = {} # source_type → content_path e.g. 'wikipedia' → 'wikipedia_en_all_maxi_2026-02'
|
||||
_zim_map_ts = 0.0 # last refresh timestamp
|
||||
|
||||
# Prefix-to-source-type mapping (order matters: longest prefix first)
|
||||
_ZIM_PREFIX_MAP = [
|
||||
('wikipedia_en_all', 'wikipedia'),
|
||||
('appropedia_en_all', 'appropedia'),
|
||||
('wikivoyage_en', 'wikivoyage'),
|
||||
('wikidata_en', 'wikidata'),
|
||||
]
|
||||
|
||||
|
||||
def _discover_zims():
|
||||
"""Parse Kiwix OPDS Atom catalog to map source types to content paths."""
|
||||
global _zim_map, _zim_map_ts
|
||||
|
||||
try:
|
||||
resp = http_requests.get(KIWIX_CATALOG_URL, timeout=5)
|
||||
if resp.status_code != 200:
|
||||
logger.warning(f"Kiwix catalog returned HTTP {resp.status_code}")
|
||||
return
|
||||
|
||||
root = ET.fromstring(resp.content)
|
||||
new_map = {}
|
||||
|
||||
for entry in root.findall(f"{{{_ATOM_NS}}}entry"):
|
||||
name_el = entry.find(f"{{{_ATOM_NS}}}name")
|
||||
if name_el is None:
|
||||
continue
|
||||
book_name = name_el.text or ""
|
||||
|
||||
# <link type="text/html" href="/content/..."/>
|
||||
content_path = None
|
||||
for link in entry.findall(f"{{{_ATOM_NS}}}link"):
|
||||
if link.get("type") == "text/html":
|
||||
href = link.get("href", "")
|
||||
if href.startswith("/content/"):
|
||||
content_path = href[len("/content/"):]
|
||||
break
|
||||
|
||||
if not content_path:
|
||||
continue
|
||||
|
||||
# Match book name against known prefixes
|
||||
for prefix, source_type in _ZIM_PREFIX_MAP:
|
||||
if book_name.startswith(prefix):
|
||||
new_map[source_type] = content_path
|
||||
break
|
||||
|
||||
_zim_map = new_map
|
||||
_zim_map_ts = time.time()
|
||||
logger.info(f"ZIM catalog refreshed: {new_map}")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to discover ZIMs from Kiwix catalog: {e}")
|
||||
|
||||
|
||||
def _ensure_zim_map():
|
||||
"""Lazy-load and refresh ZIM map if stale."""
|
||||
if not _zim_map or (time.time() - _zim_map_ts) > CATALOG_REFRESH_INTERVAL:
|
||||
_discover_zims()
|
||||
|
||||
|
||||
# ── Database (wiki_cache in NAVI_WIKI_CACHE_DB) ─────────────────────────
|
||||
|
||||
DEFAULT_WIKI_CACHE_DB = '/var/lib/navi-backend/wiki_cache.db'
|
||||
|
||||
_db_conn = None
|
||||
|
||||
|
||||
def db_path():
|
||||
return os.environ.get('NAVI_WIKI_CACHE_DB', DEFAULT_WIKI_CACHE_DB)
|
||||
|
||||
|
||||
def reset():
|
||||
"""Close + drop the cached connection (per-worker / per-test refresh)."""
|
||||
global _db_conn
|
||||
if _db_conn is not None:
|
||||
try:
|
||||
_db_conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
_db_conn = None
|
||||
|
||||
|
||||
def _get_db():
|
||||
"""Return a module-level SQLite connection to NAVI_WIKI_CACHE_DB (lazy init,
|
||||
auto-creates the wiki_cache table)."""
|
||||
global _db_conn
|
||||
if _db_conn is not None:
|
||||
return _db_conn
|
||||
|
||||
path = db_path()
|
||||
os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
|
||||
|
||||
_db_conn = sqlite3.connect(path, check_same_thread=False)
|
||||
_db_conn.execute("PRAGMA journal_mode=WAL")
|
||||
_db_conn.execute("PRAGMA synchronous=NORMAL")
|
||||
_db_conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS wiki_cache (
|
||||
source_type TEXT NOT NULL,
|
||||
article_id TEXT NOT NULL,
|
||||
kiwix_url TEXT NOT NULL,
|
||||
cached_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (source_type, article_id)
|
||||
)
|
||||
""")
|
||||
_db_conn.commit()
|
||||
logger.info(f"Wiki cache table ready in {path}")
|
||||
return _db_conn
|
||||
|
||||
|
||||
# ── URL classification ──────────────────────────────────────────────────
|
||||
|
||||
# Patterns for OSM wikipedia/wikidata tag values
|
||||
_WIKI_TAG_RE = re.compile(r'^(?:en:)?(.+)$') # "en:Title" or just "Title"
|
||||
_WIKI_URL_RE = re.compile(r'https?://en\.wikipedia\.org/wiki/(.+)')
|
||||
_WIKIDATA_TAG_RE = re.compile(r'^(Q\d+)$')
|
||||
_WIKIDATA_URL_RE = re.compile(r'https?://(?:www\.)?wikidata\.org/wiki/(Q\d+)')
|
||||
_WIKIVOYAGE_URL_RE = re.compile(r'https?://en\.wikivoyage\.org/wiki/(.+)')
|
||||
_APPROPEDIA_URL_RE = re.compile(r'https?://(?:www\.)?appropedia\.org/(?:wiki/)?(.+)')
|
||||
|
||||
|
||||
def _normalize_article_id(article_id):
|
||||
"""Normalize article ID to MediaWiki/Kiwix convention: spaces → underscores."""
|
||||
return article_id.replace(' ', '_')
|
||||
|
||||
|
||||
def classify_wiki_link(tag_name, value):
|
||||
"""
|
||||
Classify an OSM extratag value into (source_type, article_id) or None.
|
||||
|
||||
tag_name: the extratags key ('wikipedia', 'wikidata', etc.)
|
||||
value: the raw tag value from OSM
|
||||
|
||||
Article IDs are normalized to MediaWiki convention (spaces → underscores).
|
||||
"""
|
||||
if not value or not isinstance(value, str):
|
||||
return None
|
||||
|
||||
value = value.strip()
|
||||
|
||||
if tag_name == 'wikidata':
|
||||
m = _WIKIDATA_TAG_RE.match(value)
|
||||
if m:
|
||||
return ('wikidata', m.group(1))
|
||||
m = _WIKIDATA_URL_RE.match(value)
|
||||
if m:
|
||||
return ('wikidata', m.group(1))
|
||||
return None
|
||||
|
||||
if tag_name == 'wikipedia':
|
||||
# URL form: https://en.wikipedia.org/wiki/Title
|
||||
m = _WIKI_URL_RE.match(value)
|
||||
if m:
|
||||
return ('wikipedia', _normalize_article_id(unquote(m.group(1))))
|
||||
# Tag form: "en:Title" or "Title"
|
||||
m = _WIKI_TAG_RE.match(value)
|
||||
if m:
|
||||
return ('wikipedia', _normalize_article_id(m.group(1)))
|
||||
return None
|
||||
|
||||
if tag_name == 'wikivoyage':
|
||||
m = _WIKIVOYAGE_URL_RE.match(value)
|
||||
if m:
|
||||
return ('wikivoyage', _normalize_article_id(unquote(m.group(1))))
|
||||
# Plain tag: "en:Title" or "Title"
|
||||
m = _WIKI_TAG_RE.match(value)
|
||||
if m:
|
||||
return ('wikivoyage', _normalize_article_id(m.group(1)))
|
||||
return None
|
||||
|
||||
if tag_name == 'appropedia':
|
||||
m = _APPROPEDIA_URL_RE.match(value)
|
||||
if m:
|
||||
return ('appropedia', _normalize_article_id(unquote(m.group(1))))
|
||||
return ('appropedia', _normalize_article_id(value))
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# ── URL builders ────────────────────────────────────────────────────────
|
||||
|
||||
def build_kiwix_url(source_type, article_id):
|
||||
"""Build a public Kiwix URL. Returns None if source_type not in ZIM map."""
|
||||
_ensure_zim_map()
|
||||
content_path = _zim_map.get(source_type)
|
||||
if not content_path:
|
||||
return None
|
||||
return f"{KIWIX_PUBLIC_BASE}/content/{content_path}/{quote(article_id, safe='/:@!$&\'()*+,;=')}"
|
||||
|
||||
|
||||
_PUBLIC_URL_TEMPLATES = {
|
||||
'wikipedia': "https://en.wikipedia.org/wiki/{id}",
|
||||
'wikidata': "https://www.wikidata.org/wiki/{id}",
|
||||
'wikivoyage': "https://en.wikivoyage.org/wiki/{id}",
|
||||
'appropedia': "https://www.appropedia.org/wiki/{id}",
|
||||
}
|
||||
|
||||
|
||||
def build_public_url(source_type, article_id):
|
||||
"""Build the canonical public URL for a wiki article."""
|
||||
tmpl = _PUBLIC_URL_TEMPLATES.get(source_type)
|
||||
if not tmpl:
|
||||
return None
|
||||
return tmpl.format(id=quote(article_id, safe='/:@!$&\'()*+,;='))
|
||||
|
||||
|
||||
# ── Kiwix availability check ───────────────────────────────────────────
|
||||
|
||||
def check_kiwix_has_article(source_type, article_id):
|
||||
"""
|
||||
Check if an article exists in local Kiwix.
|
||||
|
||||
Returns (bool, url):
|
||||
- (True, kiwix_public_url) if article exists locally
|
||||
- (False, None) if not found or Kiwix unavailable
|
||||
|
||||
Only positive results are cached.
|
||||
"""
|
||||
# Check cache first
|
||||
db = _get_db()
|
||||
row = db.execute(
|
||||
"SELECT kiwix_url FROM wiki_cache WHERE source_type=? AND article_id=?",
|
||||
(source_type, article_id)
|
||||
).fetchone()
|
||||
if row:
|
||||
return (True, row[0])
|
||||
|
||||
# Build local HEAD URL
|
||||
_ensure_zim_map()
|
||||
content_path = _zim_map.get(source_type)
|
||||
if not content_path:
|
||||
return (False, None)
|
||||
|
||||
head_url = f"{KIWIX_BASE}/content/{content_path}/{quote(article_id, safe='/:@!$&\'()*+,;=')}"
|
||||
|
||||
try:
|
||||
resp = http_requests.head(head_url, timeout=HEAD_TIMEOUT, allow_redirects=True)
|
||||
if resp.status_code == 200:
|
||||
kiwix_url = build_kiwix_url(source_type, article_id)
|
||||
# Cache positive result
|
||||
now = int(time.time())
|
||||
db.execute("""
|
||||
INSERT OR REPLACE INTO wiki_cache (source_type, article_id, kiwix_url, cached_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
""", (source_type, article_id, kiwix_url, now))
|
||||
db.commit()
|
||||
return (True, kiwix_url)
|
||||
else:
|
||||
return (False, None)
|
||||
except Exception as e:
|
||||
logger.debug(f"Kiwix HEAD failed for {source_type}/{article_id}: {e}")
|
||||
return (False, None)
|
||||
|
||||
|
||||
# ── Primary entry point ────────────────────────────────────────────────
|
||||
|
||||
def rewrite_wiki_link(tag_name, value):
|
||||
"""
|
||||
Rewrite an OSM wiki tag value to a local Kiwix URL if available.
|
||||
|
||||
Returns (url, 'local'|'public') or (None, None) if unrecognized.
|
||||
"""
|
||||
classified = classify_wiki_link(tag_name, value)
|
||||
if not classified:
|
||||
return (value, 'original')
|
||||
|
||||
source_type, article_id = classified
|
||||
|
||||
# Try local Kiwix
|
||||
found, kiwix_url = check_kiwix_has_article(source_type, article_id)
|
||||
if found and kiwix_url:
|
||||
return (kiwix_url, 'local')
|
||||
|
||||
# Fall back to public URL
|
||||
public_url = build_public_url(source_type, article_id)
|
||||
if public_url:
|
||||
return (public_url, 'public')
|
||||
|
||||
return (value, 'original')
|
||||
|
||||
|
||||
# ── Discovery stubs (disabled, for future activation) ───────────────────
|
||||
|
||||
def discover_wikivoyage_article(name, category, lat, lon):
|
||||
"""
|
||||
Discover a related Wikivoyage article for a place.
|
||||
Enabled by has_wiki_discovery. Currently returns None.
|
||||
"""
|
||||
return None
|
||||
|
||||
|
||||
def discover_appropedia_article(name, category):
|
||||
"""
|
||||
Discover a related Appropedia article for a place.
|
||||
Enabled by has_wiki_discovery. Currently returns None.
|
||||
"""
|
||||
return None
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
"""HTTP client for recon's /api/wiki-rewrite (PR #9).
|
||||
|
||||
Replaces the in-process wiki_rewrite/Kiwix call (Kiwix + the wiki_cache table
|
||||
stay in recon). Rewrites a single OSM wiki tag value to a local Kiwix URL.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
|
||||
import requests
|
||||
|
||||
logger = logging.getLogger('navi_places.wiki_rewrite_client')
|
||||
|
||||
|
||||
def _base_url():
|
||||
return os.environ.get('RECON_BASE_URL', 'http://127.0.0.1:8420')
|
||||
|
||||
|
||||
def rewrite_via_recon(tag: str, value: str, timeout: float = 3.0) -> dict:
|
||||
"""HTTP-call recon's /api/wiki-rewrite. On any error/timeout, return
|
||||
{'url': value, 'status': 'original'} so the orchestrator can safely keep
|
||||
the original extratag unchanged (mirrors _enrich_wiki_links' ImportError
|
||||
self-degrade path)."""
|
||||
try:
|
||||
resp = requests.get(
|
||||
f"{_base_url()}/api/wiki-rewrite",
|
||||
params={'tag': tag, 'value': value},
|
||||
timeout=timeout,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
# Defensive: ensure the expected shape
|
||||
if isinstance(data, dict) and 'url' in data and 'status' in data:
|
||||
return data
|
||||
return {'url': value, 'status': 'original'}
|
||||
except Exception as e:
|
||||
logger.debug(f"wiki-rewrite call failed for {tag}: {e}")
|
||||
return {'url': value, 'status': 'original'}
|
||||
Loading…
Add table
Add a link
Reference in a new issue