|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Build labels-wikibase.csv directly from the Wikibase ``wbgetentities`` API. |
| 3 | +
|
| 4 | +The SPARQL export (``all-multilingual-labels.rq``) times out on the endpoint and |
| 5 | +returns partial, value-misaligned results, so items are dropped and labels land |
| 6 | +in the wrong rows. The primary-database API is reliable and paginates cleanly, so |
| 7 | +this fetcher reproduces the same CSV schema authoritatively. |
| 8 | +
|
| 9 | +Items fetched are the union of every identifier already in the current export and |
| 10 | +every QID bound in a Q315 template (``data-content``/``data-entity``), so bound |
| 11 | +items dropped by the broken export are recovered. ``itemtype`` is the item's |
| 12 | +first ``P8`` value, matching the export's column. |
| 13 | +""" |
| 14 | + |
| 15 | +from __future__ import annotations |
| 16 | + |
| 17 | +import argparse |
| 18 | +import csv |
| 19 | +import json |
| 20 | +import re |
| 21 | +import sys |
| 22 | +import time |
| 23 | +import urllib.parse |
| 24 | +import urllib.request |
| 25 | +from html.parser import HTMLParser |
| 26 | +from pathlib import Path |
| 27 | +from typing import Sequence |
| 28 | + |
| 29 | +HERE = Path(__file__).resolve().parent |
| 30 | +sys.path.insert(0, str(HERE.parent)) |
| 31 | + |
| 32 | +from abstract.css_assets import DEFAULT_DATA_DIR, DEFAULT_REPO_ROOT |
| 33 | +from abstract.discover_content_migration import abstract_sources |
| 34 | +from abstract.prepare_travel_content import LANGUAGES |
| 35 | + |
| 36 | +DEFAULT_DATA = DEFAULT_DATA_DIR |
| 37 | +API = "https://jsamwrites.wikibase.cloud/w/api.php" |
| 38 | +QID = re.compile(r"Q[1-9][0-9]*") |
| 39 | + |
| 40 | + |
| 41 | +class _Bound(HTMLParser): |
| 42 | + def __init__(self) -> None: |
| 43 | + super().__init__() |
| 44 | + self.qids: set[str] = set() |
| 45 | + |
| 46 | + def handle_starttag(self, tag, attrs) -> None: |
| 47 | + values = dict(attrs) |
| 48 | + for attr in ("data-content", "data-entity"): |
| 49 | + value = values.get(attr) or "" |
| 50 | + if value.startswith("local:") and QID.fullmatch(value.removeprefix("local:")): |
| 51 | + self.qids.add(value.removeprefix("local:")) |
| 52 | + |
| 53 | + |
| 54 | +def bound_qids(repo_root: Path) -> set[str]: |
| 55 | + result: set[str] = set() |
| 56 | + for _, relative in abstract_sources(repo_root): |
| 57 | + parser = _Bound() |
| 58 | + parser.feed((repo_root / relative).read_text(encoding="utf-8")) |
| 59 | + result |= parser.qids |
| 60 | + return result |
| 61 | + |
| 62 | + |
| 63 | +def existing_identifiers(data_dir: Path) -> set[str]: |
| 64 | + path = data_dir / "labels-wikibase.csv" |
| 65 | + if not path.exists(): |
| 66 | + return set() |
| 67 | + with path.open(encoding="utf-8-sig", newline="") as source: |
| 68 | + return { |
| 69 | + row["identifier"].strip() |
| 70 | + for row in csv.DictReader(source) |
| 71 | + if QID.fullmatch(row.get("identifier", "").strip()) |
| 72 | + } |
| 73 | + |
| 74 | + |
| 75 | +def fetch(ids: list[str], pause: float) -> dict[str, dict[str, str]]: |
| 76 | + result: dict[str, dict[str, str]] = {} |
| 77 | + for start in range(0, len(ids), 50): |
| 78 | + chunk = ids[start : start + 50] |
| 79 | + query = urllib.parse.urlencode( |
| 80 | + { |
| 81 | + "action": "wbgetentities", |
| 82 | + "ids": "|".join(chunk), |
| 83 | + "props": "labels|claims", |
| 84 | + "languages": "|".join(LANGUAGES), |
| 85 | + "format": "json", |
| 86 | + } |
| 87 | + ) |
| 88 | + request = urllib.request.Request( |
| 89 | + f"{API}?{query}", headers={"User-Agent": "Q315-label-fetcher/1.0"} |
| 90 | + ) |
| 91 | + with urllib.request.urlopen(request, timeout=60) as response: |
| 92 | + entities = json.load(response)["entities"] |
| 93 | + for qid, entity in entities.items(): |
| 94 | + if "missing" in entity: |
| 95 | + continue |
| 96 | + labels = { |
| 97 | + language: entity.get("labels", {}).get(language, {}).get("value", "") |
| 98 | + for language in LANGUAGES |
| 99 | + } |
| 100 | + itemtype = "" |
| 101 | + for claim in entity.get("claims", {}).get("P8", []): |
| 102 | + value = claim.get("mainsnak", {}).get("datavalue", {}).get("value", {}) |
| 103 | + if value.get("id"): |
| 104 | + itemtype = value["id"] |
| 105 | + break |
| 106 | + result[qid] = {"identifier": qid, "itemtype": itemtype, **labels} |
| 107 | + if pause: |
| 108 | + time.sleep(pause) |
| 109 | + print(f" fetched {min(start + 50, len(ids))}/{len(ids)}", file=sys.stderr) |
| 110 | + return result |
| 111 | + |
| 112 | + |
| 113 | +def main(argv: Sequence[str] | None = None) -> int: |
| 114 | + parser = argparse.ArgumentParser(description=__doc__) |
| 115 | + parser.add_argument("--repo-root", type=Path, default=DEFAULT_REPO_ROOT) |
| 116 | + parser.add_argument("--data-dir", type=Path, default=DEFAULT_DATA) |
| 117 | + parser.add_argument( |
| 118 | + "--out", |
| 119 | + type=Path, |
| 120 | + default=DEFAULT_DATA_DIR / "labels-wikibase.csv", |
| 121 | + help="output CSV path (default: the canonical repo label store)", |
| 122 | + ) |
| 123 | + parser.add_argument("--pause", type=float, default=0.1) |
| 124 | + args = parser.parse_args(argv) |
| 125 | + |
| 126 | + repo_root = args.repo_root.resolve() |
| 127 | + ids = sorted( |
| 128 | + existing_identifiers(args.data_dir.resolve()) | bound_qids(repo_root), |
| 129 | + key=lambda value: int(value[1:]), |
| 130 | + ) |
| 131 | + print(f"Fetching {len(ids)} items from {API}", file=sys.stderr) |
| 132 | + rows = fetch(ids, args.pause) |
| 133 | + fields = ("identifier", "itemtype", *LANGUAGES) |
| 134 | + args.out.parent.mkdir(parents=True, exist_ok=True) |
| 135 | + with args.out.open("w", encoding="utf-8", newline="") as destination: |
| 136 | + writer = csv.DictWriter(destination, fieldnames=fields) |
| 137 | + writer.writeheader() |
| 138 | + for qid in ids: |
| 139 | + if qid in rows: |
| 140 | + writer.writerow(rows[qid]) |
| 141 | + print(f"Wrote {len(rows)} rows to {args.out}") |
| 142 | + return 0 |
| 143 | + |
| 144 | + |
| 145 | +if __name__ == "__main__": |
| 146 | + raise SystemExit(main()) |
0 commit comments