"""Held-out text for honest checks: the same Indian topics in all four languages
(never the India page). Used only to measure generalisation, never for the score."""
import json, pathlib, time, urllib.parse
import requests
from markdownify import markdownify
from build_corpus import UA

TOPICS = ["Delhi", "Mumbai", "Kolkata", "Chennai", "Hyderabad", "Bangalore", "Taj Mahal", "Mahatma Gandhi",
          "Jawaharlal Nehru", "B. R. Ambedkar", "Indian cuisine", "Cricket", "Hindi cinema", "Ganges", "Himalayas",
          "Indian Railways", "Constitution of India", "Indian independence movement", "Hinduism", "Buddhism",
          "Mughal Empire", "Maurya Empire", "Indian Space Research Organisation", "Monsoon", "Bengal tiger"]
OUT = pathlib.Path(__file__).resolve().parent / "heldout"

def langlinks(title):
    d = requests.get("https://en.wikipedia.org/w/api.php", params={"action": "query", "titles": title, "prop": "langlinks",
                     "lllimit": 500, "redirects": 1, "format": "json"}, headers=UA, timeout=60).json()
    p = next(iter(d["query"]["pages"].values()))
    ll = {x["lang"]: x["*"] for x in p.get("langlinks", [])}
    ll["en"] = p["title"]
    return ll

def html_md(lang, title):
    url = f"https://{lang}.wikipedia.org/api/rest_v1/page/html/{urllib.parse.quote(title.replace(' ', '_'))}"
    return markdownify(requests.get(url, headers=UA, timeout=120).text, heading_style="ATX")

OUT.mkdir(exist_ok=True)
index = {}
for t in TOPICS:
    ll = langlinks(t)
    if not all(l in ll for l in ("hi", "te", "mr")):
        print("skip (missing a language):", t); continue
    index[t] = {}
    for l in ("en", "hi", "te", "mr"):
        md = html_md(l, ll[l])
        (OUT / f"{l}__{t.replace(' ', '_')}.md").write_text(md, encoding="utf-8")
        index[t][l] = {"title": ll[l], "chars": len(md)}
        time.sleep(0.2)
    print("ok", t, {l: index[t][l]["chars"] for l in index[t]})
(OUT / "index.json").write_text(json.dumps(index, ensure_ascii=False, indent=1), encoding="utf-8")