"""Registered scorer (protocol-interim-2024.json). Reads the frozen forecasts and the 2004 values, then
fetches 2024 life expectancy at birth (Eurostat for European countries, OECD for the other OECD
members) and scores each country's forecast gain added to its own 2004 value from the same source.
Usage: score-interim-2024.py (run from experiment/lives)"""
import json, hashlib, urllib.request, csv, io
P = json.load(open("protocol-interim-2024.json"))
for f, h in P["frozen_sha256"].items():
    assert hashlib.sha256(open(f, "rb").read()).hexdigest() == h, f
F = {r["country"]: r for r in json.load(open("forecast-e0-2004.json"))["rows"]}
crisis = set(json.load(open("crisis-2004-2024.json"))["crisis"])
J04 = {}
for g, v in json.load(open("eurostat-e0-2004.json"))["values"].items(): J04[("eurostat", g)] = v
for g, v in json.load(open("oecd-e0-2004.json"))["values"].items():
    if all(v[s]["status"] != "E" for s in v): J04[("oecd", g)] = {s: v[s]["value"] for s in v}
J24 = {}
q = "".join(f"&geo={g}" for g in P["countries"]["eurostat"])
d = json.load(urllib.request.urlopen(f"https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/demo_mlexpec?format=JSON&lang=EN&unit=YR&age=Y_LT1&sex=F&sex=M&time=2024{q}", timeout=120))
dims, size = d["id"], d["size"]; inv = {k: {p: c for c, p in d["dimension"][k]["category"]["index"].items()} for k in dims}
for flat, v in d["value"].items():
    i = int(flat); pos = {}
    for k, s in zip(reversed(dims), reversed(size)): pos[k] = i % s; i //= s
    J24.setdefault(("eurostat", inv["geo"][pos["geo"]]), {})["female" if inv["sex"][pos["sex"]] == "F" else "male"] = v
req = urllib.request.Request("https://sdmx.oecd.org/public/rest/data/OECD.ELS.HD,DSD_HEALTH_STAT@DF_LE," + "/" + ".".join(["+".join(P["countries"]["oecd"]), "A", "LFEXP", "Y", "Y0", "F+M"] + [""] * 7) + "?startPeriod=2024&endPeriod=2024&dimensionAtObservation=AllDimensions",
                             headers={"Accept": "application/vnd.sdmx.data+csv; charset=utf-8", "User-Agent": "lw-engine-scorer"})
for r in csv.DictReader(io.StringIO(urllib.request.urlopen(req, timeout=120).read().decode("utf-8"))):
    if r["TIME_PERIOD"] == "2024" and r["OBS_STATUS"] != "E":
        J24.setdefault(("oecd", r["REF_AREA"]), {})["female" if r["SEX"] == "F" else "male"] = float(r["OBS_VALUE"])
res = {"countries": {}, "summary": {}}
for sex in ("female", "male"):
    errs, raw, signed = [], [], []
    for src, cs in P["countries"].items():
        for g, name in cs.items():
            k = (src, g)
            if name in crisis or k not in J04 or k not in J24 or sex not in J04[k] or sex not in J24[k]: continue
            r = F[name][sex]; pred = J04[k][sex] + (r["forecast"] - r["atOrigin"]); e = pred - J24[k][sex]
            errs.append(abs(e)); signed.append(e); raw.append(abs(r["forecast"] - J24[k][sex]))
            res["countries"].setdefault(f"{src}:{g}", {})[sex] = {"e2004": J04[k][sex], "predicted": round(pred, 2), "e2024": J24[k][sex], "error": round(e, 2)}
    n = len(errs)
    res["summary"][sex] = {"n": n, "mae": round(sum(errs) / n, 3), "meanSigned": round(sum(signed) / n, 3), "maeWithoutJumpOff": round(sum(raw) / n, 3),
                           "met": sum(errs) / n <= P["primary"]["bar_years"], "stretch": sum(errs) / n <= P["primary"]["stretch_years"]}
res["primary_met"] = all(res["summary"][s]["met"] for s in ("female", "male"))
json.dump(res, open("results-interim-2024.json", "w"), indent=1)
print(json.dumps(res["summary"]), "primary_met", res["primary_met"])