"""Sensor glucose 30 and 60 minutes ahead in people with type 1 diabetes: carrying the last value forward, a
typical model, the model fitted to the person, learned baselines per person and across people, and the fitted
model of each person with a correction learned across people in the model's own units.

Data: HUPA-UCM Diabetes Dataset (Mendeley Data 10.17632/3hbcscwz44.1, CC BY 4.0): 25 people with type 1
diabetes, sensor glucose every five minutes with the insulin delivered and the carbohydrate recorded.

    python hupa.py dev <root>          development: fit on the first 70% of each development part, check on the rest
    python hupa.py run <protocol>      the registered run on the sealed files the protocol names, once

The glucose model is the glucose backend of livingworlds/map (livingworlds_map.backends.glucose). Every arm is
given the same thing: glucose up to now, and the insulin and carbohydrate up to the forecast time (a replay
with the inputs known, not a guess at what the person will do next).
"""
from __future__ import annotations

import hashlib
import json
import sys
import time
from pathlib import Path

import numpy as np
import pandas as pd
from sklearn.ensemble import HistGradientBoostingRegressor

from livingworlds_map import validate
from livingworlds_map.backends.glucose import GlucoseModel, GlucoseParams, ResidualMemory, fit, trend, warmup_steps

STEP_MIN = 5.0
HORIZONS = (6, 12)                 #30 and 60 minutes
MIN_EVENTS = 10                    #a person is scored only with this many doses and this many meals recorded
SENSOR_MARD = 0.092                #FreeStyle Libre 2, adults (Alva et al. 2022, J. Diabetes Sci. Technol. 16:70)
TAUS = (56.0,)
KS = (160,)
PAST = ((0, 6), (6, 12), (12, 24), (24, 48))   #windows of past steps summed for the learned arm


def load(paths):
    d = pd.concat([pd.read_csv(p, sep=";") for p in paths], ignore_index=True)
    t = pd.to_datetime(d["time"])
    step = t.diff().dt.total_seconds().dropna() / 60.0
    if len(step) and not (step == STEP_MIN).all():
        raise ValueError("%s is not on an even %g-minute step" % (paths[0], STEP_MIN))
    return {
        "glucose": d["glucose"].to_numpy(float),
        #basal_rate is per step in this record: summed over a day it is of the size of the day's boluses
        "insulin": d["basal_rate"].fillna(0).to_numpy(float) + d["bolus_volume_delivered"].fillna(0).to_numpy(float),
        "bolus": d["bolus_volume_delivered"].fillna(0).to_numpy(float),
        "carbs": d["carb_input"].fillna(0).to_numpy(float),
        "minute": (t.dt.hour * 60 + t.dt.minute).to_numpy(float),
    }


def eligible(rec, upto):
    return bool((rec["bolus"][:upto] > 0).sum() >= MIN_EVENTS and (rec["carbs"][:upto] > 0).sum() >= MIN_EVENTS)


def window_sum(x, a, b):
    """Sum of x over steps t-b .. t-a-1 for every t (zero-padded before the record)."""
    c = np.concatenate([[0.0], np.cumsum(x)])
    n = len(x)
    t = np.arange(n)
    return c[np.clip(t - a, 0, n)] - c[np.clip(t - b, 0, n)]


def features(rec, h):
    g, n = rec["glucose"], len(rec["glucose"])
    cols = [g]
    for lag in range(1, 12):
        d = np.full(n, np.nan)
        d[lag:] = g[lag:] - g[:-lag]
        cols.append(d)
    for key in ("insulin", "carbs"):
        x = rec[key]
        for a, b in PAST:
            cols.append(window_sum(x, a, b))
        c = np.concatenate([[0.0], np.cumsum(x)])
        t = np.arange(n)
        cols.append(c[np.clip(t + h, 0, n)] - c[t])        #what is delivered or eaten from now to the forecast time
    ang = 2 * np.pi * rec["minute"] / 1440.0
    cols += [np.sin(ang), np.cos(ang)]
    return np.stack(cols, 1)


def shared_features(rec, h, params, f_cal, x_now, ra_now):
    """What one model can learn from across people: everything in mg/dL or in the fitted model's own units, so
    one person's record reads like another's whatever unit their insulin and carbohydrate were written in."""
    g, n = rec["glucose"], len(rec["glucose"])
    cols = [g, f_cal - g, params.si * x_now, params.kc * ra_now]
    for lag in range(1, 12):
        d = np.full(n, np.nan)
        d[lag:] = g[lag:] - g[:-lag]
        cols.append(d)
    t = np.arange(n)
    for x, gain in ((rec["insulin"], params.si), (rec["carbs"], params.kc)):
        for a, b in PAST:
            cols.append(gain * window_sum(x, a, b))
        c = np.concatenate([[0.0], np.cumsum(x)])
        cols.append(gain * (c[np.clip(t + h, 0, n)] - c[t]))
    ang = 2 * np.pi * rec["minute"] / 1440.0
    cols += [np.sin(ang), np.cos(ang)]
    return np.stack(cols, 1)


def across_people(results, h):
    """Two arms trained once on every scored person's training steps, each person weighing the same: a learned
    model on the raw record, and a learned correction of each person's fitted model in the model's own units."""
    w = np.concatenate([np.full(len(r["horizons"][h]["pool"]["y"]), 1.0 / len(r["horizons"][h]["pool"]["y"])) for r in results.values()])
    w = w / w.mean()
    cat = lambda key: np.concatenate([r["horizons"][h]["pool"][key] for r in results.values()])
    raw = HistGradientBoostingRegressor(max_iter=300, learning_rate=0.05, random_state=0).fit(cat("X"), cat("y") - cat("g"), sample_weight=w)
    cor = HistGradientBoostingRegressor(max_iter=300, learning_rate=0.05, random_state=0).fit(cat("S"), cat("y") - cat("f"), sample_weight=w)
    for r in results.values():
        hz = r["horizons"][h]
        p = hz.pop("pool")
        hz["arms"]["learned_across_people"] = p["g_te"] + raw.predict(p["X_te"])
        hz["arms"]["living_worlds"] = p["f_te"] + cor.predict(p["S_te"])


def rmse(a, b):
    return float(np.sqrt(np.mean((np.asarray(a) - np.asarray(b)) ** 2)))


def person(rec, cut, end, grid):
    """Every arm for one person: fitted on starts whose target lies before `cut`, forecast for starts in
    cut..end-h. grid is a list of (tau_days, k); returns per horizon the truth and each arm's forecast."""
    g = rec["glucose"][:end]
    rec = {k: v[:end] for k, v in rec.items()}
    n = len(g)
    model = GlucoseModel(rec["insulin"], rec["carbs"], STEP_MIN)
    warm = warmup_steps(STEP_MIN)
    idx = np.arange(n)
    fit_mask = (idx >= warm) & (idx < cut - max(HORIZONS))
    params, info = fit(model, g, horizons=HORIZONS, mask=fit_mask)
    prior = GlucoseParams()
    tr = trend(g)
    day = idx * STEP_MIN / 1440.0
    out = {"params": params.__dict__.copy(), "fit": info, "horizons": {}}
    for h in HORIZONS:
        truth = np.full(n, np.nan)
        truth[: n - h] = g[h:]
        train = (idx >= warm) & (idx < cut - h) & np.isfinite(tr)
        test = (idx >= cut) & (idx < n - h) & np.isfinite(tr)
        f_prior = model.forecast(prior, g, h)
        f_cal = model.forecast(params, g, h)
        arms = {"persistence": g[test], "physics_prior": f_prior[test], "calibrated_physics": f_cal[test]}
        X = features(rec, h)
        learned = HistGradientBoostingRegressor(max_iter=300, learning_rate=0.05, random_state=0)
        learned.fit(X[train], truth[train] - g[train])
        arms["learned"] = g[test] + learned.predict(X[test])
        #the fitted model, and a learned correction of what it leaves: the correction sees the model's own
        #forecast change and the insulin action and carbohydrate appearance it computed, beside the record
        x_now, ra_now = model.effects(params)
        XH = np.column_stack([X, f_cal - g, x_now, ra_now])
        resid = HistGradientBoostingRegressor(max_iter=300, learning_rate=0.05, random_state=0)
        resid.fit(XH[train], truth[train] - f_cal[train])
        arms["hybrid"] = f_cal[test] + resid.predict(XH[test])
        for tau, k in grid:
            mem = ResidualMemory(tau_days=tau, k=k).remember(
                g[train], tr[train], f_cal[train] - g[train], rec["minute"][train], day[train], f_cal[train], truth[train])
            arms["memory|%g|%d" % (tau, k)] = mem.correct(
                g[test], tr[test], f_cal[test] - g[test], rec["minute"][test], day[test], f_cal[test])
        S = shared_features(rec, h, params, f_cal, x_now, ra_now)
        out["horizons"][h] = {"truth": truth[test], "arms": arms, "train_steps": int(train.sum()),
                              "pool": {"X": X[train], "S": S[train], "y": truth[train], "g": g[train], "f": f_cal[train],
                                       "X_te": X[test], "S_te": S[test], "g_te": g[test], "f_te": f_cal[test]}}
    out["fit_fingerprint"] = validate._fingerprint(g[:cut])
    return out


def pool(results, h, arm):
    truth = np.concatenate([r["horizons"][h]["truth"] for r in results.values()])
    pred = np.concatenate([r["horizons"][h]["arms"][arm] for r in results.values()])
    return truth, pred


def dev(root):
    root = Path(root)
    grid = [(t, k) for t in TAUS for k in KS]
    results, skipped = {}, []
    for f in sorted((root / "dev").glob("HUPA*P.csv")):
        rec = load([f])
        n = len(rec["glucose"])
        cut = int(n * 0.7)
        if not eligible(rec, cut):
            skipped.append(f.stem)
            continue
        results[f.stem] = person(rec, cut, n, grid)
        print(f.stem, n, results[f.stem]["fit"], flush=True)
    table = {}
    for h in HORIZONS:
        across_people(results, h)
        table[h] = {arm: rmse(*pool(results, h, arm)) for arm in next(iter(results.values()))["horizons"][h]["arms"]}
    print(json.dumps({"people": len(results), "skipped": skipped, "rmse": table}, indent=1))


def sha(path):
    return hashlib.sha256(open(path, "rb").read()).hexdigest()


def run(protocol_path):
    t0 = time.time()
    proto_bytes = open(protocol_path, "rb").read()
    proto = json.loads(proto_bytes)
    root = Path(proto["data"]["root"])
    out_path = Path(protocol_path).with_name(proto["results"])
    if out_path.exists():
        raise SystemExit("%s exists: the registered run is made once" % out_path)
    for rel, h in proto["data"]["sha256"].items():
        if sha(root / rel) != h:
            raise SystemExit("%s does not match the registered hash" % rel)
    tau, k = float(proto["model"]["tau_days"]), int(proto["model"]["k"])
    arm_mem = "memory|%g|%d" % (tau, k)
    results, skipped, people = {}, [], {}
    for name in proto["people"]["files"]:
        d, s = root / "dev" / name, root / "sealed" / name
        rec_dev = load([d])
        cut = len(rec_dev["glucose"])
        if not eligible(rec_dev, cut):
            skipped.append(name)
            continue
        rec = load([d, s])
        results[name] = person(rec, cut, len(rec["glucose"]), [(tau, k)])
    if sorted(skipped) != sorted(proto["people"]["not_scored"]):
        raise SystemExit("the people left out are not the registered ones: %s" % skipped)
    res = {"protocolSha256": hashlib.sha256(proto_bytes).hexdigest(), "people": len(results), "notScored": skipped,
           "horizons": {}, "perPerson": {}}
    names = {"persistence": "persistence", "physics_prior": "physics_prior", "calibrated_physics": "calibrated_physics",
             "learned": "learned", "learned_across_people": "learned_across_people", "hybrid": "hybrid_per_person",
             arm_mem: "memory", "living_worlds": "living_worlds"}
    for h in HORIZONS:
        across_people(results, h)
    for h in HORIZONS:
        truth, _ = pool(results, h, "persistence")
        floor = SENSOR_MARD * float(np.mean(truth))
        row = {"minutes": int(h * STEP_MIN), "steps": int(len(truth)), "meanMeasured": float(np.mean(truth)), "floor": floor}
        for arm, label in names.items():
            row[label] = rmse(*pool(results, h, arm))
        _, lw = pool(results, h, "living_worlds")
        report = validate.score(lw, truth, rmse, floor, len(truth),
                                "HUPA-UCM Diabetes Dataset, sealed parts (CC BY 4.0)",
                                fit_fingerprints=[r["fit_fingerprint"] for r in results.values()])
        row["passesFloor"] = bool(report.value <= floor)
        row["band"] = {"glucose_mgdl": [float(np.min(truth)), float(np.max(truth))]}
        res["horizons"][str(int(h * STEP_MIN))] = row
    for name, r in results.items():
        p = {"params": r["params"], "atBound": r["fit"]["at_bound"], "fitRmse": r["fit"]["rmse"]}
        for h in HORIZONS:
            hz = r["horizons"][h]
            p[str(int(h * STEP_MIN))] = {"steps": int(len(hz["truth"])), "trainSteps": hz["train_steps"],
                                         **{label: rmse(hz["truth"], hz["arms"][arm]) for arm, label in names.items()}}
        res["perPerson"][name] = p
    bar_h = str(proto["bar"]["minutes"])
    b = res["horizons"][bar_h]
    res["bar"] = {"minutes": int(bar_h), "beatsCalibratedPhysics": bool(b["living_worlds"] < b["calibrated_physics"]),
                  "beatsLearned": bool(b["living_worlds"] < b["learned"]),
                  "beatsLearnedAcrossPeople": bool(b["living_worlds"] < b["learned_across_people"])}
    res["bar"]["met"] = bool(all(res["bar"][k] for k in ("beatsCalibratedPhysics", "beatsLearned", "beatsLearnedAcrossPeople")))
    res["peopleWhereLivingWorldsBeatsLearned"] = int(sum(
        1 for p in res["perPerson"].values() if p[bar_h]["living_worlds"] < p[bar_h]["learned"]))
    res["seconds"] = round(time.time() - t0, 1)
    out_path.write_text(json.dumps(res, indent=1) + "\n")
    print(json.dumps({"bar": res["bar"], "horizons": res["horizons"]}, indent=1))


if __name__ == "__main__":
    if len(sys.argv) == 3 and sys.argv[1] == "dev":
        dev(sys.argv[2])
    elif len(sys.argv) == 3 and sys.argv[1] == "run":
        run(sys.argv[2])
    else:
        raise SystemExit(__doc__)
