"""Whole-body fat, lean tissue and bone mineral from body measures, against the scan: a published formula, a
linear equation, a learned baseline and the mass-closing model of livingworlds/map.

Data: NHANES 2011-2018 (National Center for Health Statistics, public domain): whole-body dual-energy X-ray
absorptiometry (DXX), body measures (BMX) and demographics (DEMO), people aged 8 to 59.

    python nhanes.py dev <root>          development: fit on 2011-2014, check on 2015-2016
    python nhanes.py run <protocol>      the registered run on the sealed cycle the protocol names, once

The model is the body-composition backend of livingworlds/map (livingworlds_map.backends.body_composition).
"""
from __future__ import annotations

import hashlib
import json
import sys
from pathlib import Path

import numpy as np
import pandas as pd
from sklearn.ensemble import HistGradientBoostingRegressor

from livingworlds_map import validate
from livingworlds_map.backends.body_composition import FIELDS, Measures, fit, predict

ISCD_FAT, ISCD_LEAN = 0.03, 0.02   #minimum acceptable precision of a scan, total fat and total lean mass
                                   #(Hangartner et al. 2013, J. Clin. Densitom. 16:520)
RAW = ["BMXWT", "BMXHT", "BMXWAIST", "BMXARMC", "BMXARML", "BMXLEG"]


def cycle(root, letter):
    root = Path(root)
    d = pd.read_sas(root / ("DXX_%s.xpt" % letter))
    b = pd.read_sas(root / ("BMX_%s.xpt" % letter))
    m = pd.read_sas(root / ("DEMO_%s.xpt" % letter))
    x = d.merge(b, on="SEQN").merge(m[["SEQN", "RIAGENDR", "RIDAGEYR"]], on="SEQN")
    #a valid whole-body scan with all three tissues, and every measure taken
    keep = (x["DXAEXSTS"] == 1) & x[["DXDTOFAT", "DXDTOLE", "DXDTOBMC"] + RAW + ["RIAGENDR", "RIDAGEYR"]].notna().all(axis=1)
    x = x[keep].reset_index(drop=True)
    meas = Measures(sex=(x["RIAGENDR"] == 1).to_numpy(float), age=x["RIDAGEYR"].to_numpy(float),
                    weight_kg=x["BMXWT"].to_numpy(float), height_cm=x["BMXHT"].to_numpy(float),
                    waist_cm=x["BMXWAIST"].to_numpy(float), arm_circ_cm=x["BMXARMC"].to_numpy(float),
                    arm_len_cm=x["BMXARML"].to_numpy(float), leg_len_cm=x["BMXLEG"].to_numpy(float))
    truth = {"fat": x["DXDTOFAT"].to_numpy(float) / 1000, "lean": x["DXDTOLE"].to_numpy(float) / 1000,
             "bone": x["DXDTOBMC"].to_numpy(float) / 1000}
    return meas, truth


def join(parts):
    meas = Measures(**{f: np.concatenate([np.asarray(getattr(p[0], f), float) for p in parts]) for f in FIELDS})
    truth = {t: np.concatenate([p[1][t] for p in parts]) for t in ("fat", "lean", "bone")}
    return meas, truth


def matrix(meas):
    return np.column_stack([np.asarray(getattr(meas, f), float) for f in FIELDS])


def published_fat(meas):
    """Body fat from the body mass index, age and sex (Deurenberg, Weststrate and Seidell 1991, Br. J. Nutr.
    65:105): adults 1.20 BMI + 0.23 age - 10.8 sex - 5.4, children to 15 years 1.51 BMI - 0.70 age - 3.6 sex + 1.4,
    in percent of weight, sex 1 for male."""
    m = meas.arrays()
    bmi = m["weight_kg"] / (m["height_cm"] / 100.0) ** 2
    adult = 1.20 * bmi + 0.23 * m["age"] - 10.8 * m["sex"] - 5.4
    child = 1.51 * bmi - 0.70 * m["age"] - 3.6 * m["sex"] + 1.4
    pct = np.where(m["age"] <= 15, child, adult)
    return np.clip(pct, 0, 100) / 100.0 * m["weight_kg"]


def rmse(a, b):
    return float(np.sqrt(np.mean((np.asarray(a) - np.asarray(b)) ** 2)))


def arms(train, test):
    (m_tr, y_tr), (m_te, y_te) = train, test
    coef = fit(m_tr, y_tr["fat"], y_tr["lean"], y_tr["bone"])
    lw = predict(m_te, coef)
    X_tr, X_te = matrix(m_tr), matrix(m_te)
    out = {"living_worlds": lw, "published": {"fat": published_fat(m_te)}, "linear": {}, "learned": {}}
    for tissue in ("fat", "lean", "bone"):
        pred = np.empty(len(X_te))
        for sx in (0.0, 1.0):
            a, b = X_tr[:, 0] == sx, X_te[:, 0] == sx
            A = np.column_stack([np.ones(a.sum()), X_tr[a, 1:]])
            beta, *_ = np.linalg.lstsq(A, y_tr[tissue][a], rcond=None)
            pred[b] = np.column_stack([np.ones(b.sum()), X_te[b, 1:]]) @ beta
        out["linear"][tissue] = pred
        g = HistGradientBoostingRegressor(max_iter=400, learning_rate=0.05, random_state=0)
        out["learned"][tissue] = g.fit(X_tr, y_tr[tissue]).predict(X_te)
    return coef, out


def regimes(meas):
    m = meas.arrays()
    bmi = m["weight_kg"] / (m["height_cm"] / 100.0) ** 2
    age = np.where(m["age"] < 18, "8-17", np.where(m["age"] < 40, "18-39", "40-59"))
    sex = np.where(m["sex"] == 1, "male", "female")
    size = np.where(bmi < 25, "bmi<25", np.where(bmi < 30, "bmi25-30", "bmi30+"))
    return {"sex": sex, "age": age, "bmi": size}


def table(out, truth, meas):
    res = {}
    for tissue in ("fat", "lean", "bone"):
        res[tissue] = {arm: rmse(p[tissue], truth[tissue]) for arm, p in out.items() if tissue in p}
        res[tissue]["meanMeasured"] = float(np.mean(truth[tissue]))
    per = {}
    for kind, labels in regimes(meas).items():
        for lab in np.unique(labels):
            w = labels == lab
            per["%s=%s" % (kind, lab)] = {"people": int(w.sum()), "meanMeasuredFat": float(np.mean(truth["fat"][w])),
                                          **{arm: rmse(p["fat"][w], truth["fat"][w]) for arm, p in out.items()}}
    res["fatByRegime"] = per
    lw = out["living_worlds"]
    res["closure"] = float(np.abs(lw["fat"] + lw["lean"] + lw["bone"] - lw["total"]).max())
    res["bias"] = {t: float(np.mean(lw[t] - truth[t])) for t in ("fat", "lean", "bone")}
    return res


def dev(root):
    train = join([cycle(root, "G"), cycle(root, "H")])
    test = cycle(root, "I")
    coef, out = arms(train, test)
    res = table(out, test[1], test[0])
    print(json.dumps({"train": len(train[1]["fat"]), "check": len(test[1]["fat"]), **res}, indent=1))


def sha(path):
    return hashlib.sha256(open(path, "rb").read()).hexdigest()


def run(protocol_path):
    proto_bytes = open(protocol_path, "rb").read()
    proto = json.loads(proto_bytes)
    root = Path(proto["data"]["root"])
    out_path = Path(protocol_path).with_name(proto["results"])
    if out_path.exists():
        raise SystemExit("%s exists: the registered run is made once" % out_path)
    for rel, h in proto["data"]["sha256"].items():
        if sha(root / rel) != h:
            raise SystemExit("%s does not match the registered hash" % rel)
    train = join([cycle(root, c) for c in proto["data"]["development"]])
    test = cycle(root, proto["data"]["sealed"])
    coef, out = arms(train, test)
    res = table(out, test[1], test[0])
    floor = ISCD_FAT * res["fat"]["meanMeasured"]
    report = validate.score(out["living_worlds"]["fat"], test[1]["fat"], rmse, floor, len(test[1]["fat"]),
                            "NHANES 2017-2018 whole-body DXA (public domain)",
                            fit_fingerprints=[validate._fingerprint(train[1]["fat"])])
    m = test[0].arrays()
    bmi = m["weight_kg"] / (m["height_cm"] / 100.0) ** 2
    f = res["fat"]
    result = {
        "protocolSha256": hashlib.sha256(proto_bytes).hexdigest(),
        "developmentPeople": int(len(train[1]["fat"])), "sealedPeople": int(len(test[1]["fat"])),
        **res,
        "floor": {"fat": floor, "lean": ISCD_LEAN * res["lean"]["meanMeasured"]},
        "passesFloor": bool(report.value <= floor),
        "band": {"age_years": [float(m["age"].min()), float(m["age"].max())],
                 "weight_kg": [float(m["weight_kg"].min()), float(m["weight_kg"].max())],
                 "height_cm": [float(m["height_cm"].min()), float(m["height_cm"].max())],
                 "waist_cm": [float(m["waist_cm"].min()), float(m["waist_cm"].max())],
                 "bmi": [float(bmi.min()), float(bmi.max())]},
        "bar": {"beatsPublished": bool(f["living_worlds"] < f["published"]), "beatsLinear": bool(f["living_worlds"] < f["linear"])},
        "coefficients": coef,
    }
    result["bar"]["met"] = bool(result["bar"]["beatsPublished"] and result["bar"]["beatsLinear"])
    out_path.write_text(json.dumps(result, indent=1) + "\n")
    print(json.dumps({k: result[k] for k in ("bar", "fat", "lean", "bone", "floor", "passesFloor", "sealedPeople")}, indent=1))


if __name__ == "__main__":
    if len(sys.argv) == 3 and sys.argv[1] == "dev":
        dev(sys.argv[2])
    elif len(sys.argv) == 3 and sys.argv[1] == "run":
        run(sys.argv[2])
    else:
        raise SystemExit(__doc__)
