"""
TIER 2 — contribution C3 (calibration + decision-curve) and C5 (SHAP), plus C3 benchmarks.

CRITICAL: the 1:2 matched design fixes the case fraction at 1/3. Probabilities from that
design are NOT population-calibrated, so every probability-scale output here is produced
(a) on the matched (sampling) scale, and (b) after prior correction to a target prevalence.

pi0 is NOT known from the data. It is taken from a documented, data-derived anchor
(the observed community KPSP at-risk proportion) and a sensitivity range is reported.
The authors MUST justify pi0 before this enters the manuscript (todo item T2.11).
"""
import json
import warnings
import numpy as np
import pandas as pd

warnings.filterwarnings("ignore")
from sklearn.calibration import CalibratedClassifierCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import brier_score_loss, roc_auc_score
from sklearn.model_selection import RepeatedStratifiedKFold
from sklearn.base import clone

from common import FEATURES, RISK_FEATURES, build_arms

RNG = 42
N_SPLITS, N_REPEATS = 5, 5
OUT = "/home/hermes/journal-revision/analysis"

matched = pd.read_csv(f"{OUT}/matched_set.csv")
X = matched[RISK_FEATURES].copy()
X = X.fillna(X.median(numeric_only=True))
y = matched["case"].astype(int)

pi_s = float(y.mean())                      # sampling (matched) prevalence
pi0_anchor = 0.0619                         # community KPSP at-risk proportion (data-derived)
pi0_grid = [0.02, 0.05, pi0_anchor, 0.10, 0.15, 0.20]

print(f"sampling prevalence pi_s = {pi_s:.4f}  (1:{round(1/pi_s - 1)} matching)")
print(f"anchor pi0 (community KPSP at-risk) = {pi0_anchor}")


def prior_correct(p, pi0, pi_s):
    """King & Zeng / Pozzolo-style prior correction from sampling to population prevalence."""
    p = np.clip(np.asarray(p, dtype=float), 1e-6, 1 - 1e-6)
    odds = p / (1 - p)
    factor = (pi0 / (1 - pi0)) / (pi_s / (1 - pi_s))
    odds_c = odds * factor
    return odds_c / (1 + odds_c)


def rf():
    return RandomForestClassifier(n_estimators=500, min_samples_leaf=2, max_features="sqrt",
                                  class_weight="balanced", random_state=RNG, n_jobs=1)


# ---------------------------------------------------------------- out-of-fold probabilities
cv = RepeatedStratifiedKFold(n_splits=N_SPLITS, n_repeats=N_REPEATS, random_state=RNG)
oof = np.full((N_REPEATS, len(y)), np.nan)
for r, (tr, te) in enumerate(cv.split(X, y)):
    m = rf().fit(X.iloc[tr], y.iloc[tr])
    oof[r // N_SPLITS, te] = m.predict_proba(X.iloc[te])[:, 1]
p_raw = np.nanmean(oof, axis=0)          # mean OOF probability across repeats
np.save(f"{OUT}/oof_prob_rf.npy", p_raw)
np.save(f"{OUT}/oof_y.npy", y.values)

# ---------------------------------------------------------------- calibration
def calib_stats(p, y_true):
    pc = np.clip(p, 1e-6, 1 - 1e-6)
    logit = np.log(pc / (1 - pc))
    lr = LogisticRegression(penalty=None, max_iter=5000).fit(logit.reshape(-1, 1), y_true)
    slope = float(lr.coef_[0][0])
    intercept = float(lr.intercept_[0])
    # intercept with slope fixed at 1
    lr2 = LogisticRegression(penalty=None, max_iter=5000, fit_intercept=False).fit(
        logit.reshape(-1, 1), y_true)
    return {"calibration_slope": round(slope, 4),
            "calibration_intercept": round(intercept, 4),
            "brier": round(float(brier_score_loss(y_true, pc)), 4),
            "auc": round(float(roc_auc_score(y_true, pc)), 4),
            "mean_predicted": round(float(pc.mean()), 4),
            "observed_rate": round(float(np.mean(y_true)), 4)}


cal = {"matched_scale": calib_stats(p_raw, y.values)}
for pi0 in pi0_grid:
    cal[f"prior_corrected_pi0={pi0}"] = calib_stats(prior_correct(p_raw, pi0, pi_s), y.values)

# ---------------------------------------------------------------- risk strata
strata = {}
p_anchor = prior_correct(p_raw, pi0_anchor, pi_s)
q = pd.qcut(p_anchor, 3, labels=["low", "medium", "high"], duplicates="drop")
for lab in q.categories:
    sel = q == lab
    strata[str(lab)] = {
        "n": int(sel.sum()),
        "cases": int(y[sel].sum()),
        "observed_risk": round(float(y[sel].mean()), 4),
        "mean_predicted": round(float(p_anchor[sel].mean()), 4),
        "ppv": round(float(y[sel].mean()), 4),
    }

# ---------------------------------------------------------------- decision curve
def net_benefit(p, y_true, thresh):
    out = []
    n = len(y_true)
    for t in thresh:
        pred = p >= t
        tp = int(((pred == 1) & (y_true == 1)).sum())
        fp = int(((pred == 1) & (y_true == 0)).sum())
        nb = tp / n - (fp / n) * (t / (1 - t))
        # treat-all / treat-none
        prev = y_true.mean()
        nb_all = prev - (1 - prev) * (t / (1 - t))
        out.append({"threshold": round(float(t), 3), "model": round(float(nb), 5),
                    "screen_all": round(float(nb_all), 5), "screen_none": 0.0})
    return out


thr = np.round(np.arange(0.05, 0.66, 0.05), 3)
nb_raw = net_benefit(p_raw, y.values, thr)
nb_cor = net_benefit(p_anchor, y.values, thr)

# analytic KPSP comparator (published Se 0.60 / Sp 0.92) at each threshold
kpsp_cmp = []
for t in thr:
    se, sp = 0.60, 0.92
    # net benefit of a fixed-Se/Sp test at threshold t  (Vickers)
    nb_test = se * pi_s - (1 - sp) * (1 - pi_s) * (t / (1 - t))
    kpsp_cmp.append({"threshold": round(float(t), 3), "kpsp_analytic": round(float(nb_test), 5)})

# ---------------------------------------------------------------- SHAP
shap_info = {}
try:
    import shap
    model_full = rf().fit(X, y)
    expl = shap.TreeExplainer(model_full)
    sv = expl.shap_values(X)
    if isinstance(sv, list):
        sv = sv[1]
    sv = np.asarray(sv)
    if sv.ndim == 3:            # shap >=0.4x returns (n_samples, n_features, n_classes)
        sv = sv[:, :, 1]
    mean_abs = np.abs(sv).mean(axis=0)
    order = np.argsort(mean_abs)[::-1]
    shap_info["global_top8"] = [{"feature": X.columns[i], "mean_abs_shap": round(float(mean_abs[i]), 4)}
                                for i in order[:8]]
    # worked vignette: the highest-scoring case
    case_pos = np.where(y.values == 1)[0]
    top_case = case_pos[np.argmax(p_raw[case_pos])]
    contrib = sv[top_case]
    ord_c = np.argsort(np.abs(contrib))[::-1][:6]
    shap_info["vignette"] = {
        "row_index": int(top_case),
        "predicted_probability": round(float(p_raw[top_case]), 4),
        "true_label": int(y.values[top_case]),
        "top_contributions": [{"feature": X.columns[i],
                               "shap_value": round(float(contrib[i]), 4),
                               "feature_value": (round(float(X.iloc[top_case, i]), 3)
                                                 if pd.notna(X.iloc[top_case, i]) else None)}
                              for i in ord_c],
    }
    np.save(f"{OUT}/shap_values.npy", sv)
except Exception as e:  # noqa: BLE001
    shap_info["error"] = f"{type(e).__name__}: {e}"

# ---------------------------------------------------------------- benchmarks
def cv_prob_metrics(est, name):
    oof_p = np.full((N_REPEATS, len(y)), np.nan)
    for r, (tr, te) in enumerate(cv.split(X, y)):
        m = clone(est).fit(X.iloc[tr], y.iloc[tr])
        oof_p[r // N_SPLITS, te] = m.predict_proba(X.iloc[te])[:, 1]
    pp = np.nanmean(oof_p, axis=0)
    return {"model": name, **calib_stats(pp, y.values)}


bench = [cv_prob_metrics(rf(), "RandomForest")]
try:
    from xgboost import XGBClassifier
    bench.append(cv_prob_metrics(
        XGBClassifier(n_estimators=400, learning_rate=0.05, max_depth=3, subsample=0.9,
                      colsample_bytree=0.9, eval_metric="logloss", random_state=RNG), "XGBoost"))
except Exception as e:  # noqa: BLE001
    bench.append({"model": "XGBoost", "error": f"{type(e).__name__}: {e}"})
bench.append(cv_prob_metrics(
    CalibratedClassifierCV(LogisticRegression(max_iter=5000), method="sigmoid", cv=5),
    "Calibrated Logistic Regression"))

payload = {
    "sampling_prevalence": round(pi_s, 4),
    "pi0_anchor": pi0_anchor,
    "pi0_grid": pi0_grid,
    "calibration": cal,
    "risk_strata_pi0_anchor": strata,
    "net_benefit_matched_scale": nb_raw,
    "net_benefit_prior_corrected": nb_cor,
    "kpsp_analytic_comparator": kpsp_cmp,
    "shap": shap_info,
    "benchmarks": bench,
    "caveat": ("pi0 is an assumption from a data-derived anchor and MUST be justified by the "
               "authors (T2.11). The KPSP comparator is analytic (published Se/Sp) because KPSP "
               "was not administered in the clinic arm, so no empirical KPSP-only arm exists."),
}
with open(f"{OUT}/tier2_decision.json", "w") as fh:
    json.dump(payload, fh, indent=2, ensure_ascii=False)
print(json.dumps(payload, indent=2, ensure_ascii=False))
