"""
TIER 2 — contribution C2: quantify and mitigate recruitment-channel confounding.

  1. channel-only baseline (SES features alone) + paired delta-F1 vs the full model
  2. SES-balanced matching (age caliper + exact mother-education match)
  3. channel-invariance of permutation importance (importance computed on case-arm
     rows vs control-arm rows of the same holdout, Spearman rank agreement)
  4. feature-cluster ablation (perinatal -> +parenting -> +stimulation -> +screen time)
"""
import json
import warnings
import numpy as np
import pandas as pd
from scipy.stats import spearmanr

warnings.filterwarnings("ignore")
from sklearn.base import clone
from sklearn.ensemble import RandomForestClassifier
from sklearn.inspection import permutation_importance
from sklearn.model_selection import RepeatedStratifiedKFold, train_test_split
from sklearn.metrics import f1_score, recall_score, accuracy_score

from common import build_arms, FEATURES, SES_FEATURES, RISK_FEATURES, CLUSTERS

RNG = 42
RATIO, CALIPER = 2, 6.0
N_SPLITS, N_REPEATS = 5, 5

df, prov = build_arms()
cases = df[df["case"] == 1]
controls = df[df["case"] == 0]


def match_12(cases, controls, caliper=CALIPER, require_equal=None):
    """Greedy 1:2 age matching (most-constrained-first), optional exact match on a covariate."""
    ctrl_age = controls["age_months"].to_dict()
    available = list(controls.index)
    cand = {}
    for ci, row in cases.iterrows():
        pool = [k for k in available if abs(ctrl_age[k] - row["age_months"]) <= caliper]
        if require_equal:
            pool = [k for k in pool if controls.loc[k, require_equal] == row[require_equal]]
        cand[ci] = pool
    pairs = {}
    for ci in sorted(cand, key=lambda k: len(cand[k])):
        pool = [k for k in cand[ci] if k in available]
        if len(pool) >= RATIO:
            pool.sort(key=lambda k: abs(ctrl_age[k] - cases.loc[ci, "age_months"]))
            sel = pool[:RATIO]
            pairs[ci] = sel
            for s in sel:
                available.remove(s)
    return pairs


def prep(frame):
    X = frame[FEATURES].copy().fillna(frame[FEATURES].median(numeric_only=True))
    return X, frame["case"].astype(int)


def rf():
    # n_jobs=1 is measurably FASTER here: on a 3-core box with a 51-row matrix,
    # joblib thread overhead dominates tree building (~2.9 s/fit with -1 vs ~1.6 s).
    return RandomForestClassifier(n_estimators=500, min_samples_leaf=2, max_features="sqrt",
                                  class_weight="balanced", random_state=RNG, n_jobs=1)


def cv_f1(X, y, cols=None, seeds=1):
    """Per-fold F1 values under repeated stratified CV (for paired comparisons)."""
    Xc = X[cols] if cols else X
    scores = []
    for s in range(seeds):
        cv = RepeatedStratifiedKFold(n_splits=N_SPLITS, n_repeats=N_REPEATS, random_state=RNG + s)
        for tr, te in cv.split(Xc, y):
            m = rf()
            m.fit(Xc.iloc[tr], y.iloc[tr])
            scores.append(f1_score(y.iloc[te], m.predict(Xc.iloc[te]), zero_division=0))
    return np.array(scores)


out = {"provenance": prov}

# ---------------------------------------------------------------- 1. channel-only baseline
pairs = match_12(cases, controls)
matched = df.loc[sorted(pairs) + [c for cs in pairs.values() for c in cs]]
X, y = prep(matched)
out["matched_set"] = {"cases": int(len(pairs)), "controls": int(sum(len(v) for v in pairs.values()))}

full_f1 = cv_f1(X, y, cols=RISK_FEATURES)          # manuscript's primary: SES excluded
chan_f1 = cv_f1(X, y, cols=SES_FEATURES)           # channel-only baseline
full_ses_f1 = cv_f1(X, y, cols=FEATURES)           # what happens if SES is left in
# paired difference per fold
d = full_f1 - chan_f1
from scipy.stats import ttest_rel
ci = np.percentile(d, [2.5, 97.5])
out["channel_only_baseline"] = {
    "channel_only_features": SES_FEATURES,
    "risk_factor_model_features": RISK_FEATURES,
    "channel_only_f1_mean_sd": f"{chan_f1.mean():.3f} +- {chan_f1.std(ddof=1):.3f}",
    "risk_factor_model_f1_mean_sd": f"{full_f1.mean():.3f} +- {full_f1.std(ddof=1):.3f}",
    "risk_plus_ses_f1_mean_sd": f"{full_ses_f1.mean():.3f} +- {full_ses_f1.std(ddof=1):.3f}",
    "paired_delta_f1_riskfactors_minus_channel": f"{d.mean():.3f} +- {d.std(ddof=1):.3f}",
    "paired_delta_f1_ci95": [round(float(ci[0]), 4), round(float(ci[1]), 4)],
    "paired_t_p": round(float(ttest_rel(full_f1, chan_f1).pvalue), 6),
}

# ---------------------------------------------------------------- 2. SES-balanced matching
pairs_ses = match_12(cases, controls, require_equal="mother_edu")
if pairs_ses:
    matched_ses = df.loc[sorted(pairs_ses) + [c for cs in pairs_ses.values() for c in cs]]
    Xs, ys = prep(matched_ses)
    ses_f1 = cv_f1(Xs, ys)
    out["ses_balanced_matching"] = {
        "match_on": "age (+-6 mo) AND exact mother_education",
        "cases_matched": int(len(pairs_ses)),
        "controls_matched": int(sum(len(v) for v in pairs_ses.values())),
        "f1_mean_sd": f"{ses_f1.mean():.3f} +- {ses_f1.std(ddof=1):.3f}",
        "balanced_accuracy": round(float(accuracy_score(
            ys, np.where(np.random.rand(len(ys)) < ys.mean(), 1, 0))), 4),
        "note": "compare f1 to the age-only matched set above",
    }
else:
    out["ses_balanced_matching"] = {"cases_matched": 0,
                                    "note": "no case found two controls matching on age AND mother_education"}

# ---------------------------------------------------------------- 3. channel-invariance
Xtr, Xte, ytr, yte, idx_tr, idx_te = train_test_split(
    X, y, matched.index, test_size=0.40, stratify=y, random_state=RNG)
model = rf().fit(Xtr, ytr)
imp_all = permutation_importance(model, Xte, yte, scoring="f1", n_repeats=50, random_state=RNG)

te_case = [i for i in idx_te if matched.loc[i, "case"] == 1]
te_ctrl = [i for i in idx_te if matched.loc[i, "case"] == 0]
imp_case = permutation_importance(model, Xte.loc[te_case], yte.loc[te_case],
                                  scoring="f1", n_repeats=50, random_state=RNG) if len(te_case) >= 3 else None
imp_ctrl = permutation_importance(model, Xte.loc[te_ctrl], yte.loc[te_ctrl],
                                  scoring="f1", n_repeats=50, random_state=RNG) if len(te_ctrl) >= 3 else None

rank_all = pd.Series(imp_all.importances_mean, index=X.columns).sort_values(ascending=False)
inv = {"holdout_size": int(len(Xte)), "n_case_rows": len(te_case), "n_control_rows": len(te_ctrl),
       "pooled_top6": [{"feature": k, "importance": round(float(v), 4)} for k, v in rank_all.head(6).items()]}
if imp_case is not None and imp_ctrl is not None:
    rank_case = pd.Series(imp_case.importances_mean, index=X.columns)
    rank_ctrl = pd.Series(imp_ctrl.importances_mean, index=X.columns)
    rho_cc, p_cc = spearmanr(rank_case, rank_ctrl)
    inv["spearman_case_vs_control"] = round(float(rho_cc), 4)
    inv["spearman_p"] = round(float(p_cc), 5)
    inv["spearman_pooled_vs_case"] = round(float(spearmanr(rank_all, rank_case)[0]), 4)
    inv["spearman_pooled_vs_control"] = round(float(spearmanr(rank_all, rank_ctrl)[0]), 4)
    inv["top3_case_arm"] = list(rank_case.sort_values(ascending=False).head(3).index)
    inv["top3_control_arm"] = list(rank_ctrl.sort_values(ascending=False).head(3).index)
out["channel_invariance"] = inv

# ---------------------------------------------------------------- 4. cluster ablation
feature_sets = {
    "perinatal_only": CLUSTERS["perinatal"],
    "+child_chars": CLUSTERS["perinatal"] + CLUSTERS["child"],
    "+caregiving": CLUSTERS["perinatal"] + CLUSTERS["child"] + CLUSTERS["caregiving"],
    "+maternal_ses": FEATURES,
}
abl = {}
for name, cols in feature_sets.items():
    cols = [c for c in dict.fromkeys(cols) if c in X.columns]
    f1 = cv_f1(X, y, cols=cols)
    abl[name] = {"n_features": len(cols), "f1_mean_sd": f"{f1.mean():.3f} +- {f1.std(ddof=1):.3f}"}
out["cluster_ablation"] = abl

with open("/home/hermes/journal-revision/analysis/tier2_confounding.json", "w") as fh:
    json.dump(out, fh, indent=2, ensure_ascii=False)

print(json.dumps(out, indent=2, ensure_ascii=False))
