"""
Re-test the channel-confounding claim using the AUTHORS' exact pipeline and features
(not my reconstructed feature set), on the corrected 17+34 matched set.

Models compared (identical pipeline, only the feature list changes):
  A. their risk factors  NUM + CAT
  B. channel-only        SES  (edu_mom, edu_dad, income, mom_work)
  C. risk factors + SES
"""
import json
import warnings
import numpy as np
import pandas as pd
from scipy.stats import ttest_rel
from sklearn.compose import ColumnTransformer
from sklearn.ensemble import RandomForestClassifier
from sklearn.impute import SimpleImputer
from sklearn.metrics import f1_score
from sklearn.model_selection import RepeatedStratifiedKFold
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OneHotEncoder, StandardScaler

warnings.filterwarnings("ignore")
RES = "/home/hermes/journal-revision/laptop/ProjekHibah/analysis/results"
RND = 42
OUT = "/home/hermes/journal-revision/analysis"

pos = pd.read_csv(f"{RES}/cleaned_posyandu.csv")
klinik = pd.read_csv(f"{RES}/cleaned_klinik.csv")

NUM = ['age_months','birth_order','n_under5','maternal_age_cat','anc','birth_weight','lbw',
       'premature','cried_birth','ebf','smoker_home','screen_time',
       'par_authoritative','par_authoritarian','par_permissive','stim_score',
       'preg_any','preg_unknown','ill_seizure','ill_hosp','ill_recurrent']
CAT = ['sex','caregiver']
SES_NUM = ['edu_mom','edu_dad','income']
SES_CAT = ['mom_work']

# rebuild the corrected matched set (same matching algorithm, only truly matched cases kept)
cases = klinik[klinik.age_months <= 72].copy().reset_index(drop=True)
avail = pos.copy()
midx, kept = [], []
for i, c in cases.iterrows():
    pool = avail[(avail.age_months - c.age_months).abs() <= 6]
    take = pool.sample(n=min(2, len(pool)), random_state=RND)
    if len(take):
        midx.extend(take.index.tolist())
        kept.append(i)
        avail = avail.drop(take.index)
controls = pos.loc[sorted(set(midx))]
matched = pd.concat([controls.assign(case=0), cases.loc[kept].assign(case=1)], ignore_index=True)
print("corrected matched set:", len(matched), "rows | cases:", int(matched.case.sum()),
      "controls:", int((1 - matched.case).sum()))

rskf = RepeatedStratifiedKFold(n_splits=5, n_repeats=5, random_state=RND)
SPLITS = list(rskf.split(matched, matched["case"]))


def pipe(num_cols, cat_cols):
    pre = ColumnTransformer([
        ("num", Pipeline([("imp", SimpleImputer(strategy="median"))]), num_cols),
        ("cat", Pipeline([("imp", SimpleImputer(strategy="most_frequent")),
                          ("oh", OneHotEncoder(handle_unknown="ignore"))]), cat_cols)])
    return Pipeline([("pre", pre),
                     ("clf", RandomForestClassifier(random_state=RND, class_weight="balanced",
                                                    n_estimators=500, min_samples_leaf=2,
                                                    max_features="sqrt"))])


def fold_f1(num_cols, cat_cols):
    y = matched["case"].astype(int)
    X = matched[num_cols + cat_cols]
    scores = []
    for tr, te in SPLITS:
        m = pipe(num_cols, cat_cols).fit(X.iloc[tr], y.iloc[tr])
        scores.append(f1_score(y.iloc[te], m.predict(X.iloc[te]), zero_division=0))
    return np.array(scores)


A = fold_f1(NUM, CAT)
B = fold_f1(SES_NUM, SES_CAT)
C = fold_f1(NUM + SES_NUM, CAT + SES_CAT)

d = A - B
ci = np.percentile(d, [2.5, 97.5])
out = {
    "matched_set": {"rows": int(len(matched)), "cases": int(matched.case.sum()),
                    "controls": int((1 - matched.case).sum())},
    "A_their_risk_factors": f"{A.mean():.3f} +- {A.std(ddof=1):.3f}",
    "B_channel_only_SES_4vars": f"{B.mean():.3f} +- {B.std(ddof=1):.3f}",
    "C_risk_plus_SES": f"{C.mean():.3f} +- {C.std(ddof=1):.3f}",
    "paired_A_minus_B": f"{d.mean():.3f} +- {d.std(ddof=1):.3f}",
    "paired_ci95": [round(float(ci[0]), 4), round(float(ci[1]), 4)],
    "paired_t_p": round(float(ttest_rel(A, B).pvalue), 6),
}
with open(f"{OUT}/verify_confounding_theirs.json", "w") as fh:
    json.dump(out, fh, indent=2)
print(json.dumps(out, indent=2))
