"""
Reproduce the authors' own pipeline exactly, then show the corrected result.

Uses THEIR cleaned feature files (so the feature set, imputation, one-hot handling and
model hyper-parameters are identical), and their own matching algorithm from
matched_final.py — including the defect that retains unmatched cases.

Run A : matched set exactly as their code builds it  (23 cases + 34 controls = 57)
Run B : matched set restricted to genuinely matched cases (17 + 34 = 51)   <- corrected
"""
import json
import warnings
import numpy as np
import pandas as pd
from sklearn.compose import ColumnTransformer
from sklearn.ensemble import RandomForestClassifier
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, confusion_matrix, f1_score, precision_score, recall_score
from sklearn.model_selection import RepeatedStratifiedKFold, StratifiedKFold, cross_validate, cross_val_predict
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OneHotEncoder, StandardScaler
from sklearn.tree import DecisionTreeClassifier

warnings.filterwarnings("ignore")
RES = "/home/hermes/journal-revision/laptop/ProjekHibah/analysis/results"
RND = 42
OUT = "/home/hermes/journal-revision/analysis"

pos = pd.read_csv(f"{RES}/cleaned_posyandu.csv")
klinik = pd.read_csv(f"{RES}/cleaned_klinik.csv")

NUM = ['age_months','birth_order','n_under5','maternal_age_cat','anc','birth_weight','lbw',
       'premature','cried_birth','ebf','smoker_home','screen_time',
       'par_authoritative','par_authoritarian','par_permissive','stim_score',
       'preg_any','preg_unknown','ill_seizure','ill_hosp','ill_recurrent']
CAT = ['sex','caregiver']


def build_pipe(clf, scale=False):
    steps = [("imp", SimpleImputer(strategy="median"))]
    if scale:
        steps.append(("sc", StandardScaler()))
    pre = ColumnTransformer([
        ("num", Pipeline(steps), NUM),
        ("cat", Pipeline([("imp", SimpleImputer(strategy="most_frequent")),
                          ("oh", OneHotEncoder(handle_unknown="ignore"))]), CAT)])
    return Pipeline([("pre", pre), ("clf", clf)])


CLF = {
    "Logistic Regression": (LogisticRegression(max_iter=2000, random_state=RND), True),
    "Decision Tree": (DecisionTreeClassifier(random_state=RND, class_weight="balanced"), False),
    "Random Forest": (RandomForestClassifier(random_state=RND, class_weight="balanced",
                                             n_estimators=500, min_samples_leaf=2,
                                             max_features="sqrt"), False),
}
SC = {"accuracy": "accuracy", "precision": "precision", "recall": "recall", "f1": "f1"}
rskf = RepeatedStratifiedKFold(n_splits=5, n_repeats=5, random_state=RND)


def evaluate(X, y):
    res = {}
    for name, (clf, scale) in CLF.items():
        cv = cross_validate(build_pipe(clf, scale), X, y, cv=rskf, scoring=SC, n_jobs=1)
        res[name] = {m: f"{cv['test_'+m].mean():.3f} +- {cv['test_'+m].std():.3f}" for m in SC}
    return res


# ---------------------------------------------------------------- replicate THEIR matching
cases = klinik[klinik.age_months <= 72].copy().reset_index(drop=True)
avail = pos.copy()
matched_idx, unmatched_ages, matched_case_pos = [], [], []
for i, c in cases.iterrows():
    pool = avail[(avail.age_months - c.age_months).abs() <= 6]
    take = pool.sample(n=min(2, len(pool)), random_state=RND)
    if len(take) == 0:
        unmatched_ages.append(float(c.age_months))
    else:
        matched_idx.extend(take.index.tolist())
        matched_case_pos.append(i)          # <- bookkeeping their code does NOT do
        avail = avail.drop(take.index)

matched_controls = pos.loc[sorted(set(matched_idx))]

# Run A — exactly their construction (all 23 cases retained)
matched_A = pd.concat([matched_controls.assign(case=0), cases.assign(case=1)], ignore_index=True)
# Run B — corrected: only the cases that actually got controls
cases_B = cases.loc[matched_case_pos].copy()
matched_B = pd.concat([matched_controls.assign(case=0), cases_B.assign(case=1)], ignore_index=True)

out = {
    "their_matching_metadata": {
        "cases_total": int(len(cases)), "cases_matched": int(len(matched_case_pos)),
        "cases_unmatched": len(unmatched_ages), "unmatched_case_ages": unmatched_ages,
        "controls_matched": int(len(matched_controls)),
        "case_age_mean": round(float(cases.age_months.mean()), 1),
        "ctrl_age_mean": round(float(matched_controls.age_months.mean()), 1)},
    "run_A_their_construction": {
        "rows": int(len(matched_A)), "cases": int(matched_A.case.sum()),
        "controls": int((1 - matched_A.case).sum()),
        "cv": evaluate(matched_A[NUM + CAT], matched_A["case"].astype(int)),
    },
    "run_B_corrected": {
        "rows": int(len(matched_B)), "cases": int(matched_B.case.sum()),
        "controls": int((1 - matched_B.case).sum()),
        "cv": evaluate(matched_B[NUM + CAT], matched_B["case"].astype(int)),
    },
}

# pooled OOF confusion matrix for RF on both runs (their reported matrix came from Run A)
for tag, key, frame in [("A", "run_A_their_construction", matched_A),
                        ("B", "run_B_corrected", matched_B)]:
    Xm, ym = frame[NUM + CAT], frame["case"].astype(int)
    rf = build_pipe(RandomForestClassifier(random_state=RND, class_weight="balanced",
                                           n_estimators=500, min_samples_leaf=2,
                                           max_features="sqrt"))
    oof = cross_val_predict(rf, Xm, ym, cv=StratifiedKFold(5, shuffle=True, random_state=RND), n_jobs=1)
    tn, fp, fn, tp = confusion_matrix(ym, oof).ravel()
    out[key]["rf_pooled_oof"] = {
        "TP": int(tp), "FN": int(fn), "FP": int(fp), "TN": int(tn),
        "accuracy": round(float(accuracy_score(ym, oof)), 3),
        "precision": round(float(precision_score(ym, oof)), 3),
        "recall": round(float(recall_score(ym, oof)), 3),
        "f1": round(float(f1_score(ym, oof)), 3),
    }

out["manuscript_table2"] = {
    "Logistic Regression": "0.839 +- 0.079 / 0.859 / 0.740 / 0.777",
    "Decision Tree": "0.731 +- 0.112 / 0.669 / 0.628 / 0.629",
    "Random Forest": "0.852 +- 0.082 / 0.890 / 0.750 / 0.791"}

with open(f"{OUT}/reproduce_theirs.json", "w") as fh:
    json.dump(out, fh, indent=2)
print(json.dumps(out, indent=2))
