"""
Shared feature construction for both study arms.

Community arm : /home/hermes/journal-revision/data/dataset.xlsx  (posyandu, controls)
Clinic arm    : /home/hermes/journal-revision/data/clinic.xlsx   (therapy clinic, cases)

Both arms use the same ordinal vocabularies, verified against the unique-value dumps,
so one mapping table serves both. Column lookup is by substring match so that the
differing header prefixes ("...SEBELUM anak mulai terapi") do not break anything.
"""
from __future__ import annotations
import numpy as np
import pandas as pd

COMMUNITY_XLSX = "/home/hermes/journal-revision/data/dataset.xlsx"
CLINIC_XLSX = "/home/hermes/journal-revision/data/clinic.xlsx"

# ---------------------------------------------------------------- scale maps (data-driven)
ORD_PARENT = {"Tidak Pernah": 0, "Kadang-kadang": 1, "Sering": 2, "Selalu": 3}
ORD_STIM = {"Tidak Pernah": 0, "Jarang": 1, "Sering": 2, "Setiap Hari": 3}
ORD_SCREEN = {"Tidak pernah": 0, "Kurang dari 1 jam": 1, "1 – 3 jam": 2, "Lebih dari 3 jam": 3}
ORD_EDU = {"Tidak sekolah / SD": 0, "SMP": 1, "SMA / SMK": 2, "Diploma / Sarjana atau lebih": 3}
ORD_INCOME = {"Kurang dari Rp1.500.000": 0, "Rp1.500.000 – Rp3.000.000": 1,
              "Rp3.000.000 – Rp5.000.000": 2, "Lebih dari Rp5.000.000": 3}
ORD_MAGE = {"Kurang dari 20 tahun": 0, "20 – 35 tahun": 1, "Lebih dari 35 tahun": 2}
ORD_ANC = {"Tidak pernah": 0, "1 – 3 kali": 1, "4 kali atau lebih": 2}

# the 9 parenting items, in questionnaire order; same 3x3 conceptual grouping in both arms
PARENT_AUTHORITATIVE = [0, 1, 2]     # praise, explain, comfort
PARENT_AUTHORITARIAN = [3, 4, 5]     # shout, hit, demand-obedience
PARENT_PERMISSIVE = [6, 7, 8]        # let-anything, indulge, hard-to-discipline
STIM_ITEMS = list(range(10))         # all ten stimulation items

FEATURES = [
    "age_months", "male", "birth_order", "under5_household",
    "birth_weight", "low_birth_weight", "premature", "no_cry_at_birth",
    "exclusive_breastfeeding", "household_smoking", "mother_edu", "father_edu",
    "mother_works_outside", "income", "mother_age_at_birth", "anc_visits",
    "primary_caregiver_mother", "screen_time",
    "ever_hospitalised", "ever_seizure", "recurrent_illness",
    "preg_hypertension", "preg_anaemia", "preg_malnutrition",
    "parent_authoritative", "parent_authoritarian", "parent_permissive",
    "stimulation_total",
]

CLUSTERS = {
    "child": ["age_months", "male", "birth_order", "under5_household", "birth_weight",
              "low_birth_weight", "premature", "no_cry_at_birth", "exclusive_breastfeeding",
              "household_smoking"],
    "perinatal": ["premature", "no_cry_at_birth", "birth_weight", "low_birth_weight",
                  "ever_hospitalised", "ever_seizure", "recurrent_illness"],
    "maternal_ses": ["mother_edu", "father_edu", "mother_works_outside", "income",
                     "mother_age_at_birth", "anc_visits", "preg_hypertension",
                     "preg_anaemia", "preg_malnutrition"],
    "caregiving": ["parent_authoritative", "parent_authoritarian", "parent_permissive",
                   "stimulation_total", "screen_time", "primary_caregiver_mother"],
}
SES_FEATURES = ["mother_edu", "father_edu", "mother_works_outside", "income"]
# Manuscript's PRIMARY model explicitly EXCLUDES socioeconomic features
# ("these features were therefore excluded from the primary model").
RISK_FEATURES = [f for f in FEATURES if f not in SES_FEATURES]


def _col(df: pd.DataFrame, *needles: str, exclude: tuple[str, ...] = ()) -> str:
    """Find the first column containing every needle substring (case-insensitive)."""
    for c in df.columns:
        low = str(c).lower()
        if all(n.lower() in low for n in needles) and not any(e.lower() in low for e in exclude):
            return c
    raise KeyError(f"no column matching {needles}")


def _ordinal_series(df: pd.DataFrame, col: str, mapping: dict) -> pd.Series:
    s = df[col].astype(str).str.strip()
    return s.map({k.strip(): v for k, v in mapping.items()})


def _parent_stim(df: pd.DataFrame, parent_prefix: str, stim_prefix: str):
    """Return (parenting 9-item frame, stimulation 10-item frame) as ordinal scores."""
    pcols = [c for c in df.columns if parent_prefix.lower() in str(c).lower()]
    scols = [c for c in df.columns if stim_prefix.lower() in str(c).lower()]
    if len(pcols) != 9:
        raise ValueError(f"expected 9 parenting items, found {len(pcols)}")
    if len(scols) != 10:
        raise ValueError(f"expected 10 stimulation items, found {len(scols)}")
    P = df[pcols].apply(lambda c: _ordinal_series(df, c.name, ORD_PARENT))
    S = df[scols].apply(lambda c: _ordinal_series(df, c.name, ORD_STIM))
    return P, S


def _assemble(df: pd.DataFrame, P: pd.DataFrame, S: pd.DataFrame, age_months: pd.Series,
              bw_col: str, case: int) -> pd.DataFrame:
    X = pd.DataFrame(index=df.index)
    X["age_months"] = pd.to_numeric(age_months, errors="coerce")
    X["male"] = df[_col(df, "jenis kelamin")].astype(str).str.lower().str.startswith("laki").astype(int)
    X["birth_order"] = pd.to_numeric(df[_col(df, "anak ke-berapa")], errors="coerce")
    X["under5_household"] = pd.to_numeric(df[_col(df, "jumlah anak usia di bawah 5")], errors="coerce")

    bw = pd.to_numeric(df[bw_col].astype(str).str.replace(r"[^0-9.]", "", regex=True), errors="coerce")
    bw = bw.where(bw < 100) .mul(1000).fillna(bw) if (bw.dropna() < 100).any() else bw  # tolerate kg entries
    X["birth_weight"] = bw
    X["low_birth_weight"] = (bw < 2500).astype(float).where(bw.notna())
    X["premature"] = df[_col(df, "lahir pada usia kehamilan")].astype(str).str.contains("Kurang bulan").astype(int)
    X["no_cry_at_birth"] = (df[_col(df, "langsung menangis")].astype(str).str.strip() == "Tidak").astype(int)
    X["exclusive_breastfeeding"] = (df[_col(df, "asi saja")].astype(str).str.strip() == "Ya").astype(int)
    X["household_smoking"] = (df[_col(df, "merokok di dalam rumah")].astype(str).str.strip() == "Ya").astype(int)
    X["mother_edu"] = _ordinal_series(df, _col(df, "pendidikan terakhir ibu"), ORD_EDU)
    X["father_edu"] = _ordinal_series(df, _col(df, "pendidikan terakhir ayah"), ORD_EDU)
    occ = df[_col(df, "pekerjaan ibu")].astype(str).str.lower()
    X["mother_works_outside"] = occ.str.contains("bekerja").astype(int)
    X["income"] = _ordinal_series(df, _col(df, "penghasilan keluarga"), ORD_INCOME)
    X["mother_age_at_birth"] = _ordinal_series(df, _col(df, "usia ibu saat melahirkan"), ORD_MAGE)
    X["anc_visits"] = _ordinal_series(df, _col(df, "periksa kehamilan", "anc"), ORD_ANC)
    X["primary_caregiver_mother"] = (df[_col(df, "paling sering mengasuh")].astype(str).str.strip() == "Ibu").astype(int)
    X["screen_time"] = _ordinal_series(df, _col(df, "menonton layar"), ORD_SCREEN)

    ill = df[_col(df, "pernah mengalami hal berikut")].astype(str)
    X["ever_hospitalised"] = ill.str.contains("Dirawat inap").astype(int)
    X["ever_seizure"] = ill.str.contains("Kejang").astype(int)
    X["recurrent_illness"] = ill.str.contains("Sakit berulang").astype(int)

    preg = df[_col(df, "masalah saat kehamilan")].astype(str)
    X["preg_hypertension"] = preg.str.contains("Darah tinggi", case=False).astype(int)
    X["preg_anaemia"] = preg.str.contains("anemia|kurang darah", case=False).astype(int)
    X["preg_malnutrition"] = preg.str.contains("KEK|kurang gizi", case=False).astype(int)

    X["parent_authoritative"] = P.iloc[:, PARENT_AUTHORITATIVE].sum(axis=1, min_count=3)
    X["parent_authoritarian"] = P.iloc[:, PARENT_AUTHORITARIAN].sum(axis=1, min_count=3)
    X["parent_permissive"] = P.iloc[:, PARENT_PERMISSIVE].sum(axis=1, min_count=3)
    X["stimulation_total"] = S.sum(axis=1, min_count=10)

    X["case"] = case
    return X[FEATURES + ["case"]]


def build_arms(verbose: bool = False):
    """Return (combined DataFrame, provenance dict)."""
    # ---------- community (controls) ----------
    co = pd.read_excel(COMMUNITY_XLSX, sheet_name="Form Responses 1")
    n_co_raw = len(co)
    sick = co[_col(co, "kondisi anak")].astype(str).str.lower().str.contains("sakit")
    co1 = co[~sick].copy()
    key = (co1[_col(co1, "inisial anak")].astype(str).str.strip() + "|"
           + co1[_col(co1, "tanggal lahir anak")].astype(str))
    dup = key.duplicated(keep="first")
    co2 = co1[~dup].copy()
    P, S = _parent_stim(co2, "kebiasaan pengasuhan", "seberapa sering hal berikut")
    Xco = _assemble(co2, P, S, co2[_col(co2, "usia anak saat pemeriksaan")],
                    _col(co2, "berat lahir menurut buku kia"), case=0)
    # fall back to the mother-reported weight where the KIA record is missing
    bw_mother = pd.to_numeric(co2[_col(co2, "berat badan anak saat lahir menurut ibu")], errors="coerce")
    Xco["birth_weight"] = Xco["birth_weight"].fillna(bw_mother)
    Xco["low_birth_weight"] = (Xco["birth_weight"] < 2500).astype(float).where(Xco["birth_weight"].notna())

    # ---------- clinic (cases) ----------
    cl = pd.read_excel(CLINIC_XLSX, sheet_name="Form Responses 1")
    n_cl_raw = len(cl)
    dx = cl[_col(cl, "keluhan / diagnosis utama")].astype(str)
    excl_down = dx.str.contains("Down syndrome", case=False)
    excl_nodx = dx.str.contains("Belum ada diagnosis pasti", case=False)
    keep = ~(excl_down | excl_nodx)
    cl2 = cl[keep].copy()
    ts = pd.to_datetime(cl2[_col(cl2, "timestamp")])
    dob = pd.to_datetime(cl2[_col(cl2, "tanggal lahir anak")])
    age_cl = (ts - dob).dt.days / 30.4375
    P2, S2 = _parent_stim(cl2, "kebiasaan pengasuhan", "seberapa sering ayah/bunda")
    Xcl = _assemble(cl2, P2, S2, age_cl,
                    _col(cl2, "berat badan anak saat lahir"), case=1)

    combined = pd.concat([Xcl, Xco], ignore_index=True)
    prov = {
        "community_raw": n_co_raw,
        "community_excluded_acute_illness": int(sick.sum()),
        "community_excluded_duplicates": int(dup.sum()),
        "community_controls": int(len(Xco)),
        "clinic_raw": n_cl_raw,
        "clinic_excluded_down": int(excl_down.sum()),
        "clinic_excluded_no_diagnosis": int(excl_nodx.sum()),
        "clinic_cases": int(len(Xcl)),
        "clinic_cases_le72m": int((age_cl <= 72).sum()),
    }
    if verbose:
        print(prov)
    return combined, prov


if __name__ == "__main__":
    df, prov = build_arms(verbose=True)
    print(df.groupby("case")["age_months"].describe().to_string())
    print()
    print("missing per feature (all arms):")
    print(df[FEATURES].isna().sum().to_string())
