#!/usr/bin/env python3
"""heldout_card_test.py — the pre-registered held-out test of the whole Re-check card (survives-a-recheck, Stage 3).

27 findings re-run by the Social Sciences Replication Project or Many Labs 2, as used by Hoogeveen, Sarafoglou &
Wagenmakers (2020). None is in the 2015 OSC sample the card's signs came from. Signs were coded and frozen before any
outcome was read (`data/hoogeveen2020/heldout_signs.csv`, sha256 checked below):
  weak pass    original p not below .001 (as published: SSRP supplementary table, or FReD's row paired with the re-run)
  interaction  coded INTERACTION by a sealed model from the plain description, names removed
  doubted      fewer than half of ~125 lay readers (description only, no PhD in psychology) predicted it would replicate

Scored by the rules in `PREREG — survives-a-recheck — Held-out card test — 2026-10-27.md`.

    python3 heldout_card_test.py
"""
from __future__ import annotations

import hashlib
import json
import os
import sys

import pandas as pd
from scipy.stats import fisher_exact

HERE = os.path.dirname(os.path.abspath(__file__))
DATA = os.path.join(HERE, "data", "hoogeveen2020")
SIGNS = os.path.join(DATA, "heldout_signs.csv")
STUDIES = os.path.join(DATA, "DescriptionAndStatisticsPerStudy.csv")
SIGNS_SHA = "0875008358e65f7476a769b45fbaed0717023d8853206a89cdeaea268ba28eb4"
STUDIES_SHA = "7fba68c006acd4f781d9c4dbc7045fdac986a7df662dbb9160bd0e1661240f40"
RESULTS = os.path.join(HERE, "heldout-card-test-results.json")


def sha(p: str) -> str:
    return hashlib.sha256(open(p, "rb").read()).hexdigest()


def rate(g: pd.DataFrame) -> tuple[int, int]:
    return int(g["held"].sum()), int(len(g))


def main() -> int:
    if sha(SIGNS) != SIGNS_SHA or sha(STUDIES) != STUDIES_SHA:
        print("HASH MISMATCH: the frozen sign table or the study file changed")
        return 1
    signs = pd.read_csv(SIGNS)
    out = pd.read_csv(STUDIES)[["Study number", "RS outcome"]].rename(columns={"Study number": "study"})
    d = signs.merge(out, on="study", how="left")
    d["held"] = d["RS outcome"].astype(int)
    # coding check fixed in the prereg: Karpicke & Blunt (study 10) replicated (ledger A02c), so it must read 1
    if int(d.loc[d["study"] == 10, "held"].iloc[0]) != 1:
        d["held"] = 1 - d["held"]
        print("coding check: study 10 read 0, so the outcome coding was reversed (as pre-registered)")
    res = {"n": int(len(d)), "by_action": {}, "by_signs": {}}
    print(f"Held-out card test · {len(d)} findings (SSRP + Many Labs 2), signs frozen before outcomes\n")
    for a in ("act", "try small", "wait"):
        k, n = rate(d[d["action"] == a])
        res["by_action"][a] = {"held": k, "n": n, "rate": k / n if n else None}
    for s in sorted(d["signs"].unique()):
        k, n = rate(d[d["signs"] == s])
        res["by_signs"][str(int(s))] = {"held": k, "n": n, "rate": k / n if n else None}
    act, tr, wait = (res["by_action"][a] for a in ("act", "try small", "wait"))
    diff = act["rate"] - wait["rate"]
    fp = float(fisher_exact([[act["held"], act["n"] - act["held"]], [wait["held"], wait["n"] - wait["held"]]])[1])
    monotone = act["rate"] >= tr["rate"] >= wait["rate"]
    verdict = "CARD SORTS ON HELD-OUT CLAIMS" if (diff >= 0.20 and monotone) else "NOT SHOWN ON HELD-OUT CLAIMS"
    res.update({"act_minus_wait": diff, "fisher_p": fp, "monotone": bool(monotone), "verdict": verdict})
    print("by action: " + " · ".join(f"{a} {v['held']}/{v['n']} ({v['rate'] * 100:.0f}%)" for a, v in res["by_action"].items()))
    print("by signs:  " + " · ".join(f"{s} signs {v['held']}/{v['n']}" for s, v in res["by_signs"].items()))
    print(f"act - wait {diff * 100:+.0f} points, Fisher p = {fp:.3g}, monotone = {monotone} -> {verdict}\n")

    # descriptive comparisons fixed in the prereg (no pass/fail)
    base = d["held"].mean()
    rules = {
        "p-value only (strong -> holds)": (~d["weak_pass"]).astype(int),
        "lay majority only (most predict it replicates -> holds)": (~d["doubted"]).astype(int),
        "always predict the commoner outcome": pd.Series([int(base >= 0.5)] * len(d)),
    }
    res["comparisons"] = {}
    for name, pred in rules.items():
        acc = float((pred.values == d["held"].values).mean())
        res["comparisons"][name] = acc
        print(f"accuracy, {name}: {acc * 100:.0f}% (n = {len(d)})")
    decided = d[d["action"] != "try small"]
    acc_card = float(((decided["action"] == "act").astype(int).values == decided["held"].values).mean()) if len(decided) else None
    res["comparisons"]["card, act vs wait only (try small abstains)"] = {"accuracy": acc_card, "coverage": len(decided) / len(d)}
    print(f"accuracy, card act vs wait (try small abstains): {acc_card * 100:.0f}% on {len(decided)} of {len(d)} decided")
    print(f"base rate held: {base * 100:.0f}% ({int(d['held'].sum())}/{len(d)})")
    json.dump(res, open(RESULTS, "w"), indent=2, default=float)
    print(f"\nwrote {os.path.basename(RESULTS)}")
    return 0


if __name__ == "__main__":
    sys.exit(main())