#!/usr/bin/env python3
"""score.py — planted-numbers: H1–H3 under PREREG d0a017213ffb.

Primary = first trace (traces.jsonl) with the third read's final calls applied to the audit disagreements
(third_read.jsonl). The second independent trace (traces2/) is scored the same way and reported beside it,
as the prereg requires when audit agreement < 80%. The headline line printed last is the Stage 0 number.

    python3 score.py            # both traces
"""
import json, math, sys
from collections import Counter
from pathlib import Path
HERE = Path(__file__).resolve().parent
def load(p):
    p = HERE / p
    return [json.loads(l) for l in p.read_text().splitlines() if l.strip()] if p.exists() else []
key = json.loads((HERE / "blind" / "key.json").read_text())
Q = {q["id"]: q for q in json.loads((HERE / "questions.json").read_text())["questions"]}
fc = {f["label"]: f for f in load("factcheck.jsonl")}
BAD = {"DEAD END", "MISMATCH", "NOT ON PAGE", "NO LINK / BROKEN"}

def wilson(k, n, z=1.96):
    if n == 0: return (0, 0)
    p = k/n; d = 1+z*z/n; c = p+z*z/(2*n); h = z*math.sqrt(p*(1-p)/n+z*z/(4*n*n))
    return (round(100*(c-h)/d), round(100*(c+h)/d))
def fmt(k, n):
    lo, hi = wilson(k, n); return f"{k}/{n} ({round(100*k/n) if n else 0}%, 95% CI {lo}–{hi}%)"

def score(name, tr):
    print(f"\n=== {name} ===")
    scored = {l: t for l, t in tr.items() if t["class"] not in ("NO STAT", "VOID-CASE")}
    n = len(scored)
    print("classes:", dict(Counter(t["class"] for t in tr.values())))
    traced = sum(t["class"] == "TRACED" for t in scored.values())
    click = sum(bool(t.get("click_pass")) for t in scored.values())
    gap = sum(bool(t.get("click_pass")) and t["class"] != "TRACED" for t in scored.values())
    print(f"H1 TRACED: {fmt(traced, n)}")
    print(f"CLICK PASS: {fmt(click, n)}")
    print(f"H2 gap: {fmt(gap, n)} -> {'MAY SAY click check missed many (>=6)' if gap >= 6 else 'BELOW 6: click check did well'}")
    bad = [l for l, t in scored.items() if t["class"] in BAD]; good = [l for l, t in scored.items() if t["class"] == "TRACED"]
    fb = sum(fc[l]["flagged"] for l in bad); fg = sum(fc[l]["flagged"] for l in good)
    rb = fb/len(bad) if bad else 0; rg = fg/len(good) if good else 0
    print(f"H3 flags BAD {fmt(fb, len(bad))} · GOOD {fmt(fg, len(good))}")
    safe = [l for l in scored if l in fc and fc[l]["verdict"] == "SAFE TO USE"]
    print(f"SAFE TO USE verdicts on scored statistics: {len(safe)}, of which TRACED: {sum(scored[l]['class']=='TRACED' for l in safe)}")
    print("H3:", "VOID (BAD<5)" if len(bad) < 5 else ("RELIABLE FILTER" if rb >= .7 and (rg == 0 or rb >= 2*rg) else "NOT A RELIABLE FILTER"))
    nt = [l for l, t in scored.items() if t["class"] != "TRACED"]
    cf = [l for l in nt if not scored[l].get("click_pass")]
    print(f"not traced: {len(nt)} · of which failed on the cited page itself (not on page or mismatch): {len(cf)}")
    print("  cited-page failures by class:", dict(Counter(scored[l]["class"] for l in cf)))
    print("stateglobe in trail:", sum(bool(t.get("stateglobe")) for t in tr.values()),
          "| list page in trail:", sum(bool(t.get("list_page")) for t in tr.values()))
    for lab, f in (("subject", lambda l: key[l]["subject"]), ("group", lambda l: Q[key[l]["qid"]]["group"]),
                   ("scope", lambda l: Q[key[l]["qid"]]["scope"])):
        for v in sorted({f(l) for l in scored}):
            ls = [l for l in scored if f(l) == v]
            print(f"  {lab}={v}: TRACED {fmt(sum(scored[l]['class']=='TRACED' for l in ls), len(ls))}")
    return traced, n

t1 = {t["label"]: dict(t) for t in load("traces.jsonl")}
third = load("third_read.jsonl")
for r in third:
    t1[r["label"]].update({"class": r["class"], "click_pass": r["click_pass"], "third_read": True})
h1 = score(f"PRIMARY: first trace + third read on {len(third)} audit disagreements", t1)
t2rows = load("traces2/traces2_1.jsonl") + load("traces2/traces2_2.jsonl") + load("traces2/traces2_3.jsonl") + load("traces2/traces2_4.jsonl")
if t2rows:
    t2 = {t["label"]: t for t in t2rows}
    score("SECOND independent trace", t2)
    common = [l for l in t1 if l in t2]
    agree = sum(t1[l]["class"] == t2[l]["class"] for l in common)
    print(f"\ntrace-vs-trace agreement: {agree}/{len(common)}")
    for l in common:
        if t1[l]["class"] != t2[l]["class"]: print(f"  {l}: primary {t1[l]['class']} | second {t2[l]['class']}")
print(f"\nHEADLINE traced {h1[0]}/{h1[1]}")