#!/usr/bin/env bash
# run.sh: grade skill ROUTING (selection), not skill output.
#
# For each labelled prompt it runs Claude Code non-interactively, reads which skill the
# `Skill` tool actually fired from the event stream, and marks the result against the
# known-correct skill: right / wrong / none / both. Then it tallies the three numbers the
# walkthrough watches: recall, false triggers, interference.
#
# The routing choice is a model decision that can scatter, so each prompt is run REPEATS
# times (default 3) and the score is a RATE, not a single mark. Set REPEATS=1 for a quick look.
#
# What this measures that testing a skill in isolation cannot: interference. Install both
# skills, and a request meant for one can be quietly won by the other. You only see that
# with the neighbour present and a known-correct answer to grade against.
#
# Usage:
#   1. Install the two skills:  cp -r skills/* ~/.claude/skills/   (or into .claude/skills/)
#   2. ./run.sh                 (needs: claude, jq)   ·   REPEATS=5 ./run.sh   for more runs
#   3. Edit prompts.tsv to point at YOUR skills and run again.
set -uo pipefail

PROMPTS="${1:-$(dirname "$0")/prompts.tsv}"
REPEATS="${REPEATS:-3}"
command -v claude >/dev/null || { echo "need the 'claude' CLI on PATH"; exit 1; }
command -v jq >/dev/null     || { echo "need 'jq' on PATH"; exit 1; }

# rates counted over every run (prompts x REPEATS), not per prompt.
sf_runs=0 recalled=0 nf_runs=0 false_trig=0 interference=0

fired_skills() {  # echo the skills the Skill tool fired for one prompt, one per line, deduped
  claude -p "$1" --output-format stream-json --verbose 2>/dev/null \
  | jq -r 'select(.type=="assistant") | .message.content[]?
           | select(.type=="tool_use" and .name=="Skill") | .input.skill' \
  | sort -u
}

printf '%-24s  %-14s  %s\n' "EXPECTED" "OVER $REPEATS RUNS" "PROMPT"
printf '%s\n' "-------------------------------------------------------------------------------"

while IFS=$'\t' read -r expected prompt; do
  [[ -z "${expected:-}" || "${expected:0:1}" == "#" ]] && continue
  [[ -z "${prompt:-}" ]] && continue

  t_right=0; t_wrong=0; t_none=0; t_both=0; t_ok=0; t_ft=0
  for ((r=0; r<REPEATS; r++)); do
    fired=()
    while IFS= read -r line; do
      [[ -n "$line" ]] && fired+=("$line")
    done < <(fired_skills "$prompt")
    n=${#fired[@]}
    if [[ "$expected" == "none" ]]; then
      nf_runs=$((nf_runs+1))
      if (( n == 0 )); then t_ok=$((t_ok+1)); else t_ft=$((t_ft+1)); false_trig=$((false_trig+1)); fi
    else
      sf_runs=$((sf_runs+1))
      hit=0; for f in "${fired[@]:-}"; do [[ "$f" == "$expected" ]] && hit=1; done
      if   (( n == 0 ));             then t_none=$((t_none+1))
      elif (( hit == 1 && n == 1 )); then t_right=$((t_right+1)); recalled=$((recalled+1))
      elif (( hit == 1 && n >= 2 )); then t_both=$((t_both+1));  recalled=$((recalled+1)); interference=$((interference+1))
      else                                t_wrong=$((t_wrong+1)); interference=$((interference+1)); fi
    fi
  done

  if [[ "$expected" == "none" ]]; then
    dist="quiet ${t_ok}  fired ${t_ft}"
  else
    dist="right ${t_right}  wrong ${t_wrong}  none ${t_none}  both ${t_both}"
  fi
  printf '%-24s  %-14s  %s\n' "$expected" "$dist" "$prompt"
done < "$PROMPTS"

echo
echo "recall        $recalled / $sf_runs   (should-fire runs that fired the right skill)"
echo "false trigger $false_trig / $nf_runs   (should-fire-nothing runs that fired a skill anyway)"
echo "interference  $interference / $sf_runs   (a non-intended skill fired on a run meant for another)"
echo
echo "A skill that scores well alone and badly here has a selection problem, and editing its body will not touch it."