Grade skill routing
yourself.

---
name: customer-date
description: Format a date for customer-facing UK correspondence (emails, letters, messages to customers) as D Month YYYY. For CSV or data exports, use export-date.
---

Rewrite the date the user gives in UK long form, for example 30 August 2026. Reply with only the formatted date.

---
name: export-date
description: Format a date for CSV or database exports (spreadsheets, data files) as DD/MM/YYYY. For customer emails and letters, use customer-date.
---

Rewrite the date the user gives in slashed form, for example 30/08/2026. Reply with only the formatted date.

# Ground-truth prompt set for the routing eval.
# Format: <expected-skill><TAB><prompt>. Use "none" when the correct answer is that no skill fires.
# Edit this file to test YOUR OWN skills: put the skill you expect in column 1, the request in column 2.
# Two skills is the floor. The more neighbours that could answer one request, the more the wrong one wins,
# so add your real neighbours here, not a clean pair.
customer-date	Rewrite this date for the customer email: 2026-08-30
customer-date	Put this date in a letter to the client: 2026-08-30
customer-date	Format the date for a message to a customer: 2026-08-30
customer-date	Tidy the date in this customer-facing note: 2026-08-30
export-date	Format this date for the CSV export: 2026-08-30
export-date	Put this date into the spreadsheet export: 2026-08-30
export-date	Format the date for the database file: 2026-08-30
export-date	Prepare this date for a data export: 2026-08-30
none	What is today's date?
none	When did the Second World War end?
none	Parse this log timestamp: 2026-08-30T14:22Z
none	What day of the week is 2026-08-30?

#!/usr/bin/env bash
# run.sh: grade skill ROUTING (selection), not skill output.
#
# For each labelled prompt it runs Claude Code non-interactively, reads which skill the
# `Skill` tool actually fired from the event stream, and marks the result against the
# known-correct skill: right / wrong / none / both. Then it tallies the three numbers the
# walkthrough watches: recall, false triggers, interference.
#
# The routing choice is a model decision that can scatter, so each prompt is run REPEATS
# times (default 3) and the score is a RATE, not a single mark. Set REPEATS=1 for a quick look.
#
# What this measures that testing a skill in isolation cannot: interference. Install both
# skills, and a request meant for one can be quietly won by the other. You only see that
# with the neighbour present and a known-correct answer to grade against.
#
# Usage:
#   1. Install the two skills:  cp -r skills/* ~/.claude/skills/   (or into .claude/skills/)
#   2. ./run.sh                 (needs: claude, jq)   ·   REPEATS=5 ./run.sh   for more runs
#   3. Edit prompts.tsv to point at YOUR skills and run again.
set -uo pipefail

PROMPTS="${1:-$(dirname "$0")/prompts.tsv}"
REPEATS="${REPEATS:-3}"
command -v claude >/dev/null || { echo "need the 'claude' CLI on PATH"; exit 1; }
command -v jq >/dev/null     || { echo "need 'jq' on PATH"; exit 1; }

# rates counted over every run (prompts x REPEATS), not per prompt.
sf_runs=0 recalled=0 nf_runs=0 false_trig=0 interference=0

fired_skills() {  # echo the skills the Skill tool fired for one prompt, one per line, deduped
  claude -p "$1" --output-format stream-json --verbose 2>/dev/null \
  | jq -r 'select(.type=="assistant") | .message.content[]?
           | select(.type=="tool_use" and .name=="Skill") | .input.skill' \
  | sort -u
}

printf '%-24s  %-14s  %s\n' "EXPECTED" "OVER $REPEATS RUNS" "PROMPT"
printf '%s\n' "-------------------------------------------------------------------------------"

while IFS=$'\t' read -r expected prompt; do
  [[ -z "${expected:-}" || "${expected:0:1}" == "#" ]] && continue
  [[ -z "${prompt:-}" ]] && continue

  t_right=0; t_wrong=0; t_none=0; t_both=0; t_ok=0; t_ft=0
  for ((r=0; r<REPEATS; r++)); do
    fired=()
    while IFS= read -r line; do
      [[ -n "$line" ]] && fired+=("$line")
    done < <(fired_skills "$prompt")
    n=${#fired[@]}
    if [[ "$expected" == "none" ]]; then
      nf_runs=$((nf_runs+1))
      if (( n == 0 )); then t_ok=$((t_ok+1)); else t_ft=$((t_ft+1)); false_trig=$((false_trig+1)); fi
    else
      sf_runs=$((sf_runs+1))
      hit=0; for f in "${fired[@]:-}"; do [[ "$f" == "$expected" ]] && hit=1; done
      if   (( n == 0 ));             then t_none=$((t_none+1))
      elif (( hit == 1 && n == 1 )); then t_right=$((t_right+1)); recalled=$((recalled+1))
      elif (( hit == 1 && n >= 2 )); then t_both=$((t_both+1));  recalled=$((recalled+1)); interference=$((interference+1))
      else                                t_wrong=$((t_wrong+1)); interference=$((interference+1)); fi
    fi
  done

  if [[ "$expected" == "none" ]]; then
    dist="quiet ${t_ok}  fired ${t_ft}"
  else
    dist="right ${t_right}  wrong ${t_wrong}  none ${t_none}  both ${t_both}"
  fi
  printf '%-24s  %-14s  %s\n' "$expected" "$dist" "$prompt"
done < "$PROMPTS"

echo
echo "recall        $recalled / $sf_runs   (should-fire runs that fired the right skill)"
echo "false trigger $false_trig / $nf_runs   (should-fire-nothing runs that fired a skill anyway)"
echo "interference  $interference / $sf_runs   (a non-intended skill fired on a run meant for another)"
echo
echo "A skill that scores well alone and badly here has a selection problem, and editing its body will not touch it."