Grade skill routing
yourself.

---
name: customer-date
description: Format a date for customer-facing UK correspondence (emails, letters, messages to customers) as D Month YYYY. For CSV or data exports, use export-date.
---

Rewrite the date the user gives in UK long form, for example 30 August 2026. Reply with only the formatted date.

---
name: export-date
description: Format a date for CSV or database exports (spreadsheets, data files) as DD/MM/YYYY. For customer emails and letters, use customer-date.
---

Rewrite the date the user gives in slashed form, for example 30/08/2026. Reply with only the formatted date.

# Ground-truth prompt set for the routing eval.
# Format: <expected-skill><TAB><prompt>. Use "none" when the correct answer is that no skill fires.
# To test your own skills, put the skill you expect in column 1 and the request in column 2.
# Two skills is the floor. Add the real neighbours your skill may compete with.
customer-date	Rewrite this date for the customer email: 2026-08-30
customer-date	Put this date in a letter to the client: 2026-08-30
customer-date	Format the date for a message to a customer: 2026-08-30
customer-date	Tidy the date in this customer-facing note: 2026-08-30
export-date	Format this date for the CSV export: 2026-08-30
export-date	Put this date into the spreadsheet export: 2026-08-30
export-date	Format the date for the database file: 2026-08-30
export-date	Prepare this date for a data export: 2026-08-30
none	What is today's date?
none	When did the Second World War end?
none	Parse this log timestamp: 2026-08-30T14:22Z
none	What day of the week is 2026-08-30?

---
name: long-date
description: Format a date.
---

Rewrite the date the user gives in UK long form, for example 30 August 2026. Reply with only the formatted date.

---
name: slash-date
description: Format a date.
---

Rewrite the date the user gives in slashed form, for example 30/08/2026. Reply with only the formatted date.

# The failing pair: long-date and slash-date share one vague description, "Format a date."
# Same requests and same ground truth as prompts.tsv: customer wording wants the long form,
# export wording wants the slashed form, and the last four should fire neither skill.
# Run it with:  SKILLS=skills-vague ./run.sh prompts-vague.tsv
long-date	Rewrite this date for the customer email: 2026-08-30
long-date	Put this date in a letter to the client: 2026-08-30
long-date	Format the date for a message to a customer: 2026-08-30
long-date	Tidy the date in this customer-facing note: 2026-08-30
slash-date	Format this date for the CSV export: 2026-08-30
slash-date	Put this date into the spreadsheet export: 2026-08-30
slash-date	Format the date for the database file: 2026-08-30
slash-date	Prepare this date for a data export: 2026-08-30
none	What is today's date?
none	When did the Second World War end?
none	Parse this log timestamp: 2026-08-30T14:22Z
none	What day of the week is 2026-08-30?

#!/usr/bin/env bash
# run.sh: grade skill ROUTING (selection) against a labelled prompt set.
#
# For each labelled prompt it runs Claude Code non-interactively, reads which skill the
# `Skill` tool actually fired from the event stream, and marks the result against the
# known-correct skill: right / wrong / none / both. Each row also names the skills that
# fired, so a `wrong` shows which neighbour took the request. Then it tallies three
# numbers: recall, false triggers and interference.
#
# Interference is the number a one-skill trigger rate cannot give you: which skill won
# when a neighbour that could answer the same request was installed as well.
#
# Routing is a model decision that can scatter, so each prompt runs REPEATS times
# (default 3) and every score is a rate over all runs. REPEATS=1 gives a quick look.
#
# Usage, from the unzipped folder:
#   OUT=routing-$(date +%F).txt ./run.sh                   demo: the sharp pair in skills/
#   SKILLS=skills-vague ./run.sh prompts-vague.tsv          the failing pair, same requests
#   SKILLS=~/my-skills OUT=mine.txt ./run.sh my-prompts.tsv your own skill folders + labelled prompts
#   SKILLS=installed ./run.sh my-prompts.tsv                grade what is already installed where you run it
#
# OUT=<file> writes the output to that file as well as the screen and keeps the exit status.
# (Piping into tee reports tee's status instead, unless you run `set -o pipefail` first.)
#
# SKILLS is a folder of skill folders (<name>/SKILL.md). They are copied into a throwaway
# project made with mktemp and Claude runs there, so nothing is written to ~/.claude/skills.
# Your personal and plugin skills still load, as they do in real use, and count as neighbours.
# A skill under test with the same name as one in ~/.claude/skills would be shadowed by the
# personal copy, so run.sh stops if it finds one.
#
# Every row of the prompt file is checked before any Claude session starts: a TAB between
# label and prompt, no spaces in the label, and (when SKILLS is a folder) a label that is
# "none" or one of the skills being tested.
#
# Settings: REPEATS (3) · MODEL (your CLI default) · OUT (none) · MIN_RECALL_PCT (90)
#           MAX_FALSE (0) · MAX_INTERFERENCE (0)
# Exit status: 0 gate passed · 1 gate failed · 2 the run could not be graded or OUT could not be written
# Needs: claude (logged in) and jq. Cost: one Claude Code session per prompt per repeat.
set -uo pipefail

here="$(cd "$(dirname "$0")" && pwd)"
PROMPTS="${1:-$here/prompts.tsv}"
REPEATS="${REPEATS:-3}"
MODEL="${MODEL:-}"
OUT="${OUT:-}"
SKILLS="${SKILLS:-$here/skills}"
MIN_RECALL_PCT="${MIN_RECALL_PCT:-90}"
MAX_FALSE="${MAX_FALSE:-0}"
MAX_INTERFERENCE="${MAX_INTERFERENCE:-0}"

die() { echo "run.sh: $*" >&2; exit 2; }
command -v claude >/dev/null || die "need the 'claude' CLI on PATH"
command -v jq >/dev/null     || die "need 'jq' on PATH"
# a number above 999999999 (leading zeros aside) can wrap round in shell arithmetic, so refuse it before any sum
big='^0*[1-9][0-9]{9,}$'
[[ "$REPEATS" =~ $big ]]          && die "REPEATS is too large (the most is 999999999)"
[[ "$MAX_FALSE" =~ $big ]]        && die "MAX_FALSE is too large (the most is 999999999)"
[[ "$MAX_INTERFERENCE" =~ $big ]] && die "MAX_INTERFERENCE is too large (the most is 999999999)"
[[ "$REPEATS" =~ ^[0-9]+$ ]] && (( 10#$REPEATS >= 1 )) || die "REPEATS must be a whole number of 1 or more"
[[ "$MIN_RECALL_PCT" =~ ^[0-9]+$ && ! "$MIN_RECALL_PCT" =~ $big ]] && (( 10#$MIN_RECALL_PCT <= 100 )) \
  || die "MIN_RECALL_PCT must be a whole number from 0 to 100"
[[ "$MAX_FALSE" =~ ^[0-9]+$ ]]        || die "MAX_FALSE must be a whole number"
[[ "$MAX_INTERFERENCE" =~ ^[0-9]+$ ]] || die "MAX_INTERFERENCE must be a whole number"
# read every number as base 10, so a leading zero (08) is not taken as octal
REPEATS=$((10#$REPEATS)); MIN_RECALL_PCT=$((10#$MIN_RECALL_PCT))
MAX_FALSE=$((10#$MAX_FALSE)); MAX_INTERFERENCE=$((10#$MAX_INTERFERENCE))
[[ "$PROMPTS" == /* ]] || PROMPTS="$PWD/$PROMPTS"
[[ -r "$PROMPTS" ]] || die "cannot read the prompt file $PROMPTS"
if [[ -n "$OUT" ]]; then
  [[ "$OUT" == /* ]] || OUT="$PWD/$OUT"
  [[ "$OUT" != */ && ! -d "$OUT" ]] || die "cannot write OUT=$OUT (OUT must be a file, and this is a folder)"
  [[ -d "$(dirname "$OUT")" && -w "$(dirname "$OUT")" ]] || die "cannot write OUT=$OUT (its folder is missing or read-only)"
  [[ ! -e "$OUT" || -w "$OUT" ]] || die "cannot write OUT=$OUT"
fi

skill_names_in() {  # the folder names and frontmatter names of the skills under $1 (any depth up to $2)
  find -L "$1" -mindepth 2 -maxdepth "$2" -name SKILL.md 2>/dev/null | while IFS= read -r f; do
    basename "$(dirname "$f")"
    awk 'NR==1{if($0!~/^---/)exit;next} /^---/{exit} /^name:/{sub(/^name:[ \t]*/,"");sub(/[ \t\r]+$/,"");print;exit}' "$f"
  done | grep -v '^$' | sort -u
}

# scratch space for this run; removed on exit, whatever happens
work="$(mktemp -d "${TMPDIR:-/tmp}/skill-routing-eval.XXXXXX")" || die "mktemp failed"
trap 'rm -rf "$work"' EXIT

personal="$(skill_names_in "$HOME/.claude/skills" 4)"
if [[ "$SKILLS" == "installed" ]]; then
  skills_label="already installed in $PWD"
  allowed=""
  known="$( { skill_names_in "$PWD/.claude/skills" 2; printf '%s\n' "$personal"; } | sort -u)"
else
  [[ "$SKILLS" == /* ]] || SKILLS="$PWD/$SKILLS"
  ls "$SKILLS"/*/SKILL.md >/dev/null 2>&1 \
    || die "no <name>/SKILL.md folders in $SKILLS (download the zip, or point SKILLS at your skills)"
  testing="$(skill_names_in "$SKILLS" 2)"
  clash="$(comm -12 <(printf '%s\n' "$testing") <(printf '%s\n' "$personal" | sort -u) | grep -v '^$')"
  if [[ -n "$clash" ]]; then
    die "$(echo $clash) is also installed in ~/.claude/skills. Claude Code loads a personal skill \
over a project skill with the same name, so the copy under test would be shadowed. Move the \
personal copy out of ~/.claude/skills or rename the skill under test, or run with SKILLS=installed \
to grade the installed one."
  fi
  allowed="$(printf '%s\nnone\n' "$testing")"
  known=""
  mkdir -p "$work/project/.claude/skills"
  cp -R "$SKILLS"/. "$work/project/.claude/skills/"
  skills_label="$(cd "$SKILLS"/ && ls -d */ | tr -d / | tr '\n' ' ')(copied into a temp project)"
fi

# check every row before any Claude session starts
TAB=$'\t'; CR=$'\r'
labels=(); prompts=(); bad=(); notes=(); lineno=0
while IFS= read -r line || [[ -n "$line" ]]; do
  lineno=$((lineno+1))
  line=${line%${CR}}
  [[ -z "${line//[[:space:]]/}" || "${line:0:1}" == "#" ]] && continue
  if [[ $line != *${TAB}* ]]; then
    bad+=("line $lineno: no TAB between the label and the prompt"); continue; fi
  label=${line%%${TAB}*}; prompt=${line#*${TAB}}
  while [[ "${prompt:0:1}" == "$TAB" ]]; do prompt=${prompt:1}; done
  if [[ -z "$label" || "$label" =~ [[:space:]] ]]; then
    bad+=("line $lineno: the label '$label' is empty or contains a space"); continue; fi
  if [[ -z "${prompt//[[:space:]]/}" ]]; then
    bad+=("line $lineno: no prompt after the TAB"); continue; fi
  if [[ -n "$allowed" ]] && ! grep -qxF -- "$label" <<<"$allowed"; then
    bad+=("line $lineno: '$label' is neither 'none' nor a skill in $SKILLS"); continue; fi
  if [[ -n "$known" && "$label" != "none" && "$label" != *:* ]] && ! grep -qxF -- "$label" <<<"$known"; then
    note="note: '$label' is not in ./.claude/skills or ~/.claude/skills; fine for a plugin or managed skill, otherwise check the spelling"
    [[ " ${notes[*]:-} " == *"$note"* ]] || notes+=("$note"); fi
  labels+=("$label"); prompts+=("$prompt")
done < "$PROMPTS"
if (( ${#bad[@]} > 0 )); then
  { echo "run.sh: ${#bad[@]} bad row(s) in $PROMPTS; nothing was run:"; printf '  %s\n' "${bad[@]}"; } >&2
  exit 2
fi
(( ${#labels[@]} > 0 )) || die "no rows read from $PROMPTS (each row is <skill><TAB><prompt>)"
[[ "$SKILLS" == "installed" ]] || cd "$work/project" || die "cannot enter the temp project"

fired_skills() {  # one line per skill the Skill tool fired for this prompt (deduped), or __ERROR__
  local out names
  out=$(claude -p "$1" ${MODEL:+--model "$MODEL"} --output-format stream-json --verbose \
          --no-session-persistence </dev/null 2>>"$work/stderr.log")
  if [[ $? -ne 0 ]] || ! jq -e 'select(.type=="result") | (.is_error | not)' <<<"$out" >/dev/null 2>&1; then
    jq -r 'select(.type=="result") | .result // empty' <<<"$out" 2>/dev/null | head -1 > "$work/error"
    [[ -s "$work/error" ]] || tail -1 "$work/stderr.log" > "$work/error" 2>/dev/null
    echo "__ERROR__"; return
  fi
  [[ -s "$work/model" ]] || jq -r 'select(.type=="system" and .subtype=="init") | .model // empty' \
                              <<<"$out" > "$work/model" 2>/dev/null
  # A skill fired when the stream holds a Skill call for it and that call's result is not an error.
  #   the call:   an assistant event, a message.content item with type "tool_use" and name "Skill";
  #               the skill is its input.skill and the call is known by its id
  #   its result: a user event, a message.content item with type "tool_result" whose tool_use_id
  #               is the call's id; is_error true means the call failed and no skill was loaded
  # A call answered with an error does not count as fired. A call with no result means the stream
  # is cut short, and an assistant event whose content is not a list, or a content item (assistant or
  # user) that is not an object with a type, a result whose is_error is not true or false, or two
  # Skill calls with the same id, is a shape this reader does not know: all of these stop the
  # run, since passing over them would score the prompt as
  # "nothing fired". The stream is read whole (jq -s) so each call can be matched with its result.
  names=$(jq -rs 'try (
      [ .[] | select(.type=="user") | .message.content
        | if type=="array" then .[] else empty end
        | if type!="object" or (.type|type)!="string" then error("an item is not a typed object") else . end
        | select(.type=="tool_result") ] as $results
      | if ($results | map(select(has("is_error") and (.is_error | type) != "boolean")) | length) > 0
        then error("is_error is not true or false") else . end
      | [ .[] | select(.type=="assistant") | .message.content | if type=="array" then .[] else empty end
          | select(type=="object" and .type=="tool_use" and .name=="Skill") | .id ] as $ids
      | if ($ids | length) != ($ids | unique | length) then error("two Skill calls share an id") else . end
      | .[] | select(.type=="assistant") | .message.content
      | if type=="array" then .[] else error("content is not a list") end
      | if type!="object" or (.type|type)!="string" then error("an item is not a typed object") else . end
      | select(.type=="tool_use" and .name=="Skill") | . as $call
      | [ $results[] | select(.tool_use_id == $call.id) ] as $r
      | if ($call.input.skill | type=="string" and length>0 | not) then "__NOSKILL__"
        elif ($call.id | type) != "string" or ($r | length) == 0 then "__NORESULT__"
        elif ($r | map(select(.is_error == true)) | length) > 0 then empty
        else $call.input.skill end )
    catch "__UNREAD__"' <<<"$out") || names="__UNREAD__"
  if grep -qx '__UNREAD__' <<<"$names"; then
    echo "an event in the stream has a shape run.sh cannot read, so the event stream's format may have changed (see README: read one decision by hand)" > "$work/error"
    echo "__ERROR__"; return
  fi
  if grep -qx '__NOSKILL__' <<<"$names"; then
    echo "a Skill tool call carried no input.skill, so the event stream's format may have changed (see README: read one decision by hand)" > "$work/error"
    echo "__ERROR__"; return
  fi
  if grep -qx '__NORESULT__' <<<"$names"; then
    echo "a Skill tool call has no result in the stream, so the stream may be cut short or its format may have changed (see README: read one decision by hand)" > "$work/error"
    echo "__ERROR__"; return
  fi
  [[ -n "$names" ]] && sort -u <<<"$names"
}

main() {
  local sf_runs=0 recalled=0 nf_runs=0 false_trig=0 interference=0 i r n hit f gate
  echo "skill routing eval · $(date '+%Y-%m-%d %H:%M %Z') · $(claude --version </dev/null 2>/dev/null | head -1)"
  echo "prompts: $PROMPTS"
  echo "skills:  $skills_label"
  echo "repeats: $REPEATS · model requested: ${MODEL:-CLI default}"
  (( ${#notes[@]} > 0 )) && printf '%s\n' "${notes[@]}"
  echo
  printf '%-24s  %-36s  %-36s  %s\n' "EXPECTED" "OVER $REPEATS RUNS" "FIRED (RUNS)" "PROMPT"
  printf '%s\n' "------------------------------------------------------------------------------------------------------------------------------"

  for ((i=0; i<${#labels[@]}; i++)); do
    local expected="${labels[i]}" prompt="${prompts[i]}" allfired=""
    local t_right=0 t_wrong=0 t_none=0 t_both=0 t_ok=0 t_ft=0
    for ((r=0; r<REPEATS; r++)); do
      fired=()
      while IFS= read -r line; do
        [[ -n "$line" ]] && fired+=("$line")
      done < <(fired_skills "$prompt")
      if [[ "${fired[0]:-}" == "__ERROR__" ]]; then
        echo
        echo "run.sh could not grade: $prompt"
        echo "  $(cat "$work/error" 2>/dev/null)"
        echo "Nothing from this run is scored. For a login or usage message, run claude and /login, or wait for usage to reset."
        exit 2
      fi
      n=${#fired[@]}
      (( n > 0 )) && allfired+="$(printf '%s\n' "${fired[@]}")"$'\n'
      if [[ "$expected" == "none" ]]; then
        nf_runs=$((nf_runs+1))
        if (( n == 0 )); then t_ok=$((t_ok+1)); else t_ft=$((t_ft+1)); false_trig=$((false_trig+1)); fi
      else
        sf_runs=$((sf_runs+1))
        # Skills copied from a folder are local, so only the exact name is the skill under test: a
        # plugin's plugin:name with the same name is a neighbour. With SKILLS=installed the expected
        # skill may be a plugin's, which fires as plugin:name, so either spelling in column 1 matches.
        hit=0
        for f in "${fired[@]:-}"; do
          if [[ "$f" == "$expected" ]] || [[ "$SKILLS" == "installed" && "$f" == *":$expected" ]]; then hit=1; fi
        done
        if   (( n == 0 ));             then t_none=$((t_none+1))
        elif (( hit == 1 && n == 1 )); then t_right=$((t_right+1)); recalled=$((recalled+1))
        elif (( hit == 1 && n >= 2 )); then t_both=$((t_both+1));  recalled=$((recalled+1)); interference=$((interference+1))
        else                                t_wrong=$((t_wrong+1)); interference=$((interference+1)); fi
      fi
    done

    if [[ "$expected" == "none" ]]; then
      dist="quiet ${t_ok}  fired ${t_ft}"
    else
      dist="right ${t_right}  wrong ${t_wrong}  none ${t_none}  both ${t_both}"
    fi
    firedcol="$(printf '%s' "$allfired" | grep -v '^$' | sort | uniq -c \
                | awk '{printf "%s%s (%s)", (NR>1 ? ", " : ""), $2, $1}')"
    printf '%-24s  %-36s  %-36s  %s\n' "$expected" "$dist" "${firedcol:--}" "$prompt"
  done

  echo
  echo "recall        $recalled / $sf_runs   (should-fire runs that fired the right skill)"
  echo "false trigger $false_trig / $nf_runs   (should-fire-nothing runs that fired a skill anyway)"
  echo "interference  $interference / $sf_runs   (a non-intended skill fired on a run meant for another)"
  echo "model used:   $(cat "$work/model" 2>/dev/null)"
  echo
  echo "A skill that scores well alone and badly here has a selection problem, and editing its body will not touch it."

  # release gate: a non-zero exit when routing crosses the thresholds above
  gate=0
  if (( sf_runs > 0 && recalled * 100 < MIN_RECALL_PCT * sf_runs )); then
    echo "GATE FAIL  recall below ${MIN_RECALL_PCT}%"; gate=1; fi
  if (( false_trig > MAX_FALSE )); then
    echo "GATE FAIL  false triggers above $MAX_FALSE"; gate=1; fi
  if (( interference > MAX_INTERFERENCE )); then
    echo "GATE FAIL  interference above $MAX_INTERFERENCE"; gate=1; fi
  # a pass names only what was tested: a measure with no rows behind it cannot have met its threshold
  if (( gate == 0 )); then
    if (( sf_runs == 0 )); then
      echo "GATE PASS  false triggers <= $MAX_FALSE (recall and interference not tested: no row expects a skill)"
    elif (( nf_runs == 0 )); then
      echo "GATE PASS  recall >= ${MIN_RECALL_PCT}%, interference <= $MAX_INTERFERENCE (false triggers not tested: no row is labelled none)"
    else
      echo "GATE PASS  recall >= ${MIN_RECALL_PCT}%, false triggers <= $MAX_FALSE, interference <= $MAX_INTERFERENCE"
    fi
  fi
  return $gate
}

if [[ -n "$OUT" ]]; then
  main | tee "$OUT"
  codes=("${PIPESTATUS[@]}")
  (( codes[1] == 0 )) || die "could not write OUT=$OUT, so that file is missing or incomplete"
  exit "${codes[0]}"
fi
main
exit $?