{
  "schema": 1,
  "index": 1,
  "name": "index-1-2026-11-20-run2",
  "dry_run": false,
  "run_date": "2026-11-20",
  "started_utc": "2026-11-20T19:20:33+00:00",
  "finished_utc": "2026-11-20T19:33:00+00:00",
  "raw_file": "runs/index-1-2026-11-20-run2.txt",
  "jsonl_file": "runs/index-1-2026-11-20-run2.jsonl",
  "results_file": "runs/index-1-2026-11-20-run2-results.json",
  "harness": "scripts/index_run.py",
  "commit": "378fee41125546195af96a4c8aa6f7ff511256c2",
  "anthropic": "1.2.0",
  "pydantic": "2.13.5",
  "python": "3.12.16",
  "default_model": "claude-opus-5",
  "models_listed": [
    "claude-fable-5",
    "claude-fable-5-1",
    "claude-fable-5-2",
    "claude-haiku-4-5-20251001",
    "claude-haiku-5-5",
    "claude-opus-4-5-20251101",
    "claude-opus-4-6",
    "claude-opus-4-7",
    "claude-opus-4-8",
    "claude-opus-5",
    "claude-opus-5-5",
    "claude-sonnet-4-5-20250929",
    "claude-sonnet-4-6",
    "claude-sonnet-5",
    "claude-sonnet-5-5"
  ],
  "method": {
    "index": 1,
    "default_model": "claude-opus-5",
    "suites": [
      {
        "id": "route-single",
        "items": 10,
        "calls_per_item": 1,
        "source": "scripts/ch2_course_collapse.py",
        "key": "KEY (ch2_collapse.py's answer key)",
        "measures": "Routing a support ticket to one of four queues with one direct call.",
        "scoring": "1 point when the reply, stripped and lower-cased, is exactly the key's queue."
      },
      {
        "id": "route-chain",
        "items": 10,
        "calls_per_item": 3,
        "source": "scripts/ch2_course_collapse.py",
        "key": "KEY (ch2_collapse.py's answer key)",
        "measures": "The same routing done as a three-call chain (summarize, list candidates, pick one).",
        "scoring": "1 point when the last call's reply, stripped and lower-cased, is exactly the key's queue."
      },
      {
        "id": "route-typed",
        "items": 10,
        "calls_per_item": 1,
        "source": "scripts/ch3_course_route_typed.py",
        "key": "KEY (ch2_collapse.py's answer key)",
        "measures": "The same routing with a typed output (messages.parse into one of the four queue names).",
        "scoring": "1 point when the call returns a typed queue equal to the key's. A raise or None is a failure."
      },
      {
        "id": "agent",
        "items": 5,
        "calls_per_item": null,
        "runs": 6,
        "source": "scripts/ch4_course_agent.py",
        "key": "GOLD in scripts/ch7_grader_rungs.py",
        "measures": "A two-tool inventory agent (lookup_stock, calculate) answering questions, up to five steps each.",
        "scoring": "1 point when the final answer, ended on end_turn, holds the gold value as a number (ch9_grade.py's rule), or for the keyboards question says keyboards are the larger stock (ch7_grader_rungs.py's directional rule). The open-ended reorder-policy question is run and kept but not scored. A step cap or a stop on max_tokens is a failure."
      }
    ],
    "items_scored": 35,
    "tickets": 10,
    "queues": [
      "billing",
      "auth",
      "bug",
      "general"
    ],
    "agent_questions": 6,
    "limits": {
      "single_max_tokens": 1024,
      "chain_max_tokens": 1024,
      "typed_max_tokens": 1000,
      "agent_max_tokens": 2048,
      "agent_max_steps": 5,
      "max_calls_per_model": 80,
      "token_ceiling_per_model": 120000
    },
    "settings": "Each model is called as a developer would call it by default: no system prompt, no temperature, no thinking or effort setting, one pass per item, the course's max_tokens.",
    "ranking": "By task score, highest first. Equal scores are ordered by cost per run, lowest first, then by median latency. A model whose pass did not finish is listed apart, unranked.",
    "dimensions": {
      "quality": "task score",
      "cost": "cost per run",
      "latency": "median seconds per call",
      "safety": "not in Issue 1: no safety test set is wired into this harness yet"
    },
    "planned_models": [
      {
        "id": "claude-fable-5-2",
        "name": "Claude Fable 5.2",
        "why": "latest lineup, models overview page"
      },
      {
        "id": "claude-opus-5-5",
        "name": "Claude Opus 5.5",
        "why": "latest lineup, models overview page"
      },
      {
        "id": "claude-sonnet-5-5",
        "name": "Claude Sonnet 5.5",
        "why": "latest lineup, models overview page"
      },
      {
        "id": "claude-haiku-5-5",
        "name": "Claude Haiku 5.5",
        "why": "latest lineup, models overview page"
      },
      {
        "id": "claude-opus-5",
        "name": "Claude Opus 5",
        "why": "the model Course 1 runs on (BOOK.md, Course Module 1 harness row)"
      },
      {
        "id": "claude-opus-4-6",
        "name": "Claude Opus 4.6",
        "why": "the model the book's Ch 2 and Ch 3 runs were made on"
      },
      {
        "id": "claude-haiku-4-5-20251001",
        "name": "Claude Haiku 4.5",
        "why": "the small model the book's Ch 13 runs were made on"
      }
    ],
    "not_planned": [
      {
        "name": "Claude Mythos 5, 5.1 and 5.2",
        "reason": "limited availability (verification programme); not expected on this key"
      },
      {
        "name": "Claude Fable 5 and 5.1, Opus 4.5, 4.7 and 4.8, Sonnet 4.5, 4.6 and 5",
        "reason": "kept out of Issue 1 to keep the run small; each is one line to add"
      }
    ],
    "prices": {
      "source_url": "https://platform.claude.com/docs/en/about-claude/pricing",
      "read_on": "2026-11-20",
      "currency": "USD",
      "unit": "per million tokens",
      "note": "Base input and output prices, standard (non-batch) API. Thinking tokens are billed as output tokens and are inside usage.output_tokens. Claude Haiku 5.5's price applies to prompts up to 100,000 tokens; a longer prompt makes its cost unpriced, never guessed.",
      "per_mtok": {
        "claude-fable-5-2": {
          "input": 8.0,
          "output": 40.0,
          "cache_write_5m": 10.0,
          "cache_read": 0.2
        },
        "claude-opus-5-5": {
          "input": 4.0,
          "output": 20.0,
          "cache_write_5m": 5.0,
          "cache_read": 0.2
        },
        "claude-sonnet-5-5": {
          "input": 2.0,
          "output": 10.0,
          "cache_write_5m": 2.5,
          "cache_read": 0.1
        },
        "claude-haiku-5-5": {
          "input": 0.1,
          "output": 0.5,
          "cache_write_5m": 0.125,
          "cache_read": 0.01,
          "max_prompt_tokens": 100000
        },
        "claude-opus-5": {
          "input": 5.0,
          "output": 25.0,
          "cache_write_5m": 6.25,
          "cache_read": 0.5
        },
        "claude-opus-4-6": {
          "input": 5.0,
          "output": 25.0,
          "cache_write_5m": 6.25,
          "cache_read": 0.5
        },
        "claude-haiku-4-5-20251001": {
          "input": 1.0,
          "output": 5.0,
          "cache_write_5m": 1.25,
          "cache_read": 0.1
        }
      }
    },
    "unmeasured_providers": [
      {
        "provider": "OpenAI",
        "reason": "no API key in this repository; not run and not estimated"
      },
      {
        "provider": "Google",
        "reason": "no API key in this repository; not run and not estimated"
      },
      {
        "provider": "Meta (Llama, open weights)",
        "reason": "no endpoint or key in this repository; not run and not estimated"
      },
      {
        "provider": "Mistral AI",
        "reason": "no API key in this repository; not run and not estimated"
      },
      {
        "provider": "DeepSeek",
        "reason": "no API key in this repository; not run and not estimated"
      }
    ],
    "harness": "scripts/index_run.py",
    "example": {
      "ticket": "I was charged twice for the same month.",
      "key": "billing",
      "single_prompt": "Route this support ticket to exactly one queue.\n\nbilling  - charges, invoices, refunds, plan renewals, anything about money\nauth     - sign-in, passwords, sessions, lockouts, account access\nbug      - the software is behaving incorrectly: crashes, errors, broken features\ngeneral  - anything else, including permissions, sharing, and \"who do I ask\"\n\nReply with one word: billing, auth, bug, or general.\n\nTicket: I was charged twice for the same month.",
      "chain_prompts": [
        "Summarize this support ticket in one sentence:\n\nI was charged twice for the same month.",
        "Which of these queues could own this? billing, auth, bug, general\n\n<the summary>",
        "Pick exactly one queue from this list. Reply with one word.\n\n<the candidates>"
      ],
      "typed_prompt": "Route this support ticket to exactly one of: billing, auth, bug, general.\n\nI was charged twice for the same month.",
      "typed_output": "queue: Literal[\"billing\", \"auth\", \"bug\", \"general\"]",
      "agent_question": "What is the total value of the webcams on hand?",
      "agent_gold": "186",
      "agent_tools": [
        "lookup_stock",
        "calculate"
      ]
    }
  },
  "models": [
    {
      "model": "claude-fable-5-2",
      "status": "measured",
      "stopped": null,
      "task_score": {
        "correct": 33,
        "scored": 35,
        "planned": 35
      },
      "by_suite": {
        "route-single": {
          "correct": 10,
          "scored": 10,
          "planned": 10,
          "failures": 0
        },
        "route-chain": {
          "correct": 9,
          "scored": 10,
          "planned": 10,
          "failures": 0
        },
        "route-typed": {
          "correct": 9,
          "scored": 10,
          "planned": 10,
          "failures": 0
        },
        "agent": {
          "correct": 5,
          "scored": 5,
          "planned": 5,
          "failures": 0
        }
      },
      "cost_usd_per_run": 0.327712,
      "cost_note": null,
      "tokens": {
        "input": 14209,
        "output": 5351,
        "thinking": 2131
      },
      "calls": 62,
      "latency_median_s": 1.073,
      "latency_calls_timed": 62,
      "failures": {
        "items": 0,
        "kinds": {}
      },
      "stop_reasons": {
        "end_turn": 56,
        "tool_use": 6
      },
      "name": "Claude Fable 5.2",
      "run_started_utc": "2026-11-20T19:20:33+00:00",
      "run_finished_utc": "2026-11-20T19:22:12+00:00",
      "rank": 4
    },
    {
      "model": "claude-opus-5-5",
      "status": "measured",
      "stopped": null,
      "task_score": {
        "correct": 33,
        "scored": 35,
        "planned": 35
      },
      "by_suite": {
        "route-single": {
          "correct": 10,
          "scored": 10,
          "planned": 10,
          "failures": 0
        },
        "route-chain": {
          "correct": 9,
          "scored": 10,
          "planned": 10,
          "failures": 0
        },
        "route-typed": {
          "correct": 9,
          "scored": 10,
          "planned": 10,
          "failures": 0
        },
        "agent": {
          "correct": 5,
          "scored": 5,
          "planned": 5,
          "failures": 0
        }
      },
      "cost_usd_per_run": 0.17366,
      "cost_note": null,
      "tokens": {
        "input": 14730,
        "output": 5737,
        "thinking": 1803
      },
      "calls": 62,
      "latency_median_s": 1.196,
      "latency_calls_timed": 62,
      "failures": {
        "items": 0,
        "kinds": {}
      },
      "stop_reasons": {
        "end_turn": 56,
        "tool_use": 6
      },
      "name": "Claude Opus 5.5",
      "run_started_utc": "2026-11-20T19:22:12+00:00",
      "run_finished_utc": "2026-11-20T19:23:55+00:00",
      "rank": 3
    },
    {
      "model": "claude-sonnet-5-5",
      "status": "measured",
      "stopped": null,
      "task_score": {
        "correct": 33,
        "scored": 35,
        "planned": 35
      },
      "by_suite": {
        "route-single": {
          "correct": 10,
          "scored": 10,
          "planned": 10,
          "failures": 0
        },
        "route-chain": {
          "correct": 9,
          "scored": 10,
          "planned": 10,
          "failures": 0
        },
        "route-typed": {
          "correct": 9,
          "scored": 10,
          "planned": 10,
          "failures": 0
        },
        "agent": {
          "correct": 5,
          "scored": 5,
          "planned": 5,
          "failures": 0
        }
      },
      "cost_usd_per_run": 0.09058,
      "cost_note": null,
      "tokens": {
        "input": 15800,
        "output": 5898,
        "thinking": 1421
      },
      "calls": 63,
      "latency_median_s": 1.134,
      "latency_calls_timed": 63,
      "failures": {
        "items": 0,
        "kinds": {}
      },
      "stop_reasons": {
        "end_turn": 56,
        "tool_use": 7
      },
      "name": "Claude Sonnet 5.5",
      "run_started_utc": "2026-11-20T19:23:55+00:00",
      "run_finished_utc": "2026-11-20T19:25:32+00:00",
      "rank": 2
    },
    {
      "model": "claude-haiku-5-5",
      "status": "measured",
      "stopped": null,
      "task_score": {
        "correct": 34,
        "scored": 35,
        "planned": 35
      },
      "by_suite": {
        "route-single": {
          "correct": 10,
          "scored": 10,
          "planned": 10,
          "failures": 0
        },
        "route-chain": {
          "correct": 9,
          "scored": 10,
          "planned": 10,
          "failures": 0
        },
        "route-typed": {
          "correct": 10,
          "scored": 10,
          "planned": 10,
          "failures": 0
        },
        "agent": {
          "correct": 5,
          "scored": 5,
          "planned": 5,
          "failures": 0
        }
      },
      "cost_usd_per_run": 0.003271,
      "cost_note": null,
      "tokens": {
        "input": 14568,
        "output": 3629,
        "thinking": 1021
      },
      "calls": 63,
      "latency_median_s": 0.648,
      "latency_calls_timed": 63,
      "failures": {
        "items": 0,
        "kinds": {}
      },
      "stop_reasons": {
        "end_turn": 56,
        "tool_use": 7
      },
      "name": "Claude Haiku 5.5",
      "run_started_utc": "2026-11-20T19:25:32+00:00",
      "run_finished_utc": "2026-11-20T19:26:22+00:00",
      "rank": 1
    },
    {
      "model": "claude-opus-5",
      "status": "measured",
      "stopped": null,
      "task_score": {
        "correct": 32,
        "scored": 35,
        "planned": 35
      },
      "by_suite": {
        "route-single": {
          "correct": 10,
          "scored": 10,
          "planned": 10,
          "failures": 0
        },
        "route-chain": {
          "correct": 9,
          "scored": 10,
          "planned": 10,
          "failures": 0
        },
        "route-typed": {
          "correct": 9,
          "scored": 10,
          "planned": 10,
          "failures": 0
        },
        "agent": {
          "correct": 4,
          "scored": 5,
          "planned": 5,
          "failures": 0
        }
      },
      "cost_usd_per_run": 0.39091,
      "cost_note": null,
      "tokens": {
        "input": 18937,
        "output": 11849,
        "thinking": 3699
      },
      "calls": 62,
      "latency_median_s": 1.331,
      "latency_calls_timed": 62,
      "failures": {
        "items": 0,
        "kinds": {}
      },
      "stop_reasons": {
        "end_turn": 56,
        "tool_use": 6
      },
      "name": "Claude Opus 5",
      "run_started_utc": "2026-11-20T19:26:22+00:00",
      "run_finished_utc": "2026-11-20T19:30:04+00:00",
      "rank": 6
    },
    {
      "model": "claude-opus-4-6",
      "status": "measured",
      "stopped": null,
      "task_score": {
        "correct": 28,
        "scored": 35,
        "planned": 35
      },
      "by_suite": {
        "route-single": {
          "correct": 10,
          "scored": 10,
          "planned": 10,
          "failures": 0
        },
        "route-chain": {
          "correct": 4,
          "scored": 10,
          "planned": 10,
          "failures": 0
        },
        "route-typed": {
          "correct": 9,
          "scored": 10,
          "planned": 10,
          "failures": 0
        },
        "agent": {
          "correct": 5,
          "scored": 5,
          "planned": 5,
          "failures": 0
        }
      },
      "cost_usd_per_run": 0.135495,
      "cost_note": null,
      "tokens": {
        "input": 15484,
        "output": 2323,
        "thinking": null
      },
      "calls": 64,
      "latency_median_s": 1.632,
      "latency_calls_timed": 64,
      "failures": {
        "items": 0,
        "kinds": {}
      },
      "stop_reasons": {
        "end_turn": 56,
        "tool_use": 8
      },
      "name": "Claude Opus 4.6",
      "run_started_utc": "2026-11-20T19:30:04+00:00",
      "run_finished_utc": "2026-11-20T19:32:10+00:00",
      "rank": 7
    },
    {
      "model": "claude-haiku-4-5-20251001",
      "status": "measured",
      "stopped": null,
      "task_score": {
        "correct": 32,
        "scored": 35,
        "planned": 35
      },
      "by_suite": {
        "route-single": {
          "correct": 10,
          "scored": 10,
          "planned": 10,
          "failures": 0
        },
        "route-chain": {
          "correct": 8,
          "scored": 10,
          "planned": 10,
          "failures": 0
        },
        "route-typed": {
          "correct": 9,
          "scored": 10,
          "planned": 10,
          "failures": 0
        },
        "agent": {
          "correct": 5,
          "scored": 5,
          "planned": 5,
          "failures": 0
        }
      },
      "cost_usd_per_run": 0.02718,
      "cost_note": null,
      "tokens": {
        "input": 15670,
        "output": 2302,
        "thinking": null
      },
      "calls": 64,
      "latency_median_s": 0.668,
      "latency_calls_timed": 64,
      "failures": {
        "items": 0,
        "kinds": {}
      },
      "stop_reasons": {
        "end_turn": 56,
        "tool_use": 8
      },
      "name": "Claude Haiku 4.5",
      "run_started_utc": "2026-11-20T19:32:10+00:00",
      "run_finished_utc": "2026-11-20T19:33:00+00:00",
      "rank": 5
    }
  ]
}
