From c363397aad5ce11c5b3ca038ab6bac599ef1f7ac Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Sun, 9 Aug 2026 11:07:44 -0700
Subject: [PATCH] Site: serving-sensitivity note on the leaderboard +
 methodology copy fixes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Leaderboard (US view): labeled callout for the Claude thinking
  sensitivity — forced tool_choice suppresses extended thinking; links
  the published supplement and the v2 plan (#139). Board unchanged.
- Methodology Task card: answers come via a forced answer-schema tool
  call, and a few models are served one output per request — replaces
  the stale 'in one response, with no tool use' claim.
- Dataset v1.1 description: 'identical forced-tool request' replaces
  'unconfigured provider defaults'.
- Sensitivity doc Status: next board version commits to tool_choice
  auto (expedited, gated on roster-wide probes); manuscript table note
  moved to future tense pending the next refreeze.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 app/src/components/Methodology.tsx      |  7 ++++--
 app/src/components/ModelLeaderboard.tsx | 33 +++++++++++++++++++++++++
 app/src/data.versions.json              |  2 +-
 sensitivity/claude-thinking-2026-08.md  | 13 +++++-----
 4 files changed, 46 insertions(+), 9 deletions(-)

diff --git a/app/src/components/Methodology.tsx b/app/src/components/Methodology.tsx
index 730151b..859e11b 100644
--- a/app/src/components/Methodology.tsx
+++ b/app/src/components/Methodology.tsx
@@ -130,8 +130,11 @@ export default function Methodology({
       <div className="grid lg:grid-cols-2 gap-4 mt-8">
         <SectionCard title="Task">
           Each model sees the same household description and must return all
-          scored outputs plus a short explanation for each output in one
-          response, with no tool use. The exact provider-specific prompts are
+          scored outputs plus a short explanation for each output, with no
+          external tools — no calculator, search, or PolicyEngine access.
+          Answers come back through a forced answer-schema tool call; a few
+          models are served one output per request for parse reliability, per
+          the repo&apos;s model cards. The exact provider-specific prompts are
           visible in the scenario explorer, so you can inspect the contract
           instead of inferring it.
         </SectionCard>
diff --git a/app/src/components/ModelLeaderboard.tsx b/app/src/components/ModelLeaderboard.tsx
index a327c52..169529c 100644
--- a/app/src/components/ModelLeaderboard.tsx
+++ b/app/src/components/ModelLeaderboard.tsx
@@ -273,6 +273,39 @@ export default function ModelLeaderboard({
       >
         Model rankings
       </h2>
+      {selectedView === "us" && (
+        <div
+          className="card mt-5 px-5 py-4 animate-fade-up"
+          style={{ animationDelay: "120ms" }}
+        >
+          <div className="text-[10px] uppercase tracking-[0.14em] text-text-muted font-medium">
+            Serving sensitivity · August 2026
+          </div>
+          <p className="mt-2 text-sm leading-relaxed text-text-secondary">
+            Claude models skip extended thinking when the answer tool call is
+            forced, as it is in the identical request this board holds every
+            model to; other reasoning-by-default providers reason regardless.
+            Re-run with <code>tool_choice: auto</code>, Claude Fable 5 scores
+            86.9 (would rank #2), Claude Opus 5 85.6 (#3), and Claude Sonnet 5
+            80.2 (#8). The board below is unchanged — those runs sit beside it
+            as a{" "}
+            <a
+              href="https://github.com/PolicyEngine/policybench/blob/main/sensitivity/claude-thinking-2026-08.md"
+              className="text-primary hover:underline"
+            >
+              labeled sensitivity
+            </a>{" "}
+            — and the{" "}
+            <a
+              href="https://github.com/PolicyEngine/policybench/issues/139"
+              className="text-primary hover:underline"
+            >
+              next board version
+            </a>{" "}
+            moves every model to auto.
+          </p>
+        </div>
+      )}
       <div
         role="region"
         aria-labelledby="leaderboard-heading"
diff --git a/app/src/data.versions.json b/app/src/data.versions.json
index f444a4f..85b7f30 100644
--- a/app/src/data.versions.json
+++ b/app/src/data.versions.json
@@ -5,7 +5,7 @@
     {
       "id": "1.1",
       "label": "1.1",
-      "description": "Corrected ground truth (policyengine-us 1.755.4); every model at unconfigured provider defaults - 29 models",
+      "description": "Corrected ground truth (policyengine-us 1.755.4); identical forced-tool request for every model - 29 models",
       "snapshotLabel": null,
       "artifact": {
         "pointer": "live"
diff --git a/sensitivity/claude-thinking-2026-08.md b/sensitivity/claude-thinking-2026-08.md
index 209f4ca..2502464 100644
--- a/sensitivity/claude-thinking-2026-08.md
+++ b/sensitivity/claude-thinking-2026-08.md
@@ -58,12 +58,13 @@ identical request, and this run sits beside it as a labeled sensitivity,
 not in it. Each run's predictions are attached to the
 `dashboard-data-20260805` release as
 `sensitivity-claude-{fable,opus,sonnet}-5-thinking-predictions.csv.gz`. The manuscript's
-serving-configuration table documents the interaction. A future board
-version may move every model to `tool_choice: "auto"` so each provider's
-default reasoning posture engages under a still-identical request shape;
-that change requires roster-wide probes confirming reliable tool calling
-under `auto` and would ship as a versioned re-run, not an edit to existing
-scores.
+serving-configuration table will document the interaction at its next
+refreeze. The next board version moves every model to
+`tool_choice: "auto"` so each provider's default reasoning posture engages
+under a still-identical request shape — expedited, gated on roster-wide
+probes confirming reliable tool calling under `auto`, and shipped as a
+versioned re-run, never an edit to existing scores; the plan is
+[policybench#139](https://github.com/PolicyEngine/policybench/issues/139).
 
 ## Reproducing