{
  "api_version": "1",
  "updated": "2026-10-29",
  "latest_event": "2026-10-28",
  "rights": "© Frontier Log, all rights reserved.",
  "story": {
    "slug": "who-leads-terminal-bench-4",
    "title": "Who Leads Terminal-Bench 4.0 Depends on Who Ran It",
    "description": "Vals AI ranks Claude Opus 5.5 first on Terminal-Bench 4.0. Artificial Analysis, running the same harness, ranks Kimi K3.1 Pro first.",
    "url": "https://frontierlog.ai/stories/who-leads-terminal-bench-4",
    "date": "2026-10-30",
    "image": "https://frontierlog.ai/og/story-who-leads-terminal-bench-4.png",
    "event": "2026-10-28-gemini-4-carbon",
    "tags": [
      "coding",
      "agents"
    ],
    "steps": [],
    "tables": [
      {
        "title": "Terminal-Bench 4.0, each lab's own report",
        "unit": "%",
        "rows": [
          {
            "event": "2026-09-28-claude-sonnet-5-5",
            "date": "2026-09-28",
            "provider": "anthropic",
            "value": 70.6,
            "printed": "70.6%",
            "source": "https://www.anthropic.com/claude-sonnet-5-5",
            "where": "benchmark table, row 'Terminal-Bench 4.0', column 'Sonnet 5.5'"
          },
          {
            "event": "2026-09-22-claude-opus-5-5",
            "date": "2026-09-22",
            "provider": "anthropic",
            "value": 66.4,
            "printed": "66.4%",
            "source": "https://www.anthropic.com/claude-opus-5-5",
            "where": "benchmark table, row 'Terminal-Bench 4.0', column 'Opus 5.5'"
          },
          {
            "event": "2026-10-24-kimi-k3-1-pro",
            "date": "2026-10-24",
            "provider": "moonshot",
            "value": 65.7,
            "printed": "65.7",
            "source": "https://www.kimi.ai/blog/kimi-k3-1-pro",
            "where": "'Benchmarks' table (rendered by the page's script), section 'Coding', row 'Terminal-Bench 4.0', column 'Kimi K3.1 Pro (max)'"
          },
          {
            "event": "2026-09-03-gpt-6-astra",
            "date": "2026-09-03",
            "provider": "openai",
            "value": 57.9,
            "printed": "57.9%",
            "source": "https://openai.com/index/gpt-6-astra",
            "where": "Coding table, row 'Terminal-Bench 4.0', column 'GPT‑6 Astra'"
          },
          {
            "event": "2026-09-01-claude-fable-5-1-mythos-5-1",
            "date": "2026-09-01",
            "provider": "anthropic",
            "value": 55.8,
            "printed": "55.8%",
            "source": "https://www.anthropic.com/claude-fable-and-mythos-5-1",
            "where": "benchmark table, row 'Terminal-Bench 4.0', column 'Fable 5.1' (the cell also prints a Mythos 5.1 figure)"
          }
        ]
      },
      {
        "title": "Terminal-Bench 4.0, run by Vals AI",
        "unit": "%",
        "rows": [
          {
            "event": "2026-09-22-claude-opus-5-5",
            "date": "2026-09-22",
            "provider": "anthropic",
            "value": 65.15,
            "printed": null,
            "source": "https://www.vals.ai/benchmarks/terminal-bench-4",
            "where": "'Terminal-Bench 4.0 leaderboard' table, row 'Claude Opus 5.5', column 'Score' (harness Mini-SWE-agent)"
          },
          {
            "event": "2026-09-28-claude-sonnet-5-5",
            "date": "2026-09-28",
            "provider": "anthropic",
            "value": 64.14,
            "printed": null,
            "source": "https://www.vals.ai/benchmarks/terminal-bench-4",
            "where": "'Terminal-Bench 4.0 leaderboard' table, row 'Claude Sonnet 5.5', column 'Score' (harness Mini-SWE-agent)"
          },
          {
            "event": "2026-10-24-kimi-k3-1-pro",
            "date": "2026-10-24",
            "provider": "moonshot",
            "value": 63.64,
            "printed": null,
            "source": "https://www.vals.ai/benchmarks/terminal-bench-4",
            "where": "'Terminal-Bench 4.0 leaderboard' table, row 'Kimi K3.1 Pro', column 'Score' (harness Mini-SWE-agent)"
          },
          {
            "event": "2026-10-28-gemini-4-carbon",
            "date": "2026-10-28",
            "provider": "google",
            "value": 63.13,
            "printed": null,
            "source": "https://www.vals.ai/benchmarks/terminal-bench-4",
            "where": "'Terminal-Bench 4.0 leaderboard' table, row 'Gemini 4 Carbon', column 'Score' (harness Mini-SWE-agent)"
          },
          {
            "event": "2026-09-03-gpt-6-astra",
            "date": "2026-09-03",
            "provider": "openai",
            "value": 59.6,
            "printed": null,
            "source": "https://www.vals.ai/benchmarks/terminal-bench-4",
            "where": "'Terminal-Bench 4.0 leaderboard' table, row 'GPT-6 Astra', column 'Score' (harness Mini-SWE-agent)"
          },
          {
            "event": "2026-09-01-claude-fable-5-1-mythos-5-1",
            "date": "2026-09-01",
            "provider": "anthropic",
            "value": 58.08,
            "printed": null,
            "source": "https://www.vals.ai/benchmarks/terminal-bench-4",
            "where": "'Terminal-Bench 4.0 leaderboard' table, row 'Claude Fable 5.1', column 'Score' (harness Mini-SWE-agent)"
          }
        ]
      },
      {
        "title": "Terminal-Bench 4.0, Vals AI's standard error",
        "unit": " points",
        "rows": [
          {
            "event": "2026-09-22-claude-opus-5-5",
            "date": "2026-09-22",
            "provider": "anthropic",
            "value": 0.0,
            "printed": null,
            "source": "https://www.vals.ai/benchmarks/terminal-bench-4",
            "where": "'Terminal-Bench 4.0 leaderboard' table, row 'Claude Opus 5.5', the ± beside its score"
          },
          {
            "event": "2026-09-28-claude-sonnet-5-5",
            "date": "2026-09-28",
            "provider": "anthropic",
            "value": 1.01,
            "printed": null,
            "source": "https://www.vals.ai/benchmarks/terminal-bench-4",
            "where": "'Terminal-Bench 4.0 leaderboard' table, row 'Claude Sonnet 5.5', the ± beside its score"
          },
          {
            "event": "2026-10-24-kimi-k3-1-pro",
            "date": "2026-10-24",
            "provider": "moonshot",
            "value": 1.75,
            "printed": null,
            "source": "https://www.vals.ai/benchmarks/terminal-bench-4",
            "where": "'Terminal-Bench 4.0 leaderboard' table, row 'Kimi K3.1 Pro', the ± beside its score"
          },
          {
            "event": "2026-10-28-gemini-4-carbon",
            "date": "2026-10-28",
            "provider": "google",
            "value": 2.2,
            "printed": null,
            "source": "https://www.vals.ai/benchmarks/terminal-bench-4",
            "where": "'Terminal-Bench 4.0 leaderboard' table, row 'Gemini 4 Carbon', the ± beside its score"
          },
          {
            "event": "2026-09-03-gpt-6-astra",
            "date": "2026-09-03",
            "provider": "openai",
            "value": 4.4,
            "printed": null,
            "source": "https://www.vals.ai/benchmarks/terminal-bench-4",
            "where": "'Terminal-Bench 4.0 leaderboard' table, row 'GPT-6 Astra', the ± beside its score"
          },
          {
            "event": "2026-09-01-claude-fable-5-1-mythos-5-1",
            "date": "2026-09-01",
            "provider": "anthropic",
            "value": 3.31,
            "printed": null,
            "source": "https://www.vals.ai/benchmarks/terminal-bench-4",
            "where": "'Terminal-Bench 4.0 leaderboard' table, row 'Claude Fable 5.1', the ± beside its score"
          }
        ]
      },
      {
        "title": "Terminal-Bench 4.0, run by Artificial Analysis",
        "unit": "%",
        "rows": [
          {
            "event": "2026-10-24-kimi-k3-1-pro",
            "date": "2026-10-24",
            "provider": "moonshot",
            "value": 67.2,
            "printed": null,
            "source": "https://artificialanalysis.ai/evaluations/terminalbench-4-0",
            "where": "'Terminal-Bench 4.0: Score' chart, bar 'Kimi K3.1 Pro'"
          },
          {
            "event": "2026-10-28-gemini-4-carbon",
            "date": "2026-10-28",
            "provider": "google",
            "value": 65.2,
            "printed": null,
            "source": "https://artificialanalysis.ai/evaluations/terminalbench-4-0",
            "where": "'Terminal-Bench 4.0: Score' chart, bar 'Gemini 4 Carbon (High)'"
          },
          {
            "event": "2026-09-28-claude-sonnet-5-5",
            "date": "2026-09-28",
            "provider": "anthropic",
            "value": 63.6,
            "printed": null,
            "source": "https://artificialanalysis.ai/evaluations/terminalbench-4-0",
            "where": "'Terminal-Bench 4.0: Score' chart, bar 'Claude Sonnet 5.5 (Max, Default Fallback)'"
          },
          {
            "event": "2026-09-22-claude-opus-5-5",
            "date": "2026-09-22",
            "provider": "anthropic",
            "value": 59.6,
            "printed": null,
            "source": "https://artificialanalysis.ai/evaluations/terminalbench-4-0",
            "where": "'Terminal-Bench 4.0: Score' chart, bar 'Claude Opus 5.5 (Max, Default Fallback)'"
          },
          {
            "event": "2026-09-03-gpt-6-astra",
            "date": "2026-09-03",
            "provider": "openai",
            "value": 59.1,
            "printed": null,
            "source": "https://artificialanalysis.ai/evaluations/terminalbench-4-0",
            "where": "'Terminal-Bench 4.0: Score' chart, bar 'GPT-6 Astra (Max)'"
          },
          {
            "event": "2026-09-01-claude-fable-5-1-mythos-5-1",
            "date": "2026-09-01",
            "provider": "anthropic",
            "value": 52.0,
            "printed": null,
            "source": "https://artificialanalysis.ai/evaluations/terminalbench-4-0",
            "where": "'Terminal-Bench 4.0: Score' chart, bar 'Claude Fable 5.1 (Max, Default Fallback)'"
          }
        ]
      }
    ],
    "cites": [
      {
        "url": "https://www.vals.ai/benchmarks/terminal-bench-4",
        "by": "Vals AI",
        "medium": "benchmark page"
      },
      {
        "url": "https://www.vals.ai/models/anthropic_claude-opus-5-5",
        "by": "Vals AI",
        "medium": "model page"
      },
      {
        "url": "https://www.vals.ai/models/kimi_kimi-k3.1-pro",
        "by": "Vals AI",
        "medium": "model page"
      },
      {
        "url": "https://www.vals.ai/models/google_gemini-4-carbon",
        "by": "Vals AI",
        "medium": "model page"
      },
      {
        "url": "https://artificialanalysis.ai/evaluations/terminalbench-4-0",
        "by": "Artificial Analysis",
        "medium": "evaluation page"
      },
      {
        "url": "https://artificialanalysis.ai/methodology/intelligence-benchmarking",
        "by": "Artificial Analysis",
        "medium": "methodology page"
      }
    ],
    "sources": [
      "https://www.anthropic.com/claude-sonnet-5-5",
      "https://www.anthropic.com/claude-opus-5-5",
      "https://www.kimi.ai/blog/kimi-k3-1-pro",
      "https://openai.com/index/gpt-6-astra",
      "https://www.anthropic.com/claude-fable-and-mythos-5-1",
      "https://www.vals.ai/benchmarks/terminal-bench-4",
      "https://www.vals.ai/models/anthropic_claude-opus-5-5",
      "https://www.vals.ai/models/kimi_kimi-k3.1-pro",
      "https://www.vals.ai/models/google_gemini-4-carbon",
      "https://artificialanalysis.ai/evaluations/terminalbench-4-0",
      "https://artificialanalysis.ai/methodology/intelligence-benchmarking"
    ],
    "body": [
      {
        "figure": "main"
      },
      {
        "p": "Vals AI and Artificial Analysis both publish Terminal-Bench 4.0 scores run with the same agent harness, mini-swe-agent, on all 66 of the benchmark's tasks. On Oct 29 they put different models first. [Vals AI](https://www.vals.ai/benchmarks/terminal-bench-4) ranks Claude Opus 5.5 first at 65.15%, and [Artificial Analysis](https://artificialanalysis.ai/evaluations/terminalbench-4-0) ranks Kimi K3.1 Pro first at 67.2%, with Opus 5.5 fourth at 59.6%. Of the scores the labs reported for their own models, the highest is the 70.6% [Anthropic reported](https://www.anthropic.com/claude-sonnet-5-5) for Claude Sonnet 5.5."
      },
      {
        "h": "Same harness, different limits"
      },
      {
        "p": "Vals AI's [methodology](https://www.vals.ai/benchmarks/terminal-bench-4) runs each model through the benchmark three times and reports the mean, with error bars showing \"the standard error across those three runs\". It says \"there is no step or cost limit\", and each task has an 8-hour agent time limit. Artificial Analysis's [methodology](https://artificialanalysis.ai/methodology/intelligence-benchmarking) reports pass@1 \"averaged over 3 repeats per task\", limits the agent to 500 steps and follows each task's own timeout. Its page prints no error bars."
      },
      {
        "p": "Vals AI ran Opus 5.5 at \"max\" effort, with Claude Opus 5 and Claude Opus 4.8 \"as server-side fallbacks for refusals\", its [model page](https://www.vals.ai/models/anthropic_claude-opus-5-5) says. It ran Kimi K3.1 Pro at max effort ([model page](https://www.vals.ai/models/kimi_kimi-k3.1-pro)) and Gemini 4 Carbon at high ([model page](https://www.vals.ai/models/google_gemini-4-carbon)). Artificial Analysis labels its Claude runs \"Max, Default Fallback\" and its Carbon run \"High\"."
      },
      {
        "h": "Within Vals AI's error bars"
      },
      {
        "p": "Vals AI's top four sit within 2.02 points: Opus 5.5 at 65.15% (±0.00), Sonnet 5.5 at 64.14% (±1.01), Kimi K3.1 Pro at 63.64% (±1.75) and Gemini 4 Carbon at 63.13% (±2.20). GPT-6 Astra's 59.60% carries the widest error bar, ±4.40."
      },
      {
        "p": "Vals AI's own launch write-up of Sep 22, on its [Opus 5.5 model page](https://www.vals.ai/models/anthropic_claude-opus-5-5) printed a Terminal-Bench 4.0 score of 61.62% for the model; its leaderboard now prints 65.15%."
      },
      {
        "h": "The labs' own reports"
      },
      {
        "p": "Anthropic reported 66.4% for [Opus 5.5](https://www.anthropic.com/claude-opus-5-5) at \"xhigh\" effort, with a standard error of ±2.6 points and its production safeguards enabled, and 55.8% for [Fable 5.1](https://www.anthropic.com/claude-fable-and-mythos-5-1). Moonshot reported 65.7% for [Kimi K3.1 Pro](https://www.kimi.ai/blog/kimi-k3-1-pro), evaluated with the Claude Code harness. OpenAI reported 57.9% for [GPT-6 Astra](https://openai.com/index/gpt-6-astra). Google printed no Terminal-Bench 4.0 score of its own for Gemini 4 Carbon: its launch table used Artificial Analysis's 65.2% ([Gemini 4 Carbon's Table Moves Rivals' Scores](https://frontierlog.ai/stories/gemini-4-carbon-aa-scores))."
      },
      {
        "p": "This story quotes Vals AI's and Artificial Analysis's published figures directly, an exception to Frontier Log's rule of recording only the labs' own reports, and makes no claim about which ranking is right. For how much the harness alone can move a score, see [Does the Harness Version Change a Model's Score?](https://frontierlog.ai/stories/does-harness-version-change-score)."
      }
    ],
    "models": [
      "claude-opus-5-5",
      "claude-sonnet-5-5",
      "kimi-k3-1-pro",
      "gemini-4-carbon",
      "gpt-6-astra",
      "claude-fable-5-1"
    ],
    "stories": [
      "gemini-4-carbon-aa-scores",
      "does-harness-version-change-score"
    ]
  }
}