{
  "api_version": "1",
  "updated": "2026-11-10",
  "latest_event": "2026-11-09",
  "rights": "© Frontier Log, all rights reserved.",
  "story": {
    "slug": "who-runs-automationbench",
    "title": "AutomationBench Scores Depend on Who Runs the Test",
    "description": "Labs' AutomationBench figures come from their own runs, others' tables, Zapier or Artificial Analysis, and sit up to 35.6 points apart with partial credit.",
    "url": "https://frontierlog.ai/stories/who-runs-automationbench",
    "date": "2026-11-08",
    "image": "https://frontierlog.ai/og/story-who-runs-automationbench-spread.png",
    "event": "2026-11-05-inkling-2-small",
    "tags": [
      "agents"
    ],
    "images": [
      "https://frontierlog.ai/og/story-who-runs-automationbench-spread.png",
      "https://frontierlog.ai/og/story-who-runs-automationbench-kimi.png",
      "https://frontierlog.ai/og/story-who-runs-automationbench-labels.png"
    ],
    "steps": [],
    "tables": [
      {
        "title": "AutomationBench, run by the model's own lab",
        "unit": "%",
        "rows": [
          {
            "event": "2026-07-16-kimi-k3",
            "date": "2026-07-16",
            "provider": "moonshot",
            "value": 30.8,
            "printed": "30.8",
            "source": "https://www.kimi.ai/blog/kimi-k3",
            "where": "'Full Benchmark Table', row 'Automation Bench' (Agentic), column 'Kimi K3 (max)'"
          },
          {
            "event": "2026-08-14-glm-5-3",
            "date": "2026-08-14",
            "provider": "zai",
            "value": 48.2,
            "printed": "48.2",
            "source": "https://z.ai/blog/glm-5.3",
            "where": "'Performance across comparison models' table, row 'AutomationBench v1.0.6', column 'GLM-5.3'"
          },
          {
            "event": "2026-10-17-deepseek-v4-1-flash-1017",
            "date": "2026-10-17",
            "provider": "deepseek",
            "value": 60.5,
            "printed": "60.5",
            "source": "https://api-docs.deepseek.com/news/news261017/",
            "where": "benchmark table image (first image), row 'Automation-Bench V1.0.6', column 'DeepSeek V4.1-Flash-1017'"
          },
          {
            "event": "2026-11-04-hy4",
            "date": "2026-11-04",
            "provider": "tencent",
            "value": 56.2,
            "printed": "56.2",
            "source": "https://hy.tencent.ai/research/hy4",
            "where": "'Appendix: benchmark results' table image, row 'AutomationBench v1.0.6', column 'Hy4'"
          },
          {
            "event": "2026-10-19-minimax-m3-1-flash",
            "date": "2026-10-19",
            "provider": "minimax",
            "value": 62.8,
            "printed": "62.8",
            "source": "https://www.minimax.io/blog/minimax-m31f",
            "where": "benchmark table image (first image), row 'Automation-Bench v1.0.6', column 'MiniMax M3.1 Flash'"
          },
          {
            "event": "2026-09-02-muse-spark-1-3",
            "date": "2026-09-02",
            "provider": "meta",
            "value": 49.6,
            "printed": "49.6",
            "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3",
            "where": "scorecard (image), row 'AutomationBench', column 'Muse Spark 1.3 (max)'"
          }
        ]
      },
      {
        "title": "AutomationBench, Zapier's private-set leaderboard",
        "unit": "%",
        "rows": [
          {
            "event": "2026-07-16-kimi-k3",
            "date": "2026-07-16",
            "provider": "moonshot",
            "value": 22.7,
            "printed": null,
            "source": "https://zapier.com/benchmarks",
            "where": "AutomationBench leaderboard, private set, read 2026-11-08, row 'Kimi K3' (22.68%)"
          },
          {
            "event": "2026-07-09-gpt-5-6",
            "date": "2026-07-09",
            "provider": "openai",
            "value": 28.8,
            "printed": null,
            "source": "https://zapier.com/benchmarks",
            "where": "AutomationBench leaderboard, private set, read 2026-11-08, row 'GPT-5.6 Sol (Max)' (28.77%)"
          },
          {
            "event": "2026-07-24-claude-opus-5",
            "date": "2026-07-24",
            "provider": "anthropic",
            "value": 26.9,
            "printed": null,
            "source": "https://zapier.com/benchmarks",
            "where": "AutomationBench leaderboard, private set, read 2026-11-08, row 'Claude Opus 5 (Max)' (26.94%)"
          },
          {
            "event": "2026-05-28-claude-opus-4-8",
            "date": "2026-05-28",
            "provider": "anthropic",
            "value": 16.4,
            "printed": null,
            "source": "https://zapier.com/benchmarks",
            "where": "AutomationBench leaderboard, private set, read 2026-11-08, row 'Claude Opus 4.8 (Max)' (16.44%)"
          },
          {
            "event": "2026-09-22-claude-opus-5-5",
            "date": "2026-09-22",
            "provider": "anthropic",
            "value": 42.5,
            "printed": null,
            "source": "https://zapier.com/benchmarks",
            "where": "AutomationBench leaderboard, private set, read 2026-11-08, row 'Claude Opus 5.5 (default fallbacks, Max)' (42.47%)"
          },
          {
            "event": "2026-09-02-gemini-3-8-flash-and-3-8-flash",
            "date": "2026-09-02",
            "provider": "google",
            "value": 29.7,
            "printed": null,
            "source": "https://zapier.com/benchmarks",
            "where": "AutomationBench leaderboard, private set, read 2026-11-08, rows 'Gemini 3.8 Flash (Medium)' and '(High)' (29.68%)"
          },
          {
            "event": "2026-10-07-claude-haiku-5-5",
            "date": "2026-10-07",
            "provider": "anthropic",
            "value": 12.0,
            "printed": null,
            "source": "https://zapier.com/benchmarks",
            "where": "AutomationBench leaderboard, private set, read 2026-11-08, row 'Claude Haiku 5.5 (default fallbacks, Max)' (12.02%), marked 'Initial results based on the launch version of Haiku 5.5'"
          },
          {
            "event": "2026-09-22-claude-opus-5-5",
            "date": "2026-09-22",
            "provider": "anthropic",
            "value": 40.0,
            "printed": null,
            "source": "https://www.anthropic.com/claude-opus-5-5-system-card",
            "where": "section on AutomationBench: 'On AutomationBench's leaderboard, which measures performance on a private held-out evaluation set, Claude Opus 5.5 at max effort scored 40.0%'. The Claude Opus 5.5 post's footnote 2 says these runs 'were performed without fallback models'. Release named in the Claude Sonnet 5.5 system card, section 8.14.6, footnote 23: 'Results are on release 1.0.6 of Zapier's private evaluation set', which prints the same figure: '(40.0% without the re-run, as reported in the Claude Opus 5.5 System Card)'"
          },
          {
            "event": "2026-09-01-claude-fable-5-1-mythos-5-1",
            "date": "2026-09-01",
            "provider": "anthropic",
            "value": 31.4,
            "printed": null,
            "source": "https://zapier.com/benchmarks",
            "where": "AutomationBench leaderboard, private set, read 2026-11-08, row 'Claude Fable 5.1 (with Opus 5 Fallback)' (31.4%)"
          },
          {
            "event": "2026-09-01-claude-fable-5-1-mythos-5-1",
            "date": "2026-09-01",
            "provider": "anthropic",
            "value": 22.4,
            "printed": null,
            "source": "https://zapier.com/benchmarks",
            "where": "AutomationBench leaderboard, private set, read 2026-11-08, row 'Claude Fable 5.1 (Max)' (22.37%)"
          },
          {
            "event": "2026-09-03-gpt-6-astra",
            "date": "2026-09-03",
            "provider": "openai",
            "value": 41.4,
            "printed": null,
            "source": "https://zapier.com/benchmarks",
            "where": "AutomationBench leaderboard, private set, read 2026-11-08, row 'GPT-6 Astra (Max)' (41.40%)"
          },
          {
            "event": "2026-10-28-gemini-4-carbon",
            "date": "2026-10-28",
            "provider": "google",
            "value": 53.9,
            "printed": null,
            "source": "https://zapier.com/benchmarks",
            "where": "AutomationBench leaderboard, private set, read 2026-11-08, row 'Gemini 4 Carbon (High)' (53.88%)"
          },
          {
            "event": "2026-09-02-muse-spark-1-3",
            "date": "2026-09-02",
            "provider": "meta",
            "value": 20.7,
            "printed": null,
            "source": "https://zapier.com/benchmarks",
            "where": "AutomationBench leaderboard, private set, read 2026-11-08, row 'Muse Spark 1.3 (Max)' (20.7%)"
          }
        ]
      },
      {
        "title": "AutomationBench, Zapier's public-set table, in other labs' tables",
        "unit": "%",
        "rows": [
          {
            "event": "2026-10-17-deepseek-v4-1-flash-1017",
            "date": "2026-10-17",
            "provider": "deepseek",
            "value": 46.7,
            "printed": null,
            "source": "https://api-docs.deepseek.com/news/news261017/",
            "where": "benchmark table (image), row 'Automation-Bench V1.0.6', column 'Kimi-K3'"
          },
          {
            "event": "2026-10-17-deepseek-v4-1-flash-1017",
            "date": "2026-10-17",
            "provider": "deepseek",
            "value": 45.8,
            "printed": null,
            "source": "https://api-docs.deepseek.com/news/news261017/",
            "where": "benchmark table (image), row 'Automation-Bench V1.0.6', column 'GPT-5.6-Sol'"
          },
          {
            "event": "2026-10-17-deepseek-v4-1-flash-1017",
            "date": "2026-10-17",
            "provider": "deepseek",
            "value": 50.3,
            "printed": null,
            "source": "https://api-docs.deepseek.com/news/news261017/",
            "where": "benchmark table (image), row 'Automation-Bench V1.0.6', column 'Opus-5'"
          },
          {
            "event": "2026-08-14-glm-5-3",
            "date": "2026-08-14",
            "provider": "zai",
            "value": 41.0,
            "printed": null,
            "source": "https://z.ai/blog/glm-5.3",
            "where": "benchmark table, row 'AutomationBench v1.0.6', column 'Claude Opus 4.8'"
          }
        ]
      },
      {
        "title": "AutomationBench, run by a rival lab",
        "unit": "%",
        "rows": [
          {
            "event": "2026-08-28-hy4-preview",
            "date": "2026-08-28",
            "provider": "tencent",
            "value": 45.5,
            "printed": null,
            "source": "https://hy.tencent.ai/research/hy4-preview",
            "where": "appendix table, row 'AutomationBench (v1.0.6)', column 'Kimi K3', starred figure"
          },
          {
            "event": "2026-07-16-kimi-k3",
            "date": "2026-07-16",
            "provider": "moonshot",
            "value": 29.7,
            "printed": null,
            "source": "https://www.kimi.ai/blog/kimi-k3",
            "where": "full benchmark table, row 'Automation Bench', column 'GPT 5.6 Sol (max)'"
          },
          {
            "event": "2026-08-28-hy4-preview",
            "date": "2026-08-28",
            "provider": "tencent",
            "value": 39.9,
            "printed": null,
            "source": "https://hy.tencent.ai/research/hy4-preview",
            "where": "appendix table, row 'AutomationBench (v1.0.6)', column 'GPT 5.6 Sol', starred figure"
          },
          {
            "event": "2026-08-28-hy4-preview",
            "date": "2026-08-28",
            "provider": "tencent",
            "value": 48.7,
            "printed": null,
            "source": "https://hy.tencent.ai/research/hy4-preview",
            "where": "appendix table, row 'AutomationBench (v1.0.6)', column 'Claude Opus 5', starred figure"
          },
          {
            "event": "2026-07-16-kimi-k3",
            "date": "2026-07-16",
            "provider": "moonshot",
            "value": 27.2,
            "printed": null,
            "source": "https://www.kimi.ai/blog/kimi-k3",
            "where": "full benchmark table, row 'Automation Bench', column 'Claude Opus 4.8 (max)'"
          },
          {
            "event": "2026-11-04-hy4",
            "date": "2026-11-04",
            "provider": "tencent",
            "value": 54.1,
            "printed": null,
            "source": "https://hy.tencent.ai/research/hy4",
            "where": "appendix table, row 'AutomationBench v1.0.6', column 'GLM-5.3', starred figure"
          },
          {
            "event": "2026-11-04-hy4",
            "date": "2026-11-04",
            "provider": "tencent",
            "value": 54.3,
            "printed": null,
            "source": "https://hy.tencent.ai/research/hy4",
            "where": "appendix table, row 'AutomationBench v1.0.6', column 'DeepSeek-V4.1-Flash-1017', starred figure"
          },
          {
            "event": "2026-11-04-hy4",
            "date": "2026-11-04",
            "provider": "tencent",
            "value": 65.6,
            "printed": null,
            "source": "https://hy.tencent.ai/research/hy4",
            "where": "appendix table, row 'AutomationBench v1.0.6', column 'Kimi K3.1 Pro', starred figure"
          },
          {
            "event": "2026-11-04-hy4",
            "date": "2026-11-04",
            "provider": "tencent",
            "value": 58.8,
            "printed": null,
            "source": "https://hy.tencent.ai/research/hy4",
            "where": "appendix table, row 'AutomationBench v1.0.6', column 'Claude Opus 5.5', starred figure"
          },
          {
            "event": "2026-11-04-hy4",
            "date": "2026-11-04",
            "provider": "tencent",
            "value": 51.4,
            "printed": null,
            "source": "https://hy.tencent.ai/research/hy4",
            "where": "appendix table, row 'AutomationBench v1.0.6', column 'GPT-6.1 Sol', starred figure"
          }
        ]
      },
      {
        "title": "AutomationBench, run by Artificial Analysis",
        "unit": "%",
        "rows": [
          {
            "event": "2026-11-05-inkling-2-small",
            "date": "2026-11-05",
            "provider": "thinking-machines",
            "value": 58.3,
            "printed": null,
            "source": "https://thinkingmachines.ai/model-card/inkling-2-small/",
            "where": "benchmark table, row 'AutomationBench v1.0.6', column 'Kimi K3'"
          },
          {
            "event": "2026-11-05-inkling-2-small",
            "date": "2026-11-05",
            "provider": "thinking-machines",
            "value": 62.2,
            "printed": null,
            "source": "https://thinkingmachines.ai/model-card/inkling-2-small/",
            "where": "benchmark table, row 'AutomationBench v1.0.6', column 'GLM 5.3'"
          },
          {
            "event": "2026-11-05-inkling-2-small",
            "date": "2026-11-05",
            "provider": "thinking-machines",
            "value": 66.6,
            "printed": null,
            "source": "https://thinkingmachines.ai/model-card/inkling-2-small/",
            "where": "benchmark table, row 'AutomationBench v1.0.6', column 'DeepSeek-V4.1-Flash-1017'"
          },
          {
            "event": "2026-11-05-inkling-2-small",
            "date": "2026-11-05",
            "provider": "thinking-machines",
            "value": 59.9,
            "printed": null,
            "source": "https://thinkingmachines.ai/model-card/inkling-2-small/",
            "where": "benchmark table, row 'AutomationBench v1.0.6', column 'Gemini 3.8 Flash'"
          },
          {
            "event": "2026-11-05-inkling-2-small",
            "date": "2026-11-05",
            "provider": "thinking-machines",
            "value": 35.4,
            "printed": null,
            "source": "https://thinkingmachines.ai/model-card/inkling-2-small/",
            "where": "benchmark table, row 'AutomationBench v1.0.6', column 'Claude Haiku 5.5'"
          },
          {
            "event": "2026-10-24-kimi-k3-1-pro",
            "date": "2026-10-24",
            "provider": "moonshot",
            "value": 73.3,
            "printed": "73.3",
            "source": "https://www.kimi.ai/blog/kimi-k3-1-pro",
            "where": "'Benchmarks' table (rendered by the page's script), section 'Agentic', row 'Automation Bench-AA', column 'Kimi K3.1 Pro (max)'; runner: the table's notes, 'AutomationBench-AA scores are cited from https://artificialanalysis.ai/'"
          },
          {
            "event": "2026-10-24-kimi-k3-1-pro",
            "date": "2026-10-24",
            "provider": "moonshot",
            "value": 69.5,
            "printed": null,
            "source": "https://www.kimi.ai/blog/kimi-k3-1-pro",
            "where": "'Benchmarks' table, section 'Agentic', row 'Automation Bench-AA', column 'Claude Opus 5.5'; notes: 'AutomationBench-AA scores are cited from https://artificialanalysis.ai/'"
          },
          {
            "event": "2026-10-24-kimi-k3-1-pro",
            "date": "2026-10-24",
            "provider": "moonshot",
            "value": 59.4,
            "printed": null,
            "source": "https://www.kimi.ai/blog/kimi-k3-1-pro",
            "where": "'Benchmarks' table, row 'Automation Bench-AA', column 'Claude Fable 5.1 (max, with fallback)'"
          },
          {
            "event": "2026-10-24-kimi-k3-1-pro",
            "date": "2026-10-24",
            "provider": "moonshot",
            "value": 68.5,
            "printed": null,
            "source": "https://www.kimi.ai/blog/kimi-k3-1-pro",
            "where": "'Benchmarks' table, row 'Automation Bench-AA', column 'GPT-6 Astra (max, with fallback)'"
          },
          {
            "event": "2026-09-29-gpt-6-1-sol",
            "date": "2026-09-29",
            "provider": "openai",
            "value": 64.9,
            "printed": null,
            "source": "https://artificialanalysis.ai/evaluations/automationbench-aa",
            "where": "AutomationBench-AA results, read 2026-11-08, row 'GPT-6.1 Sol (max)' (64.9%)"
          },
          {
            "event": "2026-10-28-gemini-4-carbon",
            "date": "2026-10-28",
            "provider": "google",
            "value": 77.8,
            "printed": null,
            "source": "https://artificialanalysis.ai/evaluations/automationbench-aa",
            "where": "AutomationBench-AA results, read 2026-11-08, row 'Gemini 4 Carbon (high)' (77.8%)"
          },
          {
            "event": "2026-11-04-hy4",
            "date": "2026-11-04",
            "provider": "tencent",
            "value": 65.6,
            "printed": null,
            "source": "https://artificialanalysis.ai/evaluations/automationbench-aa",
            "where": "AutomationBench-AA results, read 2026-11-08, row 'Tencent Hy4' (65.6%)"
          },
          {
            "event": "2026-10-19-minimax-m3-1-flash",
            "date": "2026-10-19",
            "provider": "minimax",
            "value": 72.5,
            "printed": null,
            "source": "https://artificialanalysis.ai/evaluations/automationbench-aa",
            "where": "AutomationBench-AA results, read 2026-11-08, row 'MiniMax-M3.1-Flash' (72.5%)"
          }
        ]
      },
      {
        "title": "AutomationBench, labelled 'Automation Bench-AA' in Moonshot AI's Kimi K3.1 Pro post",
        "unit": "%",
        "rows": [
          {
            "event": "2026-10-24-kimi-k3-1-pro",
            "date": "2026-10-24",
            "provider": "moonshot",
            "value": 66.6,
            "printed": null,
            "source": "https://www.kimi.ai/blog/kimi-k3-1-pro",
            "where": "'Benchmarks' table, row 'Automation Bench-AA', column 'DeepSeek-V4.1-Flash-1017 (max)'"
          },
          {
            "event": "2026-10-24-kimi-k3-1-pro",
            "date": "2026-10-24",
            "provider": "moonshot",
            "value": 62.2,
            "printed": null,
            "source": "https://www.kimi.ai/blog/kimi-k3-1-pro",
            "where": "'Benchmarks' table, row 'Automation Bench-AA', column 'GLM-5.3 (max)'"
          }
        ]
      }
    ],
    "sources": [
      "https://api-docs.deepseek.com/news/news260813/",
      "https://api-docs.deepseek.com/news/news261017/",
      "https://artificialanalysis.ai/evaluations/automationbench-aa",
      "https://artificialanalysis.ai/methodology/intelligence-benchmarking",
      "https://blog.google/innovation-and-ai/models-and-research/gemini-models/introducing-gemini-4-carbon/",
      "https://github.com/zapier/AutomationBench",
      "https://hy.tencent.ai/research/hy4",
      "https://hy.tencent.ai/research/hy4-preview",
      "https://longcat.ai/blog/longcat-2.5",
      "https://openai.com/index/gpt-5-6",
      "https://openai.com/index/introducing-gpt-6-sol-and-luna",
      "https://qwen.ai/blog?id=qwen3.8",
      "https://research.meta.ai/blog/introducing-muse-spark-1-3",
      "https://storage.googleapis.com/deepmind-media/gemini/gemini_4_argon_model_evaluation.pdf",
      "https://storage.googleapis.com/deepmind-media/gemini/gemini_4_carbon_model_evaluation.pdf",
      "https://thinkingmachines.ai/model-card/inkling-2-small/",
      "https://thinkingmachines.ai/news/inkling-2-small/",
      "https://www.anthropic.com/claude-opus-4-8-system-card",
      "https://www.anthropic.com/claude-opus-5-5",
      "https://www.anthropic.com/claude-opus-5-5-system-card",
      "https://www.anthropic.com/claude-opus-5-system-card",
      "https://www.anthropic.com/claude-sonnet-5-5-system-card",
      "https://www.kimi.ai/blog/kimi-k3",
      "https://www.kimi.ai/blog/kimi-k3-1-pro",
      "https://www.minimax.io/blog/minimax-m31f",
      "https://z.ai/blog/glm-5.3",
      "https://zapier.com/benchmarks"
    ],
    "updates": [],
    "body": [
      {
        "figure": "spread"
      },
      {
        "p": "When a lab launches a model, its benchmark table puts the model beside its rivals. On [AutomationBench](https://frontierlog.ai/benchmark/automationbench-1-0-6), Zapier's test of business workflows, those figures come from four places. The lab may run the test itself or print a figure someone else published. It may also cite Zapier's own runs, or use Artificial Analysis, a third party that didn't make the benchmark. The same model can get very different figures depending on which one a table uses. [Kimi K3](https://frontierlog.ai/model/kimi-k3) is at 22.7% on [Zapier's leaderboard](https://zapier.com/benchmarks), 30.8% in Moonshot AI's own post and 58.3% in Thinking Machines' card for [Inkling 2 Small](https://frontierlog.ai/model/inkling-2-small) on Nov 5."
      },
      {
        "h": "The lab runs the test"
      },
      {
        "p": "In its Kimi K3 post, Moonshot AI ran every model on AutomationBench itself. Its [Jul 16 post](https://www.kimi.ai/blog/kimi-k3) says \"All models are evaluated on the 600-task public subset, following the official GitHub setup in all other respects.\" Tencent's Hy4 tables mix its own runs with figures it took from others, and mark its own with a star: \"* indicating our own testing,\" says its [Nov 4 post](https://hy.tencent.ai/research/hy4). Meituan's [LongCat-2.5 post](https://longcat.ai/blog/longcat-2.5) marks the other way round. Its starred figures \"are from public technical report, blog posts or the official benchmark leaderboards, otherwise are self-run and are indicative only.\""
      },
      {
        "p": "A lab that runs a rival chooses the task set, the harness and the settings. Its figure can differ from the rival lab's own. Tencent's run puts [DeepSeek-V4.1-Flash-1017](https://frontierlog.ai/model/deepseek-v4-1-flash-1017) at 54.3%, and DeepSeek's [own post](https://api-docs.deepseek.com/news/news261017/) gives 60.5%. Tencent's run puts [GLM-5.3](https://frontierlog.ai/model/glm-5-3) at 54.1%, and [Z.ai's post](https://z.ai/blog/glm-5.3) gives 48.2%. Tencent's [Aug 28 table](https://hy.tencent.ai/research/hy4-preview) had run GLM-5.3 at 49.4%. Its Nov 4 notes say \"Certain Hy4-preview benchmark scores may differ from previously reported results following updates to the evaluation harness, judge model, and anti-hacking mechanisms.\""
      },
      {
        "h": "Printing someone else's figure"
      },
      {
        "p": "Some tables print figures identical to ones another party published, without naming the source. DeepSeek's [Aug 13 table](https://api-docs.deepseek.com/news/news260813/) and Alibaba's [Qwen3.8-Max post](https://qwen.ai/blog?id=qwen3.8) give [Claude Opus 4.8](https://frontierlog.ai/model/claude-opus-4-8) 27.2% on the public set, the same figure as Moonshot AI's run. Alibaba also gives [GPT-5.6 Sol](https://frontierlog.ai/model/gpt-5-6-sol) Moonshot AI's 29.7%. Alibaba's footnote says \"Automation-Bench: Evaluated on the 600-task public subset.\""
      },
      {
        "p": "Later tables print other public-set figures for the same models, identical to the table on [Zapier's GitHub page](https://github.com/zapier/AutomationBench). DeepSeek, Z.ai, MiniMax, Meituan and Tencent give GPT-5.6 Sol 45.8%, and Z.ai and Meituan give Claude Opus 4.8 41.0%. Zapier's page says those pass rates are on \"the 600-task public benchmark,\" with \"each model run at its highest available reasoning effort.\""
      },
      {
        "p": "Anthropic says its [competitor figures](https://www.anthropic.com/claude-opus-5-system-card) \"are drawn from the respective developers' published system cards or benchmark leaderboards.\" OpenAI's [GPT-6 Sol and Luna post](https://openai.com/index/introducing-gpt-6-sol-and-luna) says \"Evaluations of competitor models were taken from publicly available reports.\""
      },
      {
        "h": "The benchmark's maker runs it"
      },
      {
        "p": "Zapier keeps a second set of tasks private. Its [leaderboard](https://zapier.com/benchmarks) says its scores \"run against a held-out private evaluation set.\" The GitHub page says the private set \"is purposely harder and is not released, so scores you measure locally may not match the official leaderboard 1:1.\" Anthropic and Google report private-set figures. Anthropic's [Claude Opus 5.5 post](https://www.anthropic.com/claude-opus-5-5) says its \"AutomationBench results were run and reported by Zapier.\" Zapier ran them without fallback models, so safeguard interventions counted as failures. Anthropic says \"this resulted in a lower score than Claude Opus 5.5 would achieve in practice.\" Google's [Gemini 4 Carbon methodology](https://storage.googleapis.com/deepmind-media/gemini/gemini_4_carbon_model_evaluation.pdf) says its results \"use the private set and are sourced from the official Zapier public leaderboard.\" A [separate story](https://frontierlog.ai/stories/automationbench-private-and-public-sets) sets out which labs report which set."
      },
      {
        "p": "Zapier's private-set figures change over time. The GitHub page says: \"When we make changes that affect scores, we try to rerun all models on the benchmark and update scores when they change outside run to run variance.\" OpenAI's [GPT-5.6 post](https://openai.com/index/gpt-5-6) gave GPT-5.6 Sol 18.1% and doesn't name a task set. Zapier's leaderboard now shows 28.8%, the figure Anthropic's Opus 5.5 post prints. Anthropic's [Claude Opus 5 card](https://www.anthropic.com/claude-opus-5-system-card) gave Claude Opus 5 26.0% and the leaderboard now shows 26.9%. Its [Claude Opus 4.8 card](https://www.anthropic.com/claude-opus-4-8-system-card) gave Opus 4.8 15.5%, its Opus 5 card gave 17.0%, and the leaderboard now shows 16.4%. The leaderboard says \"Run-to-run variance is typically within 1%.\" Zapier's leaderboard now lists Claude Opus 5.5 at 42.5% and Claude Haiku 5.5 at 12.0% with \"default fallbacks\", where Anthropic's Opus 5.5 post gave 40.0% from runs without them. For Haiku 5.5 it notes \"Initial results based on the launch version of Haiku 5.5.\" Zapier's leaderboard lists Claude Fable 5.1 twice: 22.4% alone, and 31.4% \"with Opus 5 Fallback\", where \"Opus 5 handled ~40% of tasks (260 of 657)\"."
      },
      {
        "h": "A third party runs it"
      },
      {
        "p": "Thinking Machines takes its figures from outside runs, for its own model as well as its rivals. Its [launch post](https://thinkingmachines.ai/news/inkling-2-small/) says \"To improve consistency, we rely on externally reported evaluations for both internal and external models when applicable,\" and lists AutomationBench-AA under Artificial Analysis. Its [model card](https://thinkingmachines.ai/model-card/inkling-2-small/) gives Kimi K3 58.3%, GLM-5.3 62.2% and DeepSeek-V4.1-Flash-1017 66.6%. Each is higher than the figure in the model's own lab's post. Artificial Analysis's [methodology page](https://artificialanalysis.ai/methodology/intelligence-benchmarking) says \"We evaluate a private 657-task held-out split from AutomationBench dataset version 1.0.6.\" It also scores tasks differently from Zapier's pass rates. A task that breaks no guardrail \"receives the percentage of objectives the model completed.\""
      },
      {
        "p": "Thinking Machines' [model card](https://thinkingmachines.ai/model-card/inkling-2-small/) labels its row \"AutomationBench v1.0.6\". Moonshot AI's [Kimi K3.1 Pro post](https://www.kimi.ai/blog/kimi-k3-1-pro) prints the same figures for DeepSeek-V4.1-Flash-1017 and GLM-5.3, 66.6% and 62.2%, under \"Automation Bench-AA\". It says \"AutomationBench-AA scores are cited from https://artificialanalysis.ai/\". Tencent's [Hy4 post](https://hy.tencent.ai/research/hy4) uses the label \"AutomationBench v1.0.6\" for its own runs of the same two models, 54.3% and 54.1%, on \"600 public tasks released by Zapier\". Anthropic's [Claude Opus 5.5 post](https://www.anthropic.com/claude-opus-5-5) and Google's [Gemini 4 Carbon post](https://blog.google/innovation-and-ai/models-and-research/gemini-models/introducing-gemini-4-carbon/) also label their row AutomationBench, for Zapier's private-set pass rates. GPT-6 Astra and Claude Opus 5.5 score 41.4% and 42.5% in Google's table, and 68.5% and 69.5% in Artificial Analysis's figures."
      },
      {
        "figure": "labels"
      },
      {
        "h": "Versions differ too"
      },
      {
        "p": "Most tables name AutomationBench v1.0.6: DeepSeek's Oct 17 table, Z.ai's, Tencent's, MiniMax's, Meituan's and Thinking Machines' card. Moonshot AI's Jul 16 post, DeepSeek's Aug 13 table and Alibaba's Qwen3.8-Max table don't name a version. Z.ai adds that it evaluates \"incorporating the fix for the null-type handling issue introduced in PR #13.\" Its post gives GLM-5.3 48.2%, and DeepSeek's and MiniMax's tables print 48.8%. Zapier released v1.0.6 on Jul 31. Its [changelog](https://github.com/zapier/AutomationBench) says the release \"corrected additional task fairness (public and private)\" and that \"Private tasks were made a bit harder to compensate for bug fixes.\" On the public set, Zapier's own table moved Claude Opus 4.8 from 30.3% to 41.0%, GPT-5.6 Sol from 29.2% to 45.8% and Claude Fable 5 from 25.8% to 46.2% between v1.0.5 (Jul 16) and v1.0.6. The images draw only v1.0.6 figures, so they leave out Moonshot AI's 30.8% for Kimi K3, from its Jul 16 post."
      },
      {
        "h": "Which labs use which"
      },
      {
        "p": "Across all benchmarks, the posts this story cites show each approach below. Every lab runs its own models too. The benchmark makers and third parties named are the ones each lab names in those posts or in its results on this site."
      },
      {
        "table": {
          "cols": [
            "Lab",
            "Runs rivals",
            "Prints others' figures",
            "Benchmark maker runs",
            "Third party runs"
          ],
          "rows": [
            [
              "OpenAI",
              "-",
              "[Yes](https://openai.com/index/introducing-gpt-6-sol-and-luna)",
              "Zapier, ARC Prize Foundation",
              "-"
            ],
            [
              "Anthropic",
              "-",
              "[Yes](https://www.anthropic.com/claude-opus-5-system-card)",
              "Zapier, Cognition, Scale AI, ARC Prize Foundation",
              "-"
            ],
            [
              "Google",
              "[Yes](https://storage.googleapis.com/deepmind-media/gemini/gemini_4_argon_model_evaluation.pdf)",
              "[Yes](https://storage.googleapis.com/deepmind-media/gemini/gemini_4_carbon_model_evaluation.pdf)",
              "Zapier, Cognition, Surge AI, ARC Prize Foundation, [Datacurve, Proximal, Vals AI](https://storage.googleapis.com/deepmind-media/gemini/gemini_4_argon_model_evaluation.pdf), [Andon Labs](https://storage.googleapis.com/deepmind-media/gemini/gemini_4_carbon_model_evaluation.pdf)",
              "[Artificial Analysis](https://storage.googleapis.com/deepmind-media/gemini/gemini_4_carbon_model_evaluation.pdf)"
            ],
            [
              "Meta",
              "-",
              "-",
              "Scale AI, Datacurve",
              "-"
            ],
            [
              "Moonshot AI",
              "[Yes](https://www.kimi.ai/blog/kimi-k3)",
              "[Yes](https://www.kimi.ai/blog/kimi-k3)",
              "[Datacurve, the FrontierSWE leaderboard](https://www.kimi.ai/blog/kimi-k3)",
              "[Artificial Analysis](https://www.kimi.ai/blog/kimi-k3)"
            ],
            [
              "Tencent",
              "[Yes](https://hy.tencent.ai/research/hy4)",
              "[Yes](https://hy.tencent.ai/research/hy4)",
              "-",
              "-"
            ],
            [
              "Meituan",
              "[Yes](https://longcat.ai/blog/longcat-2.5)",
              "[Yes](https://longcat.ai/blog/longcat-2.5)",
              "-",
              "-"
            ],
            [
              "MiniMax",
              "-",
              "[Yes](https://www.minimax.io/blog/minimax-m31f)",
              "-",
              "[Artificial Analysis](https://www.minimax.io/blog/minimax-m31f)"
            ],
            [
              "DeepSeek",
              "-",
              "[Same figures](https://api-docs.deepseek.com/news/news260813/)",
              "-",
              "-"
            ],
            [
              "Alibaba",
              "-",
              "[Yes](https://qwen.ai/blog?id=qwen3.8)",
              "[The FrontierSWE leaderboard](https://qwen.ai/blog?id=qwen3.8)",
              "-"
            ],
            [
              "Z.ai",
              "-",
              "[Same figures](https://z.ai/blog/glm-5.3)",
              "[Proximal](https://z.ai/blog/glm-5.3)",
              "[Artificial Analysis](https://z.ai/blog/glm-5.3)"
            ],
            [
              "Thinking Machines",
              "-",
              "-",
              "Scale AI, Datacurve, [Surge AI](https://thinkingmachines.ai/news/inkling-2-small/)",
              "[Artificial Analysis, Vals AI](https://thinkingmachines.ai/news/inkling-2-small/)"
            ],
            [
              "Mistral AI",
              "-",
              "-",
              "-",
              "Artificial Analysis, All Hands AI"
            ]
          ]
        }
      },
      {
        "p": "\"Same figures\" marks a table that prints figures identical to ones another party published, without naming their source. A dash means we found no such case in the posts this story cites."
      },
      {
        "figure": "kimi"
      }
    ],
    "events": [
      "2026-07-09-gpt-5-6",
      "2026-07-16-kimi-k3",
      "2026-08-03-qwen3-8-max",
      "2026-08-13-deepseek-v4-pro-ga",
      "2026-08-14-glm-5-3",
      "2026-08-28-hy4-preview",
      "2026-09-02-muse-spark-1-3",
      "2026-09-22-claude-opus-5-5",
      "2026-09-22-gpt-6-sol-and-luna",
      "2026-10-17-deepseek-v4-1-flash-1017",
      "2026-10-19-minimax-m3-1-flash",
      "2026-10-24-kimi-k3-1-pro",
      "2026-10-28-gemini-4-carbon",
      "2026-11-01-longcat-2-5",
      "2026-11-04-hy4",
      "2026-11-05-inkling-2-small"
    ],
    "stories": [
      "the-harness-is-part-of-the-score",
      "china-leads-on-automationbench",
      "gemini-4-carbon-aa-scores",
      "automationbench-private-and-public-sets",
      "labs-run-benchmarks-different-ways"
    ],
    "models": [
      "kimi-k3",
      "gpt-5-6-sol",
      "claude-opus-5",
      "claude-opus-4-8",
      "glm-5-3",
      "deepseek-v4-1-flash-1017",
      "inkling-2-small",
      "muse-spark-1-3",
      "minimax-m3-1-flash",
      "kimi-k3-1-pro",
      "hy4"
    ]
  }
}