{
  "api_version": "1",
  "updated": "2026-11-10",
  "latest_event": "2026-11-09",
  "rights": "© Frontier Log, all rights reserved.",
  "story": {
    "slug": "automationbench-private-and-public-sets",
    "title": "OpenAI, Anthropic and Google Report the Harder AutomationBench Set",
    "description": "OpenAI, Anthropic and Google report Zapier's runs on AutomationBench's private set. The other labs that name a set report their own runs on the public set.",
    "url": "https://frontierlog.ai/stories/automationbench-private-and-public-sets",
    "date": "2026-11-08",
    "image": "https://frontierlog.ai/og/story-automationbench-private-and-public-sets-sets.png",
    "event": "2026-10-28-gemini-4-carbon",
    "tags": [
      "agents"
    ],
    "images": [
      "https://frontierlog.ai/og/story-automationbench-private-and-public-sets-sets.png",
      "https://frontierlog.ai/og/story-automationbench-private-and-public-sets-drop.png"
    ],
    "steps": [],
    "tables": [
      {
        "title": "AutomationBench private set, the labs' reports",
        "unit": "%",
        "rows": [
          {
            "event": "2026-10-28-gemini-4-carbon",
            "date": "2026-10-28",
            "provider": "google",
            "value": 53.9,
            "printed": "53.9%",
            "source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/introducing-gemini-4-carbon/",
            "where": "benchmark table (image), row 'AutomationBench', 'Zapier', column 'Gemini 4 Carbon'"
          },
          {
            "event": "2026-09-28-claude-sonnet-5-5",
            "date": "2026-09-28",
            "provider": "anthropic",
            "value": 44.7,
            "printed": "44.7",
            "source": "https://www.anthropic.com/claude-sonnet-5-5-system-card",
            "where": "Section 8.1, Table 8.1.A, row 'AutomationBench', column 'Claude Sonnet 5.5'"
          },
          {
            "event": "2026-09-03-gpt-6-astra",
            "date": "2026-09-03",
            "provider": "openai",
            "value": 41.4,
            "printed": "41.4%",
            "source": "https://openai.com/index/gpt-6-astra",
            "where": "Professional table, row 'AutomationBench', column 'GPT‑6 Astra'"
          }
        ]
      },
      {
        "title": "AutomationBench public set, the labs' reports",
        "unit": "%",
        "rows": [
          {
            "event": "2026-10-19-minimax-m3-1-flash",
            "date": "2026-10-19",
            "provider": "minimax",
            "value": 62.8,
            "printed": "62.8",
            "source": "https://www.minimax.io/blog/minimax-m31f",
            "where": "benchmark table image (first image), row 'Automation-Bench v1.0.6', column 'MiniMax M3.1 Flash'"
          },
          {
            "event": "2026-11-04-hy4",
            "date": "2026-11-04",
            "provider": "tencent",
            "value": 56.2,
            "printed": "56.2",
            "source": "https://hy.tencent.ai/research/hy4",
            "where": "'Appendix: benchmark results' table image, row 'AutomationBench v1.0.6', column 'Hy4'"
          },
          {
            "event": "2026-09-10-deepseek-v4-1-flash",
            "date": "2026-09-10",
            "provider": "deepseek",
            "value": 54.8,
            "printed": "54.8",
            "source": "https://api-docs.deepseek.com/news/news260910/",
            "where": "benchmark table image (second image), row 'Automation-Bench', column 'DeepSeek V4.1-Flash' (the first chart image, v4.1_260910_benchmark.png, prints the same value label)"
          },
          {
            "event": "2026-11-01-longcat-2-5",
            "date": "2026-11-01",
            "provider": "meituan",
            "value": 51.8,
            "printed": "51.8",
            "source": "https://longcat.ai/blog/longcat-2.5",
            "where": "'Evaluations' section table at the end of the post (rendered by the page's script; load https://longcat.ai/blog/longcat-2.5 in a browser and scroll to the end), row 'AutomationBench v1.0.6' under 'General Agent', column 'LongCat-2.5'"
          },
          {
            "event": "2026-09-02-muse-spark-1-3",
            "date": "2026-09-02",
            "provider": "meta",
            "value": 49.6,
            "printed": "49.6",
            "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3",
            "where": "scorecard (image), row 'AutomationBench', column 'Muse Spark 1.3 (max)'"
          },
          {
            "event": "2026-11-04-ripple-65b-preview",
            "date": "2026-11-04",
            "provider": "ricursive",
            "value": 36.4,
            "printed": "36.4",
            "source": "https://www.ricursive.com/ripple",
            "where": "Bar chart grid under the intro, panel 'Automation / AutomationBench (Public)', value label on the first (dark blue, Ricursive logo) bar"
          },
          {
            "event": "2026-07-16-kimi-k3",
            "date": "2026-07-16",
            "provider": "moonshot",
            "value": 30.8,
            "printed": "30.8",
            "source": "https://www.kimi.ai/blog/kimi-k3",
            "where": "'Full Benchmark Table', row 'Automation Bench' (Agentic), column 'Kimi K3 (max)'"
          },
          {
            "event": "2026-08-03-qwen3-8-max",
            "date": "2026-08-03",
            "provider": "alibaba",
            "value": 27.3,
            "printed": "27.3",
            "source": "https://qwen.ai/blog?id=qwen3.8",
            "where": "'Full Benchmark Table' (first, language/agent HTML table), General Agent section, row 'Automation-Bench (Pass@1)', column 'Qwen3.8-Max'"
          }
        ]
      },
      {
        "title": "AutomationBench public set, run by Zapier",
        "unit": "%",
        "rows": [
          {
            "event": "2026-07-24-claude-opus-5",
            "date": "2026-07-24",
            "provider": "anthropic",
            "value": 50.3,
            "printed": null,
            "source": "https://github.com/zapier/AutomationBench/blob/4a8e106/README.md",
            "where": "Zapier's README, 'Public vs. Official Scores', table of pass rates on the 600-task public benchmark, row 'Claude Opus 5', reasoning effort max"
          },
          {
            "event": "2026-07-16-kimi-k3",
            "date": "2026-07-16",
            "provider": "moonshot",
            "value": 46.67,
            "printed": null,
            "source": "https://github.com/zapier/AutomationBench/blob/4a8e106/README.md",
            "where": "Zapier's README, 'Public vs. Official Scores', table of pass rates on the 600-task public benchmark, row 'Kimi K3', reasoning effort max"
          },
          {
            "event": "2026-06-09-claude-fable-5-mythos-5",
            "date": "2026-06-09",
            "provider": "anthropic",
            "value": 46.17,
            "printed": null,
            "source": "https://github.com/zapier/AutomationBench/blob/4a8e106/README.md",
            "where": "Zapier's README, 'Public vs. Official Scores', table of pass rates on the 600-task public benchmark, row 'Claude Fable 5', reasoning effort max"
          },
          {
            "event": "2026-07-09-gpt-5-6",
            "date": "2026-07-09",
            "provider": "openai",
            "value": 45.83,
            "printed": null,
            "source": "https://github.com/zapier/AutomationBench/blob/4a8e106/README.md",
            "where": "Zapier's README, 'Public vs. Official Scores', table of pass rates on the 600-task public benchmark, row 'GPT-5.6 Sol', reasoning effort max"
          },
          {
            "event": "2026-07-21-gemini-3-6-flash-and-3-5-flash",
            "date": "2026-07-21",
            "provider": "google",
            "value": 45.0,
            "printed": null,
            "source": "https://github.com/zapier/AutomationBench/blob/4a8e106/README.md",
            "where": "Zapier's README, 'Public vs. Official Scores', table of pass rates on the 600-task public benchmark, row 'Gemini 3.6 Flash', reasoning effort high"
          },
          {
            "event": "2026-05-28-claude-opus-4-8",
            "date": "2026-05-28",
            "provider": "anthropic",
            "value": 41.0,
            "printed": null,
            "source": "https://github.com/zapier/AutomationBench/blob/4a8e106/README.md",
            "where": "Zapier's README, 'Public vs. Official Scores', table of pass rates on the 600-task public benchmark, row 'Claude Opus 4.8', reasoning effort max"
          },
          {
            "event": "2026-05-19-gemini-3-5-flash",
            "date": "2026-05-19",
            "provider": "google",
            "value": 38.33,
            "printed": null,
            "source": "https://github.com/zapier/AutomationBench/blob/4a8e106/README.md",
            "where": "Zapier's README, 'Public vs. Official Scores', table of pass rates on the 600-task public benchmark, row 'Gemini 3.5 Flash', reasoning effort high"
          },
          {
            "event": "2026-07-09-gpt-5-6",
            "date": "2026-07-09",
            "provider": "openai",
            "value": 37.17,
            "printed": null,
            "source": "https://github.com/zapier/AutomationBench/blob/4a8e106/README.md",
            "where": "Zapier's README, 'Public vs. Official Scores', table of pass rates on the 600-task public benchmark, row 'GPT-5.6 Terra', reasoning effort max"
          },
          {
            "event": "2026-06-30-claude-sonnet-5",
            "date": "2026-06-30",
            "provider": "anthropic",
            "value": 34.67,
            "printed": null,
            "source": "https://github.com/zapier/AutomationBench/blob/4a8e106/README.md",
            "where": "Zapier's README, 'Public vs. Official Scores', table of pass rates on the 600-task public benchmark, row 'Claude Sonnet 5', reasoning effort max"
          },
          {
            "event": "2026-06-16-glm-5-2",
            "date": "2026-06-16",
            "provider": "zai",
            "value": 26.17,
            "printed": null,
            "source": "https://github.com/zapier/AutomationBench/blob/4a8e106/README.md",
            "where": "Zapier's README, 'Public vs. Official Scores', table of pass rates on the 600-task public benchmark, row 'GLM 5.2', reasoning effort max"
          }
        ]
      },
      {
        "title": "AutomationBench private set, Zapier's leaderboard",
        "unit": "%",
        "rows": [
          {
            "event": "2026-07-24-claude-opus-5",
            "date": "2026-07-24",
            "provider": "anthropic",
            "value": 26.94,
            "printed": null,
            "source": "https://zapier.com/benchmarks",
            "where": "Zapier's leaderboard, release 1.0.6, row 'Claude Opus 5 (Max)'"
          },
          {
            "event": "2026-07-16-kimi-k3",
            "date": "2026-07-16",
            "provider": "moonshot",
            "value": 22.68,
            "printed": null,
            "source": "https://zapier.com/benchmarks",
            "where": "Zapier's leaderboard, release 1.0.6, row 'Kimi K3'"
          },
          {
            "event": "2026-06-09-claude-fable-5-mythos-5",
            "date": "2026-06-09",
            "provider": "anthropic",
            "value": 17.05,
            "printed": null,
            "source": "https://zapier.com/benchmarks",
            "where": "Zapier's leaderboard, release 1.0.6, row 'Claude Fable 5.0 (Max)'"
          },
          {
            "event": "2026-07-09-gpt-5-6",
            "date": "2026-07-09",
            "provider": "openai",
            "value": 28.77,
            "printed": null,
            "source": "https://zapier.com/benchmarks",
            "where": "Zapier's leaderboard, release 1.0.6, row 'GPT-5.6 Sol (Max)'"
          },
          {
            "event": "2026-07-21-gemini-3-6-flash-and-3-5-flash",
            "date": "2026-07-21",
            "provider": "google",
            "value": 17.05,
            "printed": null,
            "source": "https://zapier.com/benchmarks",
            "where": "Zapier's leaderboard, release 1.0.6, row 'Gemini 3.6 Flash (High)'"
          },
          {
            "event": "2026-05-28-claude-opus-4-8",
            "date": "2026-05-28",
            "provider": "anthropic",
            "value": 16.44,
            "printed": null,
            "source": "https://zapier.com/benchmarks",
            "where": "Zapier's leaderboard, release 1.0.6, row 'Claude Opus 4.8 (Max)'"
          },
          {
            "event": "2026-05-19-gemini-3-5-flash",
            "date": "2026-05-19",
            "provider": "google",
            "value": 13.39,
            "printed": null,
            "source": "https://zapier.com/benchmarks",
            "where": "Zapier's leaderboard, release 1.0.6, row 'Gemini 3.5 Flash (High)'"
          },
          {
            "event": "2026-07-09-gpt-5-6",
            "date": "2026-07-09",
            "provider": "openai",
            "value": 23.59,
            "printed": null,
            "source": "https://zapier.com/benchmarks",
            "where": "Zapier's leaderboard, release 1.0.6, row 'GPT-5.6 Terra (Max)'"
          },
          {
            "event": "2026-06-30-claude-sonnet-5",
            "date": "2026-06-30",
            "provider": "anthropic",
            "value": 10.65,
            "printed": null,
            "source": "https://zapier.com/benchmarks",
            "where": "Zapier's leaderboard, release 1.0.6, row 'Claude Sonnet 5 (Max)'"
          },
          {
            "event": "2026-06-16-glm-5-2",
            "date": "2026-06-16",
            "provider": "zai",
            "value": 7.76,
            "printed": null,
            "source": "https://zapier.com/benchmarks",
            "where": "Zapier's leaderboard, release 1.0.6, row 'GLM 5.2 (Max)'"
          },
          {
            "event": "2026-09-02-muse-spark-1-3",
            "date": "2026-09-02",
            "provider": "meta",
            "value": 20.7,
            "printed": null,
            "source": "https://zapier.com/benchmarks",
            "where": "Zapier's leaderboard, release 1.0.6, row 'Muse Spark 1.3 (Max)'"
          }
        ]
      }
    ],
    "cites": [
      {
        "url": "https://zapier.com/benchmarks",
        "by": "Zapier",
        "medium": "AutomationBench leaderboard, release 1.0.6, read Nov 7, 2026",
        "date": "2026-11-07"
      },
      {
        "url": "https://github.com/zapier/AutomationBench/blob/4a8e106/README.md",
        "by": "Zapier",
        "medium": "AutomationBench README, release 1.0.6",
        "date": "2026-08-04"
      },
      {
        "url": "https://github.com/zapier/AutomationBench/blob/6d21054/CHANGELOG.md",
        "by": "Zapier",
        "medium": "AutomationBench changelog",
        "date": "2026-08-04"
      },
      {
        "url": "https://deepmind.google/models/evals-methodology/gemini-4-carbon",
        "by": "Google",
        "medium": "evaluation methodology",
        "date": "2026-10-28"
      },
      {
        "url": "https://deepmind.google/models/evals-methodology/gemini-3-7-flash",
        "by": "Google",
        "medium": "evaluation methodology",
        "date": "2026-08-13"
      }
    ],
    "sources": [
      "https://api-docs.deepseek.com/news/news260910/",
      "https://blog.google/innovation-and-ai/models-and-research/gemini-models/introducing-gemini-3-7-flash/",
      "https://blog.google/innovation-and-ai/models-and-research/gemini-models/introducing-gemini-4-carbon/",
      "https://deepmind.google/models/evals-methodology/gemini-3-7-flash",
      "https://deepmind.google/models/evals-methodology/gemini-4-carbon",
      "https://github.com/zapier/AutomationBench/blob/4a8e106/README.md",
      "https://github.com/zapier/AutomationBench/blob/6d21054/CHANGELOG.md",
      "https://hy.tencent.ai/research/hy4",
      "https://longcat.ai/blog/longcat-2.5",
      "https://openai.com/index/gpt-6-astra",
      "https://openai.com/index/introducing-gpt-6-1-sol",
      "https://qwen.ai/blog?id=qwen3.8",
      "https://research.meta.ai/blog/introducing-muse-spark-1-3",
      "https://thinkingmachines.ai/news/inkling-2-small/",
      "https://www.anthropic.com/claude-opus-5-5",
      "https://www.anthropic.com/claude-sonnet-5-5-system-card",
      "https://www.kimi.ai/blog/kimi-k3",
      "https://www.kimi.ai/blog/kimi-k3-1-pro",
      "https://www.minimax.io/blog/minimax-m31f",
      "https://www.ricursive.com/ripple",
      "https://z.ai/blog/glm-5.3-flash",
      "https://zapier.com/benchmarks"
    ],
    "updates": [],
    "body": [
      {
        "figure": "sets"
      },
      {
        "p": "[AutomationBench](https://frontierlog.ai/benchmark/automationbench), Zapier's test of AI agents on business workflows, has two task sets. Zapier scores its [leaderboard](https://zapier.com/benchmarks) on a private set it doesn't release, and its [README](https://github.com/zapier/AutomationBench/blob/4a8e106/README.md) says that set \"is purposely harder\" than the 600 public tasks anyone can run. In their launch posts, OpenAI, Anthropic and Google report Zapier's runs on the private set. The other labs that name a set give their own runs on the public set, and no lab reports both."
      },
      {
        "h": "Zapier keeps one set private"
      },
      {
        "p": "Zapier's leaderboard page says its \"scores run against a held-out private evaluation set\" and that \"the public 600-task set is for research and experimentation\". The README adds that scores measured on the public set \"may not match the official leaderboard 1:1\". In release 1.0.6, dated Jul 31, Zapier's [changelog](https://github.com/zapier/AutomationBench/blob/6d21054/CHANGELOG.md) says \"Private tasks were made a bit harder to compensate for bug fixes\"."
      },
      {
        "h": "Who reports which set"
      },
      {
        "p": "Google's [Gemini 4 Carbon post](https://blog.google/innovation-and-ai/models-and-research/gemini-models/introducing-gemini-4-carbon/) of Oct 28 gives 53.9%, and its [evaluation methodology](https://deepmind.google/models/evals-methodology/gemini-4-carbon) says the results \"use the private set and are sourced from the official Zapier public leaderboard\". Anthropic's [Claude Sonnet 5.5 system card](https://www.anthropic.com/claude-sonnet-5-5-system-card) gives 44.7% and says the results \"are on release 1.0.6 of Zapier's private evaluation set\". OpenAI's [GPT-6 Astra post](https://openai.com/index/gpt-6-astra) of Sep 3 gives 41.4% and doesn't name a set. Its [GPT-6.1 Sol post](https://openai.com/index/introducing-gpt-6-1-sol) of Sep 29 shows the same figure in its chart and links it to Zapier's leaderboard."
      },
      {
        "p": "The public-set scores come from the labs' own runs. [MiniMax M3.1 Flash](https://www.minimax.io/blog/minimax-m31f) scored 62.8% on Oct 19, \"evaluated on the 600 publicly released tasks\". Tencent's [Hy4](https://hy.tencent.ai/research/hy4) scored 56.2% on Nov 4, and DeepSeek's [V4.1-Flash](https://api-docs.deepseek.com/news/news260910/) scored 54.8% on Sep 10. Meituan's [LongCat-2.5](https://longcat.ai/blog/longcat-2.5) scored 51.8% and Meta's [Muse Spark 1.3](https://research.meta.ai/blog/introducing-muse-spark-1-3) 49.6%, which Meta ran on \"the public v3 task set\". Ricursive's [Ripple-65B Preview](https://www.ricursive.com/ripple), Moonshot's [Kimi K3](https://www.kimi.ai/blog/kimi-k3) and Alibaba's [Qwen3.8-Max](https://qwen.ai/blog?id=qwen3.8) also report the public set."
      },
      {
        "p": "Most labs name the set beside the figure or in a linked document. Some name only a release: Z.ai's GLM-5.3 posts say \"v1.0.6\", and DeepSeek's Oct 17 post says \"V1.0.6\". DeepSeek's Oct 17 post gives 60.5% for V4.1-Flash-1017, and that result and Z.ai's are filed under [set not stated](https://frontierlog.ai/benchmark/automationbench/unstated). Moonshot's [Kimi K3.1 Pro post](https://www.kimi.ai/blog/kimi-k3-1-pro) and Thinking Machines' [Inkling 2 Small post](https://thinkingmachines.ai/news/inkling-2-small/) report AutomationBench-AA, a version Artificial Analysis runs."
      },
      {
        "h": "Rivals' scores come from the same set"
      },
      {
        "p": "The labs' comparison tables mostly keep to one set. Anthropic and Google print rivals' figures from Zapier's private leaderboard. DeepSeek, MiniMax and LongCat print Zapier's public-set runs from its README, or the rivals' own public-set figures. Tencent's [Hy4 post](https://hy.tencent.ai/research/hy4) gives its own public-set runs of its rivals, marked \"our own testing\", and gives Claude Opus 5.5 58.8% there. Anthropic's [Claude Opus 5.5 page](https://www.anthropic.com/claude-opus-5-5) says its figures for Opus 5, GPT-5.6 Sol and GPT-6 Astra \"come from Zapier's public leaderboard\"."
      },
      {
        "p": "Some rival figures don't match any published score. Z.ai's [GLM-5.3-Flash post](https://z.ai/blog/glm-5.3-flash) of Aug 26 prints 52.3% for Gemini 3.7 Flash. Google's [own figure](https://blog.google/innovation-and-ai/models-and-research/gemini-models/introducing-gemini-3-7-flash/) is 30.4%, and its [evaluation methodology](https://deepmind.google/models/evals-methodology/gemini-3-7-flash) says that figure uses the private set. Zapier's README doesn't list the model. Meta's Muse Spark 1.3 scorecard gives GPT-5.6 Sol 46.7%, where Zapier's README gives 45.83%."
      },
      {
        "h": "The same models score lower on the private set"
      },
      {
        "figure": "drop"
      },
      {
        "p": "Zapier's README lists its own public-set runs of 10 models at release 1.0.6. Each of them scores lower on the private leaderboard, by 14 to 29 points. Claude Opus 5 scored 50.3% on the public set and 26.9% on the private set. The widest gap is Claude Fable 5's, 46.2% against 17.1%, and the README notes its public score \"is after the July 2026 stricter classifier\"."
      },
      {
        "p": "Kimi K3 has two public-set figures. Moonshot gave 30.8% on Jul 16, before release 1.0.6, and Zapier's 1.0.6 README gives 46.7%. Zapier's leaderboard also lists Muse Spark 1.3 at 20.7% on the private set, against Meta's 49.6% on the public set."
      },
      {
        "p": "Our earlier story, [China Leads on AutomationBench](https://frontierlog.ai/stories/china-leads-on-automationbench), compared scores from both sets."
      }
    ],
    "events": [
      "2026-07-16-kimi-k3",
      "2026-08-03-qwen3-8-max",
      "2026-08-13-gemini-3-7-flash",
      "2026-08-26-glm-5-3-flash",
      "2026-09-02-muse-spark-1-3",
      "2026-09-03-gpt-6-astra",
      "2026-09-10-deepseek-v4-1-flash",
      "2026-09-22-claude-opus-5-5",
      "2026-09-29-gpt-6-1-sol",
      "2026-10-19-minimax-m3-1-flash",
      "2026-10-24-kimi-k3-1-pro",
      "2026-10-28-gemini-4-carbon",
      "2026-11-01-longcat-2-5",
      "2026-11-04-hy4",
      "2026-11-04-ripple-65b-preview",
      "2026-11-05-inkling-2-small"
    ],
    "stories": [
      "china-leads-on-automationbench",
      "labs-run-benchmarks-different-ways",
      "who-runs-automationbench"
    ],
    "models": [
      "gemini-4-carbon",
      "claude-sonnet-5-5",
      "gpt-6-astra",
      "minimax-m3-1-flash",
      "hy4",
      "deepseek-v4-1-flash",
      "longcat-2-5",
      "muse-spark-1-3",
      "ripple-65b-preview",
      "kimi-k3",
      "qwen3-8-max"
    ]
  }
}