{
  "api_version": "1",
  "updated": "2026-11-10",
  "latest_event": "2026-11-09",
  "rights": "© Frontier Log, all rights reserved.",
  "benchmark": {
    "slug": "automationbench-no-release",
    "name": "AutomationBench, no release named",
    "family": "automationbench",
    "unit": "%",
    "higher_is_better": true,
    "status": "active",
    "active": true,
    "superseded_by": null,
    "retired": null,
    "benchmark_url": "https://zapier.com/benchmarks",
    "introduced": "2026-04-20",
    "creator": "Zapier",
    "introduced_source": "https://zapier.com/blog/introducing-automationbench/",
    "settings": [
      {
        "setting": "private",
        "name": "Private set",
        "url": "https://frontierlog.ai/benchmark/automationbench-no-release",
        "results": 12
      },
      {
        "setting": "public",
        "name": "Public set",
        "url": "https://frontierlog.ai/benchmark/automationbench-no-release/public",
        "results": 10
      }
    ]
  },
  "total": 22,
  "results": [
    {
      "id": "gpt-5-5:automationbench-no-release:private:1",
      "model": "gpt-5-5",
      "model_name": "GPT-5.5",
      "model_date": "2026-04-23",
      "event": "https://frontierlog.ai/event/2026-04-23-gpt-5-5",
      "benchmark": "automationbench-no-release",
      "benchmark_name": "AutomationBench, no release named",
      "setting": "private",
      "setting_name": "Private set",
      "score": 12.9,
      "unit": "%",
      "printed": "12.9%",
      "reported_by": "openai",
      "run_by": "Zapier",
      "label": "GPT-5.5",
      "source": "https://openai.com/index/gpt-5-6",
      "source_title": "GPT-5.6: Frontier intelligence that scales with your ambition",
      "source_date": "2026-07-09",
      "where": "tool-use table (columns GPT-5.6 Sol, Terra, Luna, GPT-5.5, ...), row 'AutomationBench', column 'GPT-5.5'",
      "own": false,
      "primary": true,
      "parallel": false,
      "notes": "",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-19",
      "corroborated_by": [
        {
          "url": "https://www.anthropic.com/news/claude-fable-5-mythos-5",
          "name": "Claude Fable 5 and Mythos 5",
          "title": "Claude Fable 5 and Claude Mythos 5",
          "lab": "anthropic",
          "date": "2026-06-09",
          "printed": "12.9%",
          "where": "Benchmark table image, row \"Tool use AutomationBench\", column \"GPT 5.5\""
        }
      ],
      "source_capture": "https://frontierlog.ai/captures/90318aa23c794381.png"
    },
    {
      "id": "deepseek-v4-pro:automationbench-no-release:public:1",
      "model": "deepseek-v4-pro",
      "model_name": "DeepSeek-V4-Pro",
      "model_date": "2026-04-24",
      "event": "https://frontierlog.ai/event/2026-04-24-deepseek-v4-preview",
      "benchmark": "automationbench-no-release",
      "benchmark_name": "AutomationBench, no release named",
      "setting": "public",
      "setting_name": "Public set",
      "score": 12.8,
      "unit": "%",
      "printed": "12.8",
      "reported_by": "deepseek",
      "run_by": null,
      "label": "DeepSeek-V4-Pro-Preview",
      "source": "https://api-docs.deepseek.com/news/news260813/",
      "source_title": "DeepSeek-V4-Pro GA Release",
      "source_date": "2026-08-13",
      "where": "later report: DeepSeek-V4-Pro GA post, table image, row 'AutomationBench (Public)', column 'DeepSeek-V4-Pro-Preview'",
      "own": false,
      "primary": true,
      "parallel": false,
      "notes": "",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-20",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/51578c812ffc7347.png",
      "source_image_original": "https://api-docs.deepseek.com/img/v4_260813_benchmark_table_en.png"
    },
    {
      "id": "qwen3-7-max:automationbench-no-release:public:1",
      "model": "qwen3-7-max",
      "model_name": "Qwen3.7-Max",
      "model_date": "2026-05-20",
      "event": "https://frontierlog.ai/event/2026-05-20-qwen3-7-max",
      "benchmark": "automationbench-no-release",
      "benchmark_name": "AutomationBench, no release named",
      "setting": "public",
      "setting_name": "Public set",
      "score": 14.2,
      "unit": "%",
      "printed": "14.2",
      "reported_by": "alibaba",
      "run_by": null,
      "label": "Qwen3.7-Max",
      "source": "https://qwen.ai/blog?id=qwen3.8",
      "source_title": "Qwen3.8-Max: A New Bar for Coding and Cowork",
      "source_date": "2026-08-03",
      "where": "Qwen3.8 post, 'Full Benchmark Table', row 'Automation-Bench (Pass@1)', column 'Qwen3.7-Max'",
      "own": false,
      "primary": true,
      "parallel": false,
      "notes": "pass@1",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-19",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_capture": "https://frontierlog.ai/captures/10d02db2e5d51286.png"
    },
    {
      "id": "claude-opus-4-8:automationbench-no-release:private:1",
      "model": "claude-opus-4-8",
      "model_name": "Claude Opus 4.8",
      "model_date": "2026-05-28",
      "event": "https://frontierlog.ai/event/2026-05-28-claude-opus-4-8",
      "benchmark": "automationbench-no-release",
      "benchmark_name": "AutomationBench, no release named",
      "setting": "private",
      "setting_name": "Private set",
      "score": 15.5,
      "unit": "%",
      "printed": "15.5",
      "reported_by": "anthropic",
      "run_by": "Zapier",
      "label": "Claude Opus 4.8",
      "source": "https://www.anthropic.com/claude-opus-4-8-system-card",
      "source_title": "Claude Opus 4.8 System Card",
      "source_date": "2026-05-28",
      "where": "Section 8.1, Table 8.1.A (evaluation summary), row 'Automation Bench', column 'Claude Opus 4.8'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "max effort",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-19",
      "corroborated_by": [
        {
          "url": "https://www.anthropic.com/claude-fable-5-mythos-5-system-card",
          "name": "Claude Fable 5 and Mythos 5 card",
          "title": "Claude Fable 5 and Claude Mythos 5 System Card",
          "lab": "anthropic",
          "date": "2026-06-09",
          "printed": "15.5",
          "where": "Table 8.1.A, row AutomationBench, column Claude Opus 4.8 (also 8.17.9 text: 'Claude Opus 4.8 (max effort) at 15.5%')"
        },
        {
          "url": "https://www.anthropic.com/news/claude-fable-5-mythos-5",
          "name": "Claude Fable 5 and Mythos 5",
          "title": "Claude Fable 5 and Claude Mythos 5",
          "lab": "anthropic",
          "date": "2026-06-09",
          "printed": "15.5%",
          "where": "Benchmark table image, row Tool use AutomationBench, column Claude Opus 4.8"
        }
      ],
      "source_capture": "https://frontierlog.ai/captures/37133ade6e699bde.png"
    },
    {
      "id": "claude-opus-4-8:automationbench-no-release:private:2",
      "model": "claude-opus-4-8",
      "model_name": "Claude Opus 4.8",
      "model_date": "2026-05-28",
      "event": "https://frontierlog.ai/event/2026-05-28-claude-opus-4-8",
      "benchmark": "automationbench-no-release",
      "benchmark_name": "AutomationBench, no release named",
      "setting": "private",
      "setting_name": "Private set",
      "score": 17.0,
      "unit": "%",
      "printed": "17.0%",
      "reported_by": "anthropic",
      "run_by": "Zapier",
      "label": "Opus 4.8",
      "source": "https://www.anthropic.com/news/claude-opus-5",
      "source_title": "Introducing Claude Opus 5",
      "source_date": "2026-07-24",
      "where": "benchmark table (image), row 'Business workflows, AutomationBench', column 'Opus 4.8'",
      "own": false,
      "primary": false,
      "parallel": false,
      "notes": "max effort; set named in the Claude Opus 5 system card, which prints the same figure",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-19",
      "corroborated_by": [
        {
          "url": "https://www.anthropic.com/claude-opus-5-system-card",
          "name": "Claude Opus 5 card",
          "title": "Claude Opus 5 System Card",
          "lab": "anthropic",
          "date": "2026-07-24",
          "printed": "17.0",
          "where": "Table 8.1.A, row AutomationBench, column Opus 4.8; 8.13.7 text: 'Claude Opus 4.8 (max effort) at 17.0%'"
        }
      ],
      "source_image": "https://frontierlog.ai/sources/924165396ca89130.png",
      "source_image_original": "https://www-cdn.anthropic.com/images/4zrzovbb/website/a8fb4f77a9fe240e6f27f3bdc47a137f3c74a29d-2600x2578.png"
    },
    {
      "id": "claude-fable-5:automationbench-no-release:private:1",
      "model": "claude-fable-5",
      "model_name": "Claude Fable 5",
      "model_date": "2026-06-09",
      "event": "https://frontierlog.ai/event/2026-06-09-claude-fable-5-mythos-5",
      "benchmark": "automationbench-no-release",
      "benchmark_name": "AutomationBench, no release named",
      "setting": "private",
      "setting_name": "Private set",
      "score": 17.4,
      "unit": "%",
      "printed": "17.4",
      "reported_by": "anthropic",
      "run_by": "Zapier",
      "label": "Fable 5",
      "source": "https://www.anthropic.com/claude-fable-5-mythos-5-system-card",
      "source_title": "Claude Fable 5 and Claude Mythos 5 System Card",
      "source_date": "2026-06-09",
      "where": "Section 8.1, Table 8.1.A, row 'AutomationBench', column 'Fable 5'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "max effort",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-19",
      "corroborated_by": [
        {
          "url": "https://www.anthropic.com/claude-opus-5-system-card",
          "name": "Claude Opus 5 card",
          "title": "Claude Opus 5 System Card",
          "lab": "anthropic",
          "date": "2026-07-24",
          "printed": "17.4",
          "where": "Table 8.1.A, row AutomationBench, column Fable 5 (also 8.13.7 text: 'Claude Fable 5 at 17.4%')"
        },
        {
          "url": "https://www.anthropic.com/news/claude-opus-5",
          "name": "Claude Opus 5",
          "title": "Introducing Claude Opus 5",
          "lab": "anthropic",
          "date": "2026-07-24",
          "printed": "17.4%",
          "where": "Benchmark table image, row Business workflows AutomationBench, column Fable 5"
        }
      ],
      "source_capture": "https://frontierlog.ai/captures/d9b0b430494cc63d.png"
    },
    {
      "id": "claude-fable-5:automationbench-no-release:private:2",
      "model": "claude-fable-5",
      "model_name": "Claude Fable 5",
      "model_date": "2026-06-09",
      "event": "https://frontierlog.ai/event/2026-06-09-claude-fable-5-mythos-5",
      "benchmark": "automationbench-no-release",
      "benchmark_name": "AutomationBench, no release named",
      "setting": "private",
      "setting_name": "Private set",
      "score": 17.1,
      "unit": "%",
      "printed": "17.1%",
      "reported_by": "anthropic",
      "run_by": "Zapier",
      "label": "Fable 5",
      "source": "https://www.anthropic.com/claude-fable-and-mythos-5-1",
      "source_title": "Introducing Claude Fable 5.1 and Claude Mythos 5.1",
      "source_date": "2026-09-01",
      "where": "benchmark table (HTML), row 'Business workflows, AutomationBench', column 'Fable 5'",
      "own": false,
      "primary": false,
      "parallel": false,
      "notes": "tasks blocked by safeguards scored zero; set named in the Claude Fable 5.1 system card, which prints 17.05",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-19",
      "corroborated_by": [
        {
          "url": "https://www.anthropic.com/claude-fable-5-1-mythos-5-1-system-card",
          "name": "Claude Fable 5.1 and Mythos 5.1 card",
          "title": "Claude Fable 5.1 and Claude Mythos 5.1 System Card",
          "lab": "anthropic",
          "date": "2026-09-01",
          "printed": "17.1",
          "where": "Table 8.1.A, row AutomationBench, column Claude Fable 5/Mythos 5; note under Figure 8.14.3.A: 'Fable 5 scored a zero on AutomationBench' where safeguards intervened (8.15.6 text prints 17.05%)"
        }
      ],
      "source_capture": "https://frontierlog.ai/captures/f381a9ad69b68272.png"
    },
    {
      "id": "claude-sonnet-5:automationbench-no-release:private:1",
      "model": "claude-sonnet-5",
      "model_name": "Claude Sonnet 5",
      "model_date": "2026-06-30",
      "event": "https://frontierlog.ai/event/2026-06-30-claude-sonnet-5",
      "benchmark": "automationbench-no-release",
      "benchmark_name": "AutomationBench, no release named",
      "setting": "private",
      "setting_name": "Private set",
      "score": 13.5,
      "unit": "%",
      "printed": "13.5",
      "reported_by": "anthropic",
      "run_by": "Zapier",
      "label": "Claude Sonnet 5",
      "source": "https://www.anthropic.com/claude-sonnet-5-system-card",
      "source_title": "Claude Sonnet 5 System Card",
      "source_date": "2026-06-30",
      "where": "Section 8.1, Table 8.1.A, row 'AutomationBench', column 'Claude Sonnet 5'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "max effort",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-20",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_capture": "https://frontierlog.ai/captures/83869c127dd1cbc7.png"
    },
    {
      "id": "gpt-5-6-luna:automationbench-no-release:private:1",
      "model": "gpt-5-6-luna",
      "model_name": "GPT-5.6 Luna",
      "model_date": "2026-07-09",
      "event": "https://frontierlog.ai/event/2026-07-09-gpt-5-6",
      "benchmark": "automationbench-no-release",
      "benchmark_name": "AutomationBench, no release named",
      "setting": "private",
      "setting_name": "Private set",
      "score": 14.9,
      "unit": "%",
      "printed": "14.9%",
      "reported_by": "openai",
      "run_by": "Zapier",
      "label": "GPT‑5.6 Luna",
      "source": "https://openai.com/index/gpt-5-6",
      "source_title": "GPT-5.6: Frontier intelligence that scales with your ambition",
      "source_date": "2026-07-09",
      "where": "tool-use table (columns GPT-5.6 Sol, Terra, Luna, GPT-5.5, ...), row 'AutomationBench', column 'GPT‑5.6 Luna'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-26",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_capture": "https://frontierlog.ai/captures/90318aa23c794381.png"
    },
    {
      "id": "gpt-5-6-sol:automationbench-no-release:private:1",
      "model": "gpt-5-6-sol",
      "model_name": "GPT-5.6 Sol",
      "model_date": "2026-07-09",
      "event": "https://frontierlog.ai/event/2026-07-09-gpt-5-6",
      "benchmark": "automationbench-no-release",
      "benchmark_name": "AutomationBench, no release named",
      "setting": "private",
      "setting_name": "Private set",
      "score": 18.1,
      "unit": "%",
      "printed": "18.1%",
      "reported_by": "openai",
      "run_by": "Zapier",
      "label": "GPT-5.6 Sol",
      "source": "https://openai.com/index/gpt-5-6",
      "source_title": "GPT-5.6: Frontier intelligence that scales with your ambition",
      "source_date": "2026-07-09",
      "where": "tool-use table (columns GPT-5.6 Sol, Terra, Luna, GPT-5.5, ...), row 'AutomationBench', column 'GPT-5.6 Sol'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-19",
      "corroborated_by": [
        {
          "url": "https://www.anthropic.com/news/claude-opus-5",
          "name": "Claude Opus 5",
          "title": "Introducing Claude Opus 5",
          "lab": "anthropic",
          "date": "2026-07-24",
          "printed": "18.1%",
          "where": "Benchmark table image, row \"Business workflows AutomationBench\", column \"GPT-5.6 Sol\""
        },
        {
          "url": "https://openai.com/index/gpt-6-astra",
          "name": "GPT-6 Astra",
          "title": "GPT-6 Astra: A new generation of intelligence",
          "lab": "openai",
          "date": "2026-09-03",
          "printed": "18.1%",
          "where": "Professional table, row 'AutomationBench', column 'GPT-5.6 Sol'"
        }
      ],
      "source_capture": "https://frontierlog.ai/captures/90318aa23c794381.png"
    },
    {
      "id": "gpt-5-6-terra:automationbench-no-release:private:1",
      "model": "gpt-5-6-terra",
      "model_name": "GPT-5.6 Terra",
      "model_date": "2026-07-09",
      "event": "https://frontierlog.ai/event/2026-07-09-gpt-5-6",
      "benchmark": "automationbench-no-release",
      "benchmark_name": "AutomationBench, no release named",
      "setting": "private",
      "setting_name": "Private set",
      "score": 15.2,
      "unit": "%",
      "printed": "15.2%",
      "reported_by": "openai",
      "run_by": "Zapier",
      "label": "GPT‑5.6 Terra",
      "source": "https://openai.com/index/gpt-5-6",
      "source_title": "GPT-5.6: Frontier intelligence that scales with your ambition",
      "source_date": "2026-07-09",
      "where": "tool-use table (columns GPT-5.6 Sol, Terra, Luna, GPT-5.5, ...), row 'AutomationBench', column 'GPT‑5.6 Terra'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-26",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_capture": "https://frontierlog.ai/captures/90318aa23c794381.png"
    },
    {
      "id": "kimi-k3:automationbench-no-release:public:1",
      "model": "kimi-k3",
      "model_name": "Kimi K3",
      "model_date": "2026-07-16",
      "event": "https://frontierlog.ai/event/2026-07-16-kimi-k3",
      "benchmark": "automationbench-no-release",
      "benchmark_name": "AutomationBench, no release named",
      "setting": "public",
      "setting_name": "Public set",
      "score": 30.8,
      "unit": "%",
      "printed": "30.8",
      "reported_by": "moonshot",
      "run_by": null,
      "label": "Kimi K3 (max)",
      "source": "https://www.kimi.ai/blog/kimi-k3",
      "source_title": "Kimi K3: Open Frontier Intelligence",
      "source_date": "2026-07-16",
      "where": "'Full Benchmark Table', row 'Automation Bench' (Agentic), column 'Kimi K3 (max)'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "max reasoning effort; release not named, post predates release 1.0.6",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-20",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_capture": "https://frontierlog.ai/captures/fd6adf73f1b281b7.png"
    },
    {
      "id": "claude-opus-5:automationbench-no-release:private:1",
      "model": "claude-opus-5",
      "model_name": "Claude Opus 5",
      "model_date": "2026-07-24",
      "event": "https://frontierlog.ai/event/2026-07-24-claude-opus-5",
      "benchmark": "automationbench-no-release",
      "benchmark_name": "AutomationBench, no release named",
      "setting": "private",
      "setting_name": "Private set",
      "score": 26.0,
      "unit": "%",
      "printed": "26.0%",
      "reported_by": "anthropic",
      "run_by": "Zapier",
      "label": "Opus 5",
      "source": "https://www.anthropic.com/news/claude-opus-5",
      "source_title": "Introducing Claude Opus 5",
      "source_date": "2026-07-24",
      "where": "benchmark table (image), row 'Business workflows, AutomationBench', column 'Opus 5'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "max effort; set named in the Claude Opus 5 system card, which prints the same figure",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-19",
      "corroborated_by": [
        {
          "url": "https://www.anthropic.com/claude-opus-5-system-card",
          "name": "Claude Opus 5 card",
          "title": "Claude Opus 5 System Card",
          "lab": "anthropic",
          "date": "2026-07-24",
          "printed": "26.0",
          "where": "Table 8.1.A, row AutomationBench, column Opus 5 (also 8.13.7 text: 'Claude Opus 5 (max effort) scored 26.0%')"
        }
      ],
      "source_image": "https://frontierlog.ai/sources/924165396ca89130.png",
      "source_image_original": "https://www-cdn.anthropic.com/images/4zrzovbb/website/a8fb4f77a9fe240e6f27f3bdc47a137f3c74a29d-2600x2578.png"
    },
    {
      "id": "claude-opus-5:automationbench-no-release:private:2",
      "model": "claude-opus-5",
      "model_name": "Claude Opus 5",
      "model_date": "2026-07-24",
      "event": "https://frontierlog.ai/event/2026-07-24-claude-opus-5",
      "benchmark": "automationbench-no-release",
      "benchmark_name": "AutomationBench, no release named",
      "setting": "private",
      "setting_name": "Private set",
      "score": 24.0,
      "unit": "%",
      "printed": "24%",
      "reported_by": "anthropic",
      "run_by": "Zapier",
      "label": "Claude Opus 5 (medium effort)",
      "source": "https://www.anthropic.com/claude-opus-5-system-card",
      "source_title": "Claude Opus 5 System Card",
      "source_date": "2026-07-24",
      "where": "Section 8.13.7 'AutomationBench', paragraph after the benchmark description, the medium-effort figure",
      "own": true,
      "primary": false,
      "parallel": false,
      "notes": "medium effort",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-20",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": []
    },
    {
      "id": "qwen3-8-max:automationbench-no-release:public:1",
      "model": "qwen3-8-max",
      "model_name": "Qwen3.8-Max",
      "model_date": "2026-08-03",
      "event": "https://frontierlog.ai/event/2026-08-03-qwen3-8-max",
      "benchmark": "automationbench-no-release",
      "benchmark_name": "AutomationBench, no release named",
      "setting": "public",
      "setting_name": "Public set",
      "score": 27.3,
      "unit": "%",
      "printed": "27.3",
      "reported_by": "alibaba",
      "run_by": null,
      "label": "Qwen3.8-Max",
      "source": "https://qwen.ai/blog?id=qwen3.8",
      "source_title": "Qwen3.8-Max: A New Bar for Coding and Cowork",
      "source_date": "2026-08-03",
      "where": "'Full Benchmark Table' (first, language/agent HTML table), General Agent section, row 'Automation-Bench (Pass@1)', column 'Qwen3.8-Max'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "pass@1",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-19",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_capture": "https://frontierlog.ai/captures/10d02db2e5d51286.png"
    },
    {
      "id": "muse-spark-1-2:automationbench-no-release:public:1",
      "model": "muse-spark-1-2",
      "model_name": "Muse Spark 1.2",
      "model_date": "2026-08-05",
      "event": "https://frontierlog.ai/event/2026-08-05-muse-spark-1-2",
      "benchmark": "automationbench-no-release",
      "benchmark_name": "AutomationBench, no release named",
      "setting": "public",
      "setting_name": "Public set",
      "score": 38.2,
      "unit": "%",
      "printed": "38.2",
      "reported_by": "meta",
      "run_by": null,
      "label": "Muse Spark 1.2 (xhigh)",
      "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3",
      "source_title": "Introducing Muse Spark 1.3",
      "source_date": "2026-09-02",
      "where": "Muse Spark 1.3 scorecard (image), row 'AutomationBench', column 'Muse Spark 1.2 (xhigh)'",
      "own": false,
      "primary": true,
      "parallel": false,
      "notes": "xhigh effort; pass@1; the post's evaluation methodology names the \"public v3 task set\"",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-20",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/f3ebe2a54d79082c.webp",
      "source_image_original": "https://research.meta.ai/articles/introducing-muse-1-3/benchmarks/benchmark-scorecard-v6.webp"
    },
    {
      "id": "deepseek-v4-pro-0813:automationbench-no-release:public:1",
      "model": "deepseek-v4-pro-0813",
      "model_name": "DeepSeek-V4-Pro-0813",
      "model_date": "2026-08-13",
      "event": "https://frontierlog.ai/event/2026-08-13-deepseek-v4-pro-ga",
      "benchmark": "automationbench-no-release",
      "benchmark_name": "AutomationBench, no release named",
      "setting": "public",
      "setting_name": "Public set",
      "score": 31.8,
      "unit": "%",
      "printed": "31.8",
      "reported_by": "deepseek",
      "run_by": null,
      "label": "DeepSeek-V4-Pro-0813",
      "source": "https://api-docs.deepseek.com/news/news260813/",
      "source_title": "DeepSeek-V4-Pro GA Release",
      "source_date": "2026-08-13",
      "where": "table image, row 'AutomationBench (Public)', column 'DeepSeek-V4-Pro-0813'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-20",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/51578c812ffc7347.png",
      "source_image_original": "https://api-docs.deepseek.com/img/v4_260813_benchmark_table_en.png"
    },
    {
      "id": "deepseek-v4-pro-0813:automationbench-no-release:public:2",
      "model": "deepseek-v4-pro-0813",
      "model_name": "DeepSeek-V4-Pro-0813",
      "model_date": "2026-08-13",
      "event": "https://frontierlog.ai/event/2026-08-13-deepseek-v4-pro-ga",
      "benchmark": "automationbench-no-release",
      "benchmark_name": "AutomationBench, no release named",
      "setting": "public",
      "setting_name": "Public set",
      "score": 43.2,
      "unit": "%",
      "printed": "43.2",
      "reported_by": "deepseek",
      "run_by": null,
      "label": "DeepSeek V4-Pro 0813",
      "source": "https://api-docs.deepseek.com/news/news260910/",
      "source_title": "DeepSeek-V4.1-Flash: Smarter, Faster, More Efficient",
      "source_date": "2026-09-10",
      "where": "later report: DeepSeek-V4.1-Flash post, benchmark table image (second image), row 'Automation-Bench', column 'DeepSeek V4-Pro 0813'",
      "own": false,
      "primary": false,
      "parallel": false,
      "notes": "v1.0.6; set named in the technical report the post links, which prints the same figure",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-19",
      "corroborated_by": [
        {
          "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash",
          "name": "DeepSeek-V4.1-Flash card",
          "title": "deepseek-ai/DeepSeek-V4.1-Flash model card",
          "lab": "deepseek",
          "date": "2026-09-10",
          "printed": "43.2",
          "where": "'Comparison with frontier models (Max reasoning effort)' table, row 'AutomationBench (Pass@1)', column 'DS-V4-Pro'"
        }
      ],
      "source_image": "https://frontierlog.ai/sources/19d3b83b1b9e5f5b.png",
      "source_image_original": "https://api-docs.deepseek.com/img/v4.1_260910_benchmark_table_en.png"
    },
    {
      "id": "deepseek-v4-flash-vision-exp:automationbench-no-release:public:1",
      "model": "deepseek-v4-flash-vision-exp",
      "model_name": "DeepSeek-V4-Flash-Vision-Exp",
      "model_date": "2026-08-21",
      "event": "https://frontierlog.ai/event/2026-08-21-deepseek-v4-flash-vision-exp",
      "benchmark": "automationbench-no-release",
      "benchmark_name": "AutomationBench, no release named",
      "setting": "public",
      "setting_name": "Public set",
      "score": 25.7,
      "unit": "%",
      "printed": "25.7",
      "reported_by": "deepseek",
      "run_by": null,
      "label": "DeepSeek V4-Flash-Vision-Exp",
      "source": "https://api-docs.deepseek.com/news/news260821/",
      "source_title": "DeepSeek-V4-Flash-Vision-Exp Release: Multimodal API Now Live",
      "source_date": "2026-08-21",
      "where": "table image, section 'Text-Based Agent Evaluation', row 'AutomationBench (Public)', column 'DeepSeek V4-Flash-Vision-Exp'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "DeepSeek Harness minimal mode",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-20",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/4b3cca9a4075fb05.png",
      "source_image_original": "https://api-docs.deepseek.com/img/v4_260821_benchmark_en.png"
    },
    {
      "id": "claude-fable-5-1:automationbench-no-release:private:1",
      "model": "claude-fable-5-1",
      "model_name": "Claude Fable 5.1",
      "model_date": "2026-09-01",
      "event": "https://frontierlog.ai/event/2026-09-01-claude-fable-5-1-mythos-5-1",
      "benchmark": "automationbench-no-release",
      "benchmark_name": "AutomationBench, no release named",
      "setting": "private",
      "setting_name": "Private set",
      "score": 31.4,
      "unit": "%",
      "printed": "31.4%",
      "reported_by": "anthropic",
      "run_by": "Zapier",
      "label": "Fable 5.1",
      "source": "https://www.anthropic.com/claude-fable-and-mythos-5-1",
      "source_title": "Introducing Claude Fable 5.1 and Claude Mythos 5.1",
      "source_date": "2026-09-01",
      "where": "benchmark table (HTML), row 'Business workflows, AutomationBench', column 'Fable 5.1'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "max effort; with production safeguards; set named in the Claude Fable 5.1 system card, which prints the same figure",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-19",
      "corroborated_by": [
        {
          "url": "https://www.anthropic.com/claude-fable-5-1-mythos-5-1-system-card",
          "name": "Claude Fable 5.1 and Mythos 5.1 card",
          "title": "Claude Fable 5.1 and Claude Mythos 5.1 System Card",
          "lab": "anthropic",
          "date": "2026-09-01",
          "printed": "31.4",
          "where": "Table 8.1.A Capability evaluation summary, row AutomationBench, column Claude Fable 5.1/Mythos 5.1 (also 8.15.6 text: 'Claude Fable 5.1 at max effort scored 31.4%')"
        },
        {
          "url": "https://www.anthropic.com/claude-opus-5-5-system-card",
          "name": "Claude Opus 5.5 card",
          "title": "Claude Opus 5.5 System Card",
          "lab": "anthropic",
          "date": "2026-09-22",
          "printed": "31.4",
          "where": "Table 8.1.A, row AutomationBench, column Claude Fable 5.1 (also 8.14.6 text: 'Claude Fable 5.1 at max effort at 31.4%')"
        },
        {
          "url": "https://www.anthropic.com/claude-opus-5-5",
          "name": "Claude Opus 5.5",
          "title": "Introducing Claude Opus 5.5",
          "lab": "anthropic",
          "date": "2026-09-22",
          "printed": "31.4%",
          "where": "Benchmark table, row Business workflows AutomationBench, column Fable 5.1"
        }
      ],
      "source_capture": "https://frontierlog.ai/captures/f381a9ad69b68272.png"
    },
    {
      "id": "muse-spark-1-3:automationbench-no-release:public:1",
      "model": "muse-spark-1-3",
      "model_name": "Muse Spark 1.3",
      "model_date": "2026-09-02",
      "event": "https://frontierlog.ai/event/2026-09-02-muse-spark-1-3",
      "benchmark": "automationbench-no-release",
      "benchmark_name": "AutomationBench, no release named",
      "setting": "public",
      "setting_name": "Public set",
      "score": 49.6,
      "unit": "%",
      "printed": "49.6",
      "reported_by": "meta",
      "run_by": null,
      "label": "Muse Spark 1.3 (max)",
      "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3",
      "source_title": "Introducing Muse Spark 1.3",
      "source_date": "2026-09-02",
      "where": "scorecard (image), row 'AutomationBench', column 'Muse Spark 1.3 (max)'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "max effort; pass@1; the post's evaluation methodology names the \"public v3 task set\"",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-20",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/f3ebe2a54d79082c.webp",
      "source_image_original": "https://research.meta.ai/articles/introducing-muse-1-3/benchmarks/benchmark-scorecard-v6.webp"
    },
    {
      "id": "ripple-65b-preview:automationbench-no-release:public:1",
      "model": "ripple-65b-preview",
      "model_name": "Ripple-65B Preview",
      "model_date": "2026-11-04",
      "event": "https://frontierlog.ai/event/2026-11-04-ripple-65b-preview",
      "benchmark": "automationbench-no-release",
      "benchmark_name": "AutomationBench, no release named",
      "setting": "public",
      "setting_name": "Public set",
      "score": 36.4,
      "unit": "%",
      "printed": "36.4",
      "reported_by": "ricursive",
      "run_by": null,
      "label": "Ripple-65B Preview",
      "source": "https://www.ricursive.com/ripple",
      "source_title": "Introducing Ripple-65B Preview, a Frontier Open-Weight Model",
      "source_date": "2026-11-04",
      "where": "Bar chart grid under the intro, panel 'Automation / AutomationBench (Public)', value label on the first (dark blue, Ricursive logo) bar",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "OpenCode harness and maximum effort per the model card",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-11-07",
      "corroborated_by": [
        {
          "url": "https://huggingface.co/ricursiveAI/Ripple-65B-Preview",
          "name": "Ripple-65B Preview card",
          "title": "ricursiveAI/Ripple-65B-Preview model card",
          "lab": "ricursive",
          "date": "2026-11-04",
          "printed": "36.4",
          "where": "Instruct Model Benchmarks table (README), row 'AutomationBench (Public)', column 'Ripple-65B Preview'"
        }
      ],
      "source_capture": "https://frontierlog.ai/captures/135142a8b70649dd.png"
    }
  ]
}