{
  "api_version": "1",
  "updated": "2026-11-10",
  "latest_event": "2026-11-09",
  "rights": "© Frontier Log, all rights reserved.",
  "benchmark": {
    "slug": "automationbench-1-0-6",
    "name": "AutomationBench 1.0.6",
    "family": "automationbench",
    "unit": "%",
    "higher_is_better": true,
    "status": "active",
    "active": true,
    "superseded_by": null,
    "retired": null,
    "benchmark_url": "https://zapier.com/benchmarks",
    "introduced": "2026-07-31",
    "creator": "Zapier",
    "introduced_source": "https://github.com/zapier/AutomationBench/blob/main/CHANGELOG.md",
    "settings": [
      {
        "setting": "private",
        "name": "Private set",
        "url": "https://frontierlog.ai/benchmark/automationbench-1-0-6",
        "results": 22
      },
      {
        "setting": "public",
        "name": "Public set",
        "url": "https://frontierlog.ai/benchmark/automationbench-1-0-6/public",
        "results": 5
      },
      {
        "setting": "unstated",
        "name": "Set not stated",
        "url": "https://frontierlog.ai/benchmark/automationbench-1-0-6/unstated",
        "results": 10
      },
      {
        "setting": "aa",
        "name": "AutomationBench-AA",
        "url": "https://frontierlog.ai/benchmark/automationbench-1-0-6/aa",
        "results": 5
      }
    ]
  },
  "total": 42,
  "results": [
    {
      "id": "mistral-medium-3-5:automationbench-1-0-6:aa:1",
      "model": "mistral-medium-3-5",
      "model_name": "Mistral Medium 3.5",
      "model_date": "2026-05-22",
      "event": "https://frontierlog.ai/event/2026-05-22-mistral-medium-3-5",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "aa",
      "setting_name": "AutomationBench-AA",
      "score": 6.3,
      "unit": "%",
      "printed": "6.3",
      "reported_by": "mistral",
      "run_by": "Artificial Analysis",
      "label": "Mistral Medium 3.5",
      "source": "https://mistral.ai/news/mistral-large-4/",
      "source_title": "Introducing Mistral Large 4",
      "source_date": "2026-10-06",
      "where": "later report: Mistral Large 4 post, 'Agentic Workflows' section, chart tab 'AA - AutomationBench' (image titled 'Artificial Analysis - AutomationBench'), value label on the 'Mistral Medium 3.5' bar",
      "own": false,
      "primary": true,
      "parallel": false,
      "notes": "",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-11-08",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/06418443cbcaa235.webp",
      "source_image_original": "https://mistral.ai/_astro/artificial-analysis---automationbench-v3_Z267Ag9.webp?dpl=6aee0a780db43cb5574b1948"
    },
    {
      "id": "glm-5-2:automationbench-1-0-6:unstated:1",
      "model": "glm-5-2",
      "model_name": "GLM-5.2",
      "model_date": "2026-06-16",
      "event": "https://frontierlog.ai/event/2026-06-16-glm-5-2",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "unstated",
      "setting_name": "Set not stated",
      "score": 26.2,
      "unit": "%",
      "printed": "26.2",
      "reported_by": "zai",
      "run_by": null,
      "label": "GLM-5.2",
      "source": "https://z.ai/blog/glm-5.3",
      "source_title": "GLM-5.3: Frontier Coding with Emergent Cyber Capabilities",
      "source_date": "2026-08-14",
      "where": "'Performance across comparison models' table, row 'AutomationBench v1.0.6', column 'GLM-5.2'",
      "own": false,
      "primary": true,
      "parallel": false,
      "notes": "AutomationBench v1.0.6",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-19",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_capture": "https://frontierlog.ai/captures/86a1ebfddd94c33e.png"
    },
    {
      "id": "claude-sonnet-5:automationbench-1-0-6:private:1",
      "model": "claude-sonnet-5",
      "model_name": "Claude Sonnet 5",
      "model_date": "2026-06-30",
      "event": "https://frontierlog.ai/event/2026-06-30-claude-sonnet-5",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "private",
      "setting_name": "Private set",
      "score": 10.7,
      "unit": "%",
      "printed": "10.7",
      "reported_by": "anthropic",
      "run_by": "Zapier",
      "label": "Claude Sonnet 5",
      "source": "https://www.anthropic.com/claude-sonnet-5-5-system-card",
      "source_title": "Claude Sonnet 5.5 System Card",
      "source_date": "2026-09-28",
      "where": "Section 8.1, Table 8.1.A, row 'AutomationBench', column 'Claude Sonnet 5'",
      "own": false,
      "primary": true,
      "parallel": false,
      "notes": "release 1.0.6",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-19",
      "corroborated_by": [
        {
          "url": "https://deepmind.google/models/model-cards/gemini-3-7-flash/",
          "name": "Gemini 3.7 Flash card",
          "title": "Gemini 3.7 Flash model card",
          "lab": "google",
          "date": "2026-08-13",
          "printed": "10.7%",
          "where": "Results table, row AutomationBench (Private set), column Claude Sonnet 5"
        }
      ],
      "source_capture": "https://frontierlog.ai/captures/53b68efd4b35b2e3.png"
    },
    {
      "id": "hy3:automationbench-1-0-6:unstated:1",
      "model": "hy3",
      "model_name": "Hy3",
      "model_date": "2026-07-06",
      "event": "https://frontierlog.ai/event/2026-07-06-hy3",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "unstated",
      "setting_name": "Set not stated",
      "score": 16.1,
      "unit": "%",
      "printed": "16.1",
      "reported_by": "tencent",
      "run_by": null,
      "label": "Hy3",
      "source": "https://hy.tencent.ai/research/hy4-preview",
      "source_title": "Introducing Hy4 preview",
      "source_date": "2026-08-28",
      "where": "later report: Hy4 preview post, 'Appendix: Benchmark Results' table image, row 'AutomationBench (v1.0.6)', column 'Hy3'",
      "own": false,
      "primary": true,
      "parallel": false,
      "notes": "AutomationBench v1.0.6; highest reasoning setting",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-11-07",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/02c751bf51c52c52.jpg",
      "source_image_original": "https://hy-model-ap-prod-1258344703.cos.accelerate.myqcloud.com/llm-blog/default/9a5ea02f/2026082812342438_ecb98509e92e4be02a40ca358414133a.jpg?q-sign-algorithm=sha1&q-ak=AKIDRl074nOsGdJ9zjMsCRWP3ShmgS3VtX4S&q-sign-time=1787891670;2103251670&q-key-time=1787891670;2103251670&q-header-list=&q-url-param-list=&q-signature=b6d0c3c4d6d7e0e5219d177ec730735185ccdd1f"
    },
    {
      "id": "gemini-3-6-flash:automationbench-1-0-6:private:1",
      "model": "gemini-3-6-flash",
      "model_name": "Gemini 3.6 Flash",
      "model_date": "2026-07-21",
      "event": "https://frontierlog.ai/event/2026-07-21-gemini-3-6-flash-and-3-5-flash",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "private",
      "setting_name": "Private set",
      "score": 17.0,
      "unit": "%",
      "printed": "17.0%",
      "reported_by": "google",
      "run_by": "Zapier",
      "label": "Gemini 3.6 Flash",
      "source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/introducing-gemini-3-7-flash/",
      "source_title": "Introducing Gemini 3.7 Flash",
      "source_date": "2026-08-13",
      "where": "benchmark table (image), row 'AutomationBench (Private set)', column 'Gemini 3.6 Flash'",
      "own": false,
      "primary": true,
      "parallel": false,
      "notes": "",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-19",
      "corroborated_by": [
        {
          "url": "https://deepmind.google/models/model-cards/gemini-3-7-flash/",
          "name": "Gemini 3.7 Flash card",
          "title": "Gemini 3.7 Flash model card",
          "lab": "google",
          "date": "2026-08-13",
          "printed": "17.0%",
          "where": "Results table, row AutomationBench (Private set), column Gemini 3.6 Flash"
        }
      ],
      "source_image": "https://frontierlog.ai/sources/73ca55228187a14c.png",
      "source_image_original": "https://storage.googleapis.com/gweb-uniblog-publish-prod/images/gemini-3-7-flash__evals__benchmarks-table__2x.original.png"
    },
    {
      "id": "claude-opus-5:automationbench-1-0-6:private:1",
      "model": "claude-opus-5",
      "model_name": "Claude Opus 5",
      "model_date": "2026-07-24",
      "event": "https://frontierlog.ai/event/2026-07-24-claude-opus-5",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "private",
      "setting_name": "Private set",
      "score": 26.9,
      "unit": "%",
      "printed": "26.9%",
      "reported_by": "anthropic",
      "run_by": "Zapier",
      "label": "Opus 5",
      "source": "https://www.anthropic.com/claude-fable-and-mythos-5-1",
      "source_title": "Introducing Claude Fable 5.1 and Claude Mythos 5.1",
      "source_date": "2026-09-01",
      "where": "benchmark table (HTML), row 'Business workflows, AutomationBench', column 'Opus 5'",
      "own": false,
      "primary": true,
      "parallel": false,
      "notes": "max effort; set named in the Claude Fable 5.1 system card, which prints the same figure",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-19",
      "corroborated_by": [
        {
          "url": "https://www.anthropic.com/claude-fable-5-1-mythos-5-1-system-card",
          "name": "Claude Fable 5.1 and Mythos 5.1 card",
          "title": "Claude Fable 5.1 and Claude Mythos 5.1 System Card",
          "lab": "anthropic",
          "date": "2026-09-01",
          "printed": "26.9",
          "where": "Table 8.1.A, row AutomationBench, column Claude Opus 5 (also 8.15.6 text: 'Claude Opus 5 at max effort at 26.9%')"
        },
        {
          "url": "https://www.anthropic.com/claude-opus-5-5-system-card",
          "name": "Claude Opus 5.5 card",
          "title": "Claude Opus 5.5 System Card",
          "lab": "anthropic",
          "date": "2026-09-22",
          "printed": "26.9",
          "where": "Table 8.1.A, row AutomationBench, column Claude Opus 5 (also 8.14.6 text: 'Claude Opus 5 at 26.9%')"
        },
        {
          "url": "https://www.anthropic.com/claude-opus-5-5",
          "name": "Claude Opus 5.5",
          "title": "Introducing Claude Opus 5.5",
          "lab": "anthropic",
          "date": "2026-09-22",
          "printed": "26.9%",
          "where": "Benchmark table, row Business workflows AutomationBench, column Opus 5"
        }
      ],
      "source_capture": "https://frontierlog.ai/captures/f381a9ad69b68272.png"
    },
    {
      "id": "gemini-3-7-flash:automationbench-1-0-6:private:1",
      "model": "gemini-3-7-flash",
      "model_name": "Gemini 3.7 Flash",
      "model_date": "2026-08-13",
      "event": "https://frontierlog.ai/event/2026-08-13-gemini-3-7-flash",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "private",
      "setting_name": "Private set",
      "score": 30.4,
      "unit": "%",
      "printed": "30.4%",
      "reported_by": "google",
      "run_by": "Zapier",
      "label": "Gemini 3.7 Flash",
      "source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/introducing-gemini-3-7-flash/",
      "source_title": "Introducing Gemini 3.7 Flash",
      "source_date": "2026-08-13",
      "where": "benchmark table (image), row 'AutomationBench (Private set)', column 'Gemini 3.7 Flash'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-19",
      "corroborated_by": [
        {
          "url": "https://deepmind.google/models/model-cards/gemini-3-7-flash/",
          "name": "Gemini 3.7 Flash card",
          "title": "Gemini 3.7 Flash model card",
          "lab": "google",
          "date": "2026-08-13",
          "printed": "30.4%",
          "where": "Results table, row AutomationBench (Private set), column Gemini 3.7 Flash"
        }
      ],
      "source_image": "https://frontierlog.ai/sources/73ca55228187a14c.png",
      "source_image_original": "https://storage.googleapis.com/gweb-uniblog-publish-prod/images/gemini-3-7-flash__evals__benchmarks-table__2x.original.png"
    },
    {
      "id": "glm-5-3:automationbench-1-0-6:unstated:1",
      "model": "glm-5-3",
      "model_name": "GLM-5.3",
      "model_date": "2026-08-14",
      "event": "https://frontierlog.ai/event/2026-08-14-glm-5-3",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "unstated",
      "setting_name": "Set not stated",
      "score": 48.2,
      "unit": "%",
      "printed": "48.2",
      "reported_by": "zai",
      "run_by": null,
      "label": "GLM-5.3",
      "source": "https://z.ai/blog/glm-5.3",
      "source_title": "GLM-5.3: Frontier Coding with Emergent Cyber Capabilities",
      "source_date": "2026-08-14",
      "where": "'Performance across comparison models' table, row 'AutomationBench v1.0.6', column 'GLM-5.3'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "AutomationBench v1.0.6 with a fix for null-type handling",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-19",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_capture": "https://frontierlog.ai/captures/86a1ebfddd94c33e.png"
    },
    {
      "id": "glm-5-3-flash:automationbench-1-0-6:unstated:1",
      "model": "glm-5-3-flash",
      "model_name": "GLM-5.3-Flash",
      "model_date": "2026-08-26",
      "event": "https://frontierlog.ai/event/2026-08-26-glm-5-3-flash",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "unstated",
      "setting_name": "Set not stated",
      "score": 48.8,
      "unit": "%",
      "printed": "48.8",
      "reported_by": "zai",
      "run_by": null,
      "label": "GLM-5.3-Flash",
      "source": "https://z.ai/blog/glm-5.3-flash",
      "source_title": "GLM-5.3-Flash: Frontier Intelligence, Flash Cost",
      "source_date": "2026-08-26",
      "where": "main benchmark table (second table), AGENTIC block, row 'AutomationBench v1.0.6', column 'GLM-5.3-Flash'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "AutomationBench v1.0.6 with a fix for null-type handling",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-19",
      "corroborated_by": [
        {
          "url": "https://huggingface.co/zai-org/GLM-5.3-Flash",
          "name": "GLM-5.3-Flash card",
          "title": "zai-org/GLM-5.3-Flash model card",
          "lab": "zai",
          "date": "2026-08-26",
          "printed": "48.8",
          "where": "bench_53 image, AutomationBench panel, GLM-5.3-Flash bar; footnote: v1.0.6 with null-type fix (PR #13)"
        },
        {
          "url": "https://docs.z.ai/guides/llm/glm-5.3-flash",
          "name": "GLM-5.3-Flash card",
          "title": "GLM-5.3-Flash/FlashX - Overview",
          "lab": "zai",
          "date": "2026-08-26",
          "printed": "48.8",
          "where": "chart, AutomationBench panel, GLM-5.3-Flash bar; also text: '48.8 vs. 26.2 on AutomationBench'"
        },
        {
          "url": "https://github.com/zai-org/GLM-5",
          "name": "GLM-5 card",
          "title": "GLM-5: From Vibe Coding to Agentic Engineering (zai-org/GLM-5 GitHub repository)",
          "lab": "zai",
          "date": "2026-02-12",
          "printed": "48.8",
          "where": "README, bench_53 image, AutomationBench panel, GLM-5.3-Flash bar"
        }
      ],
      "source_capture": "https://frontierlog.ai/captures/23dc8480c23f5792.png"
    },
    {
      "id": "hy4-preview:automationbench-1-0-6:unstated:1",
      "model": "hy4-preview",
      "model_name": "Hy4 preview",
      "model_date": "2026-08-28",
      "event": "https://frontierlog.ai/event/2026-08-28-hy4-preview",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "unstated",
      "setting_name": "Set not stated",
      "score": 32.1,
      "unit": "%",
      "printed": "32.1",
      "reported_by": "tencent",
      "run_by": null,
      "label": "Hy4 preview",
      "source": "https://hy.tencent.ai/research/hy4-preview",
      "source_title": "Introducing Hy4 preview",
      "source_date": "2026-08-28",
      "where": "'Appendix: Benchmark Results' table image, row 'AutomationBench (v1.0.6)', column 'Hy4 preview'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "AutomationBench v1.0.6",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-11-08",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/02c751bf51c52c52.jpg",
      "source_image_original": "https://hy-model-ap-prod-1258344703.cos.accelerate.myqcloud.com/llm-blog/default/9a5ea02f/2026082812342438_ecb98509e92e4be02a40ca358414133a.jpg?q-sign-algorithm=sha1&q-ak=AKIDRl074nOsGdJ9zjMsCRWP3ShmgS3VtX4S&q-sign-time=1787891670;2103251670&q-key-time=1787891670;2103251670&q-header-list=&q-url-param-list=&q-signature=b6d0c3c4d6d7e0e5219d177ec730735185ccdd1f"
    },
    {
      "id": "hy4-preview:automationbench-1-0-6:public:1",
      "model": "hy4-preview",
      "model_name": "Hy4 preview",
      "model_date": "2026-08-28",
      "event": "https://frontierlog.ai/event/2026-08-28-hy4-preview",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "public",
      "setting_name": "Public set",
      "score": 36.3,
      "unit": "%",
      "printed": "36.3",
      "reported_by": "tencent",
      "run_by": null,
      "label": "Hy4 preview",
      "source": "https://hy.tencent.ai/research/hy4",
      "source_title": "Introducing Hy4",
      "source_date": "2026-11-04",
      "where": "later report: Hy4 post, 'Appendix: benchmark results' table image, row 'AutomationBench v1.0.6', column 'Hy4 preview'",
      "own": false,
      "primary": true,
      "parallel": false,
      "notes": "re-evaluated at Hy4 launch; AutomationBench v1.0.6; official method",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-11-08",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/8fc24d0f588e6069.jpg",
      "source_image_original": "https://hy-model-ap-prod-1258344703.cos.ap-guangzhou.myqcloud.com/llm-blog/default/9a5ea02f/2026110402023072_85b416b6484a88ef96ef6edda563b560.jpg?q-sign-algorithm=sha1&q-ak=AKIDRl074nOsGdJ9zjMsCRWP3ShmgS3VtX4S&q-sign-time=1793728954;2109088954&q-key-time=1793728954;2109088954&q-header-list=&q-url-param-list=&q-signature=747236201b67dd4e9872ed806c8de224d0ca57c1"
    },
    {
      "id": "gpt-6-astra:automationbench-1-0-6:private:1",
      "model": "gpt-6-astra",
      "model_name": "GPT-6 Astra",
      "model_date": "2026-09-03",
      "event": "https://frontierlog.ai/event/2026-09-03-gpt-6-astra",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "private",
      "setting_name": "Private set",
      "score": 41.4,
      "unit": "%",
      "printed": "41.4%",
      "reported_by": "openai",
      "run_by": "Zapier",
      "label": "GPT‑6 Astra",
      "source": "https://openai.com/index/gpt-6-astra",
      "source_title": "GPT-6 Astra: A new generation of intelligence",
      "source_date": "2026-09-03",
      "where": "Professional table, row 'AutomationBench', column 'GPT‑6 Astra'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "highest score at any effort; set named in the GPT-6.1 Sol post, which prints the same figure",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-16",
      "check": "human",
      "corroborated": true,
      "corroborated_on": "2026-10-19",
      "corroborated_by": [
        {
          "url": "https://www.anthropic.com/claude-opus-5-5-system-card",
          "name": "Claude Opus 5.5 card",
          "title": "Claude Opus 5.5 System Card",
          "lab": "anthropic",
          "date": "2026-09-22",
          "printed": "41.4",
          "where": "Table 8.1.A (p.174), row AutomationBench, column GPT-6 Astra"
        },
        {
          "url": "https://www.anthropic.com/claude-opus-5-5",
          "name": "Claude Opus 5.5",
          "title": "Introducing Claude Opus 5.5",
          "lab": "anthropic",
          "date": "2026-09-22",
          "printed": "41.4%",
          "where": "HTML benchmark table under 'Performance and cost-effectiveness', row AutomationBench² (run and reported by Zapier), column GPT-6 Astra"
        },
        {
          "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-4-argon/",
          "name": "Gemini 4 Argon",
          "title": "Gemini 4 Argon: our next era of frontier intelligence",
          "lab": "google",
          "date": "2026-09-30",
          "printed": "41.4%",
          "where": "Benchmark table image; row 'AutomationBench, Score'; column gpt-6-astra"
        }
      ],
      "source_capture": "https://frontierlog.ai/captures/8618c81cd990eeed.png"
    },
    {
      "id": "gpt-6-astra:automationbench-1-0-6:private:2",
      "model": "gpt-6-astra",
      "model_name": "GPT-6 Astra",
      "model_date": "2026-09-03",
      "event": "https://frontierlog.ai/event/2026-09-03-gpt-6-astra",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "private",
      "setting_name": "Private set",
      "score": 30.3,
      "unit": "%",
      "printed": "30.3%",
      "reported_by": "openai",
      "run_by": "Zapier",
      "label": "GPT-6 Astra (low)",
      "source": "https://openai.com/index/introducing-gpt-6-sol-and-luna",
      "source_title": "Introducing GPT-6 Sol and Luna",
      "source_date": "2026-09-22",
      "where": "AutomationBench table (Model (and effort) / Score / Cost per task), row 'GPT-6 Astra (low)', column 'Score'",
      "own": false,
      "primary": false,
      "parallel": false,
      "notes": "low effort; AutomationBench 1.0.6",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-20",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_capture": "https://frontierlog.ai/captures/60118f343b9598a5.png"
    },
    {
      "id": "gpt-6-astra:automationbench-1-0-6:private:3",
      "model": "gpt-6-astra",
      "model_name": "GPT-6 Astra",
      "model_date": "2026-09-03",
      "event": "https://frontierlog.ai/event/2026-09-03-gpt-6-astra",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "private",
      "setting_name": "Private set",
      "score": 34.1,
      "unit": "%",
      "printed": "34.1%",
      "reported_by": "openai",
      "run_by": "Zapier",
      "label": "GPT-6 Astra (Medium)",
      "source": "https://openai.com/index/introducing-gpt-6-1-sol",
      "source_title": "Introducing GPT-6.1 Sol",
      "source_date": "2026-09-29",
      "where": "AutomationBench chart under \"Professional work\" (score vs cost per task; no value labels): point 'GPT-6 Astra', effort 'Medium', read from the point's aria-label (load the page in a browser and hover or inspect the SVG point)",
      "own": false,
      "primary": false,
      "parallel": false,
      "notes": "medium effort; AutomationBench 1.0.6",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-11-07",
      "corroborated_by": [
        {
          "url": "https://openai.com/index/introducing-gpt-6-sol-and-luna",
          "name": "GPT-6 Sol and Luna",
          "title": "Introducing GPT-6 Sol and Luna",
          "lab": "openai",
          "date": "2026-09-22",
          "printed": "34.1%",
          "where": "AutomationBench chart, point aria-label \"cost: $1.27; score: 34%; modelLabel: GPT-6 Astra; Model: GPT-6 Astra; Effort: medium; Cost per task: $1.27; Score: 34.1%\""
        }
      ],
      "source_capture": "https://frontierlog.ai/captures/53d8c9ca12612a78.png"
    },
    {
      "id": "gpt-6-astra:automationbench-1-0-6:private:4",
      "model": "gpt-6-astra",
      "model_name": "GPT-6 Astra",
      "model_date": "2026-09-03",
      "event": "https://frontierlog.ai/event/2026-09-03-gpt-6-astra",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "private",
      "setting_name": "Private set",
      "score": 37.1,
      "unit": "%",
      "printed": "37.1%",
      "reported_by": "openai",
      "run_by": "Zapier",
      "label": "GPT-6 Astra (High)",
      "source": "https://openai.com/index/introducing-gpt-6-1-sol",
      "source_title": "Introducing GPT-6.1 Sol",
      "source_date": "2026-09-29",
      "where": "AutomationBench chart under \"Professional work\" (score vs cost per task; no value labels): point 'GPT-6 Astra', effort 'High', read from the point's aria-label (load the page in a browser and hover or inspect the SVG point)",
      "own": false,
      "primary": false,
      "parallel": false,
      "notes": "high effort; AutomationBench 1.0.6",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-11-07",
      "corroborated_by": [
        {
          "url": "https://openai.com/index/introducing-gpt-6-sol-and-luna",
          "name": "GPT-6 Sol and Luna",
          "title": "Introducing GPT-6 Sol and Luna",
          "lab": "openai",
          "date": "2026-09-22",
          "printed": "37.1%",
          "where": "AutomationBench chart, point aria-label \"cost: $1.44; score: 37%; modelLabel: GPT-6 Astra; Model: GPT-6 Astra; Effort: high; Cost per task: $1.44; Score: 37.1%\""
        }
      ],
      "source_capture": "https://frontierlog.ai/captures/53d8c9ca12612a78.png"
    },
    {
      "id": "gpt-6-astra:automationbench-1-0-6:private:5",
      "model": "gpt-6-astra",
      "model_name": "GPT-6 Astra",
      "model_date": "2026-09-03",
      "event": "https://frontierlog.ai/event/2026-09-03-gpt-6-astra",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "private",
      "setting_name": "Private set",
      "score": 39.0,
      "unit": "%",
      "printed": "39.0%",
      "reported_by": "openai",
      "run_by": "Zapier",
      "label": "GPT-6 Astra (Xhigh)",
      "source": "https://openai.com/index/introducing-gpt-6-1-sol",
      "source_title": "Introducing GPT-6.1 Sol",
      "source_date": "2026-09-29",
      "where": "AutomationBench chart under \"Professional work\" (score vs cost per task; no value labels): point 'GPT-6 Astra', effort 'Xhigh', read from the point's aria-label (load the page in a browser and hover or inspect the SVG point)",
      "own": false,
      "primary": false,
      "parallel": false,
      "notes": "xhigh effort; AutomationBench 1.0.6",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-11-07",
      "corroborated_by": [
        {
          "url": "https://www.anthropic.com/claude-opus-5-5",
          "name": "Claude Opus 5.5",
          "title": "Introducing Claude Opus 5.5",
          "lab": "anthropic",
          "date": "2026-09-22",
          "printed": "39",
          "where": "\"AutomationBench\" accuracy-vs-cost chart, series gpt6astra, point Xhigh (chart data in the page source); footnote 2 says the GPT-6 Astra figures come from Zapier's public leaderboard"
        },
        {
          "url": "https://openai.com/index/introducing-gpt-6-sol-and-luna",
          "name": "GPT-6 Sol and Luna",
          "title": "Introducing GPT-6 Sol and Luna",
          "lab": "openai",
          "date": "2026-09-22",
          "printed": "39.0%",
          "where": "AutomationBench chart, point aria-label \"cost: $1.50; score: 39%; modelLabel: GPT-6 Astra; Model: GPT-6 Astra; Effort: xhigh; Cost per task: $1.50; Score: 39.0%\""
        }
      ],
      "source_capture": "https://frontierlog.ai/captures/53d8c9ca12612a78.png"
    },
    {
      "id": "deepseek-v4-1-flash:automationbench-1-0-6:public:1",
      "model": "deepseek-v4-1-flash",
      "model_name": "DeepSeek-V4.1-Flash",
      "model_date": "2026-09-10",
      "event": "https://frontierlog.ai/event/2026-09-10-deepseek-v4-1-flash",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "public",
      "setting_name": "Public set",
      "score": 54.8,
      "unit": "%",
      "printed": "54.8",
      "reported_by": "deepseek",
      "run_by": null,
      "label": "DeepSeek V4.1-Flash",
      "source": "https://api-docs.deepseek.com/news/news260910/",
      "source_title": "DeepSeek-V4.1-Flash: Smarter, Faster, More Efficient",
      "source_date": "2026-09-10",
      "where": "benchmark table image (second image), row 'Automation-Bench', column 'DeepSeek V4.1-Flash' (the first chart image, v4.1_260910_benchmark.png, prints the same value label)",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "v1.0.6; set named in the technical report the post links, which prints the same figure",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-19",
      "corroborated_by": [
        {
          "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash",
          "name": "DeepSeek-V4.1-Flash card",
          "title": "deepseek-ai/DeepSeek-V4.1-Flash model card",
          "lab": "deepseek",
          "date": "2026-09-10",
          "printed": "54.8",
          "where": "'Comparison with frontier models (Max reasoning effort)' table, row 'AutomationBench (Pass@1)', column 'DS-V4.1-Flash'"
        },
        {
          "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017",
          "name": "DeepSeek-V4.1-Flash-1017 card",
          "title": "deepseek-ai/DeepSeek-V4.1-Flash-1017 model card",
          "lab": "deepseek",
          "date": "2026-10-17",
          "printed": "54.8",
          "where": "Table, row 'Automation-Bench v1.0.6', column 'DS-V4.1-Flash'"
        }
      ],
      "source_image": "https://frontierlog.ai/sources/19d3b83b1b9e5f5b.png",
      "source_image_original": "https://api-docs.deepseek.com/img/v4.1_260910_benchmark_table_en.png"
    },
    {
      "id": "claude-opus-5-5:automationbench-1-0-6:private:1",
      "model": "claude-opus-5-5",
      "model_name": "Claude Opus 5.5",
      "model_date": "2026-09-22",
      "event": "https://frontierlog.ai/event/2026-09-22-claude-opus-5-5",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "private",
      "setting_name": "Private set",
      "score": 40.0,
      "unit": "%",
      "printed": "40.0%",
      "reported_by": "anthropic",
      "run_by": "Zapier",
      "label": "Opus 5.5",
      "source": "https://www.anthropic.com/claude-opus-5-5",
      "source_title": "Introducing Claude Opus 5.5",
      "source_date": "2026-09-22",
      "where": "benchmark table, row 'AutomationBench', column 'Opus 5.5'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "without fallback models; set and release 1.0.6 named in the Claude Sonnet 5.5 system card, which prints the same figure",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-16",
      "check": "human",
      "corroborated": true,
      "corroborated_on": "2026-10-19",
      "corroborated_by": [
        {
          "url": "https://www.anthropic.com/claude-opus-5-5-system-card",
          "name": "Claude Opus 5.5 card",
          "title": "Claude Opus 5.5 System Card",
          "lab": "anthropic",
          "date": "2026-09-22",
          "printed": "40.0",
          "where": "Table 8.1.A (p.174), row AutomationBench, column Claude Opus 5.5"
        },
        {
          "url": "https://www.anthropic.com/claude-sonnet-5-5-system-card",
          "name": "Claude Sonnet 5.5 card",
          "title": "Claude Sonnet 5.5 System Card",
          "lab": "anthropic",
          "date": "2026-09-28",
          "printed": "40.0%",
          "where": "Section 8.14.6 (p.137) sentence: \"(40.0% without the re-run, as reported in the Claude Opus 5.5 System Card)\""
        }
      ],
      "source_capture": "https://frontierlog.ai/captures/ca3ad55d7ec61f3c.png"
    },
    {
      "id": "claude-opus-5-5:automationbench-1-0-6:private:2",
      "model": "claude-opus-5-5",
      "model_name": "Claude Opus 5.5",
      "model_date": "2026-09-22",
      "event": "https://frontierlog.ai/event/2026-09-22-claude-opus-5-5",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "private",
      "setting_name": "Private set",
      "score": 42.5,
      "unit": "%",
      "printed": "42.5",
      "reported_by": "anthropic",
      "run_by": "Zapier",
      "label": "Claude Opus 5.5",
      "source": "https://www.anthropic.com/claude-sonnet-5-5-system-card",
      "source_title": "Claude Sonnet 5.5 System Card",
      "source_date": "2026-09-28",
      "where": "Section 8.1, Table 8.1.A, row 'AutomationBench', column 'Claude Opus 5.5'",
      "own": false,
      "primary": false,
      "parallel": false,
      "notes": "refused tasks re-run with default fallbacks; release 1.0.6",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-20",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_capture": "https://frontierlog.ai/captures/53b68efd4b35b2e3.png"
    },
    {
      "id": "gpt-6-sol:automationbench-1-0-6:private:1",
      "model": "gpt-6-sol",
      "model_name": "GPT-6 Sol",
      "model_date": "2026-09-22",
      "event": "https://frontierlog.ai/event/2026-09-22-gpt-6-sol-and-luna",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "private",
      "setting_name": "Private set",
      "score": 33.2,
      "unit": "%",
      "printed": "33.2%",
      "reported_by": "openai",
      "run_by": "Zapier",
      "label": "GPT-6 Sol (xhigh)",
      "source": "https://openai.com/index/introducing-gpt-6-sol-and-luna",
      "source_title": "Introducing GPT-6 Sol and Luna",
      "source_date": "2026-09-22",
      "where": "AutomationBench table (Model (and effort) / Score / Cost per task), row 'GPT-6 Sol (xhigh)', column 'Score'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "xhigh effort; AutomationBench 1.0.6",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-20",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_capture": "https://frontierlog.ai/captures/60118f343b9598a5.png"
    },
    {
      "id": "claude-sonnet-5-5:automationbench-1-0-6:private:1",
      "model": "claude-sonnet-5-5",
      "model_name": "Claude Sonnet 5.5",
      "model_date": "2026-09-28",
      "event": "https://frontierlog.ai/event/2026-09-28-claude-sonnet-5-5",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "private",
      "setting_name": "Private set",
      "score": 44.7,
      "unit": "%",
      "printed": "44.7",
      "reported_by": "anthropic",
      "run_by": "Zapier",
      "label": "Claude Sonnet 5.5",
      "source": "https://www.anthropic.com/claude-sonnet-5-5-system-card",
      "source_title": "Claude Sonnet 5.5 System Card",
      "source_date": "2026-09-28",
      "where": "Section 8.1, Table 8.1.A, row 'AutomationBench', column 'Claude Sonnet 5.5'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "release 1.0.6; max effort; default fallbacks on",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-20",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_capture": "https://frontierlog.ai/captures/53b68efd4b35b2e3.png"
    },
    {
      "id": "gpt-6-1-sol:automationbench-1-0-6:private:1",
      "model": "gpt-6-1-sol",
      "model_name": "GPT-6.1 Sol",
      "model_date": "2026-09-29",
      "event": "https://frontierlog.ai/event/2026-09-29-gpt-6-1-sol",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "private",
      "setting_name": "Private set",
      "score": 24.7,
      "unit": "%",
      "printed": "24.7%",
      "reported_by": "openai",
      "run_by": "Zapier",
      "label": "GPT-6.1 Sol (Low)",
      "source": "https://openai.com/index/introducing-gpt-6-1-sol",
      "source_title": "Introducing GPT-6.1 Sol",
      "source_date": "2026-09-29",
      "where": "AutomationBench chart under \"Professional work\" (score vs cost per task; no value labels): point 'GPT-6.1 Sol', effort 'Low', read from the point's aria-label (load the page in a browser and hover or inspect the SVG point)",
      "own": true,
      "primary": false,
      "parallel": false,
      "notes": "low effort; AutomationBench 1.0.6",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-11-08",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_capture": "https://frontierlog.ai/captures/53d8c9ca12612a78.png"
    },
    {
      "id": "gpt-6-1-sol:automationbench-1-0-6:private:2",
      "model": "gpt-6-1-sol",
      "model_name": "GPT-6.1 Sol",
      "model_date": "2026-09-29",
      "event": "https://frontierlog.ai/event/2026-09-29-gpt-6-1-sol",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "private",
      "setting_name": "Private set",
      "score": 31.7,
      "unit": "%",
      "printed": "31.7%",
      "reported_by": "openai",
      "run_by": "Zapier",
      "label": "GPT-6.1 Sol (Medium)",
      "source": "https://openai.com/index/introducing-gpt-6-1-sol",
      "source_title": "Introducing GPT-6.1 Sol",
      "source_date": "2026-09-29",
      "where": "AutomationBench chart under \"Professional work\" (score vs cost per task; no value labels): point 'GPT-6.1 Sol', effort 'Medium', read from the point's aria-label (load the page in a browser and hover or inspect the SVG point)",
      "own": true,
      "primary": false,
      "parallel": false,
      "notes": "medium effort; AutomationBench 1.0.6",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-11-08",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_capture": "https://frontierlog.ai/captures/53d8c9ca12612a78.png"
    },
    {
      "id": "gpt-6-1-sol:automationbench-1-0-6:private:3",
      "model": "gpt-6-1-sol",
      "model_name": "GPT-6.1 Sol",
      "model_date": "2026-09-29",
      "event": "https://frontierlog.ai/event/2026-09-29-gpt-6-1-sol",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "private",
      "setting_name": "Private set",
      "score": 33.2,
      "unit": "%",
      "printed": "33.2%",
      "reported_by": "openai",
      "run_by": "Zapier",
      "label": "GPT-6.1 Sol (High)",
      "source": "https://openai.com/index/introducing-gpt-6-1-sol",
      "source_title": "Introducing GPT-6.1 Sol",
      "source_date": "2026-09-29",
      "where": "AutomationBench chart under \"Professional work\" (score vs cost per task; no value labels): point 'GPT-6.1 Sol', effort 'High', read from the point's aria-label (load the page in a browser and hover or inspect the SVG point)",
      "own": true,
      "primary": false,
      "parallel": false,
      "notes": "high effort; AutomationBench 1.0.6",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-11-08",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_capture": "https://frontierlog.ai/captures/53d8c9ca12612a78.png"
    },
    {
      "id": "gpt-6-1-sol:automationbench-1-0-6:private:4",
      "model": "gpt-6-1-sol",
      "model_name": "GPT-6.1 Sol",
      "model_date": "2026-09-29",
      "event": "https://frontierlog.ai/event/2026-09-29-gpt-6-1-sol",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "private",
      "setting_name": "Private set",
      "score": 35.5,
      "unit": "%",
      "printed": "35.5%",
      "reported_by": "openai",
      "run_by": "Zapier",
      "label": "GPT-6.1 Sol (Xhigh)",
      "source": "https://openai.com/index/introducing-gpt-6-1-sol",
      "source_title": "Introducing GPT-6.1 Sol",
      "source_date": "2026-09-29",
      "where": "AutomationBench chart under \"Professional work\" (score vs cost per task; no value labels): point 'GPT-6.1 Sol', effort 'Xhigh', read from the point's aria-label (load the page in a browser and hover or inspect the SVG point)",
      "own": true,
      "primary": false,
      "parallel": false,
      "notes": "xhigh effort; AutomationBench 1.0.6",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-11-08",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_capture": "https://frontierlog.ai/captures/53d8c9ca12612a78.png"
    },
    {
      "id": "gpt-6-1-sol:automationbench-1-0-6:private:5",
      "model": "gpt-6-1-sol",
      "model_name": "GPT-6.1 Sol",
      "model_date": "2026-09-29",
      "event": "https://frontierlog.ai/event/2026-09-29-gpt-6-1-sol",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "private",
      "setting_name": "Private set",
      "score": 36.1,
      "unit": "%",
      "printed": "36.1%",
      "reported_by": "openai",
      "run_by": "Zapier",
      "label": "GPT-6.1 Sol (Max)",
      "source": "https://openai.com/index/introducing-gpt-6-1-sol",
      "source_title": "Introducing GPT-6.1 Sol",
      "source_date": "2026-09-29",
      "where": "AutomationBench chart under \"Professional work\" (score vs cost per task; no value labels): point 'GPT-6.1 Sol', effort 'Max', read from the point's aria-label (load the page in a browser and hover or inspect the SVG point)",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "max effort; AutomationBench 1.0.6",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-11-08",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_capture": "https://frontierlog.ai/captures/53d8c9ca12612a78.png"
    },
    {
      "id": "gemini-4-argon:automationbench-1-0-6:private:1",
      "model": "gemini-4-argon",
      "model_name": "Gemini 4 Argon",
      "model_date": "2026-09-30",
      "event": "https://frontierlog.ai/event/2026-09-30-gemini-4-argon",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "private",
      "setting_name": "Private set",
      "score": 51.3,
      "unit": "%",
      "printed": "51.3%",
      "reported_by": "google",
      "run_by": "Zapier",
      "label": "Gemini 4 Argon",
      "source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-4-argon/",
      "source_title": "Gemini 4 Argon: our next era of frontier intelligence",
      "source_date": "2026-09-30",
      "where": "benchmark table (image), row 'AutomationBench', 'Score', column 'Gemini 4 Argon'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "set named in the evaluation methodology the post cites",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-16",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/481de187c66d4b4d.gif",
      "source_image_original": "https://storage.googleapis.com/gweb-uniblog-publish-prod/original_images/gemini-4-argon_table_blog.gif"
    },
    {
      "id": "mistral-large-4:automationbench-1-0-6:aa:1",
      "model": "mistral-large-4",
      "model_name": "Mistral Large 4",
      "model_date": "2026-10-06",
      "event": "https://frontierlog.ai/event/2026-10-06-mistral-large-4",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "aa",
      "setting_name": "AutomationBench-AA",
      "score": 59.9,
      "unit": "%",
      "printed": "59.9",
      "reported_by": "mistral",
      "run_by": "Artificial Analysis",
      "label": "Mistral Large 4 Preview",
      "source": "https://mistral.ai/news/mistral-large-4/",
      "source_title": "Introducing Mistral Large 4",
      "source_date": "2026-10-06",
      "where": "'Agentic Workflows' section, chart tab 'AA - AutomationBench' (image titled 'Artificial Analysis - AutomationBench'), value label on the 'Mistral Large 4 Preview' bar; the section's text gives the same score for AutomationBench, '657 business workflows'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-11-08",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/06418443cbcaa235.webp",
      "source_image_original": "https://mistral.ai/_astro/artificial-analysis---automationbench-v3_Z267Ag9.webp?dpl=6aee0a780db43cb5574b1948"
    },
    {
      "id": "deepseek-v4-1-flash-1017:automationbench-1-0-6:unstated:1",
      "model": "deepseek-v4-1-flash-1017",
      "model_name": "DeepSeek-V4.1-Flash-1017",
      "model_date": "2026-10-17",
      "event": "https://frontierlog.ai/event/2026-10-17-deepseek-v4-1-flash-1017",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "unstated",
      "setting_name": "Set not stated",
      "score": 60.5,
      "unit": "%",
      "printed": "60.5",
      "reported_by": "deepseek",
      "run_by": null,
      "label": "DeepSeek V4.1-Flash-1017",
      "source": "https://api-docs.deepseek.com/news/news261017/",
      "source_title": "DeepSeek-V4.1-Flash-1017: Wish-Driven Programming Made Real",
      "source_date": "2026-10-17",
      "where": "benchmark table image (first image), row 'Automation-Bench V1.0.6', column 'DeepSeek V4.1-Flash-1017'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "AutomationBench v1.0.6",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-19",
      "corroborated_by": [
        {
          "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017",
          "name": "DeepSeek-V4.1-Flash-1017 card",
          "title": "deepseek-ai/DeepSeek-V4.1-Flash-1017 model card",
          "lab": "deepseek",
          "date": "2026-10-17",
          "printed": "60.5",
          "where": "Table, row 'Automation-Bench v1.0.6', column 'DS-V4.1-Flash-1017'"
        }
      ],
      "source_image": "https://frontierlog.ai/sources/52c22094008778b4.png",
      "source_image_original": "https://api-docs.deepseek.com/img/v4.1_261017_benchmark_table_en.png"
    },
    {
      "id": "minimax-m3-1-flash:automationbench-1-0-6:public:1",
      "model": "minimax-m3-1-flash",
      "model_name": "MiniMax M3.1 Flash",
      "model_date": "2026-10-19",
      "event": "https://frontierlog.ai/event/2026-10-19-minimax-m3-1-flash",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "public",
      "setting_name": "Public set",
      "score": 62.8,
      "unit": "%",
      "printed": "62.8",
      "reported_by": "minimax",
      "run_by": null,
      "label": "MiniMax M3.1 Flash",
      "source": "https://www.minimax.io/blog/minimax-m31f",
      "source_title": "MiniMax M3.1 Flash: Redefining Agent Performance and Cost",
      "source_date": "2026-10-19",
      "where": "benchmark table image (first image), row 'Automation-Bench v1.0.6', column 'MiniMax M3.1 Flash'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "AutomationBench v1.0.6",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-26",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/5525d0bcceba0858.m31-flash-en-2-1792403273121-webp",
      "source_image_original": "https://filecdn.minimax.chat/public/m31-flash-en-2-1792403273121-webp-1.m31-flash-en-2-1792403273121-webp"
    },
    {
      "id": "minimax-m3-1-flash:automationbench-1-0-6:unstated:1",
      "model": "minimax-m3-1-flash",
      "model_name": "MiniMax M3.1 Flash",
      "model_date": "2026-10-19",
      "event": "https://frontierlog.ai/event/2026-10-19-minimax-m3-1-flash",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "unstated",
      "setting_name": "Set not stated",
      "score": 52.8,
      "unit": "%",
      "printed": "52.8%",
      "reported_by": "minimax",
      "run_by": null,
      "label": "MiniMax-M3.1-Flash (low)",
      "source": "https://www.minimax.io/blog/minimax-m31f",
      "source_title": "MiniMax M3.1 Flash: Redefining Agent Performance and Cost",
      "source_date": "2026-10-19",
      "where": "chart image 'Automation-Bench Score vs. Cost per Task (USD)', MiniMax-M3.1-Flash line, point labelled 'low'",
      "own": true,
      "primary": false,
      "parallel": false,
      "notes": "low effort; AutomationBench v1.0.6; Claude Code harness",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-26",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/2fe56b1cefe6f716.png",
      "source_image_original": "https://filecdn.minimax.chat/public/m31-flash-en-5-1792403273121.png"
    },
    {
      "id": "minimax-m3-1-flash:automationbench-1-0-6:unstated:2",
      "model": "minimax-m3-1-flash",
      "model_name": "MiniMax M3.1 Flash",
      "model_date": "2026-10-19",
      "event": "https://frontierlog.ai/event/2026-10-19-minimax-m3-1-flash",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "unstated",
      "setting_name": "Set not stated",
      "score": 57.7,
      "unit": "%",
      "printed": "57.7%",
      "reported_by": "minimax",
      "run_by": null,
      "label": "MiniMax-M3.1-Flash (medium)",
      "source": "https://www.minimax.io/blog/minimax-m31f",
      "source_title": "MiniMax M3.1 Flash: Redefining Agent Performance and Cost",
      "source_date": "2026-10-19",
      "where": "chart image 'Automation-Bench Score vs. Cost per Task (USD)', MiniMax-M3.1-Flash line, point labelled 'medium'",
      "own": true,
      "primary": false,
      "parallel": false,
      "notes": "medium effort; AutomationBench v1.0.6; Claude Code harness",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-26",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/2fe56b1cefe6f716.png",
      "source_image_original": "https://filecdn.minimax.chat/public/m31-flash-en-5-1792403273121.png"
    },
    {
      "id": "minimax-m3-1-flash:automationbench-1-0-6:unstated:3",
      "model": "minimax-m3-1-flash",
      "model_name": "MiniMax M3.1 Flash",
      "model_date": "2026-10-19",
      "event": "https://frontierlog.ai/event/2026-10-19-minimax-m3-1-flash",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "unstated",
      "setting_name": "Set not stated",
      "score": 59.2,
      "unit": "%",
      "printed": "59.2%",
      "reported_by": "minimax",
      "run_by": null,
      "label": "MiniMax-M3.1-Flash (high)",
      "source": "https://www.minimax.io/blog/minimax-m31f",
      "source_title": "MiniMax M3.1 Flash: Redefining Agent Performance and Cost",
      "source_date": "2026-10-19",
      "where": "chart image 'Automation-Bench Score vs. Cost per Task (USD)', MiniMax-M3.1-Flash line, point labelled 'high'",
      "own": true,
      "primary": false,
      "parallel": false,
      "notes": "high effort; AutomationBench v1.0.6; Claude Code harness",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-26",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/2fe56b1cefe6f716.png",
      "source_image_original": "https://filecdn.minimax.chat/public/m31-flash-en-5-1792403273121.png"
    },
    {
      "id": "minimax-m3-1-flash:automationbench-1-0-6:unstated:4",
      "model": "minimax-m3-1-flash",
      "model_name": "MiniMax M3.1 Flash",
      "model_date": "2026-10-19",
      "event": "https://frontierlog.ai/event/2026-10-19-minimax-m3-1-flash",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "unstated",
      "setting_name": "Set not stated",
      "score": 61.2,
      "unit": "%",
      "printed": "61.2%",
      "reported_by": "minimax",
      "run_by": null,
      "label": "MiniMax-M3.1-Flash (xhigh)",
      "source": "https://www.minimax.io/blog/minimax-m31f",
      "source_title": "MiniMax M3.1 Flash: Redefining Agent Performance and Cost",
      "source_date": "2026-10-19",
      "where": "chart image 'Automation-Bench Score vs. Cost per Task (USD)', MiniMax-M3.1-Flash line, point labelled 'xhigh'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "xhigh effort; AutomationBench v1.0.6; Claude Code harness",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-26",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/2fe56b1cefe6f716.png",
      "source_image_original": "https://filecdn.minimax.chat/public/m31-flash-en-5-1792403273121.png"
    },
    {
      "id": "kimi-k3-1-pro:automationbench-1-0-6:aa:1",
      "model": "kimi-k3-1-pro",
      "model_name": "Kimi K3.1 Pro",
      "model_date": "2026-10-24",
      "event": "https://frontierlog.ai/event/2026-10-24-kimi-k3-1-pro",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "aa",
      "setting_name": "AutomationBench-AA",
      "score": 73.3,
      "unit": "%",
      "printed": "73.3",
      "reported_by": "moonshot",
      "run_by": "Artificial Analysis",
      "label": "Kimi K3.1 Pro (max)",
      "source": "https://www.kimi.ai/blog/kimi-k3-1-pro",
      "source_title": "Kimi K3.1 Pro Tech Blog: Advancing Open Intelligence",
      "source_date": "2026-10-24",
      "where": "'Benchmarks' table (rendered by the page's script), section 'Agentic', row 'Automation Bench-AA', column 'Kimi K3.1 Pro (max)'; runner: the table's notes, 'AutomationBench-AA scores are cited from https://artificialanalysis.ai/'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "max reasoning effort",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-11-08",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_capture": "https://frontierlog.ai/captures/87ef377f982cf2b8.png"
    },
    {
      "id": "gemini-4-carbon:automationbench-1-0-6:private:1",
      "model": "gemini-4-carbon",
      "model_name": "Gemini 4 Carbon",
      "model_date": "2026-10-28",
      "event": "https://frontierlog.ai/event/2026-10-28-gemini-4-carbon",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "private",
      "setting_name": "Private set",
      "score": 53.9,
      "unit": "%",
      "printed": "53.9%",
      "reported_by": "google",
      "run_by": "Zapier",
      "label": "Gemini 4 Carbon",
      "source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/introducing-gemini-4-carbon/",
      "source_title": "Gemini 4 Carbon: our next frontier model",
      "source_date": "2026-10-28",
      "where": "benchmark table (image), row 'AutomationBench', 'Zapier', column 'Gemini 4 Carbon'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "set named in the evaluation methodology the post cites",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-29",
      "corroborated_by": [
        {
          "url": "https://storage.googleapis.com/deepmind-media/gemini/gemini_4_carbon_model_evaluation.pdf",
          "name": "Gemini 4 Carbon card",
          "title": "Gemini 4 Carbon Model evaluation: approach, methodology & results",
          "lab": "google",
          "date": "2026-10-28",
          "printed": "53.9%",
          "where": "Gemini 4 Carbon Model evaluation PDF, p.5 results table (raster image): row 'AutomationBench / Zapier', column 'Gemini 4 Carbon'. Methodology p.2: private set, sourced from the official Zapier public leaderboard."
        }
      ],
      "source_image": "https://frontierlog.ai/sources/bc9b1176d783fc3d.png",
      "source_image_original": "https://storage.googleapis.com/gweb-uniblog-publish-prod/images/gemini_4_carbon__evals-table.original.png"
    },
    {
      "id": "longcat-2-5:automationbench-1-0-6:public:1",
      "model": "longcat-2-5",
      "model_name": "LongCat-2.5",
      "model_date": "2026-11-01",
      "event": "https://frontierlog.ai/event/2026-11-01-longcat-2-5",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "public",
      "setting_name": "Public set",
      "score": 51.8,
      "unit": "%",
      "printed": "51.8",
      "reported_by": "meituan",
      "run_by": null,
      "label": "LongCat-2.5",
      "source": "https://longcat.ai/blog/longcat-2.5",
      "source_title": "LongCat-2.5: Native Multimodality for Agentic Tasks",
      "source_date": "2026-11-01",
      "where": "'Evaluations' section table at the end of the post (rendered by the page's script; load https://longcat.ai/blog/longcat-2.5 in a browser and scroll to the end), row 'AutomationBench v1.0.6' under 'General Agent', column 'LongCat-2.5'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "v1.0.6; official evaluation code",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-11-02",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_capture": "https://frontierlog.ai/captures/2b178d4c19a6dbe7.png"
    },
    {
      "id": "hy4:automationbench-1-0-6:public:1",
      "model": "hy4",
      "model_name": "Hy4",
      "model_date": "2026-11-04",
      "event": "https://frontierlog.ai/event/2026-11-04-hy4",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "public",
      "setting_name": "Public set",
      "score": 56.2,
      "unit": "%",
      "printed": "56.2",
      "reported_by": "tencent",
      "run_by": null,
      "label": "Hy4",
      "source": "https://hy.tencent.ai/research/hy4",
      "source_title": "Introducing Hy4",
      "source_date": "2026-11-04",
      "where": "'Appendix: benchmark results' table image, row 'AutomationBench v1.0.6', column 'Hy4'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "AutomationBench v1.0.6; official method; highest reasoning setting",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-11-05",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/8fc24d0f588e6069.jpg",
      "source_image_original": "https://hy-model-ap-prod-1258344703.cos.ap-guangzhou.myqcloud.com/llm-blog/default/9a5ea02f/2026110402023072_85b416b6484a88ef96ef6edda563b560.jpg?q-sign-algorithm=sha1&q-ak=AKIDRl074nOsGdJ9zjMsCRWP3ShmgS3VtX4S&q-sign-time=1793728954;2109088954&q-key-time=1793728954;2109088954&q-header-list=&q-url-param-list=&q-signature=747236201b67dd4e9872ed806c8de224d0ca57c1"
    },
    {
      "id": "inkling-2-small:automationbench-1-0-6:aa:1",
      "model": "inkling-2-small",
      "model_name": "Inkling 2 Small",
      "model_date": "2026-11-05",
      "event": "https://frontierlog.ai/event/2026-11-05-inkling-2-small",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "aa",
      "setting_name": "AutomationBench-AA",
      "score": 45.6,
      "unit": "%",
      "printed": "45.6%",
      "reported_by": "thinking-machines",
      "run_by": "Artificial Analysis",
      "label": "Inkling 2 Small",
      "source": "https://thinkingmachines.ai/model-card/inkling-2-small/",
      "source_title": "Inkling 2 Small Model Card",
      "source_date": "2026-11-05",
      "where": "section '5. Evaluations', evaluations table, Enterprise Operations group, row 'AutomationBench v1.0.6', column 'Inkling 2 Small' (first model column); set and runner: launch post thinkingmachines.ai/news/inkling-2-small/, list of externally reported evals right after its evaluations table, before 'Limitations'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "v1.0.6",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-11-08",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": []
    },
    {
      "id": "claude-fable-5-2:automationbench-1-0-6:private:1",
      "model": "claude-fable-5-2",
      "model_name": "Claude Fable 5.2",
      "model_date": "2026-11-09",
      "event": "https://frontierlog.ai/event/2026-11-09-claude-fable-5-2-and-mythos-5-2",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "private",
      "setting_name": "Private set",
      "score": 51.4,
      "unit": "%",
      "printed": "51.4%",
      "reported_by": "anthropic",
      "run_by": "Zapier",
      "label": "Fable 5.2",
      "source": "https://www.anthropic.com/claude-fable-and-mythos-5-2",
      "source_title": "Introducing Claude Fable 5.2 and Claude Mythos 5.2",
      "source_date": "2026-11-09",
      "where": "benchmark table (HTML), row 'Business workflows, AutomationBench', column 'Fable 5.2'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "highest available effort; safeguards on; set and release 1.0.6 named in the system card, which prints the same figure",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-11-10",
      "corroborated_by": [
        {
          "url": "https://www.anthropic.com/claude-fable-5-2-mythos-5-2-system-card",
          "name": "Claude Fable 5.2 and Mythos 5.2 card",
          "title": "Claude Fable 5.2 and Claude Mythos 5.2 System Card",
          "lab": "anthropic",
          "date": "2026-11-09",
          "printed": "51.4",
          "where": "Table 8.1.A (p. 207), row 'AutomationBench', column 'Fable 5.2'; also section 8.14.5 (p. 241): 'it scored 51.4% at max effort'"
        }
      ]
    },
    {
      "id": "claude-fable-5-2:automationbench-1-0-6:private:2",
      "model": "claude-fable-5-2",
      "model_name": "Claude Fable 5.2",
      "model_date": "2026-11-09",
      "event": "https://frontierlog.ai/event/2026-11-09-claude-fable-5-2-and-mythos-5-2",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "private",
      "setting_name": "Private set",
      "score": 39.7,
      "unit": "%",
      "printed": "39.7%",
      "reported_by": "anthropic",
      "run_by": "Zapier",
      "label": "Fable 5.2",
      "source": "https://www.anthropic.com/claude-fable-5-2-mythos-5-2-system-card",
      "source_title": "Claude Fable 5.2 and Claude Mythos 5.2 System Card",
      "source_date": "2026-11-09",
      "where": "Section 8.14.5 'AutomationBench', paragraph beginning 'Zapier evaluated Fable 5.2', the low-effort figure",
      "own": true,
      "primary": false,
      "parallel": false,
      "notes": "low effort; default fallbacks on",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "double_entry",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": []
    },
    {
      "id": "muse-ember:automationbench-1-0-6:aa:1",
      "model": "muse-ember",
      "model_name": "Muse Ember",
      "model_date": "2026-11-09",
      "event": "https://frontierlog.ai/event/2026-11-09-muse-ember",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "aa",
      "setting_name": "AutomationBench-AA",
      "score": 37.8,
      "unit": "%",
      "printed": "37.8",
      "reported_by": "meta",
      "run_by": "Artificial Analysis",
      "label": "Muse Ember (max)",
      "source": "https://research.meta.ai/blog/muse-ember",
      "source_title": "Introducing Our Smallest, Most Efficient Model: Muse Ember",
      "source_date": "2026-11-09",
      "where": "'Muse Ember evaluation results' table (HTML), row 'Agents / AutomationBench-AA', column 'Muse Ember (max)'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "max reasoning effort; pass@1",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "double_entry",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": []
    }
  ]
}