{
  "api_version": "1",
  "updated": "2026-11-10",
  "latest_event": "2026-11-09",
  "rights": "© Frontier Log, all rights reserved.",
  "benchmark": {
    "slug": "automationbench-1-0-6",
    "name": "AutomationBench 1.0.6",
    "family": "automationbench",
    "unit": "%",
    "higher_is_better": true,
    "status": "active",
    "active": true,
    "superseded_by": null,
    "retired": null,
    "benchmark_url": "https://zapier.com/benchmarks",
    "introduced": "2026-07-31",
    "creator": "Zapier",
    "introduced_source": "https://github.com/zapier/AutomationBench/blob/main/CHANGELOG.md",
    "settings": [
      {
        "setting": "private",
        "name": "Private set",
        "url": "https://frontierlog.ai/benchmark/automationbench-1-0-6",
        "results": 22
      },
      {
        "setting": "public",
        "name": "Public set",
        "url": "https://frontierlog.ai/benchmark/automationbench-1-0-6/public",
        "results": 5
      },
      {
        "setting": "unstated",
        "name": "Set not stated",
        "url": "https://frontierlog.ai/benchmark/automationbench-1-0-6/unstated",
        "results": 10
      },
      {
        "setting": "aa",
        "name": "AutomationBench-AA",
        "url": "https://frontierlog.ai/benchmark/automationbench-1-0-6/aa",
        "results": 5
      }
    ]
  },
  "total": 10,
  "results": [
    {
      "id": "glm-5-2:automationbench-1-0-6:unstated:1",
      "model": "glm-5-2",
      "model_name": "GLM-5.2",
      "model_date": "2026-06-16",
      "event": "https://frontierlog.ai/event/2026-06-16-glm-5-2",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "unstated",
      "setting_name": "Set not stated",
      "score": 26.2,
      "unit": "%",
      "printed": "26.2",
      "reported_by": "zai",
      "run_by": null,
      "label": "GLM-5.2",
      "source": "https://z.ai/blog/glm-5.3",
      "source_title": "GLM-5.3: Frontier Coding with Emergent Cyber Capabilities",
      "source_date": "2026-08-14",
      "where": "'Performance across comparison models' table, row 'AutomationBench v1.0.6', column 'GLM-5.2'",
      "own": false,
      "primary": true,
      "parallel": false,
      "notes": "AutomationBench v1.0.6",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-19",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_capture": "https://frontierlog.ai/captures/86a1ebfddd94c33e.png"
    },
    {
      "id": "hy3:automationbench-1-0-6:unstated:1",
      "model": "hy3",
      "model_name": "Hy3",
      "model_date": "2026-07-06",
      "event": "https://frontierlog.ai/event/2026-07-06-hy3",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "unstated",
      "setting_name": "Set not stated",
      "score": 16.1,
      "unit": "%",
      "printed": "16.1",
      "reported_by": "tencent",
      "run_by": null,
      "label": "Hy3",
      "source": "https://hy.tencent.ai/research/hy4-preview",
      "source_title": "Introducing Hy4 preview",
      "source_date": "2026-08-28",
      "where": "later report: Hy4 preview post, 'Appendix: Benchmark Results' table image, row 'AutomationBench (v1.0.6)', column 'Hy3'",
      "own": false,
      "primary": true,
      "parallel": false,
      "notes": "AutomationBench v1.0.6; highest reasoning setting",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-11-07",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/02c751bf51c52c52.jpg",
      "source_image_original": "https://hy-model-ap-prod-1258344703.cos.accelerate.myqcloud.com/llm-blog/default/9a5ea02f/2026082812342438_ecb98509e92e4be02a40ca358414133a.jpg?q-sign-algorithm=sha1&q-ak=AKIDRl074nOsGdJ9zjMsCRWP3ShmgS3VtX4S&q-sign-time=1787891670;2103251670&q-key-time=1787891670;2103251670&q-header-list=&q-url-param-list=&q-signature=b6d0c3c4d6d7e0e5219d177ec730735185ccdd1f"
    },
    {
      "id": "glm-5-3:automationbench-1-0-6:unstated:1",
      "model": "glm-5-3",
      "model_name": "GLM-5.3",
      "model_date": "2026-08-14",
      "event": "https://frontierlog.ai/event/2026-08-14-glm-5-3",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "unstated",
      "setting_name": "Set not stated",
      "score": 48.2,
      "unit": "%",
      "printed": "48.2",
      "reported_by": "zai",
      "run_by": null,
      "label": "GLM-5.3",
      "source": "https://z.ai/blog/glm-5.3",
      "source_title": "GLM-5.3: Frontier Coding with Emergent Cyber Capabilities",
      "source_date": "2026-08-14",
      "where": "'Performance across comparison models' table, row 'AutomationBench v1.0.6', column 'GLM-5.3'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "AutomationBench v1.0.6 with a fix for null-type handling",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-19",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_capture": "https://frontierlog.ai/captures/86a1ebfddd94c33e.png"
    },
    {
      "id": "glm-5-3-flash:automationbench-1-0-6:unstated:1",
      "model": "glm-5-3-flash",
      "model_name": "GLM-5.3-Flash",
      "model_date": "2026-08-26",
      "event": "https://frontierlog.ai/event/2026-08-26-glm-5-3-flash",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "unstated",
      "setting_name": "Set not stated",
      "score": 48.8,
      "unit": "%",
      "printed": "48.8",
      "reported_by": "zai",
      "run_by": null,
      "label": "GLM-5.3-Flash",
      "source": "https://z.ai/blog/glm-5.3-flash",
      "source_title": "GLM-5.3-Flash: Frontier Intelligence, Flash Cost",
      "source_date": "2026-08-26",
      "where": "main benchmark table (second table), AGENTIC block, row 'AutomationBench v1.0.6', column 'GLM-5.3-Flash'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "AutomationBench v1.0.6 with a fix for null-type handling",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-19",
      "corroborated_by": [
        {
          "url": "https://huggingface.co/zai-org/GLM-5.3-Flash",
          "name": "GLM-5.3-Flash card",
          "title": "zai-org/GLM-5.3-Flash model card",
          "lab": "zai",
          "date": "2026-08-26",
          "printed": "48.8",
          "where": "bench_53 image, AutomationBench panel, GLM-5.3-Flash bar; footnote: v1.0.6 with null-type fix (PR #13)"
        },
        {
          "url": "https://docs.z.ai/guides/llm/glm-5.3-flash",
          "name": "GLM-5.3-Flash card",
          "title": "GLM-5.3-Flash/FlashX - Overview",
          "lab": "zai",
          "date": "2026-08-26",
          "printed": "48.8",
          "where": "chart, AutomationBench panel, GLM-5.3-Flash bar; also text: '48.8 vs. 26.2 on AutomationBench'"
        },
        {
          "url": "https://github.com/zai-org/GLM-5",
          "name": "GLM-5 card",
          "title": "GLM-5: From Vibe Coding to Agentic Engineering (zai-org/GLM-5 GitHub repository)",
          "lab": "zai",
          "date": "2026-02-12",
          "printed": "48.8",
          "where": "README, bench_53 image, AutomationBench panel, GLM-5.3-Flash bar"
        }
      ],
      "source_capture": "https://frontierlog.ai/captures/23dc8480c23f5792.png"
    },
    {
      "id": "hy4-preview:automationbench-1-0-6:unstated:1",
      "model": "hy4-preview",
      "model_name": "Hy4 preview",
      "model_date": "2026-08-28",
      "event": "https://frontierlog.ai/event/2026-08-28-hy4-preview",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "unstated",
      "setting_name": "Set not stated",
      "score": 32.1,
      "unit": "%",
      "printed": "32.1",
      "reported_by": "tencent",
      "run_by": null,
      "label": "Hy4 preview",
      "source": "https://hy.tencent.ai/research/hy4-preview",
      "source_title": "Introducing Hy4 preview",
      "source_date": "2026-08-28",
      "where": "'Appendix: Benchmark Results' table image, row 'AutomationBench (v1.0.6)', column 'Hy4 preview'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "AutomationBench v1.0.6",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-11-08",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/02c751bf51c52c52.jpg",
      "source_image_original": "https://hy-model-ap-prod-1258344703.cos.accelerate.myqcloud.com/llm-blog/default/9a5ea02f/2026082812342438_ecb98509e92e4be02a40ca358414133a.jpg?q-sign-algorithm=sha1&q-ak=AKIDRl074nOsGdJ9zjMsCRWP3ShmgS3VtX4S&q-sign-time=1787891670;2103251670&q-key-time=1787891670;2103251670&q-header-list=&q-url-param-list=&q-signature=b6d0c3c4d6d7e0e5219d177ec730735185ccdd1f"
    },
    {
      "id": "deepseek-v4-1-flash-1017:automationbench-1-0-6:unstated:1",
      "model": "deepseek-v4-1-flash-1017",
      "model_name": "DeepSeek-V4.1-Flash-1017",
      "model_date": "2026-10-17",
      "event": "https://frontierlog.ai/event/2026-10-17-deepseek-v4-1-flash-1017",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "unstated",
      "setting_name": "Set not stated",
      "score": 60.5,
      "unit": "%",
      "printed": "60.5",
      "reported_by": "deepseek",
      "run_by": null,
      "label": "DeepSeek V4.1-Flash-1017",
      "source": "https://api-docs.deepseek.com/news/news261017/",
      "source_title": "DeepSeek-V4.1-Flash-1017: Wish-Driven Programming Made Real",
      "source_date": "2026-10-17",
      "where": "benchmark table image (first image), row 'Automation-Bench V1.0.6', column 'DeepSeek V4.1-Flash-1017'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "AutomationBench v1.0.6",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-19",
      "corroborated_by": [
        {
          "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017",
          "name": "DeepSeek-V4.1-Flash-1017 card",
          "title": "deepseek-ai/DeepSeek-V4.1-Flash-1017 model card",
          "lab": "deepseek",
          "date": "2026-10-17",
          "printed": "60.5",
          "where": "Table, row 'Automation-Bench v1.0.6', column 'DS-V4.1-Flash-1017'"
        }
      ],
      "source_image": "https://frontierlog.ai/sources/52c22094008778b4.png",
      "source_image_original": "https://api-docs.deepseek.com/img/v4.1_261017_benchmark_table_en.png"
    },
    {
      "id": "minimax-m3-1-flash:automationbench-1-0-6:unstated:1",
      "model": "minimax-m3-1-flash",
      "model_name": "MiniMax M3.1 Flash",
      "model_date": "2026-10-19",
      "event": "https://frontierlog.ai/event/2026-10-19-minimax-m3-1-flash",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "unstated",
      "setting_name": "Set not stated",
      "score": 52.8,
      "unit": "%",
      "printed": "52.8%",
      "reported_by": "minimax",
      "run_by": null,
      "label": "MiniMax-M3.1-Flash (low)",
      "source": "https://www.minimax.io/blog/minimax-m31f",
      "source_title": "MiniMax M3.1 Flash: Redefining Agent Performance and Cost",
      "source_date": "2026-10-19",
      "where": "chart image 'Automation-Bench Score vs. Cost per Task (USD)', MiniMax-M3.1-Flash line, point labelled 'low'",
      "own": true,
      "primary": false,
      "parallel": false,
      "notes": "low effort; AutomationBench v1.0.6; Claude Code harness",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-26",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/2fe56b1cefe6f716.png",
      "source_image_original": "https://filecdn.minimax.chat/public/m31-flash-en-5-1792403273121.png"
    },
    {
      "id": "minimax-m3-1-flash:automationbench-1-0-6:unstated:2",
      "model": "minimax-m3-1-flash",
      "model_name": "MiniMax M3.1 Flash",
      "model_date": "2026-10-19",
      "event": "https://frontierlog.ai/event/2026-10-19-minimax-m3-1-flash",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "unstated",
      "setting_name": "Set not stated",
      "score": 57.7,
      "unit": "%",
      "printed": "57.7%",
      "reported_by": "minimax",
      "run_by": null,
      "label": "MiniMax-M3.1-Flash (medium)",
      "source": "https://www.minimax.io/blog/minimax-m31f",
      "source_title": "MiniMax M3.1 Flash: Redefining Agent Performance and Cost",
      "source_date": "2026-10-19",
      "where": "chart image 'Automation-Bench Score vs. Cost per Task (USD)', MiniMax-M3.1-Flash line, point labelled 'medium'",
      "own": true,
      "primary": false,
      "parallel": false,
      "notes": "medium effort; AutomationBench v1.0.6; Claude Code harness",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-26",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/2fe56b1cefe6f716.png",
      "source_image_original": "https://filecdn.minimax.chat/public/m31-flash-en-5-1792403273121.png"
    },
    {
      "id": "minimax-m3-1-flash:automationbench-1-0-6:unstated:3",
      "model": "minimax-m3-1-flash",
      "model_name": "MiniMax M3.1 Flash",
      "model_date": "2026-10-19",
      "event": "https://frontierlog.ai/event/2026-10-19-minimax-m3-1-flash",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "unstated",
      "setting_name": "Set not stated",
      "score": 59.2,
      "unit": "%",
      "printed": "59.2%",
      "reported_by": "minimax",
      "run_by": null,
      "label": "MiniMax-M3.1-Flash (high)",
      "source": "https://www.minimax.io/blog/minimax-m31f",
      "source_title": "MiniMax M3.1 Flash: Redefining Agent Performance and Cost",
      "source_date": "2026-10-19",
      "where": "chart image 'Automation-Bench Score vs. Cost per Task (USD)', MiniMax-M3.1-Flash line, point labelled 'high'",
      "own": true,
      "primary": false,
      "parallel": false,
      "notes": "high effort; AutomationBench v1.0.6; Claude Code harness",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-26",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/2fe56b1cefe6f716.png",
      "source_image_original": "https://filecdn.minimax.chat/public/m31-flash-en-5-1792403273121.png"
    },
    {
      "id": "minimax-m3-1-flash:automationbench-1-0-6:unstated:4",
      "model": "minimax-m3-1-flash",
      "model_name": "MiniMax M3.1 Flash",
      "model_date": "2026-10-19",
      "event": "https://frontierlog.ai/event/2026-10-19-minimax-m3-1-flash",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "unstated",
      "setting_name": "Set not stated",
      "score": 61.2,
      "unit": "%",
      "printed": "61.2%",
      "reported_by": "minimax",
      "run_by": null,
      "label": "MiniMax-M3.1-Flash (xhigh)",
      "source": "https://www.minimax.io/blog/minimax-m31f",
      "source_title": "MiniMax M3.1 Flash: Redefining Agent Performance and Cost",
      "source_date": "2026-10-19",
      "where": "chart image 'Automation-Bench Score vs. Cost per Task (USD)', MiniMax-M3.1-Flash line, point labelled 'xhigh'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "xhigh effort; AutomationBench v1.0.6; Claude Code harness",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-26",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/2fe56b1cefe6f716.png",
      "source_image_original": "https://filecdn.minimax.chat/public/m31-flash-en-5-1792403273121.png"
    }
  ]
}