{
  "api_version": "1",
  "updated": "2026-11-10",
  "latest_event": "2026-11-09",
  "rights": "© Frontier Log, all rights reserved.",
  "benchmark": {
    "slug": "automationbench-1-0-6",
    "name": "AutomationBench 1.0.6",
    "family": "automationbench",
    "unit": "%",
    "higher_is_better": true,
    "status": "active",
    "active": true,
    "superseded_by": null,
    "retired": null,
    "benchmark_url": "https://zapier.com/benchmarks",
    "introduced": "2026-07-31",
    "creator": "Zapier",
    "introduced_source": "https://github.com/zapier/AutomationBench/blob/main/CHANGELOG.md",
    "settings": [
      {
        "setting": "private",
        "name": "Private set",
        "url": "https://frontierlog.ai/benchmark/automationbench-1-0-6",
        "results": 22
      },
      {
        "setting": "public",
        "name": "Public set",
        "url": "https://frontierlog.ai/benchmark/automationbench-1-0-6/public",
        "results": 5
      },
      {
        "setting": "unstated",
        "name": "Set not stated",
        "url": "https://frontierlog.ai/benchmark/automationbench-1-0-6/unstated",
        "results": 10
      },
      {
        "setting": "aa",
        "name": "AutomationBench-AA",
        "url": "https://frontierlog.ai/benchmark/automationbench-1-0-6/aa",
        "results": 5
      }
    ]
  },
  "total": 5,
  "results": [
    {
      "id": "hy4-preview:automationbench-1-0-6:public:1",
      "model": "hy4-preview",
      "model_name": "Hy4 preview",
      "model_date": "2026-08-28",
      "event": "https://frontierlog.ai/event/2026-08-28-hy4-preview",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "public",
      "setting_name": "Public set",
      "score": 36.3,
      "unit": "%",
      "printed": "36.3",
      "reported_by": "tencent",
      "run_by": null,
      "label": "Hy4 preview",
      "source": "https://hy.tencent.ai/research/hy4",
      "source_title": "Introducing Hy4",
      "source_date": "2026-11-04",
      "where": "later report: Hy4 post, 'Appendix: benchmark results' table image, row 'AutomationBench v1.0.6', column 'Hy4 preview'",
      "own": false,
      "primary": true,
      "parallel": false,
      "notes": "re-evaluated at Hy4 launch; AutomationBench v1.0.6; official method",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-11-08",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/8fc24d0f588e6069.jpg",
      "source_image_original": "https://hy-model-ap-prod-1258344703.cos.ap-guangzhou.myqcloud.com/llm-blog/default/9a5ea02f/2026110402023072_85b416b6484a88ef96ef6edda563b560.jpg?q-sign-algorithm=sha1&q-ak=AKIDRl074nOsGdJ9zjMsCRWP3ShmgS3VtX4S&q-sign-time=1793728954;2109088954&q-key-time=1793728954;2109088954&q-header-list=&q-url-param-list=&q-signature=747236201b67dd4e9872ed806c8de224d0ca57c1"
    },
    {
      "id": "deepseek-v4-1-flash:automationbench-1-0-6:public:1",
      "model": "deepseek-v4-1-flash",
      "model_name": "DeepSeek-V4.1-Flash",
      "model_date": "2026-09-10",
      "event": "https://frontierlog.ai/event/2026-09-10-deepseek-v4-1-flash",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "public",
      "setting_name": "Public set",
      "score": 54.8,
      "unit": "%",
      "printed": "54.8",
      "reported_by": "deepseek",
      "run_by": null,
      "label": "DeepSeek V4.1-Flash",
      "source": "https://api-docs.deepseek.com/news/news260910/",
      "source_title": "DeepSeek-V4.1-Flash: Smarter, Faster, More Efficient",
      "source_date": "2026-09-10",
      "where": "benchmark table image (second image), row 'Automation-Bench', column 'DeepSeek V4.1-Flash' (the first chart image, v4.1_260910_benchmark.png, prints the same value label)",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "v1.0.6; set named in the technical report the post links, which prints the same figure",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-19",
      "corroborated_by": [
        {
          "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash",
          "name": "DeepSeek-V4.1-Flash card",
          "title": "deepseek-ai/DeepSeek-V4.1-Flash model card",
          "lab": "deepseek",
          "date": "2026-09-10",
          "printed": "54.8",
          "where": "'Comparison with frontier models (Max reasoning effort)' table, row 'AutomationBench (Pass@1)', column 'DS-V4.1-Flash'"
        },
        {
          "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017",
          "name": "DeepSeek-V4.1-Flash-1017 card",
          "title": "deepseek-ai/DeepSeek-V4.1-Flash-1017 model card",
          "lab": "deepseek",
          "date": "2026-10-17",
          "printed": "54.8",
          "where": "Table, row 'Automation-Bench v1.0.6', column 'DS-V4.1-Flash'"
        }
      ],
      "source_image": "https://frontierlog.ai/sources/19d3b83b1b9e5f5b.png",
      "source_image_original": "https://api-docs.deepseek.com/img/v4.1_260910_benchmark_table_en.png"
    },
    {
      "id": "minimax-m3-1-flash:automationbench-1-0-6:public:1",
      "model": "minimax-m3-1-flash",
      "model_name": "MiniMax M3.1 Flash",
      "model_date": "2026-10-19",
      "event": "https://frontierlog.ai/event/2026-10-19-minimax-m3-1-flash",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "public",
      "setting_name": "Public set",
      "score": 62.8,
      "unit": "%",
      "printed": "62.8",
      "reported_by": "minimax",
      "run_by": null,
      "label": "MiniMax M3.1 Flash",
      "source": "https://www.minimax.io/blog/minimax-m31f",
      "source_title": "MiniMax M3.1 Flash: Redefining Agent Performance and Cost",
      "source_date": "2026-10-19",
      "where": "benchmark table image (first image), row 'Automation-Bench v1.0.6', column 'MiniMax M3.1 Flash'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "AutomationBench v1.0.6",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-26",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/5525d0bcceba0858.m31-flash-en-2-1792403273121-webp",
      "source_image_original": "https://filecdn.minimax.chat/public/m31-flash-en-2-1792403273121-webp-1.m31-flash-en-2-1792403273121-webp"
    },
    {
      "id": "longcat-2-5:automationbench-1-0-6:public:1",
      "model": "longcat-2-5",
      "model_name": "LongCat-2.5",
      "model_date": "2026-11-01",
      "event": "https://frontierlog.ai/event/2026-11-01-longcat-2-5",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "public",
      "setting_name": "Public set",
      "score": 51.8,
      "unit": "%",
      "printed": "51.8",
      "reported_by": "meituan",
      "run_by": null,
      "label": "LongCat-2.5",
      "source": "https://longcat.ai/blog/longcat-2.5",
      "source_title": "LongCat-2.5: Native Multimodality for Agentic Tasks",
      "source_date": "2026-11-01",
      "where": "'Evaluations' section table at the end of the post (rendered by the page's script; load https://longcat.ai/blog/longcat-2.5 in a browser and scroll to the end), row 'AutomationBench v1.0.6' under 'General Agent', column 'LongCat-2.5'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "v1.0.6; official evaluation code",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-11-02",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_capture": "https://frontierlog.ai/captures/2b178d4c19a6dbe7.png"
    },
    {
      "id": "hy4:automationbench-1-0-6:public:1",
      "model": "hy4",
      "model_name": "Hy4",
      "model_date": "2026-11-04",
      "event": "https://frontierlog.ai/event/2026-11-04-hy4",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "public",
      "setting_name": "Public set",
      "score": 56.2,
      "unit": "%",
      "printed": "56.2",
      "reported_by": "tencent",
      "run_by": null,
      "label": "Hy4",
      "source": "https://hy.tencent.ai/research/hy4",
      "source_title": "Introducing Hy4",
      "source_date": "2026-11-04",
      "where": "'Appendix: benchmark results' table image, row 'AutomationBench v1.0.6', column 'Hy4'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "AutomationBench v1.0.6; official method; highest reasoning setting",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-11-05",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/8fc24d0f588e6069.jpg",
      "source_image_original": "https://hy-model-ap-prod-1258344703.cos.ap-guangzhou.myqcloud.com/llm-blog/default/9a5ea02f/2026110402023072_85b416b6484a88ef96ef6edda563b560.jpg?q-sign-algorithm=sha1&q-ak=AKIDRl074nOsGdJ9zjMsCRWP3ShmgS3VtX4S&q-sign-time=1793728954;2109088954&q-key-time=1793728954;2109088954&q-header-list=&q-url-param-list=&q-signature=747236201b67dd4e9872ed806c8de224d0ca57c1"
    }
  ]
}