{
  "api_version": "1",
  "updated": "2026-11-10",
  "latest_event": "2026-11-09",
  "rights": "© Frontier Log, all rights reserved.",
  "benchmark": {
    "slug": "automationbench-1-0-6",
    "name": "AutomationBench 1.0.6",
    "family": "automationbench",
    "unit": "%",
    "higher_is_better": true,
    "status": "active",
    "active": true,
    "superseded_by": null,
    "retired": null,
    "benchmark_url": "https://zapier.com/benchmarks",
    "introduced": "2026-07-31",
    "creator": "Zapier",
    "introduced_source": "https://github.com/zapier/AutomationBench/blob/main/CHANGELOG.md",
    "settings": [
      {
        "setting": "private",
        "name": "Private set",
        "url": "https://frontierlog.ai/benchmark/automationbench-1-0-6",
        "results": 22
      },
      {
        "setting": "public",
        "name": "Public set",
        "url": "https://frontierlog.ai/benchmark/automationbench-1-0-6/public",
        "results": 5
      },
      {
        "setting": "unstated",
        "name": "Set not stated",
        "url": "https://frontierlog.ai/benchmark/automationbench-1-0-6/unstated",
        "results": 10
      },
      {
        "setting": "aa",
        "name": "AutomationBench-AA",
        "url": "https://frontierlog.ai/benchmark/automationbench-1-0-6/aa",
        "results": 5
      }
    ]
  },
  "total": 5,
  "results": [
    {
      "id": "mistral-medium-3-5:automationbench-1-0-6:aa:1",
      "model": "mistral-medium-3-5",
      "model_name": "Mistral Medium 3.5",
      "model_date": "2026-05-22",
      "event": "https://frontierlog.ai/event/2026-05-22-mistral-medium-3-5",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "aa",
      "setting_name": "AutomationBench-AA",
      "score": 6.3,
      "unit": "%",
      "printed": "6.3",
      "reported_by": "mistral",
      "run_by": "Artificial Analysis",
      "label": "Mistral Medium 3.5",
      "source": "https://mistral.ai/news/mistral-large-4/",
      "source_title": "Introducing Mistral Large 4",
      "source_date": "2026-10-06",
      "where": "later report: Mistral Large 4 post, 'Agentic Workflows' section, chart tab 'AA - AutomationBench' (image titled 'Artificial Analysis - AutomationBench'), value label on the 'Mistral Medium 3.5' bar",
      "own": false,
      "primary": true,
      "parallel": false,
      "notes": "",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-11-08",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/06418443cbcaa235.webp",
      "source_image_original": "https://mistral.ai/_astro/artificial-analysis---automationbench-v3_Z267Ag9.webp?dpl=6aee0a780db43cb5574b1948"
    },
    {
      "id": "mistral-large-4:automationbench-1-0-6:aa:1",
      "model": "mistral-large-4",
      "model_name": "Mistral Large 4",
      "model_date": "2026-10-06",
      "event": "https://frontierlog.ai/event/2026-10-06-mistral-large-4",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "aa",
      "setting_name": "AutomationBench-AA",
      "score": 59.9,
      "unit": "%",
      "printed": "59.9",
      "reported_by": "mistral",
      "run_by": "Artificial Analysis",
      "label": "Mistral Large 4 Preview",
      "source": "https://mistral.ai/news/mistral-large-4/",
      "source_title": "Introducing Mistral Large 4",
      "source_date": "2026-10-06",
      "where": "'Agentic Workflows' section, chart tab 'AA - AutomationBench' (image titled 'Artificial Analysis - AutomationBench'), value label on the 'Mistral Large 4 Preview' bar; the section's text gives the same score for AutomationBench, '657 business workflows'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-11-08",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/06418443cbcaa235.webp",
      "source_image_original": "https://mistral.ai/_astro/artificial-analysis---automationbench-v3_Z267Ag9.webp?dpl=6aee0a780db43cb5574b1948"
    },
    {
      "id": "kimi-k3-1-pro:automationbench-1-0-6:aa:1",
      "model": "kimi-k3-1-pro",
      "model_name": "Kimi K3.1 Pro",
      "model_date": "2026-10-24",
      "event": "https://frontierlog.ai/event/2026-10-24-kimi-k3-1-pro",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "aa",
      "setting_name": "AutomationBench-AA",
      "score": 73.3,
      "unit": "%",
      "printed": "73.3",
      "reported_by": "moonshot",
      "run_by": "Artificial Analysis",
      "label": "Kimi K3.1 Pro (max)",
      "source": "https://www.kimi.ai/blog/kimi-k3-1-pro",
      "source_title": "Kimi K3.1 Pro Tech Blog: Advancing Open Intelligence",
      "source_date": "2026-10-24",
      "where": "'Benchmarks' table (rendered by the page's script), section 'Agentic', row 'Automation Bench-AA', column 'Kimi K3.1 Pro (max)'; runner: the table's notes, 'AutomationBench-AA scores are cited from https://artificialanalysis.ai/'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "max reasoning effort",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-11-08",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_capture": "https://frontierlog.ai/captures/87ef377f982cf2b8.png"
    },
    {
      "id": "inkling-2-small:automationbench-1-0-6:aa:1",
      "model": "inkling-2-small",
      "model_name": "Inkling 2 Small",
      "model_date": "2026-11-05",
      "event": "https://frontierlog.ai/event/2026-11-05-inkling-2-small",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "aa",
      "setting_name": "AutomationBench-AA",
      "score": 45.6,
      "unit": "%",
      "printed": "45.6%",
      "reported_by": "thinking-machines",
      "run_by": "Artificial Analysis",
      "label": "Inkling 2 Small",
      "source": "https://thinkingmachines.ai/model-card/inkling-2-small/",
      "source_title": "Inkling 2 Small Model Card",
      "source_date": "2026-11-05",
      "where": "section '5. Evaluations', evaluations table, Enterprise Operations group, row 'AutomationBench v1.0.6', column 'Inkling 2 Small' (first model column); set and runner: launch post thinkingmachines.ai/news/inkling-2-small/, list of externally reported evals right after its evaluations table, before 'Limitations'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "v1.0.6",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-11-08",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": []
    },
    {
      "id": "muse-ember:automationbench-1-0-6:aa:1",
      "model": "muse-ember",
      "model_name": "Muse Ember",
      "model_date": "2026-11-09",
      "event": "https://frontierlog.ai/event/2026-11-09-muse-ember",
      "benchmark": "automationbench-1-0-6",
      "benchmark_name": "AutomationBench 1.0.6",
      "setting": "aa",
      "setting_name": "AutomationBench-AA",
      "score": 37.8,
      "unit": "%",
      "printed": "37.8",
      "reported_by": "meta",
      "run_by": "Artificial Analysis",
      "label": "Muse Ember (max)",
      "source": "https://research.meta.ai/blog/muse-ember",
      "source_title": "Introducing Our Smallest, Most Efficient Model: Muse Ember",
      "source_date": "2026-11-09",
      "where": "'Muse Ember evaluation results' table (HTML), row 'Agents / AutomationBench-AA', column 'Muse Ember (max)'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "max reasoning effort; pass@1",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "double_entry",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": []
    }
  ]
}