{
  "api_version": "1",
  "updated": "2026-11-10",
  "latest_event": "2026-11-09",
  "rights": "© Frontier Log, all rights reserved.",
  "benchmark": {
    "slug": "mcp-atlas",
    "name": "MCP Atlas",
    "family": "mcp-atlas",
    "unit": "%",
    "higher_is_better": true,
    "status": "retired",
    "active": false,
    "superseded_by": null,
    "retired": "2026-07-24",
    "benchmark_url": "https://labs.scale.com/leaderboard/mcp_atlas",
    "introduced": "2025-09-19",
    "creator": "Scale AI",
    "introduced_source": "https://scale.com/blog/mcp-atlas",
    "settings": [
      {
        "setting": "public",
        "name": "Public set (500 tasks)",
        "url": "https://frontierlog.ai/benchmark/mcp-atlas",
        "results": 26
      },
      {
        "setting": "full",
        "name": "Full set (1,000 tasks)",
        "url": "https://frontierlog.ai/benchmark/mcp-atlas/full",
        "results": 8
      },
      {
        "setting": "unstated",
        "name": "Set not stated",
        "url": "https://frontierlog.ai/benchmark/mcp-atlas/unstated",
        "results": 23
      }
    ]
  },
  "total": 8,
  "results": [
    {
      "id": "gemini-3-flash:mcp-atlas:full:1",
      "model": "gemini-3-flash",
      "model_name": "Gemini 3 Flash",
      "model_date": "2025-12-17",
      "event": "https://frontierlog.ai/event/2025-12-17-gemini-3-flash",
      "benchmark": "mcp-atlas",
      "benchmark_name": "MCP Atlas",
      "setting": "full",
      "setting_name": "Full set (1,000 tasks)",
      "score": 57.4,
      "unit": "%",
      "printed": "57.4%",
      "reported_by": "google",
      "run_by": null,
      "label": "Gemini 3 Flash Thinking",
      "source": "https://blog.google/innovation-and-ai/technology/developers-tools/build-with-gemini-3-flash/",
      "source_title": "Build with Gemini 3 Flash, frontier intelligence that scales with you",
      "source_date": "2025-12-17",
      "where": "benchmark table (image), row 'MCP Atlas', column 'Gemini 3 Flash Thinking'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "thinking",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-29",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [
        {
          "url": "https://blog.google/products-and-platforms/products/gemini/gemini-3-flash/",
          "name": "Google post",
          "title": "Gemini 3 Flash: frontier intelligence built for speed",
          "lab": "google",
          "date": "2025-12-17",
          "printed": "57.4%",
          "where": "Benchmark table image, row \"MCP Atlas\", column \"Gemini 3 Flash Thinking\""
        }
      ],
      "source_image": "https://frontierlog.ai/sources/0fb5676cc5bd5e3d.gif",
      "source_image_original": "https://storage.googleapis.com/gweb-uniblog-publish-prod/original_images/gemini-3-flash_final_benchmark-table_light_25-12-17_final.gif"
    },
    {
      "id": "muse-spark:mcp-atlas:full:1",
      "model": "muse-spark",
      "model_name": "Muse Spark",
      "model_date": "2026-04-08",
      "event": "https://frontierlog.ai/event/2026-04-08-muse-spark",
      "benchmark": "mcp-atlas",
      "benchmark_name": "MCP Atlas",
      "setting": "full",
      "setting_name": "Full set (1,000 tasks)",
      "score": 82.2,
      "unit": "%",
      "printed": "82.2",
      "reported_by": "meta",
      "run_by": "Scale AI",
      "label": "Muse Spark",
      "source": "https://ai.meta.com/blog/introducing-muse-spark-meta-model-api/",
      "source_title": "Introducing Muse Spark 1.1",
      "source_date": "2026-07-09",
      "where": "Muse Spark 1.1 benchmark table (image), row 'MCP Atlas', column 'Muse Spark'",
      "own": false,
      "primary": true,
      "parallel": false,
      "notes": "",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-20",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/fb98c97035dca178.png",
      "source_image_original": "https://lookaside.fbsbx.com/elementpath/media/?media_id=1066878142575244&version=1789412025"
    },
    {
      "id": "claude-opus-4-7:mcp-atlas:full:1",
      "model": "claude-opus-4-7",
      "model_name": "Claude Opus 4.7",
      "model_date": "2026-04-16",
      "event": "https://frontierlog.ai/event/2026-04-16-claude-opus-4-7",
      "benchmark": "mcp-atlas",
      "benchmark_name": "MCP Atlas",
      "setting": "full",
      "setting_name": "Full set (1,000 tasks)",
      "score": 77.3,
      "unit": "%",
      "printed": "77.3%",
      "reported_by": "anthropic",
      "run_by": "Scale AI",
      "label": "Opus 4.7",
      "source": "https://www.anthropic.com/news/claude-opus-4-7",
      "source_title": "Introducing Claude Opus 4.7",
      "source_date": "2026-04-16",
      "where": "benchmark table (image), row 'Scaled tool use, MCP-Atlas', column 'Opus 4.7'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "refreshed April 2026 harness; adaptive thinking, max effort",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-28",
      "corroborated_by": [
        {
          "url": "https://www.anthropic.com/claude-opus-4-7-system-card",
          "name": "Claude Opus 4.7 card",
          "title": "Claude Opus 4.7 System Card",
          "lab": "anthropic",
          "date": "2026-04-16",
          "printed": "77.3%",
          "where": "Table 8.1.A row MCP-Atlas, column Claude Opus 4.7; Section 8.10.3 \"Scale AI evaluated Claude Opus 4.7 using adaptive thinking and max effort, and found a 77.3% Pass Rate\" (refreshed harness)"
        }
      ],
      "source_image": "https://frontierlog.ai/sources/527cd462f89b6de4.png",
      "source_image_original": "https://www-cdn.anthropic.com/images/4zrzovbb/website/d434d15757c6abac1122af483617741776d5a114-2600x2638.png"
    },
    {
      "id": "gemini-3-5-flash:mcp-atlas:full:1",
      "model": "gemini-3-5-flash",
      "model_name": "Gemini 3.5 Flash",
      "model_date": "2026-05-19",
      "event": "https://frontierlog.ai/event/2026-05-19-gemini-3-5-flash",
      "benchmark": "mcp-atlas",
      "benchmark_name": "MCP Atlas",
      "setting": "full",
      "setting_name": "Full set (1,000 tasks)",
      "score": 83.6,
      "unit": "%",
      "printed": "83.6%",
      "reported_by": "google",
      "run_by": null,
      "label": "Gemini 3.5 Flash",
      "source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-5/",
      "source_title": "Gemini 3.5: frontier intelligence with action",
      "source_date": "2026-05-19",
      "where": "benchmark table (image), row 'Agentic, MCP Atlas', column 'Gemini 3.5 Flash'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-28",
      "corroborated_by": [
        {
          "url": "https://deepmind.google/models/model-cards/gemini-3-5-flash/",
          "name": "Gemini 3.6 Flash and 3.5 Flash-Lite card",
          "title": "Gemini 3.5 Flash model card",
          "lab": "google",
          "date": "2026-07-21",
          "printed": "83.6%",
          "where": "Results table (as of May 2026), row \"MCP Atlas Multi-step workflows using MCP\", column \"Gemini 3.5 Flash\""
        }
      ],
      "source_image": "https://frontierlog.ai/sources/9db2ce79d7fae814.gif",
      "source_image_original": "https://storage.googleapis.com/gweb-uniblog-publish-prod/original_images/gemini-3-5__benchmarks__light.gif"
    },
    {
      "id": "muse-spark-1-1:mcp-atlas:full:1",
      "model": "muse-spark-1-1",
      "model_name": "Muse Spark 1.1",
      "model_date": "2026-07-09",
      "event": "https://frontierlog.ai/event/2026-07-09-muse-spark-1-1",
      "benchmark": "mcp-atlas",
      "benchmark_name": "MCP Atlas",
      "setting": "full",
      "setting_name": "Full set (1,000 tasks)",
      "score": 88.1,
      "unit": "%",
      "printed": "88.1",
      "reported_by": "meta",
      "run_by": "Scale AI",
      "label": "Muse Spark 1.1",
      "source": "https://ai.meta.com/blog/introducing-muse-spark-meta-model-api/",
      "source_title": "Introducing Muse Spark 1.1",
      "source_date": "2026-07-09",
      "where": "benchmark table (image), row 'MCP Atlas', column 'Muse Spark 1.1'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-20",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/fb98c97035dca178.png",
      "source_image_original": "https://lookaside.fbsbx.com/elementpath/media/?media_id=1066878142575244&version=1789412025"
    },
    {
      "id": "muse-spark-1-2:mcp-atlas:full:1",
      "model": "muse-spark-1-2",
      "model_name": "Muse Spark 1.2",
      "model_date": "2026-08-05",
      "event": "https://frontierlog.ai/event/2026-08-05-muse-spark-1-2",
      "benchmark": "mcp-atlas",
      "benchmark_name": "MCP Atlas",
      "setting": "full",
      "setting_name": "Full set (1,000 tasks)",
      "score": 90.3,
      "unit": "%",
      "printed": "90.3%",
      "reported_by": "meta",
      "run_by": "Scale AI",
      "label": "Muse Spark 1.2",
      "source": "https://research.meta.ai/blog/introducing-muse-code-and-muse-spark-1-2",
      "source_title": "Introducing Muse Code and Muse Spark 1.2",
      "source_date": "2026-08-05",
      "where": "bar chart 'MCP Atlas' (image, General Agents tab), value label above the bar 'Muse Spark 1.2'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "the benchmark's own harness",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-20",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/fbb8f33c75c7081f.png",
      "source_image_original": "https://research.meta.ai/articles/introducing-muse-code-and-muse-spark-1-2/evaluations/mcp-atlas-v1.png"
    },
    {
      "id": "muse-spark-open-816b-a42b:mcp-atlas:full:1",
      "model": "muse-spark-open-816b-a42b",
      "model_name": "Muse Spark Open 816B-A42B",
      "model_date": "2026-10-15",
      "event": "https://frontierlog.ai/event/2026-10-15-muse-spark-open",
      "benchmark": "mcp-atlas",
      "benchmark_name": "MCP Atlas",
      "setting": "full",
      "setting_name": "Full set (1,000 tasks)",
      "score": 89.2,
      "unit": "%",
      "printed": "89.2",
      "reported_by": "meta",
      "run_by": "Scale AI",
      "label": "Muse Spark Open (xhigh)",
      "source": "https://research.meta.ai/blog/muse-spark-open",
      "source_title": "Introducing Muse Spark Open, The Most Capable US Open-Weight Model to Date",
      "source_date": "2026-10-15",
      "where": "scorecard (image), row 'MCP-Atlas', column 'Muse Spark Open (xhigh)'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "xhigh effort",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-20",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": [],
      "source_image": "https://frontierlog.ai/sources/df5a416fe3ddfaeb.png",
      "source_image_original": "https://research.meta.ai/articles/muse-spark-open/benchmarks/benchmark-scorecard-light-v6.png"
    },
    {
      "id": "inkling-2-small:mcp-atlas:full:1",
      "model": "inkling-2-small",
      "model_name": "Inkling 2 Small",
      "model_date": "2026-11-05",
      "event": "https://frontierlog.ai/event/2026-11-05-inkling-2-small",
      "benchmark": "mcp-atlas",
      "benchmark_name": "MCP Atlas",
      "setting": "full",
      "setting_name": "Full set (1,000 tasks)",
      "score": 85.4,
      "unit": "%",
      "printed": "85.4",
      "reported_by": "thinking-machines",
      "run_by": "Scale AI",
      "label": "Inkling 2 Small",
      "source": "https://thinkingmachines.ai/news/inkling-2-small/",
      "source_title": "Introducing Inkling 2 Small",
      "source_date": "2026-11-05",
      "where": "section 'Benchmarking Inkling 2 Small', main benchmark table, Search and Tools group, row 'MCP Atlas' (subtitle 'Full / Public'), column 'Inkling 2 Small' (first model column), first value in the cell (before the slash); runner: list of externally reported evals right after the main benchmark table, before 'Limitations'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "full set; effort 0.99; 1M-token trajectory limit",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-11-07",
      "corroborated_by": [
        {
          "url": "https://thinkingmachines.ai/model-card/inkling-2-small/",
          "name": "Inkling 2 Small card",
          "title": "Inkling 2 Small Model Card",
          "lab": "thinking-machines",
          "date": "2026-11-05",
          "printed": "85.4",
          "where": "Model card section 5 Evaluations, HTML table, row \"MCP Atlas, Full / Public\", column \"Inkling 2 Small\" (first column); cell \"85.4/85.0%\", first value (Full)"
        }
      ],
      "source_capture": "https://frontierlog.ai/captures/49bcda9e78812be4.png"
    }
  ]
}