{
  "api_version": "1",
  "updated": "2026-10-18",
  "latest_event": "2026-10-17",
  "rights": "© Frontier Log, all rights reserved.",
  "story": {
    "slug": "benchmark-churn",
    "title": "Who reports which benchmark",
    "description": "Which benchmarks each lab reports at launch, and how the benchmarks they share change over time.",
    "url": "https://frontierlog.ai/stories/benchmark-churn",
    "image": "https://frontierlog.ai/og/story-benchmark-churn-coding.png",
    "event": "2026-09-30-gemini-4-argon",
    "tags": [],
    "images": [
      "https://frontierlog.ai/og/story-benchmark-churn-coding.png",
      "https://frontierlog.ai/og/story-benchmark-churn-knowledge.png",
      "https://frontierlog.ai/og/story-benchmark-churn-math.png",
      "https://frontierlog.ai/og/story-benchmark-churn-agents.png",
      "https://frontierlog.ai/og/story-benchmark-churn-saturation.png",
      "https://frontierlog.ai/og/story-benchmark-churn-why.png"
    ],
    "steps": [],
    "tables": [
      {
        "title": "GPQA Diamond",
        "unit": "%",
        "rows": [
          {
            "event": "2024-03-04-claude-3-family",
            "date": "2024-03-04",
            "lab": "anthropic",
            "value": 33.3,
            "printed": "33.3%",
            "source": "https://www.anthropic.com/news/claude-3-family",
            "where": "benchmark table (image), row 'Graduate level reasoning, GPQA, Diamond', column 'Claude 3 Haiku'"
          },
          {
            "event": "2024-03-04-claude-3-family",
            "date": "2024-03-04",
            "lab": "anthropic",
            "value": 40.4,
            "printed": "40.4%",
            "source": "https://www.anthropic.com/news/claude-3-family",
            "where": "benchmark table (image), row 'Graduate level reasoning, GPQA, Diamond', column 'Claude 3 Sonnet'"
          },
          {
            "event": "2024-03-04-claude-3-family",
            "date": "2024-03-04",
            "lab": "anthropic",
            "value": 50.4,
            "printed": "50.4%",
            "source": "https://www.anthropic.com/news/claude-3-family",
            "where": "benchmark table (image), row 'Graduate level reasoning, GPQA, Diamond', column 'Claude 3 Opus'"
          },
          {
            "event": "2024-05-13-gpt-4o",
            "date": "2024-05-13",
            "lab": "openai",
            "value": 53.6,
            "printed": "53.6",
            "source": "https://openai.com/index/hello-gpt-4o",
            "where": "text evaluation bar chart (SVG), panel 'GPQA (%)', bar 'GPT-4o'"
          },
          {
            "event": "2024-06-21-claude-3-5-sonnet",
            "date": "2024-06-21",
            "lab": "anthropic",
            "value": 59.4,
            "printed": "59.4%",
            "source": "https://www.anthropic.com/news/claude-3-5-sonnet",
            "where": "benchmark table (image), row 'Graduate level reasoning, GPQA, Diamond', column 'Claude 3.5 Sonnet' (value marked *)"
          },
          {
            "event": "2024-12-11-gemini-2-0-flash",
            "date": "2024-12-11",
            "lab": "google",
            "value": 62.1,
            "printed": "62.1%",
            "source": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/google-gemini-ai-update-december-2024/",
            "where": "benchmark table (image), row 'Reasoning, GPQA (diamond)', column 'Gemini 2.0 Flash Experimental'"
          },
          {
            "event": "2025-02-05-gemini-2-0-flash-flash-lite-and-pro",
            "date": "2025-02-05",
            "lab": "google",
            "value": 64.7,
            "printed": "64.7%",
            "source": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-model-updates-february-2025/",
            "where": "benchmark table (image), row 'Reasoning, GPQA (diamond)', column 'Gemini 2.0 Pro' (experimental)"
          },
          {
            "event": "2025-02-24-claude-3-7-sonnet",
            "date": "2025-02-24",
            "lab": "anthropic",
            "value": 78.2,
            "printed": "78.2%",
            "source": "https://www.anthropic.com/news/claude-3-7-sonnet",
            "where": "benchmark table (image), row 'Graduate-level reasoning, GPQA Diamond', column 'Claude 3.7 Sonnet, 64K extended thinking', first value"
          },
          {
            "event": "2025-03-25-gemini-2-5-pro",
            "date": "2025-03-25",
            "lab": "google",
            "value": 84.0,
            "printed": "84.0%",
            "source": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-model-thinking-updates-march-2025/",
            "where": "benchmark table (image), row 'Science, GPQA diamond', 'single attempt (pass@1)', column 'Gemini 2.5 Pro Experimental (03-25)'"
          },
          {
            "event": "2025-11-12-gpt-5-1",
            "date": "2025-11-12",
            "lab": "openai",
            "value": 88.1,
            "printed": "88.1%",
            "source": "https://openai.com/index/introducing-gpt-5-2",
            "where": "first comparison table, column 'GPT‑5.1 Thinking', row 'GPQA Diamond (no tools)'"
          },
          {
            "event": "2025-11-18-gemini-3-pro",
            "date": "2025-11-18",
            "lab": "google",
            "value": 91.9,
            "printed": "91.9%",
            "source": "https://blog.google/innovation-and-ai/technology/developers-tools/gemini-3-developers/",
            "where": "benchmark table (image), row 'GPQA Diamond', 'No tools', column 'Gemini 3 Pro'"
          },
          {
            "event": "2025-12-11-gpt-5-2",
            "date": "2025-12-11",
            "lab": "openai",
            "value": 92.4,
            "printed": "92.4%",
            "source": "https://openai.com/index/introducing-gpt-5-2",
            "where": "science and math table, row 'GPQA Diamond (no tools)', column 'GPT-5.2 Thinking'"
          },
          {
            "event": "2026-02-19-gemini-3-1-pro",
            "date": "2026-02-19",
            "lab": "google",
            "value": 94.3,
            "printed": "94.3%",
            "source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-1-pro/",
            "where": "benchmark table (image), row 'GPQA Diamond', 'No tools', column 'Gemini 3.1 Pro'"
          },
          {
            "event": "2026-07-09-gpt-5-6",
            "date": "2026-07-09",
            "lab": "openai",
            "value": 94.6,
            "printed": "94.6%",
            "source": "https://openai.com/index/gpt-5-6",
            "where": "academic eval table, column 'GPT‑5.6 Sol', row 'GPQA Diamond'"
          },
          {
            "event": "2026-09-03-gpt-6-astra",
            "date": "2026-09-03",
            "lab": "openai",
            "value": 96.0,
            "printed": "96.0%",
            "source": "https://openai.com/index/gpt-6-astra",
            "where": "'Academic' table, column 'GPT‑6 Astra', row 'GPQA Diamond'"
          }
        ]
      },
      {
        "title": "SWE-bench Verified",
        "unit": "%",
        "rows": [
          {
            "event": "2024-05-13-gpt-4o",
            "date": "2024-05-13",
            "lab": "openai",
            "value": 30.7,
            "printed": "30.7%",
            "source": "https://openai.com/index/introducing-gpt-4-5",
            "where": "eval table, row 'SWE-Bench Verified (coding)*', column 'GPT‑4o'"
          },
          {
            "event": "2025-01-20-deepseek-r1",
            "date": "2025-01-20",
            "lab": "deepseek",
            "value": 49.2,
            "printed": "49.2",
            "source": "https://api-docs.deepseek.com/news/news250120/",
            "where": "bar chart image (first image), group 'SWE-bench Verified (Resolved)', first bar (legend 'DeepSeek-R1'), value label"
          },
          {
            "event": "2025-02-24-claude-3-7-sonnet",
            "date": "2025-02-24",
            "lab": "anthropic",
            "value": 62.3,
            "printed": "62.3%",
            "source": "https://www.anthropic.com/news/claude-3-7-sonnet",
            "where": "benchmark table (image), row 'Agentic coding, SWE-bench Verified', column 'Claude 3.7 Sonnet, No extended thinking', first value"
          },
          {
            "event": "2025-04-16-o3-and-o4-mini",
            "date": "2025-04-16",
            "lab": "openai",
            "value": 69.1,
            "printed": "69.1",
            "source": "https://openai.com/index/introducing-o3-and-o4-mini",
            "where": "Coding section, bar chart 'SWE-Bench Verified (n=477)', bar labelled 'o3' (3rd bar)"
          },
          {
            "event": "2025-05-22-claude-4",
            "date": "2025-05-22",
            "lab": "anthropic",
            "value": 72.5,
            "printed": "72.5%",
            "source": "https://www.anthropic.com/news/claude-4",
            "where": "'Software engineering, SWE-bench verified' bar chart (image), bar 'Opus 4', value inside the bar"
          },
          {
            "event": "2025-05-22-claude-4",
            "date": "2025-05-22",
            "lab": "anthropic",
            "value": 72.7,
            "printed": "72.7%",
            "source": "https://www.anthropic.com/news/claude-4",
            "where": "'Software engineering, SWE-bench verified' bar chart (image), bar 'Sonnet 4', value inside the bar"
          },
          {
            "event": "2025-08-05-claude-opus-4-1",
            "date": "2025-08-05",
            "lab": "anthropic",
            "value": 74.5,
            "printed": "74.5%",
            "source": "https://www.anthropic.com/news/claude-opus-4-1",
            "where": "benchmark table (image), row 'Agentic coding, SWE-bench Verified', column 'Claude Opus 4.1'"
          },
          {
            "event": "2025-08-07-gpt-5",
            "date": "2025-08-07",
            "lab": "openai",
            "value": 74.9,
            "printed": "74.9%",
            "source": "https://openai.com/index/introducing-gpt-5",
            "where": "text, opening summary of results"
          },
          {
            "event": "2025-09-29-claude-sonnet-4-5",
            "date": "2025-09-29",
            "lab": "anthropic",
            "value": 77.2,
            "printed": "77.2%",
            "source": "https://www.anthropic.com/news/claude-sonnet-4-5",
            "where": "benchmark table (image), row 'Agentic coding, SWE-bench Verified', column 'Claude Sonnet 4.5', first value"
          },
          {
            "event": "2025-11-24-claude-opus-4-5",
            "date": "2025-11-24",
            "lab": "anthropic",
            "value": 80.9,
            "printed": "80.9%",
            "source": "https://www.anthropic.com/news/claude-opus-4-5",
            "where": "benchmark table (image), row 'Agentic coding, SWE-bench Verified', column 'Opus 4.5'"
          },
          {
            "event": "2026-04-16-claude-opus-4-7",
            "date": "2026-04-16",
            "lab": "anthropic",
            "value": 87.6,
            "printed": "87.6%",
            "source": "https://www.anthropic.com/news/claude-opus-4-7",
            "where": "benchmark table (image), row 'Agentic coding, SWE-bench Verified', column 'Opus 4.7'"
          },
          {
            "event": "2026-05-28-claude-opus-4-8",
            "date": "2026-05-28",
            "lab": "anthropic",
            "value": 88.6,
            "printed": "88.6",
            "source": "https://www.anthropic.com/claude-opus-4-8-system-card",
            "where": "Section 8.1 evaluation summary table (page 194), row 'SWE-bench Verified', column 'Claude Opus 4.8'"
          },
          {
            "event": "2026-06-09-claude-fable-5-mythos-5",
            "date": "2026-06-09",
            "lab": "anthropic",
            "value": 95.0,
            "printed": "95",
            "source": "https://www.anthropic.com/claude-fable-5-mythos-5-system-card",
            "where": "Section 8.1 evaluation summary table (page 251), row 'SWE-bench Verified', column 'Fable 5'"
          }
        ]
      },
      {
        "title": "AIME 2025",
        "unit": "%",
        "rows": [
          {
            "event": "2025-03-25-gemini-2-5-pro",
            "date": "2025-03-25",
            "lab": "google",
            "value": 86.7,
            "printed": "86.7%",
            "source": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-model-thinking-updates-march-2025/",
            "where": "benchmark table image (gemini_benchmarks_cropped GIF), row 'Mathematics AIME 2025', sub-row 'single attempt (pass@1)', column 'Gemini 2.5 Pro Experimental (03-25)'"
          },
          {
            "event": "2025-04-16-o3-and-o4-mini",
            "date": "2025-04-16",
            "lab": "openai",
            "value": 88.9,
            "printed": "88.9",
            "source": "https://openai.com/index/introducing-o3-and-o4-mini",
            "where": "'What's changed' section, bar chart 'AIME 2025 / Competition Math', bar labelled 'o3 (no tools)' (3rd bar)"
          },
          {
            "event": "2025-08-07-gpt-5",
            "date": "2025-08-07",
            "lab": "openai",
            "value": 94.6,
            "printed": "94.6%",
            "source": "https://openai.com/index/introducing-gpt-5",
            "where": "'Evaluations' section, first paragraph (state of the art across math)"
          },
          {
            "event": "2025-12-11-gpt-5-2",
            "date": "2025-12-11",
            "lab": "openai",
            "value": 100.0,
            "printed": "100.0%",
            "source": "https://openai.com/index/introducing-gpt-5-2",
            "where": "headline eval table (HTML, first table), row 'AIME 2025 (no tools) / Competition math', column 'GPT-5.2 Thinking'"
          }
        ]
      },
      {
        "title": "Humanity's Last Exam, no tools",
        "unit": "%",
        "rows": [
          {
            "event": "2025-03-25-gemini-2-5-pro",
            "date": "2025-03-25",
            "lab": "google",
            "value": 18.8,
            "printed": "18.8%",
            "source": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-model-thinking-updates-march-2025/",
            "where": "benchmark table (image), row 'Reasoning & knowledge, Humanity's Last Exam (no tools)', column 'Gemini 2.5 Pro Experimental (03-25)'"
          },
          {
            "event": "2025-11-06-kimi-k2-thinking",
            "date": "2025-11-06",
            "lab": "moonshot",
            "value": 23.9,
            "printed": "23.9",
            "source": "https://www.kimi.ai/blog/kimi-k2-thinking",
            "where": "'Full Evaluations' table, row 'Humanity's Last Exam (Text-only)', sub-row 'no tools', column 'K2 Thinking'"
          },
          {
            "event": "2025-11-12-gpt-5-1",
            "date": "2025-11-12",
            "lab": "openai",
            "value": 25.7,
            "printed": "25.7%",
            "source": "https://openai.com/index/introducing-gpt-5-2",
            "where": "science and math eval table, column 'GPT-5.1 Thinking', row 'HLE (no tools)'"
          },
          {
            "event": "2025-11-18-gemini-3-pro",
            "date": "2025-11-18",
            "lab": "google",
            "value": 37.5,
            "printed": "37.5%",
            "source": "https://blog.google/innovation-and-ai/technology/developers-tools/gemini-3-developers/",
            "where": "benchmark table (image), row 'Humanity's Last Exam', 'No tools', column 'Gemini 3 Pro'"
          },
          {
            "event": "2026-02-05-claude-opus-4-6",
            "date": "2026-02-05",
            "lab": "anthropic",
            "value": 40.0,
            "printed": "40.0%",
            "source": "https://www.anthropic.com/news/claude-opus-4-6",
            "where": "benchmark table (image), row 'Multidisciplinary reasoning, Humanity's Last Exam', column 'Opus 4.6', 'without tools'"
          },
          {
            "event": "2026-02-19-gemini-3-1-pro",
            "date": "2026-02-19",
            "lab": "google",
            "value": 44.4,
            "printed": "44.4%",
            "source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-1-pro/",
            "where": "benchmark table (image), row 'Humanity's Last Exam', 'No tools', column 'Gemini 3.1 Pro, Thinking (High)'"
          },
          {
            "event": "2026-04-16-claude-opus-4-7",
            "date": "2026-04-16",
            "lab": "anthropic",
            "value": 46.9,
            "printed": "46.9%",
            "source": "https://www.anthropic.com/news/claude-opus-4-7",
            "where": "benchmark table (image), row 'Multidisciplinary reasoning, Humanity's Last Exam', column 'Opus 4.7', 'no tools'"
          },
          {
            "event": "2026-05-28-claude-opus-4-8",
            "date": "2026-05-28",
            "lab": "anthropic",
            "value": 49.8,
            "printed": "49.8%",
            "source": "https://www.anthropic.com/news/claude-opus-4-8",
            "where": "benchmark table (image), row 'Multidisciplinary reasoning, Humanity's Last Exam', column 'Opus 4.8', 'no tools'"
          },
          {
            "event": "2026-06-09-claude-fable-5-mythos-5",
            "date": "2026-06-09",
            "lab": "anthropic",
            "value": 56.5,
            "printed": "56.5%",
            "source": "https://www.anthropic.com/news/claude-opus-5",
            "where": "benchmark table (image), row 'Multidisciplinary reasoning, Humanity's Last Exam', column 'Fable 5', 'no tools'"
          },
          {
            "event": "2026-09-01-claude-fable-5-1-mythos-5-1",
            "date": "2026-09-01",
            "lab": "anthropic",
            "value": 60.9,
            "printed": "60.9%",
            "source": "https://www.anthropic.com/claude-fable-and-mythos-5-1",
            "where": "benchmark table (HTML), row 'Multidisciplinary reasoning, Humanity's Last Exam', column 'Fable 5.1', 'no tools'"
          },
          {
            "event": "2026-09-22-claude-opus-5-5",
            "date": "2026-09-22",
            "lab": "anthropic",
            "value": 64.4,
            "printed": "64.4",
            "source": "https://www.anthropic.com/claude-opus-5-5-system-card",
            "where": "Section 8.1 evaluation summary table, row 'Humanity's Last Exam, No tools', column 'Claude Opus 5.5'"
          }
        ]
      },
      {
        "title": "SWE-bench Pro",
        "unit": "%",
        "rows": [
          {
            "event": "2026-09-22-claude-opus-5-5",
            "date": "2026-09-22",
            "lab": "anthropic",
            "value": 89.9,
            "printed": "89.9",
            "source": "https://www.anthropic.com/claude-opus-5-5-system-card",
            "where": "Section 8.1 Evaluation summary, Table 8.1.A, row 'SWE-bench Pro', column 'Claude Opus 5.5'"
          }
        ]
      },
      {
        "title": "DeepSWE v1.1",
        "unit": "%",
        "rows": [
          {
            "event": "2026-09-30-gemini-4-argon",
            "date": "2026-09-30",
            "lab": "google",
            "value": 77.9,
            "printed": "77.9%",
            "source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-4-argon/",
            "where": "benchmark table (image), row 'DeepSWE v1.1', column 'Gemini 4 Argon'"
          }
        ]
      },
      {
        "title": "Terminal-Bench 2.0",
        "unit": "%",
        "rows": [
          {
            "event": "2026-04-23-gpt-5-5",
            "date": "2026-04-23",
            "lab": "openai",
            "value": 82.7,
            "printed": "82.7%",
            "source": "https://openai.com/index/introducing-gpt-5-5",
            "where": "headline eval table (HTML, first table), row 'Terminal-Bench 2.0', column 'GPT‑5.5'"
          }
        ]
      },
      {
        "title": "Terminal-Bench 4.0",
        "unit": "%",
        "rows": [
          {
            "event": "2026-09-28-claude-sonnet-5-5",
            "date": "2026-09-28",
            "lab": "anthropic",
            "value": 70.6,
            "printed": "70.6%",
            "source": "https://www.anthropic.com/claude-sonnet-5-5",
            "where": "benchmark table, row 'Terminal-Bench 4.0', column 'Sonnet 5.5'"
          }
        ]
      },
      {
        "title": "τ²-bench Telecom",
        "unit": "%",
        "rows": [
          {
            "event": "2026-02-05-claude-opus-4-6",
            "date": "2026-02-05",
            "lab": "anthropic",
            "value": 99.3,
            "printed": "99.3%",
            "source": "https://www.anthropic.com/news/claude-opus-4-6",
            "where": "benchmark table (image), row 'Agentic tool use, τ2-bench', Telecom line, column 'Opus 4.6'"
          }
        ]
      },
      {
        "title": "AutomationBench",
        "unit": "%",
        "rows": [
          {
            "event": "2026-09-30-gemini-4-argon",
            "date": "2026-09-30",
            "lab": "google",
            "value": 51.3,
            "printed": "51.3%",
            "source": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-4-argon/",
            "where": "benchmark table (image), row 'AutomationBench', 'Score', column 'Gemini 4 Argon'"
          }
        ]
      }
    ],
    "reasons": [
      {
        "bench": "aime-2025",
        "name": "AIME 2025",
        "table": "AIME 2025",
        "cause": "saturated",
        "fact": "GPT-5.2 scored 100% in Dec 2025",
        "detail": "Anthropic's system cards call it saturated",
        "introduced": "2025-02-12",
        "introduced_source": "https://maa.org/news/aime-thresholds-are-available/",
        "sources": [
          {
            "url": "https://openai.com/index/introducing-gpt-5-2",
            "label": "OpenAI, Introducing GPT-5.2 (Dec 2025)",
            "part": "fact"
          },
          {
            "url": "https://anthropic.com/claude-opus-4-7-system-card",
            "label": "Anthropic, Claude Opus 4.7 system card (Apr 2026)",
            "part": "detail"
          }
        ]
      },
      {
        "bench": "tau2-bench",
        "name": "τ²-bench Telecom",
        "table": "τ²-bench Telecom",
        "cause": "version",
        "fact": "Its successor, τ³-bench, came out in Mar 2026",
        "detail": "",
        "introduced": "2025-06-09",
        "introduced_source": "https://arxiv.org/abs/2506.07982",
        "sources": [
          {
            "url": "https://sierra.ai/resources/research/tau-3-bench",
            "label": "Sierra, τ³-bench (Mar 2026)",
            "part": "fact"
          }
        ]
      },
      {
        "bench": "swe-bench-verified",
        "name": "SWE-bench Verified",
        "table": "SWE-bench Verified",
        "cause": "flawed",
        "fact": "OpenAI, its creator, stopped reporting it in Feb 2026, citing flawed tests",
        "detail": "",
        "introduced": "2024-08-13",
        "introduced_source": "https://openai.com/index/introducing-swe-bench-verified",
        "sources": [
          {
            "url": "https://openai.com/index/why-we-no-longer-evaluate-swe-bench-verified",
            "label": "OpenAI, Why we no longer evaluate SWE-bench Verified (Feb 2026)",
            "part": "fact"
          }
        ]
      },
      {
        "bench": "terminal-bench-2-0",
        "name": "Terminal-Bench 2.0",
        "table": "Terminal-Bench 2.0",
        "cause": "version",
        "fact": "Its creators released versions 2.1 (May 2026) and 4.0 (Aug 2026)",
        "detail": "",
        "introduced": "2025-11-07",
        "introduced_source": "https://www.tbench.ai/news/announcement-2-0",
        "sources": [
          {
            "url": "https://www.tbench.ai/news/terminal-bench-2-1",
            "label": "Terminal-Bench, Terminal-Bench 2.1 (May 2026)",
            "part": "fact"
          },
          {
            "url": "https://www.tbench.ai/news/terminal-bench-4-0",
            "label": "Terminal-Bench, Terminal-Bench 4.0 (Aug 2026)",
            "part": "fact"
          }
        ]
      },
      {
        "bench": "gpqa-diamond",
        "name": "GPQA Diamond",
        "table": "GPQA Diamond",
        "cause": "saturated",
        "fact": "Anthropic called it saturated in Jun 2026",
        "detail": "",
        "introduced": "2023-11-20",
        "introduced_source": "https://arxiv.org/abs/2311.12022",
        "sources": [
          {
            "url": "https://anthropic.com/claude-fable-5-mythos-5-system-card",
            "label": "Anthropic, Claude Fable 5 and Mythos 5 system card (Jun 2026)",
            "part": "fact"
          }
        ]
      },
      {
        "bench": "swe-bench-pro",
        "name": "SWE-bench Pro",
        "table": "SWE-bench Pro",
        "cause": "flawed",
        "fact": "OpenAI estimated in Jul 2026 that about 30% of its tasks are broken",
        "detail": "",
        "introduced": "2025-09-19",
        "introduced_source": "https://scale.com/blog/swe-bench-pro",
        "sources": [
          {
            "url": "https://openai.com/index/separating-signal-from-noise-coding-evaluations",
            "label": "OpenAI, Separating signal from noise in coding evaluations (Jul 2026)",
            "part": "fact"
          }
        ]
      },
      {
        "bench": "deepswe-1-1",
        "name": "DeepSWE v1.1",
        "table": "DeepSWE v1.1",
        "cause": "renewed",
        "fact": "",
        "detail": "",
        "introduced": "2026-06-14",
        "introduced_source": "https://deepswe.datacurve.ai/blog/deepswe-v1-1",
        "sources": []
      },
      {
        "bench": "terminal-bench-4-0",
        "name": "Terminal-Bench 4.0",
        "table": "Terminal-Bench 4.0",
        "cause": "renewed",
        "fact": "",
        "detail": "",
        "introduced": "2026-08-28",
        "introduced_source": "https://www.tbench.ai/news/terminal-bench-4-0",
        "sources": []
      },
      {
        "bench": "hle",
        "name": "Humanity's Last Exam",
        "table": "Humanity's Last Exam, no tools",
        "cause": "renewed",
        "fact": "",
        "detail": "",
        "introduced": "2025-01-23",
        "introduced_source": "https://scale.com/blog/humanitys-last-exam-results",
        "sources": []
      },
      {
        "bench": "automationbench",
        "name": "AutomationBench",
        "table": "AutomationBench",
        "cause": "renewed",
        "fact": "",
        "detail": "",
        "introduced": "2026-04-20",
        "introduced_source": "https://zapier.com/blog/introducing-automationbench/",
        "sources": []
      }
    ],
    "sources": [
      "https://api-docs.deepseek.com/news/news250120/",
      "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-1-pro/",
      "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-model-thinking-updates-march-2025/",
      "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-model-updates-february-2025/",
      "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/google-gemini-ai-update-december-2024/",
      "https://blog.google/innovation-and-ai/technology/developers-tools/gemini-3-developers/",
      "https://openai.com/index/gpt-5-6",
      "https://openai.com/index/gpt-6-astra",
      "https://openai.com/index/hello-gpt-4o",
      "https://openai.com/index/introducing-gpt-4-5",
      "https://openai.com/index/introducing-gpt-5",
      "https://openai.com/index/introducing-gpt-5-2",
      "https://openai.com/index/introducing-o3-and-o4-mini",
      "https://www.anthropic.com/claude-fable-5-mythos-5-system-card",
      "https://www.anthropic.com/claude-fable-and-mythos-5-1",
      "https://www.anthropic.com/claude-opus-4-8-system-card",
      "https://www.anthropic.com/claude-opus-5-5-system-card",
      "https://www.anthropic.com/news/claude-3-5-sonnet",
      "https://www.anthropic.com/news/claude-3-7-sonnet",
      "https://www.anthropic.com/news/claude-3-family",
      "https://www.anthropic.com/news/claude-4",
      "https://www.anthropic.com/news/claude-opus-4-1",
      "https://www.anthropic.com/news/claude-opus-4-5",
      "https://www.anthropic.com/news/claude-opus-4-6",
      "https://www.anthropic.com/news/claude-opus-4-7",
      "https://www.anthropic.com/news/claude-opus-4-8",
      "https://www.anthropic.com/news/claude-opus-5",
      "https://www.anthropic.com/news/claude-sonnet-4-5",
      "https://www.kimi.ai/blog/kimi-k2-thinking",
      "https://api-docs.deepseek.com/news/news260424/",
      "https://www.kimi.ai/blog/kimi-k3",
      "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-4-argon/",
      "https://openai.com/index/introducing-gpt-5-5",
      "https://api-docs.deepseek.com/news/news260910/",
      "https://www.anthropic.com/claude-sonnet-5-5"
    ],
    "launches": [
      {
        "model": "GPT-5",
        "lab": "openai",
        "date": "2025-08-07",
        "event": "2025-08-07-gpt-5",
        "post": "https://openai.com/index/introducing-gpt-5",
        "documents": []
      },
      {
        "model": "GPT-5.2",
        "lab": "openai",
        "date": "2025-12-11",
        "event": "2025-12-11-gpt-5-2",
        "post": "https://openai.com/index/introducing-gpt-5-2",
        "documents": []
      },
      {
        "model": "GPT-5.5",
        "lab": "openai",
        "date": "2026-04-23",
        "event": "2026-04-23-gpt-5-5",
        "post": "https://openai.com/index/introducing-gpt-5-5",
        "documents": []
      },
      {
        "model": "GPT-6 Astra",
        "lab": "openai",
        "date": "2026-09-03",
        "event": "2026-09-03-gpt-6-astra",
        "post": "https://openai.com/index/gpt-6-astra",
        "documents": []
      },
      {
        "model": "Claude Opus 4",
        "lab": "anthropic",
        "date": "2025-05-22",
        "event": "2025-05-22-claude-4",
        "post": "https://www.anthropic.com/news/claude-4",
        "documents": []
      },
      {
        "model": "Claude Opus 4.5",
        "lab": "anthropic",
        "date": "2025-11-24",
        "event": "2025-11-24-claude-opus-4-5",
        "post": "https://www.anthropic.com/news/claude-opus-4-5",
        "documents": [
          "https://www.anthropic.com/claude-opus-4-5-system-card"
        ]
      },
      {
        "model": "Claude Opus 4.7",
        "lab": "anthropic",
        "date": "2026-04-16",
        "event": "2026-04-16-claude-opus-4-7",
        "post": "https://www.anthropic.com/news/claude-opus-4-7",
        "documents": [
          "https://www.anthropic.com/claude-opus-4-7-system-card"
        ]
      },
      {
        "model": "Claude Opus 5.5",
        "lab": "anthropic",
        "date": "2026-09-22",
        "event": "2026-09-22-claude-opus-5-5",
        "post": "https://www.anthropic.com/claude-opus-5-5",
        "documents": [
          "https://www.anthropic.com/claude-opus-5-5-system-card"
        ]
      },
      {
        "model": "Gemini 2.5 Pro",
        "lab": "google",
        "date": "2025-03-25",
        "event": "2025-03-25-gemini-2-5-pro",
        "post": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-model-thinking-updates-march-2025/",
        "documents": []
      },
      {
        "model": "Gemini 3 Pro",
        "lab": "google",
        "date": "2025-11-18",
        "event": "2025-11-18-gemini-3-pro",
        "post": "https://blog.google/products-and-platforms/products/gemini/gemini-3/",
        "documents": [
          "https://blog.google/innovation-and-ai/technology/developers-tools/gemini-3-developers/"
        ]
      },
      {
        "model": "Gemini 3.1 Pro",
        "lab": "google",
        "date": "2026-02-19",
        "event": "2026-02-19-gemini-3-1-pro",
        "post": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-1-pro/",
        "documents": []
      },
      {
        "model": "Gemini 4 Argon",
        "lab": "google",
        "date": "2026-09-30",
        "event": "2026-09-30-gemini-4-argon",
        "post": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-4-argon/",
        "documents": []
      },
      {
        "model": "DeepSeek R1",
        "lab": "deepseek",
        "date": "2025-01-20",
        "event": "2025-01-20-deepseek-r1",
        "post": "https://api-docs.deepseek.com/news/news250120/",
        "documents": [
          "https://github.com/deepseek-ai/DeepSeek-R1/blob/main/DeepSeek_R1.pdf"
        ]
      },
      {
        "model": "DeepSeek V3.2",
        "lab": "deepseek",
        "date": "2025-12-01",
        "event": "2025-12-01-deepseek-v3-2",
        "post": "https://api-docs.deepseek.com/news/news251201/",
        "documents": [
          "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/resolve/main/assets/paper.pdf"
        ]
      },
      {
        "model": "DeepSeek V4",
        "lab": "deepseek",
        "date": "2026-04-24",
        "event": "2026-04-24-deepseek-v4-preview",
        "post": "https://api-docs.deepseek.com/news/news260424/",
        "documents": []
      },
      {
        "model": "DeepSeek V4.1-Flash",
        "lab": "deepseek",
        "date": "2026-09-10",
        "event": "2026-09-10-deepseek-v4-1-flash",
        "post": "https://api-docs.deepseek.com/news/news260910/",
        "documents": []
      },
      {
        "model": "Qwen3",
        "lab": "alibaba",
        "date": "2025-04-28",
        "event": null,
        "post": "https://qwen.ai/blog?id=qwen3",
        "documents": []
      },
      {
        "model": "Qwen3.5",
        "lab": "alibaba",
        "date": "2026-02-15",
        "event": null,
        "post": "https://qwen.ai/blog?id=qwen3.5",
        "documents": []
      },
      {
        "model": "Qwen3.8-Max",
        "lab": "alibaba",
        "date": "2026-08-03",
        "event": "2026-08-03-qwen3-8-max",
        "post": "https://qwen.ai/blog?id=qwen3.8",
        "documents": []
      },
      {
        "model": "Kimi K2",
        "lab": "moonshot",
        "date": "2025-07-11",
        "event": "2025-07-11-kimi-k2",
        "post": "https://www.kimi.ai/blog/kimi-k2",
        "documents": []
      },
      {
        "model": "Kimi K2.5",
        "lab": "moonshot",
        "date": "2026-01-27",
        "event": "2026-01-27-kimi-k2-5",
        "post": "https://www.kimi.ai/blog/kimi-k2-5",
        "documents": []
      },
      {
        "model": "Kimi K3",
        "lab": "moonshot",
        "date": "2026-07-16",
        "event": "2026-07-16-kimi-k3",
        "post": "https://www.kimi.ai/blog/kimi-k3",
        "documents": []
      },
      {
        "model": "MiniMax M2",
        "lab": "minimax",
        "date": "2025-10-27",
        "event": null,
        "post": "https://www.minimax.io/news/minimax-m2",
        "documents": [
          "https://huggingface.co/MiniMaxAI/MiniMax-M2"
        ]
      },
      {
        "model": "MiniMax M2.5",
        "lab": "minimax",
        "date": "2026-02-12",
        "event": "2026-02-12-minimax-m2-5",
        "post": "https://www.minimax.io/news/minimax-m25",
        "documents": []
      },
      {
        "model": "MiniMax M3",
        "lab": "minimax",
        "date": "2026-06-01",
        "event": "2026-06-01-minimax-m3",
        "post": "https://www.minimax.io/blog/minimax-m3",
        "documents": []
      }
    ]
  }
}