{
  "api_version": "1",
  "updated": "2026-10-20",
  "latest_event": "2026-10-18",
  "rights": "© Frontier Log, all rights reserved.",
  "story": {
    "slug": "the-harness-is-part-of-the-score",
    "title": "The Harness Is Part of the Score",
    "description": "How a model's score on an agentic coding benchmark changes with the agent harness it runs in, from the labs' own per-harness reports.",
    "url": "https://frontierlog.ai/stories/the-harness-is-part-of-the-score",
    "date": "2026-10-20",
    "image": "https://frontierlog.ai/og/story-the-harness-is-part-of-the-score-pairs.png",
    "event": "2026-10-15-nemotron-3-5-super-vl-bolt",
    "tags": [
      "coding",
      "agents"
    ],
    "images": [
      "https://frontierlog.ai/og/story-the-harness-is-part-of-the-score-pairs.png",
      "https://frontierlog.ai/og/story-the-harness-is-part-of-the-score-nemotron.png",
      "https://frontierlog.ai/og/story-the-harness-is-part-of-the-score-effect.png"
    ],
    "steps": [],
    "tables": [
      {
        "title": "Nemotron 3.5 Super: SWE-bench Pro by harness",
        "unit": "%",
        "rows": [
          {
            "event": "2026-10-15-nemotron-3-5-super-vl-bolt",
            "date": "2026-10-15",
            "lab": "nvidia",
            "value": 49.45,
            "printed": "49.45%",
            "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4/blob/8a115b9d966adce8c059a7bcf47f49a1e57957d8/README.md",
            "where": "'Agentic Coding Benchmarks' image, 'SWE-Bench Pro' panel, value label of the 'Claude Code' bar"
          },
          {
            "event": "2026-10-15-nemotron-3-5-super-vl-bolt",
            "date": "2026-10-15",
            "lab": "nvidia",
            "value": 47.4,
            "printed": "47.40%",
            "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4/blob/8a115b9d966adce8c059a7bcf47f49a1e57957d8/README.md",
            "where": "'Agentic Coding Benchmarks' image, 'SWE-Bench Pro' panel, value label of the 'Mini-SWE agent' bar"
          },
          {
            "event": "2026-10-15-nemotron-3-5-super-vl-bolt",
            "date": "2026-10-15",
            "lab": "nvidia",
            "value": 47.26,
            "printed": "47.26%",
            "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4/blob/8a115b9d966adce8c059a7bcf47f49a1e57957d8/README.md",
            "where": "'Agentic Coding Benchmarks' image, 'SWE-Bench Pro' panel, value label of the 'Codex' bar"
          },
          {
            "event": "2026-10-15-nemotron-3-5-super-vl-bolt",
            "date": "2026-10-15",
            "lab": "nvidia",
            "value": 47.2,
            "printed": "47.20%",
            "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4/blob/8a115b9d966adce8c059a7bcf47f49a1e57957d8/README.md",
            "where": "'Agentic Coding Benchmarks' image, 'SWE-Bench Pro' panel, value label of the 'Hermes' bar"
          },
          {
            "event": "2026-10-15-nemotron-3-5-super-vl-bolt",
            "date": "2026-10-15",
            "lab": "nvidia",
            "value": 46.92,
            "printed": "46.92%",
            "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4/blob/8a115b9d966adce8c059a7bcf47f49a1e57957d8/README.md",
            "where": "'Agentic Coding Benchmarks' image, 'SWE-Bench Pro' panel, value label of the 'Pi' bar"
          },
          {
            "event": "2026-10-15-nemotron-3-5-super-vl-bolt",
            "date": "2026-10-15",
            "lab": "nvidia",
            "value": 41.04,
            "printed": "41.04%",
            "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4/blob/8a115b9d966adce8c059a7bcf47f49a1e57957d8/README.md",
            "where": "'Agentic Coding Benchmarks' image, 'SWE-Bench Pro' panel, value label of the 'OpenCode' bar"
          }
        ]
      },
      {
        "title": "Nemotron 3.5 Super: Terminal-Bench 2.1 by harness",
        "unit": "%",
        "rows": [
          {
            "event": "2026-10-15-nemotron-3-5-super-vl-bolt",
            "date": "2026-10-15",
            "lab": "nvidia",
            "value": 61.8,
            "printed": "61.80%",
            "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4/blob/8a115b9d966adce8c059a7bcf47f49a1e57957d8/README.md",
            "where": "'Agentic Coding Benchmarks' image, 'Terminal-Bench 2.1' panel, value label of the 'Codex' bar"
          },
          {
            "event": "2026-10-15-nemotron-3-5-super-vl-bolt",
            "date": "2026-10-15",
            "lab": "nvidia",
            "value": 61.42,
            "printed": "61.42%",
            "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4/blob/8a115b9d966adce8c059a7bcf47f49a1e57957d8/README.md",
            "where": "'Agentic Coding Benchmarks' image, 'Terminal-Bench 2.1' panel, value label of the 'Hermes' bar"
          },
          {
            "event": "2026-10-15-nemotron-3-5-super-vl-bolt",
            "date": "2026-10-15",
            "lab": "nvidia",
            "value": 59.55,
            "printed": "59.55%",
            "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4/blob/8a115b9d966adce8c059a7bcf47f49a1e57957d8/README.md",
            "where": "'Agentic Coding Benchmarks' image, 'Terminal-Bench 2.1' panel, value label of the 'Claude Code' bar"
          },
          {
            "event": "2026-10-15-nemotron-3-5-super-vl-bolt",
            "date": "2026-10-15",
            "lab": "nvidia",
            "value": 55.06,
            "printed": "55.06%",
            "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4/blob/8a115b9d966adce8c059a7bcf47f49a1e57957d8/README.md",
            "where": "'Agentic Coding Benchmarks' image, 'Terminal-Bench 2.1' panel, value label of the 'OpenCode' bar"
          },
          {
            "event": "2026-10-15-nemotron-3-5-super-vl-bolt",
            "date": "2026-10-15",
            "lab": "nvidia",
            "value": 54.68,
            "printed": "54.68%",
            "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4/blob/8a115b9d966adce8c059a7bcf47f49a1e57957d8/README.md",
            "where": "'Agentic Coding Benchmarks' image, 'Terminal-Bench 2.1' panel, value label of the 'Mini-SWE agent' bar"
          },
          {
            "event": "2026-10-15-nemotron-3-5-super-vl-bolt",
            "date": "2026-10-15",
            "lab": "nvidia",
            "value": 48.31,
            "printed": "48.31%",
            "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4/blob/8a115b9d966adce8c059a7bcf47f49a1e57957d8/README.md",
            "where": "'Agentic Coding Benchmarks' image, 'Terminal-Bench 2.1' panel, value label of the 'Pi' bar"
          }
        ]
      },
      {
        "title": "SWE-bench Verified by harness",
        "unit": "%",
        "rows": [
          {
            "event": "2025-10-27-minimax-m2",
            "date": "2025-10-27",
            "lab": "minimax",
            "value": 68.1,
            "printed": "68.1",
            "source": "https://www.minimax.io/news/minimax-m21",
            "where": "'Benchmarks' section, second (agent frameworks and specialized benchmarks) table image, row 'SWE-bench Verified (Droid)', column 'MiniMax-M2'"
          },
          {
            "event": "2025-10-27-minimax-m2",
            "date": "2025-10-27",
            "lab": "minimax",
            "value": 61.0,
            "printed": "61.0",
            "source": "https://www.minimax.io/news/minimax-m21",
            "where": "'Benchmarks' section, second (agent frameworks and specialized benchmarks) table image, row 'SWE-bench Verified (mini-swe-agent)', column 'MiniMax-M2'"
          },
          {
            "event": "2025-12-23-minimax-m2-1",
            "date": "2025-12-23",
            "lab": "minimax",
            "value": 71.3,
            "printed": "71.3",
            "source": "https://www.minimax.io/news/minimax-m21",
            "where": "'Benchmarks' section, second (agent frameworks and specialized benchmarks) table image, row 'SWE-bench Verified (Droid)', column 'MiniMax-M2.1'"
          },
          {
            "event": "2025-12-23-minimax-m2-1",
            "date": "2025-12-23",
            "lab": "minimax",
            "value": 67.0,
            "printed": "67.0",
            "source": "https://www.minimax.io/news/minimax-m21",
            "where": "'Benchmarks' section, second (agent frameworks and specialized benchmarks) table image, row 'SWE-bench Verified (mini-swe-agent)', column 'MiniMax-M2.1'"
          }
        ]
      },
      {
        "title": "SWE-bench Pro by harness",
        "unit": "%",
        "rows": [
          {
            "event": "2026-06-01-qwen3-7-plus",
            "date": "2026-06-01",
            "lab": "alibaba",
            "value": 57.6,
            "printed": "57.6",
            "source": "https://qwen.ai/blog?id=qwen3.7-plus",
            "where": "language benchmark table (first table), Coding Agent block, row 'SWE-Pro', column 'Qwen3.7-Plus'"
          },
          {
            "event": "2026-06-01-qwen3-7-plus",
            "date": "2026-06-01",
            "lab": "alibaba",
            "value": 55.8,
            "printed": "55.8",
            "source": "https://qwen.ai/blog?id=qwen3.8-flash-next",
            "where": "later report: Qwen3.8-Flash-Next post, Language table (first table), row 'Agentic coding / SWE-bench Pro', column 'Qwen3.7-Plus'"
          }
        ]
      },
      {
        "title": "Terminal-Bench 1.0 by harness",
        "unit": "%",
        "rows": [
          {
            "event": "2025-05-22-claude-4",
            "date": "2025-05-22",
            "lab": "anthropic",
            "value": 43.2,
            "printed": "43.2%",
            "source": "https://www.anthropic.com/news/claude-4",
            "where": "main benchmark table (HTML drawn by script), row 'Agentic terminal coding / Terminal-bench', column 'Claude Opus 4', first number of the cell"
          },
          {
            "event": "2025-05-22-claude-4",
            "date": "2025-05-22",
            "lab": "anthropic",
            "value": 39.2,
            "printed": "39.2%",
            "source": "https://www.anthropic.com/news/claude-4",
            "where": "methodology footnote 2 under the main table"
          }
        ]
      },
      {
        "title": "Terminal-Bench 2.0 by harness",
        "unit": "%",
        "rows": [
          {
            "event": "2025-12-22-glm-4-7",
            "date": "2025-12-22",
            "lab": "zai",
            "value": 41.0,
            "printed": "41.0",
            "source": "https://z.ai/blog/glm-4.7",
            "where": "benchmark table, row 'Terminal Bench 2.0', column 'GLM-4.7'"
          },
          {
            "event": "2025-12-22-glm-4-7",
            "date": "2025-12-22",
            "lab": "zai",
            "value": 32.8,
            "printed": "32.8",
            "source": "https://z.ai/blog/glm-5",
            "where": "benchmark table, row 'Terminal-Bench 2.0 Claude Code', column 'GLM-4.7 (Thinking)'"
          },
          {
            "event": "2026-02-12-glm-5",
            "date": "2026-02-12",
            "lab": "zai",
            "value": 56.2,
            "printed": "56.2",
            "source": "https://z.ai/blog/glm-5",
            "where": "benchmark table, row 'Terminal-Bench 2.0 Terminus-2', column 'GLM-5 (Thinking)'"
          },
          {
            "event": "2026-02-12-glm-5",
            "date": "2026-02-12",
            "lab": "zai",
            "value": 56.2,
            "printed": "56.2",
            "source": "https://z.ai/blog/glm-5",
            "where": "benchmark table, row 'Terminal-Bench 2.0 Claude Code', column 'GLM-5 (Thinking)'"
          }
        ]
      },
      {
        "title": "Terminal-Bench 2.1 by harness",
        "unit": "%",
        "rows": [
          {
            "event": "2026-04-07-glm-5-1",
            "date": "2026-04-07",
            "lab": "zai",
            "value": 63.5,
            "printed": "63.5",
            "source": "https://z.ai/blog/glm-5.2",
            "where": "'Full Benchmark Table', row 'Terminal Bench 2.1 Terminus-2', column 'GLM-5.1'"
          },
          {
            "event": "2026-04-07-glm-5-1",
            "date": "2026-04-07",
            "lab": "zai",
            "value": 69.0,
            "printed": "69",
            "source": "https://z.ai/blog/glm-5.2",
            "where": "'Full Benchmark Table', row 'Terminal Bench 2.1 Best Reported Harness', column 'GLM-5.1'"
          },
          {
            "event": "2026-06-16-glm-5-2",
            "date": "2026-06-16",
            "lab": "zai",
            "value": 81.0,
            "printed": "81.0",
            "source": "https://z.ai/blog/glm-5.2",
            "where": "'Full Benchmark Table', row 'Terminal Bench 2.1 Terminus-2', column 'GLM-5.2'"
          },
          {
            "event": "2026-06-16-glm-5-2",
            "date": "2026-06-16",
            "lab": "zai",
            "value": 82.7,
            "printed": "82.7",
            "source": "https://z.ai/blog/glm-5.2",
            "where": "'Full Benchmark Table', row 'Terminal Bench 2.1 Best Reported Harness', column 'GLM-5.2'"
          },
          {
            "event": "2026-07-09-muse-spark-1-1",
            "date": "2026-07-09",
            "lab": "meta",
            "value": 80.0,
            "printed": "80.0",
            "source": "https://ai.meta.com/blog/introducing-muse-spark-meta-model-api/",
            "where": "benchmark table (image), row 'Terminal-Bench 2.1', column 'Muse Spark 1.1'"
          },
          {
            "event": "2026-07-09-muse-spark-1-1",
            "date": "2026-07-09",
            "lab": "meta",
            "value": 76.2,
            "printed": "76.2%",
            "source": "https://research.meta.ai/blog/introducing-muse-code-and-muse-spark-1-2",
            "where": "bar chart 'Terminal-Bench 2.1' (image, Coding tab), value label above the bar 'Muse Spark 1.1, mini-swe-agent'"
          }
        ]
      },
      {
        "title": "DeepSWE v1.1 by harness",
        "unit": "%",
        "rows": [
          {
            "event": "2026-08-05-muse-spark-1-2",
            "date": "2026-08-05",
            "lab": "meta",
            "value": 59.3,
            "printed": "59.3%",
            "source": "https://research.meta.ai/blog/introducing-muse-code-and-muse-spark-1-2",
            "where": "bar chart 'DeepSWE 1.1' (image, Coding tab), value label above the bar 'Muse Spark 1.2, Muse Code'"
          },
          {
            "event": "2026-08-05-muse-spark-1-2",
            "date": "2026-08-05",
            "lab": "meta",
            "value": 55.0,
            "printed": "55.0",
            "source": "https://research.meta.ai/blog/introducing-muse-spark-1-3",
            "where": "Muse Spark 1.3 scorecard (image), row 'DeepSWE v1.1', column 'Muse Spark 1.2 (xhigh)'"
          },
          {
            "event": "2026-07-16-kimi-k3",
            "date": "2026-07-16",
            "lab": "moonshot",
            "value": 67.5,
            "printed": "67.5",
            "source": "https://www.kimi.ai/blog/kimi-k3",
            "where": "'Full Benchmark Table', row 'DeepSWE' (Coding), column 'Kimi K3 (max)'"
          },
          {
            "event": "2026-07-16-kimi-k3",
            "date": "2026-07-16",
            "lab": "moonshot",
            "value": 67.3,
            "printed": "67.3",
            "source": "https://www.kimi.ai/blog/kimi-k3",
            "where": "Footnotes, 'Coding benchmarks', the 'DeepSWE.' note (score on the official DeepSWE leaderboard with the mini-SWE-agent harness)"
          }
        ]
      }
    ],
    "sources": [
      "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4/blob/8a115b9d966adce8c059a7bcf47f49a1e57957d8/README.md",
      "https://www.minimax.io/news/minimax-m21",
      "https://qwen.ai/blog?id=qwen3.7-plus",
      "https://qwen.ai/blog?id=qwen3.8-flash-next",
      "https://www.anthropic.com/news/claude-4",
      "https://z.ai/blog/glm-4.7",
      "https://z.ai/blog/glm-5",
      "https://z.ai/blog/glm-5.2",
      "https://ai.meta.com/blog/introducing-muse-spark-meta-model-api/",
      "https://research.meta.ai/blog/introducing-muse-code-and-muse-spark-1-2",
      "https://research.meta.ai/blog/introducing-muse-spark-1-3",
      "https://www.kimi.ai/blog/kimi-k3"
    ]
  }
}