{
  "api_version": "1",
  "updated": "2026-10-31",
  "latest_event": "2026-10-30",
  "rights": "© Frontier Log, all rights reserved.",
  "benchmark": {
    "slug": "ifbench",
    "name": "IFBench",
    "family": "ifbench",
    "unit": "%",
    "higher_is_better": true,
    "status": "active",
    "active": true,
    "superseded_by": null,
    "retired": null,
    "benchmark_url": "https://github.com/allenai/IFBench",
    "introduced": "2025-07-03",
    "creator": "Ai2 and University of Washington",
    "introduced_source": "https://arxiv.org/abs/2507.02833",
    "settings": [
      {
        "setting": "default",
        "name": "Prompt-level or unstated",
        "url": "https://frontierlog.ai/benchmark/ifbench",
        "results": 29
      },
      {
        "setting": "instruction-loose",
        "name": "Instruction-level loose",
        "url": "https://frontierlog.ai/benchmark/ifbench/instruction-loose",
        "results": 7
      },
      {
        "setting": "prompt-avg-strict",
        "name": "Constraints met per prompt, strict",
        "url": "https://frontierlog.ai/benchmark/ifbench/prompt-avg-strict",
        "results": 3
      }
    ]
  },
  "total": 3,
  "results": [
    {
      "id": "nemotron-3-ultra-550b-a55b:ifbench:prompt-avg-strict:1",
      "model": "nemotron-3-ultra-550b-a55b",
      "model_name": "Nemotron 3 Ultra 550B-A55B",
      "model_date": "2026-06-04",
      "event": "https://frontierlog.ai/event/2026-06-04-nemotron-3-ultra",
      "benchmark": "ifbench",
      "benchmark_name": "IFBench",
      "setting": "prompt-avg-strict",
      "setting_name": "Constraints met per prompt, strict",
      "score": 82.9,
      "unit": "%",
      "printed": "82.9",
      "reported_by": "nvidia",
      "run_by": null,
      "label": "Nemotron 3 Ultra",
      "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/blob/8350289dda26a68b7d7f5c8b8ee7f815b5d1bca9/README.md",
      "source_title": "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16 model card (revision of 2026-10-15)",
      "source_date": "2026-10-15",
      "where": "later report: Nemotron 3.5 Super VL BF16 model card (launch-day revision), 'Benchmarks' table, row 'IFBench', column 'Nemotron 3 Ultra'",
      "own": false,
      "primary": false,
      "parallel": false,
      "notes": "later report",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-31",
      "corroborated_by": [
        {
          "url": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/blob/1f8953f7f0935358e3605150d22bc15df062061c/README.md",
          "name": "Nemotron 3.5 Super VL and Bolt card",
          "title": "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16 model card (revision of 2026-10-16)",
          "lab": "nvidia",
          "date": "2026-10-15",
          "printed": "82.9",
          "where": "Benchmarks table, row 'IFBench', column Nemotron 3 Ultra (revision 2026-10-16)"
        },
        {
          "url": "https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3.5-Super-and-Lightning-Technical-Report.pdf",
          "name": "Nemotron 3.5 Super VL and Bolt card",
          "title": "Nemotron 3.5 Super and Lightning: Open, Efficient Mixture-of-Experts Hybrid Mamba-Transformer Models for Agentic AI",
          "lab": "nvidia",
          "date": "2026-10-15",
          "printed": "82.94",
          "where": "later report: Nemotron 3.5 Super and Lightning technical report, Table 10 (PDF page 23), row 'IFBench', column 'N3-Ultra 550B-A55B' (second column)"
        }
      ],
      "source_capture": "https://frontierlog.ai/captures/99b6b838a462495b.png"
    },
    {
      "id": "nemotron-3-ultra-550b-a55b:ifbench:prompt-avg-strict:2",
      "model": "nemotron-3-ultra-550b-a55b",
      "model_name": "Nemotron 3 Ultra 550B-A55B",
      "model_date": "2026-06-04",
      "event": "https://frontierlog.ai/event/2026-06-04-nemotron-3-ultra",
      "benchmark": "ifbench",
      "benchmark_name": "IFBench",
      "setting": "prompt-avg-strict",
      "setting_name": "Constraints met per prompt, strict",
      "score": 82.94,
      "unit": "%",
      "printed": "82.94",
      "reported_by": "nvidia",
      "run_by": null,
      "label": "N3-Ultra 550B-A55B",
      "source": "https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3.5-Super-and-Lightning-Technical-Report.pdf",
      "source_title": "Nemotron 3.5 Super and Lightning: Open, Efficient Mixture-of-Experts Hybrid Mamba-Transformer Models for Agentic AI",
      "source_date": "2026-10-15",
      "where": "later report: Nemotron 3.5 Super and Lightning technical report, Table 10 (PDF page 23), row 'IFBench', column 'N3-Ultra 550B-A55B' (second column)",
      "own": false,
      "primary": true,
      "parallel": false,
      "notes": "later report",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-31",
      "corroborated_by": [
        {
          "url": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/blob/8350289dda26a68b7d7f5c8b8ee7f815b5d1bca9/README.md",
          "name": "Nemotron 3.5 Super VL and Bolt card",
          "title": "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16 model card (revision of 2026-10-15)",
          "lab": "nvidia",
          "date": "2026-10-15",
          "printed": "82.9",
          "where": "later report: Nemotron 3.5 Super VL BF16 model card (launch-day revision), 'Benchmarks' table, row 'IFBench', column 'Nemotron 3 Ultra'"
        }
      ],
      "source_capture": "https://frontierlog.ai/captures/ee443a24cf8af62e.png"
    },
    {
      "id": "nemotron-3-5-super-vl-120b-a12b:ifbench:prompt-avg-strict:1",
      "model": "nemotron-3-5-super-vl-120b-a12b",
      "model_name": "Nemotron 3.5 Super VL 120B-A12B",
      "model_date": "2026-10-15",
      "event": "https://frontierlog.ai/event/2026-10-15-nemotron-3-5-super-vl-bolt",
      "benchmark": "ifbench",
      "benchmark_name": "IFBench",
      "setting": "prompt-avg-strict",
      "setting_name": "Constraints met per prompt, strict",
      "score": 70.86,
      "unit": "%",
      "printed": "70.86",
      "reported_by": "nvidia",
      "run_by": null,
      "label": "Nemotron 3.5 SuperGA BF16",
      "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/blob/8350289dda26a68b7d7f5c8b8ee7f815b5d1bca9/README.md",
      "source_title": "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16 model card (revision of 2026-10-15)",
      "source_date": "2026-10-15",
      "where": "model card (launch-day revision), 'Benchmarks' table, row 'IFBench', column 'Nemotron 3.5 SuperGA BF16'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-31",
      "corroborated_by": [
        {
          "url": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/blob/1f8953f7f0935358e3605150d22bc15df062061c/README.md",
          "name": "Nemotron 3.5 Super VL and Bolt card",
          "title": "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16 model card (revision of 2026-10-16)",
          "lab": "nvidia",
          "date": "2026-10-15",
          "printed": "70.86",
          "where": "Benchmarks table, row 'IFBench', column Nemotron 3.5 SuperGA BF16 (revision 2026-10-16)"
        },
        {
          "url": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4/blob/59b8721646c38111a242da18218859fb2b454322/README.md",
          "name": "Nemotron 3.5 Super VL and Bolt card",
          "title": "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4 model card (revision of 2026-10-16)",
          "lab": "nvidia",
          "date": "2026-10-15",
          "printed": "70.86",
          "where": "BF16 vs NVFP4 table, row 'IFBench', column Nemotron 3.5 Super VL BF16"
        },
        {
          "url": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4/blob/8a115b9d966adce8c059a7bcf47f49a1e57957d8/README.md",
          "name": "Nemotron 3.5 Super VL and Bolt card",
          "title": "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4 model card (launch-day revision)",
          "lab": "nvidia",
          "date": "2026-10-15",
          "printed": "70.86",
          "where": "BF16 vs NVFP4 table, row 'IFBench', column Nemotron 3.5 Super VL BF16"
        },
        {
          "url": "https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3.5-Super-and-Lightning-Technical-Report.pdf",
          "name": "Nemotron 3.5 Super VL and Bolt card",
          "title": "Nemotron 3.5 Super and Lightning: Open, Efficient Mixture-of-Experts Hybrid Mamba-Transformer Models for Agentic AI",
          "lab": "nvidia",
          "date": "2026-10-15",
          "printed": "70.86",
          "where": "Table 10, row 'IFBench', column N3.5-Super 120B-A12B (text says fraction of constraints per prompt under strict verification)"
        }
      ],
      "source_capture": "https://frontierlog.ai/captures/99b6b838a462495b.png"
    }
  ]
}