{
  "api_version": "1",
  "updated": "2026-10-31",
  "latest_event": "2026-10-30",
  "rights": "© Frontier Log, all rights reserved.",
  "benchmark": {
    "slug": "ifbench",
    "name": "IFBench",
    "family": "ifbench",
    "unit": "%",
    "higher_is_better": true,
    "status": "active",
    "active": true,
    "superseded_by": null,
    "retired": null,
    "benchmark_url": "https://github.com/allenai/IFBench",
    "introduced": "2025-07-03",
    "creator": "Ai2 and University of Washington",
    "introduced_source": "https://arxiv.org/abs/2507.02833",
    "settings": [
      {
        "setting": "default",
        "name": "Prompt-level or unstated",
        "url": "https://frontierlog.ai/benchmark/ifbench",
        "results": 29
      },
      {
        "setting": "instruction-loose",
        "name": "Instruction-level loose",
        "url": "https://frontierlog.ai/benchmark/ifbench/instruction-loose",
        "results": 7
      },
      {
        "setting": "prompt-avg-strict",
        "name": "Constraints met per prompt, strict",
        "url": "https://frontierlog.ai/benchmark/ifbench/prompt-avg-strict",
        "results": 3
      }
    ]
  },
  "total": 7,
  "results": [
    {
      "id": "nemotron-3-nano-30b-a3b:ifbench:instruction-loose:1",
      "model": "nemotron-3-nano-30b-a3b",
      "model_name": "Nemotron 3 Nano 30B-A3B",
      "model_date": "2025-12-15",
      "event": "https://frontierlog.ai/event/2025-12-15-nemotron-3",
      "benchmark": "ifbench",
      "benchmark_name": "IFBench",
      "setting": "instruction-loose",
      "setting_name": "Instruction-level loose",
      "score": 72.17,
      "unit": "%",
      "printed": "72.17",
      "reported_by": "nvidia",
      "run_by": null,
      "label": "Nemotron 3 Nano*",
      "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16/blob/942b56629bd9b52c6813bb5f68bb5d10ba8cca68/README.md",
      "source_title": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 model card (revision of 2026-08-11)",
      "source_date": "2026-08-11",
      "where": "later report: Nemotron 3.5 Lightning model card, 'Reasoning Benchmark Evaluations' table, row 'IFBench (loose)', column 'Nemotron 3 Nano*'",
      "own": false,
      "primary": true,
      "parallel": false,
      "notes": "later re-run",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-31",
      "corroborated_by": [
        {
          "url": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16/blob/a9904d24bcc1d289a1950fa9d2b978c47cf903b9/README.md",
          "name": "Nemotron 3.5 Lightning card",
          "title": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 model card (revision of 2026-08-24)",
          "lab": "nvidia",
          "date": "2026-08-11",
          "printed": "72.17",
          "where": "benchmark table, row 'IFBench (loose)', column Nemotron 3 Nano* (revision 2026-08-24)"
        }
      ],
      "source_capture": "https://frontierlog.ai/captures/2dce41d96078aa86.png"
    },
    {
      "id": "nemotron-3-super-120b-a12b:ifbench:instruction-loose:1",
      "model": "nemotron-3-super-120b-a12b",
      "model_name": "Nemotron 3 Super 120B-A12B",
      "model_date": "2026-03-11",
      "event": "https://frontierlog.ai/event/2026-03-11-nemotron-3-super",
      "benchmark": "ifbench",
      "benchmark_name": "IFBench",
      "setting": "instruction-loose",
      "setting_name": "Instruction-level loose",
      "score": 71.92,
      "unit": "%",
      "printed": "71.92",
      "reported_by": "nvidia",
      "run_by": null,
      "label": "Nemotron 3 Super*",
      "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16/blob/942b56629bd9b52c6813bb5f68bb5d10ba8cca68/README.md",
      "source_title": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 model card (revision of 2026-08-11)",
      "source_date": "2026-08-11",
      "where": "later report: Nemotron 3.5 Lightning model card, 'Reasoning Benchmark Evaluations' table, row 'IFBench (loose)', column 'Nemotron 3 Super*'",
      "own": false,
      "primary": true,
      "parallel": false,
      "notes": "later re-run",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-31",
      "corroborated_by": [
        {
          "url": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16/blob/a9904d24bcc1d289a1950fa9d2b978c47cf903b9/README.md",
          "name": "Nemotron 3.5 Lightning card",
          "title": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 model card (revision of 2026-08-24)",
          "lab": "nvidia",
          "date": "2026-08-11",
          "printed": "71.92",
          "where": "benchmark table, row 'IFBench (loose)', column Nemotron 3 Super* (revision 2026-08-24)"
        },
        {
          "url": "https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-Technical-Report.pdf",
          "name": "Nemotron 3.5 Super VL and Bolt card",
          "title": "Nemotron 3.5 Bolt 74B-A9B 24GB: A Hybrid Vision-Language Model for Local AI",
          "lab": "nvidia",
          "date": "2026-10-15",
          "printed": "71.92",
          "where": "Table 7, row 'IFBench (instruction loose)', column Nemotron 3 Super 120B-A12B-BF16"
        }
      ],
      "source_capture": "https://frontierlog.ai/captures/2dce41d96078aa86.png"
    },
    {
      "id": "nemotron-3-5-lightning-30b-a3b:ifbench:instruction-loose:1",
      "model": "nemotron-3-5-lightning-30b-a3b",
      "model_name": "Nemotron 3.5 Lightning 30B-A3B",
      "model_date": "2026-08-11",
      "event": "https://frontierlog.ai/event/2026-08-11-nemotron-3-5-lightning",
      "benchmark": "ifbench",
      "benchmark_name": "IFBench",
      "setting": "instruction-loose",
      "setting_name": "Instruction-level loose",
      "score": 71.88,
      "unit": "%",
      "printed": "71.88",
      "reported_by": "nvidia",
      "run_by": null,
      "label": "Nemotron-3.5-Lightning-30B-A3B-BF16",
      "source": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16/blob/942b56629bd9b52c6813bb5f68bb5d10ba8cca68/README.md",
      "source_title": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 model card (revision of 2026-08-11)",
      "source_date": "2026-08-11",
      "where": "model card, 'Reasoning Benchmark Evaluations' table, row 'IFBench (loose)', column 'Nemotron-3.5-Lightning-30B-A3B-BF16'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "BF16 weights",
      "caveat": "",
      "human_verified": false,
      "human_verified_on": null,
      "check": "corroborated",
      "corroborated": true,
      "corroborated_on": "2026-10-31",
      "corroborated_by": [
        {
          "url": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16/blob/a9904d24bcc1d289a1950fa9d2b978c47cf903b9/README.md",
          "name": "Nemotron 3.5 Lightning card",
          "title": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 model card (revision of 2026-08-24)",
          "lab": "nvidia",
          "date": "2026-08-11",
          "printed": "71.88",
          "where": "benchmark table, row 'IFBench (loose)', column Nemotron-3.5-Lightning-30B-A3B-BF16 (later revision 2026-08-24)"
        },
        {
          "url": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4/blob/b14872a58afd94e2fbbc1e5a1fe4ebb7dfdc5fdd/README.md",
          "name": "Nemotron 3.5 Lightning card",
          "title": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 model card (launch-day revision)",
          "lab": "nvidia",
          "date": "2026-08-11",
          "printed": "71.88",
          "where": "BF16 vs NVFP4 table, row 'IFBench (loose)', column Nemotron-3.5-Lightning-30B-A3B-BF16"
        }
      ],
      "source_capture": "https://frontierlog.ai/captures/2dce41d96078aa86.png"
    },
    {
      "id": "nemotron-3-5-lightning-30b-a3b:ifbench:instruction-loose:2",
      "model": "nemotron-3-5-lightning-30b-a3b",
      "model_name": "Nemotron 3.5 Lightning 30B-A3B",
      "model_date": "2026-08-11",
      "event": "https://frontierlog.ai/event/2026-08-11-nemotron-3-5-lightning",
      "benchmark": "ifbench",
      "benchmark_name": "IFBench",
      "setting": "instruction-loose",
      "setting_name": "Instruction-level loose",
      "score": 72.88,
      "unit": "%",
      "printed": "72.88",
      "reported_by": "nvidia",
      "run_by": null,
      "label": "Nemotron 3.5 Lightning 30B-A3B",
      "source": "https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-Technical-Report.pdf",
      "source_title": "Nemotron 3.5 Bolt 74B-A9B 24GB: A Hybrid Vision-Language Model for Local AI",
      "source_date": "2026-10-15",
      "where": "later report: Nemotron 3.5 Bolt technical report, Table 7 (PDF page 16), row 'IFBench (instruction loose)', column 'Nemotron 3.5 Lightning 30B-A3B'",
      "own": false,
      "primary": false,
      "parallel": false,
      "notes": "later re-run",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-31",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": []
    },
    {
      "id": "nemotron-3-5-bolt:ifbench:instruction-loose:1",
      "model": "nemotron-3-5-bolt",
      "model_name": "Nemotron 3.5 Bolt",
      "model_date": "2026-10-15",
      "event": "https://frontierlog.ai/event/2026-10-15-nemotron-3-5-super-vl-bolt",
      "benchmark": "ifbench",
      "benchmark_name": "IFBench",
      "setting": "instruction-loose",
      "setting_name": "Instruction-level loose",
      "score": 68.71,
      "unit": "%",
      "printed": "68.71",
      "reported_by": "nvidia",
      "run_by": null,
      "label": "Nemotron 3.5 Bolt 74B-A9B-BF16",
      "source": "https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-Technical-Report.pdf",
      "source_title": "Nemotron 3.5 Bolt 74B-A9B 24GB: A Hybrid Vision-Language Model for Local AI",
      "source_date": "2026-10-15",
      "where": "Table 7 (PDF page 16), row 'IFBench (instruction loose)', column 'Nemotron 3.5 Bolt 74B-A9B-BF16'",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "BF16 weights",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-31",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": []
    },
    {
      "id": "nemotron-3-5-bolt:ifbench:instruction-loose:2",
      "model": "nemotron-3-5-bolt",
      "model_name": "Nemotron 3.5 Bolt",
      "model_date": "2026-10-15",
      "event": "https://frontierlog.ai/event/2026-10-15-nemotron-3-5-super-vl-bolt",
      "benchmark": "ifbench",
      "benchmark_name": "IFBench",
      "setting": "instruction-loose",
      "setting_name": "Instruction-level loose",
      "score": 66.17,
      "unit": "%",
      "printed": "66.17",
      "reported_by": "nvidia",
      "run_by": null,
      "label": "Nemotron 3.5 Bolt 74B-A9B-24GB",
      "source": "https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-Technical-Report.pdf",
      "source_title": "Nemotron 3.5 Bolt 74B-A9B 24GB: A Hybrid Vision-Language Model for Local AI",
      "source_date": "2026-10-15",
      "where": "Table 7 (PDF page 16), row 'IFBench (instruction loose)', column 'Nemotron 3.5 Bolt 74B-A9B-24GB'",
      "own": true,
      "primary": false,
      "parallel": false,
      "notes": "24GB build with 2-bit (IQ2) weights",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-31",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": []
    },
    {
      "id": "nemotron-3-5-super-vl-120b-a12b:ifbench:instruction-loose:1",
      "model": "nemotron-3-5-super-vl-120b-a12b",
      "model_name": "Nemotron 3.5 Super VL 120B-A12B",
      "model_date": "2026-10-15",
      "event": "https://frontierlog.ai/event/2026-10-15-nemotron-3-5-super-vl-bolt",
      "benchmark": "ifbench",
      "benchmark_name": "IFBench",
      "setting": "instruction-loose",
      "setting_name": "Instruction-level loose",
      "score": 73.04,
      "unit": "%",
      "printed": "73.04",
      "reported_by": "nvidia",
      "run_by": null,
      "label": "Nemotron 3.5 Super 120B-A12B-BF16",
      "source": "https://research.nvidia.com/labs/nemotron/files/NVIDIA-Nemotron-3.5-Bolt-74B-A9B-24GB-Technical-Report.pdf",
      "source_title": "Nemotron 3.5 Bolt 74B-A9B 24GB: A Hybrid Vision-Language Model for Local AI",
      "source_date": "2026-10-15",
      "where": "Nemotron 3.5 Bolt technical report, Table 7 (PDF page 16), row 'IFBench (instruction loose)', column 'Nemotron 3.5 Super 120B-A12B-BF16' (first column)",
      "own": true,
      "primary": true,
      "parallel": false,
      "notes": "",
      "caveat": "",
      "human_verified": true,
      "human_verified_on": "2026-10-31",
      "check": "human",
      "corroborated": false,
      "corroborated_on": null,
      "corroborated_by": []
    }
  ]
}