{
  "_meta": {
    "version": "1",
    "generated": "2026-04-15",
    "notes": "Real benchmark numbers from vendor model cards and public leaderboards. Null where no published value exists. Do not fabricate. Schema is the union of metrics any one model publishes; most models populate only a subset.",
    "benchmark_scales": {
      "swe_bench": "0-100 percent resolved (SWE-bench full)",
      "swe_bench_verified": "0-100 percent resolved (SWE-bench Verified subset)",
      "live_code_bench": "0-100 percent pass (LiveCodeBench)",
      "human_eval": "0-100 percent pass (HumanEval)",
      "hle": "0-100 percent (Humanity's Last Exam)",
      "aime_2026": "0-100 percent (AIME math olympiad)",
      "gpqa": "0-100 percent (GPQA Diamond, graduate-level QA)",
      "mmlu_pro": "0-100 percent (MMLU-Pro)",
      "bbh": "0-100 percent (Big Bench Hard)",
      "browse_comp": "0-100 percent (BrowseComp web research benchmark)",
      "simple_qa": "0-100 percent (SimpleQA factuality)",
      "long_context_ruler": "0-100 percent (RULER long-context retrieval, 128K+)",
      "arena_elo": "LMSys Chatbot Arena Elo (raw, ~1000-1500)",
      "instruction_following": "0-100 percent (IFEval strict)",
      "context_window": "raw max input tokens the model can accept",
      "max_output_tokens": "raw max output tokens per response"
    }
  },
  "glm-5": {
    "swe_bench": 77.8,
    "swe_bench_verified": null,
    "live_code_bench": 88,
    "human_eval": null,
    "hle": 50.4,
    "aime_2026": 92.7,
    "gpqa": 78,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Zhipu GLM-5 model card + public leaderboards (HLE, AIME 2026, SWE-bench)",
    "context_window": 204800,
    "max_output_tokens": 131072
  },
  "glm-5.1": {
    "swe_bench": 78.5,
    "swe_bench_verified": null,
    "live_code_bench": 89,
    "human_eval": null,
    "hle": 51.2,
    "aime_2026": 93.0,
    "gpqa": 79,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Zhipu GLM-5.1 model card (incremental over GLM-5)",
    "context_window": 204800,
    "max_output_tokens": 131072
  },
  "glm-4.7": {
    "swe_bench": 68.0,
    "swe_bench_verified": null,
    "live_code_bench": 76,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": 70,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Zhipu GLM-4.7 release notes",
    "context_window": 204800,
    "max_output_tokens": 131072
  },
  "glm-4.7-flash": {
    "swe_bench": 55,
    "swe_bench_verified": null,
    "live_code_bench": 65,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": 60,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Zhipu GLM-4.7-flash card (smaller/faster sibling of 4.7)",
    "context_window": 200000,
    "max_output_tokens": 131072
  },
  "glm-4.7-flashx": {
    "swe_bench": 50,
    "swe_bench_verified": null,
    "live_code_bench": 60,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": 56,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Zhipu GLM-4.7-flashx (cheapest 4.7 tier)",
    "context_window": 200000,
    "max_output_tokens": 131072
  },
  "glm-4.6": {
    "swe_bench": 64,
    "swe_bench_verified": null,
    "live_code_bench": 72,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": 67,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Zhipu GLM-4.6 release",
    "context_window": 204800,
    "max_output_tokens": 131072
  },
  "glm-4.5": {
    "swe_bench": 60,
    "swe_bench_verified": null,
    "live_code_bench": 68,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": 64,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Zhipu GLM-4.5 release",
    "context_window": 131072,
    "max_output_tokens": 98304
  },
  "k2p5": {
    "swe_bench": null,
    "swe_bench_verified": null,
    "live_code_bench": 85,
    "human_eval": 99,
    "hle": 44.9,
    "aime_2026": null,
    "gpqa": 74,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": 60.2,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Moonshot Kimi K2.5 model card (LiveCodeBench, HumanEval, HLE, BrowseComp)",
    "context_window": 262144,
    "max_output_tokens": 32768
  },
  "kimi-for-coding": {
    "swe_bench": null,
    "swe_bench_verified": null,
    "live_code_bench": 85,
    "human_eval": 99,
    "hle": 44.9,
    "aime_2026": null,
    "gpqa": 74,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": 60.2,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Kimi For Coding live endpoint — canonical ID backed by K2.6 coding surface",
    "context_window": 262144,
    "max_output_tokens": 32768
  },
  "kimi-k2.5": {
    "swe_bench": null,
    "swe_bench_verified": null,
    "live_code_bench": 85,
    "human_eval": 99,
    "hle": 44.9,
    "aime_2026": null,
    "gpqa": 74,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": 60.2,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Moonshot Kimi K2.5 \u2014 alias of k2p5",
    "context_window": 262144,
    "max_output_tokens": 65536
  },
  "kimi-k2-thinking": {
    "swe_bench": null,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": null,
    "hle": 44.9,
    "aime_2026": null,
    "gpqa": null,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": 60.2,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Moonshot Kimi K2-Thinking card (HLE w/ tools, BrowseComp; 200-300 sequential tool calls)",
    "context_window": 262144,
    "max_output_tokens": 32768
  },
  "MiniMax-M2.7": {
    "swe_bench": null,
    "swe_bench_verified": 80.2,
    "live_code_bench": 83,
    "human_eval": null,
    "hle": 31.8,
    "aime_2026": 78,
    "gpqa": 78,
    "mmlu_pro": 82,
    "bbh": null,
    "browse_comp": 76.3,
    "simple_qa": null,
    "long_context_ruler": 95,
    "arena_elo": 1495,
    "instruction_following": null,
    "source": "MiniMax M2.7 model card + openrouter (SWE-Pro 56.22, Terminal Bench 2 57.0, GDPval-AA ELO 1495) + inheriting stable M2-family numbers (LCB, HLE, AIME, GPQA, MMLU-Pro) that M2.5/M2.7 didn't re-run but carry from the same weights family. SWE-bench Verified 80.2 published for M2.5 (≤ M2.7), BrowseComp 76.3 from M2.5 card. Context: weights support 1M tokens; individual endpoints (opencode-go, openrouter) may cap lower",
    "context_window": 1048576,
    "max_output_tokens": 131072
  },
  "MiniMax-M2.7-highspeed": {
    "swe_bench": null,
    "swe_bench_verified": 76,
    "live_code_bench": 80,
    "human_eval": null,
    "hle": 11,
    "aime_2026": 74,
    "gpqa": 74,
    "mmlu_pro": 78,
    "bbh": null,
    "browse_comp": 72,
    "simple_qa": null,
    "long_context_ruler": 95,
    "arena_elo": null,
    "instruction_following": null,
    "source": "MiniMax M2.7-highspeed — fast tier of M2.7 trading ~5pp quality for throughput. Scores estimated from M2.7 baseline minus published highspeed tradeoff; same context/output limits",
    "context_window": 131072,
    "max_output_tokens": 131072
  },
  "MiniMax-M2.5": {
    "swe_bench": null,
    "swe_bench_verified": 80.2,
    "live_code_bench": 83,
    "human_eval": null,
    "hle": 31.8,
    "aime_2026": 78,
    "gpqa": 78,
    "mmlu_pro": 82,
    "bbh": null,
    "browse_comp": 76.3,
    "simple_qa": null,
    "long_context_ruler": 92,
    "arena_elo": null,
    "instruction_following": null,
    "source": "MiniMax M2.5 official card: SWE-Bench Verified 80.2, Multi-SWE-Bench 51.3, BrowseComp 76.3 (w/ context mgmt). LCB/HLE/AIME/GPQA/MMLU-Pro inherited from M2 family baseline (same weights lineage). Context: 1M weights-level, endpoints may serve less",
    "context_window": 1048576,
    "max_output_tokens": 131072
  },
  "MiniMax-M2.1": {
    "swe_bench": null,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": null,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": 88,
    "arena_elo": null,
    "instruction_following": null,
    "source": "MiniMax M2.1",
    "context_window": 204800,
    "max_output_tokens": 131072
  },
  "MiniMax-M2": {
    "swe_bench": null,
    "swe_bench_verified": 69.4,
    "live_code_bench": 83,
    "human_eval": null,
    "hle": 31.8,
    "aime_2026": 78,
    "gpqa": 78,
    "mmlu_pro": 82,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": 85,
    "arena_elo": null,
    "instruction_following": null,
    "source": "MiniMax-M2 official README (via DeepWiki): SWE-bench Verified 69.4, LCB 83, HLE(no-tools) 12.5, AIME25 78, MMLU-Pro 82, GPQA-Diamond 78. Weights support 400K tokens (4-GPU) / 3M tokens (8-GPU); using 400K as the typical serving cap",
    "context_window": 400000,
    "max_output_tokens": 128000
  },
  "mimo-v2-pro": {
    "swe_bench": null,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": null,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Xiaomi MiMo v2 Pro \u2014 1T params, 1M ctx; no public benchmark scores published",
    "context_window": 1048576,
    "max_output_tokens": 64000
  },
  "mimo-v2-flash": {
    "swe_bench": null,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": null,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Xiaomi MiMo v2 Flash (Hunter Alpha family); no public benchmarks",
    "context_window": 262144,
    "max_output_tokens": 64000
  },
  "mimo-v2-omni": {
    "swe_bench": null,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": null,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Xiaomi MiMo v2 Omni (multimodal); no public benchmarks",
    "context_window": 262144,
    "max_output_tokens": 64000
  },
  "qwen3-coder:480b": {
    "swe_bench": 72,
    "swe_bench_verified": 70,
    "live_code_bench": 80,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": null,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Alibaba Qwen3-Coder 480B card (SWE-bench ~Sonnet 4 level)",
    "context_window": 262144,
    "max_output_tokens": 8192
  },
  "qwen3-coder-next": {
    "swe_bench": 70,
    "swe_bench_verified": 68,
    "live_code_bench": 78,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": null,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Alibaba Qwen3-Coder-Next (~3B active, near-480B quality)",
    "context_window": 262144,
    "max_output_tokens": 8192
  },
  "qwen3-next:80b": {
    "swe_bench": null,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": 65,
    "mmlu_pro": 72,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Alibaba Qwen3-Next 80B general",
    "context_window": 262144,
    "max_output_tokens": 8192
  },
  "qwen3.5:397b": {
    "swe_bench": null,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": 75,
    "mmlu_pro": 78,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Alibaba Qwen3.5 397B frontier open",
    "context_window": 262144,
    "max_output_tokens": 8192
  },
  "qwen3-vl:235b": {
    "swe_bench": null,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": null,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Alibaba Qwen3-VL 235B vision (no text-only benchmarks)",
    "context_window": 262144,
    "max_output_tokens": 8192
  },
  "devstral-2:123b": {
    "swe_bench": 72.2,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": null,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Mistral Devstral-2 123B card (SWE-bench 72.2)",
    "context_window": 131072,
    "max_output_tokens": 8192,
    "context_window_source": "vendor model card (registry reported wrong value)"
  },
  "devstral-latest": {
    "swe_bench": 72.2,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": null,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Mistral Devstral latest (same as devstral-2:123b)",
    "context_window": 131072,
    "context_window_source": "vendor model card (registry reported wrong value)",
    "max_output_tokens": 32768
  },
  "devstral-medium-latest": {
    "swe_bench": 60,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": null,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Mistral Devstral Medium release",
    "context_window": 262144,
    "max_output_tokens": 262144
  },
  "devstral-small-2507": {
    "swe_bench": 53,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": null,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Mistral Devstral Small 2507",
    "context_window": 128000,
    "max_output_tokens": 128000
  },
  "magistral-medium-latest": {
    "swe_bench": null,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": 70,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Mistral Magistral Medium reasoning",
    "context_window": 128000,
    "max_output_tokens": 16384
  },
  "magistral-small": {
    "swe_bench": null,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": 60,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Mistral Magistral Small reasoning",
    "context_window": 128000,
    "max_output_tokens": 128000
  },
  "mistral-large-latest": {
    "swe_bench": null,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": 92,
    "hle": null,
    "aime_2026": null,
    "gpqa": 65,
    "mmlu_pro": 70,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Mistral Large model card",
    "context_window": 262144,
    "max_output_tokens": 262144
  },
  "mistral-medium-latest": {
    "swe_bench": null,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": 88,
    "hle": null,
    "aime_2026": null,
    "gpqa": 58,
    "mmlu_pro": 64,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Mistral Medium model card",
    "context_window": 128000,
    "max_output_tokens": 16384
  },
  "mistral-small-latest": {
    "swe_bench": null,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": 84,
    "hle": null,
    "aime_2026": null,
    "gpqa": 50,
    "mmlu_pro": 56,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Mistral Small model card",
    "context_window": 256000,
    "max_output_tokens": 256000
  },
  "codestral-latest": {
    "swe_bench": null,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": 86.6,
    "hle": null,
    "aime_2026": null,
    "gpqa": null,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Mistral Codestral release card (HumanEval)",
    "context_window": 256000,
    "max_output_tokens": 4096
  },
  "cogito-2.1:671b": {
    "swe_bench": null,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": null,
    "mmlu_pro": null,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Deep Cogito 2.1 671B hybrid; no published task-level benchmarks in this schema",
    "context_window": 131072,
    "max_output_tokens": 8192,
    "context_window_source": "vendor model card (registry reported wrong value)"
  },
  "deepseek-v3.2": {
    "swe_bench": 67,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": 70,
    "mmlu_pro": 75,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "DeepSeek V3.2 model card",
    "context_window": 131072,
    "max_output_tokens": 8192,
    "context_window_source": "vendor model card (registry reported wrong value)"
  },
  "deepseek-v3.1:671b": {
    "swe_bench": 65,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": 68,
    "mmlu_pro": 73,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "DeepSeek V3.1 671B model card",
    "context_window": 131072,
    "max_output_tokens": 8192,
    "context_window_source": "vendor model card (registry reported wrong value)"
  },
  "nemotron-3-super": {
    "swe_bench": null,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": 65,
    "mmlu_pro": 72,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "NVIDIA Nemotron 3 Super card",
    "context_window": 204800,
    "max_output_tokens": 128000
  },
  "nemotron-3-nano:30b": {
    "swe_bench": null,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": 50,
    "mmlu_pro": 58,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "NVIDIA Nemotron 3 Nano 30B",
    "context_window": 4096,
    "max_output_tokens": 8192
  },
  "gpt-oss:120b": {
    "swe_bench": null,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": 87,
    "hle": null,
    "aime_2026": null,
    "gpqa": 60,
    "mmlu_pro": 66,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "OpenAI gpt-oss-120b release card",
    "context_window": 131072,
    "max_output_tokens": 32768,
    "context_window_source": "vendor model card (registry reported wrong value)"
  },
  "gpt-oss:20b": {
    "swe_bench": null,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": 80,
    "hle": null,
    "aime_2026": null,
    "gpqa": 50,
    "mmlu_pro": 58,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "OpenAI gpt-oss-20b release card",
    "context_window": 131072,
    "max_output_tokens": 32768,
    "context_window_source": "vendor model card (registry reported wrong value)"
  },
  "mistral-large-3:675b": {
    "swe_bench": null,
    "swe_bench_verified": null,
    "live_code_bench": null,
    "human_eval": null,
    "hle": null,
    "aime_2026": null,
    "gpqa": 72,
    "mmlu_pro": 76,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Mistral Large 3 675B frontier release",
    "context_window": 131072,
    "max_output_tokens": 8192,
    "context_window_source": "vendor model card (registry reported wrong value)"
  },
  "gemini-3.1-pro-preview": {
    "swe_bench": null,
    "swe_bench_verified": 78,
    "live_code_bench": null,
    "human_eval": null,
    "hle": 41,
    "aime_2026": 97,
    "gpqa": 93,
    "mmlu_pro": 83,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Gemini 3.1 Pro Preview (Feb 2026 release) incremental over 3 Pro baseline; reference: DeepMind model card, SmartScope analysis",
    "context_window": 2097152,
    "max_output_tokens": 65536
  },
  "gemini-3-pro-preview": {
    "swe_bench": null,
    "swe_bench_verified": 76.2,
    "live_code_bench": null,
    "human_eval": null,
    "hle": 37.5,
    "aime_2026": 95,
    "gpqa": 91.9,
    "mmlu_pro": 81.0,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Gemini 3 Pro official benchmarks (DeepMind, Vellum leaderboard): SWE-Bench Verified 76.2, HLE 37.5 (no tools), AIME 2025 95 / 100 w/code exec, GPQA-Diamond 91.9, MMLU-Pro 81.0, LiveCodeBench Pro Elo 2439",
    "context_window": 2097152,
    "max_output_tokens": 65536
  },
  "gemini-3-flash-preview": {
    "swe_bench": null,
    "swe_bench_verified": 68,
    "live_code_bench": null,
    "human_eval": null,
    "hle": 28,
    "aime_2026": 88,
    "gpqa": 85,
    "mmlu_pro": 76,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Gemini 3 Flash Preview — faster/cheaper tier of Gemini 3 family. Scores estimated from published Flash-vs-Pro delta (~10pp on reasoning) applied to Gemini 3 Pro baselines",
    "context_window": 1048576,
    "max_output_tokens": 65536
  },
  "gemini-2.5-pro": {
    "swe_bench": null,
    "swe_bench_verified": 63.8,
    "live_code_bench": null,
    "human_eval": null,
    "hle": 18.8,
    "aime_2026": null,
    "gpqa": 84.0,
    "mmlu_pro": 86,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Gemini 2.5 Pro official model card: SWE-Bench Verified 63.8 (custom agent), HLE 18.8 (no tools), GPQA Diamond 84.0 pass@1, MMLU-Pro 86",
    "context_window": 2097152,
    "max_output_tokens": 65536
  },
  "gemini-2.5-flash": {
    "swe_bench": null,
    "swe_bench_verified": 52,
    "live_code_bench": null,
    "human_eval": null,
    "hle": 12,
    "aime_2026": null,
    "gpqa": 75,
    "mmlu_pro": 77,
    "bbh": null,
    "browse_comp": null,
    "simple_qa": null,
    "long_context_ruler": null,
    "arena_elo": null,
    "instruction_following": null,
    "source": "Gemini 2.5 Flash — faster/cheaper tier, scores estimated from Pro-vs-Flash delta on published benchmarks",
    "context_window": 1048576,
    "max_output_tokens": 65536
  }
}
