fix: add openai-codex provider and modern OpenAI models to MODEL_CAPABILITY_TIER and cost tables (#3070)
Closes #2885 The MODEL_CAPABILITY_TIER map in model-router.ts and the BUNDLED_COST_TABLE in model-cost-table.ts were missing all openai-codex provider models (gpt-5.1, gpt-5.2, gpt-5.3-codex, gpt-5.4, etc.) and modern OpenAI models (o4-mini, gpt-4.1, gpt-5, gpt-5-mini, gpt-5-nano, gpt-5-pro). This caused dynamic routing to treat these models as unknown (falling back to the isKnownModel guard) and cost comparisons to assign them 999 (the "unknown, assume expensive" fallback). Added 17 new model entries to MODEL_CAPABILITY_TIER across all three tiers, matching the tier assignments from the issue. Added corresponding entries to both MODEL_COST_PER_1K_INPUT (model-router.ts) and BUNDLED_COST_TABLE (model-cost-table.ts). Updated the #2192 test fixture that used gpt-5.4 as an "unknown" model since it is now known. Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
cb26d71483
commit
43ece11be7
4 changed files with 155 additions and 3 deletions
|
|
@ -33,10 +33,29 @@ export const BUNDLED_COST_TABLE: ModelCostEntry[] = [
|
|||
// OpenAI
|
||||
{ id: "gpt-4o", inputPer1k: 0.0025, outputPer1k: 0.01, updatedAt: "2025-03-15" },
|
||||
{ id: "gpt-4o-mini", inputPer1k: 0.00015, outputPer1k: 0.0006, updatedAt: "2025-03-15" },
|
||||
{ id: "gpt-4.1", inputPer1k: 0.002, outputPer1k: 0.008, updatedAt: "2026-03-29" },
|
||||
{ id: "gpt-4.1-mini", inputPer1k: 0.0004, outputPer1k: 0.0016, updatedAt: "2026-03-29" },
|
||||
{ id: "gpt-4.1-nano", inputPer1k: 0.0001, outputPer1k: 0.0004, updatedAt: "2026-03-29" },
|
||||
{ id: "gpt-5", inputPer1k: 0.01, outputPer1k: 0.04, updatedAt: "2026-03-29" },
|
||||
{ id: "gpt-5-mini", inputPer1k: 0.0003, outputPer1k: 0.0012, updatedAt: "2026-03-29" },
|
||||
{ id: "gpt-5-nano", inputPer1k: 0.0001, outputPer1k: 0.0004, updatedAt: "2026-03-29" },
|
||||
{ id: "gpt-5-pro", inputPer1k: 0.015, outputPer1k: 0.06, updatedAt: "2026-03-29" },
|
||||
{ id: "o1", inputPer1k: 0.015, outputPer1k: 0.06, updatedAt: "2025-03-15" },
|
||||
{ id: "o3", inputPer1k: 0.015, outputPer1k: 0.06, updatedAt: "2025-03-15" },
|
||||
{ id: "o4-mini", inputPer1k: 0.005, outputPer1k: 0.02, updatedAt: "2026-03-29" },
|
||||
{ id: "o4-mini-deep-research", inputPer1k: 0.005, outputPer1k: 0.02, updatedAt: "2026-03-29" },
|
||||
{ id: "gpt-4-turbo", inputPer1k: 0.01, outputPer1k: 0.03, updatedAt: "2025-03-15" },
|
||||
|
||||
// OpenAI Codex
|
||||
{ id: "gpt-5.1", inputPer1k: 0.005, outputPer1k: 0.02, updatedAt: "2026-03-29" },
|
||||
{ id: "gpt-5.1-codex-max", inputPer1k: 0.003, outputPer1k: 0.012, updatedAt: "2026-03-29" },
|
||||
{ id: "gpt-5.1-codex-mini", inputPer1k: 0.0003, outputPer1k: 0.0012, updatedAt: "2026-03-29" },
|
||||
{ id: "gpt-5.2", inputPer1k: 0.005, outputPer1k: 0.02, updatedAt: "2026-03-29" },
|
||||
{ id: "gpt-5.2-codex", inputPer1k: 0.005, outputPer1k: 0.02, updatedAt: "2026-03-29" },
|
||||
{ id: "gpt-5.3-codex", inputPer1k: 0.005, outputPer1k: 0.02, updatedAt: "2026-03-29" },
|
||||
{ id: "gpt-5.3-codex-spark", inputPer1k: 0.0003, outputPer1k: 0.0012, updatedAt: "2026-03-29" },
|
||||
{ id: "gpt-5.4", inputPer1k: 0.005, outputPer1k: 0.02, updatedAt: "2026-03-29" },
|
||||
|
||||
// Google
|
||||
{ id: "gemini-2.0-flash", inputPer1k: 0.0001, outputPer1k: 0.0004, updatedAt: "2025-03-15" },
|
||||
{ id: "gemini-flash-2.0", inputPer1k: 0.0001, outputPer1k: 0.0004, updatedAt: "2025-03-15" },
|
||||
|
|
|
|||
|
|
@ -44,6 +44,12 @@ const MODEL_CAPABILITY_TIER: Record<string, ComplexityTier> = {
|
|||
"claude-3-5-haiku-latest": "light",
|
||||
"claude-3-haiku-20240307": "light",
|
||||
"gpt-4o-mini": "light",
|
||||
"gpt-4.1-mini": "light",
|
||||
"gpt-4.1-nano": "light",
|
||||
"gpt-5-mini": "light",
|
||||
"gpt-5-nano": "light",
|
||||
"gpt-5.1-codex-mini": "light",
|
||||
"gpt-5.3-codex-spark": "light",
|
||||
"gemini-2.0-flash": "light",
|
||||
"gemini-flash-2.0": "light",
|
||||
|
||||
|
|
@ -52,6 +58,8 @@ const MODEL_CAPABILITY_TIER: Record<string, ComplexityTier> = {
|
|||
"claude-sonnet-4-5-20250514": "standard",
|
||||
"claude-3-5-sonnet-latest": "standard",
|
||||
"gpt-4o": "standard",
|
||||
"gpt-4.1": "standard",
|
||||
"gpt-5.1-codex-max": "standard",
|
||||
"gemini-2.5-pro": "standard",
|
||||
"deepseek-chat": "standard",
|
||||
|
||||
|
|
@ -59,8 +67,17 @@ const MODEL_CAPABILITY_TIER: Record<string, ComplexityTier> = {
|
|||
"claude-opus-4-6": "heavy",
|
||||
"claude-3-opus-latest": "heavy",
|
||||
"gpt-4-turbo": "heavy",
|
||||
"gpt-5": "heavy",
|
||||
"gpt-5-pro": "heavy",
|
||||
"gpt-5.1": "heavy",
|
||||
"gpt-5.2": "heavy",
|
||||
"gpt-5.2-codex": "heavy",
|
||||
"gpt-5.3-codex": "heavy",
|
||||
"gpt-5.4": "heavy",
|
||||
"o1": "heavy",
|
||||
"o3": "heavy",
|
||||
"o4-mini": "heavy",
|
||||
"o4-mini-deep-research": "heavy",
|
||||
};
|
||||
|
||||
// ─── Cost Table (per 1K input tokens, approximate USD) ───────────────────────
|
||||
|
|
@ -75,6 +92,23 @@ const MODEL_COST_PER_1K_INPUT: Record<string, number> = {
|
|||
"claude-opus-4-6": 0.015,
|
||||
"gpt-4o-mini": 0.00015,
|
||||
"gpt-4o": 0.0025,
|
||||
"gpt-4.1": 0.002,
|
||||
"gpt-4.1-mini": 0.0004,
|
||||
"gpt-4.1-nano": 0.0001,
|
||||
"gpt-5": 0.01,
|
||||
"gpt-5-mini": 0.0003,
|
||||
"gpt-5-nano": 0.0001,
|
||||
"gpt-5-pro": 0.015,
|
||||
"gpt-5.1": 0.005,
|
||||
"gpt-5.1-codex-max": 0.003,
|
||||
"gpt-5.1-codex-mini": 0.0003,
|
||||
"gpt-5.2": 0.005,
|
||||
"gpt-5.2-codex": 0.005,
|
||||
"gpt-5.3-codex": 0.005,
|
||||
"gpt-5.3-codex-spark": 0.0003,
|
||||
"gpt-5.4": 0.005,
|
||||
"o4-mini": 0.005,
|
||||
"o4-mini-deep-research": 0.005,
|
||||
"gemini-2.0-flash": 0.0001,
|
||||
"gemini-2.5-pro": 0.00125,
|
||||
"deepseek-chat": 0.00014,
|
||||
|
|
|
|||
|
|
@ -67,3 +67,37 @@ test("all cost table entries have valid data", () => {
|
|||
assert.ok(entry.updatedAt, `${entry.id} missing updatedAt`);
|
||||
}
|
||||
});
|
||||
|
||||
// ─── #2885: openai-codex and modern OpenAI models in cost table ──────────────
|
||||
|
||||
test("#2885: cost table includes openai-codex provider models", () => {
|
||||
const ids = BUNDLED_COST_TABLE.map(e => e.id);
|
||||
const codexModels = [
|
||||
"gpt-5.1", "gpt-5.1-codex-max", "gpt-5.1-codex-mini",
|
||||
"gpt-5.2", "gpt-5.2-codex", "gpt-5.3-codex", "gpt-5.3-codex-spark", "gpt-5.4",
|
||||
];
|
||||
for (const model of codexModels) {
|
||||
assert.ok(ids.includes(model), `cost table should include openai-codex model "${model}"`);
|
||||
}
|
||||
});
|
||||
|
||||
test("#2885: cost table includes modern OpenAI models", () => {
|
||||
const ids = BUNDLED_COST_TABLE.map(e => e.id);
|
||||
const newModels = [
|
||||
"o4-mini", "o4-mini-deep-research",
|
||||
"gpt-4.1", "gpt-4.1-mini", "gpt-4.1-nano",
|
||||
"gpt-5", "gpt-5-mini", "gpt-5-nano", "gpt-5-pro",
|
||||
];
|
||||
for (const model of newModels) {
|
||||
assert.ok(ids.includes(model), `cost table should include modern OpenAI model "${model}"`);
|
||||
}
|
||||
});
|
||||
|
||||
test("#2885: lookupModelCost returns costs for new models (not 999 fallback)", () => {
|
||||
const newModels = ["o4-mini", "gpt-4.1", "gpt-5", "gpt-5.4", "gpt-5.1-codex-mini"];
|
||||
for (const model of newModels) {
|
||||
const entry = lookupModelCost(model);
|
||||
assert.ok(entry, `lookupModelCost should find "${model}"`);
|
||||
assert.ok(entry.inputPer1k < 999, `${model} should have a real cost, not the 999 fallback`);
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -172,11 +172,11 @@ test("#2192: unknown model is not downgraded — respects user config", () => {
|
|||
const config = { ...defaultRoutingConfig(), enabled: true };
|
||||
const result = resolveModelForComplexity(
|
||||
makeClassification("light"),
|
||||
{ primary: "gpt-5.4", fallbacks: [] },
|
||||
{ primary: "some-future-unknown-model-v9", fallbacks: [] },
|
||||
config,
|
||||
["gpt-5.4", ...AVAILABLE_MODELS],
|
||||
["some-future-unknown-model-v9", ...AVAILABLE_MODELS],
|
||||
);
|
||||
assert.equal(result.modelId, "gpt-5.4", "unknown model should be used as-is");
|
||||
assert.equal(result.modelId, "some-future-unknown-model-v9", "unknown model should be used as-is");
|
||||
assert.equal(result.wasDowngraded, false, "should not be downgraded");
|
||||
assert.ok(result.reason.includes("not in the known tier map"), "reason should explain why");
|
||||
});
|
||||
|
|
@ -205,3 +205,68 @@ test("#2192: known model is still downgraded normally", () => {
|
|||
assert.equal(result.wasDowngraded, true, "known heavy model should still be downgraded for light tasks");
|
||||
assert.notEqual(result.modelId, "claude-opus-4-6");
|
||||
});
|
||||
|
||||
// ─── #2885: openai-codex and modern OpenAI models in tier map ────────────────
|
||||
|
||||
test("#2885: openai-codex light-tier models are recognized", () => {
|
||||
const config = { ...defaultRoutingConfig(), enabled: true };
|
||||
const lightModels = ["gpt-4.1-mini", "gpt-4.1-nano", "gpt-5-mini", "gpt-5-nano", "gpt-5.1-codex-mini", "gpt-5.3-codex-spark"];
|
||||
for (const model of lightModels) {
|
||||
const result = resolveModelForComplexity(
|
||||
makeClassification("light"),
|
||||
{ primary: model, fallbacks: [] },
|
||||
config,
|
||||
[model, ...AVAILABLE_MODELS],
|
||||
);
|
||||
// Model is known AND light-tier, so requesting light should NOT downgrade
|
||||
assert.equal(result.wasDowngraded, false, `${model} should be known as light tier (wasDowngraded)`);
|
||||
assert.equal(result.modelId, model, `${model} should be returned as-is for light tier`);
|
||||
// Verify it IS known (not hitting the unknown-model bail-out)
|
||||
assert.ok(!result.reason.includes("not in the known tier map"), `${model} should be in the known tier map`);
|
||||
}
|
||||
});
|
||||
|
||||
test("#2885: openai-codex standard-tier models are recognized", () => {
|
||||
const config = { ...defaultRoutingConfig(), enabled: true };
|
||||
const standardModels = ["gpt-4.1", "gpt-5.1-codex-max"];
|
||||
for (const model of standardModels) {
|
||||
const result = resolveModelForComplexity(
|
||||
makeClassification("standard"),
|
||||
{ primary: model, fallbacks: [] },
|
||||
config,
|
||||
[model, ...AVAILABLE_MODELS],
|
||||
);
|
||||
assert.equal(result.wasDowngraded, false, `${model} should be known as standard tier`);
|
||||
assert.equal(result.modelId, model, `${model} should be returned as-is for standard tier`);
|
||||
assert.ok(!result.reason.includes("not in the known tier map"), `${model} should be in the known tier map`);
|
||||
}
|
||||
});
|
||||
|
||||
test("#2885: openai-codex heavy-tier models are recognized", () => {
|
||||
const config = { ...defaultRoutingConfig(), enabled: true };
|
||||
const heavyModels = ["gpt-5", "gpt-5-pro", "gpt-5.1", "gpt-5.2", "gpt-5.2-codex", "gpt-5.3-codex", "gpt-5.4", "o4-mini", "o4-mini-deep-research"];
|
||||
for (const model of heavyModels) {
|
||||
const result = resolveModelForComplexity(
|
||||
makeClassification("heavy"),
|
||||
{ primary: model, fallbacks: [] },
|
||||
config,
|
||||
[model, ...AVAILABLE_MODELS],
|
||||
);
|
||||
assert.equal(result.wasDowngraded, false, `${model} should be known as heavy tier`);
|
||||
assert.equal(result.modelId, model, `${model} should be returned as-is for heavy tier`);
|
||||
assert.ok(!result.reason.includes("not in the known tier map"), `${model} should be in the known tier map`);
|
||||
}
|
||||
});
|
||||
|
||||
test("#2885: heavy openai-codex model downgrades to light for light task", () => {
|
||||
const config = { ...defaultRoutingConfig(), enabled: true };
|
||||
const result = resolveModelForComplexity(
|
||||
makeClassification("light"),
|
||||
{ primary: "gpt-5.4", fallbacks: [] },
|
||||
config,
|
||||
["gpt-5.4", "gpt-4.1-nano", ...AVAILABLE_MODELS],
|
||||
);
|
||||
assert.equal(result.wasDowngraded, true, "heavy model should downgrade for light task");
|
||||
// Should pick a light-tier model
|
||||
assert.notEqual(result.modelId, "gpt-5.4", "should not use the heavy model for light task");
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue