feat(gsd): extend flat-rate provider detection to custom/externalCli providers
The 3-entry hard-coded FLAT_RATE_PROVIDERS set in auto-model-selection.ts
treated only github-copilot/copilot/claude-code as flat-rate, so dynamic
routing would happily downgrade units on user-registered subscription
proxies and any externalCli CLI wrapper — quality loss with no cost
benefit for users whose provider charges a flat rate per request.
Make isFlatRateProvider extensible by composing three signals:
1. Built-in list (unchanged, wins first for regression safety).
2. externalCli auto-detection via ctx.modelRegistry.getProviderAuthMode()
— any CLI wrapper around the user's subscription is inherently
flat-rate.
3. User-declared `flat_rate_providers` preference for private
subscription-backed proxies, enterprise-gated deployments, and custom
CLI wrappers the built-in list doesn't know about.
Add a buildFlatRateContext() helper so every call site constructs the
context the same way and degrades gracefully when ctx/prefs/registry are
unavailable (never breaks flat-rate detection).
Thread the context through:
- resolvePreferredModelConfig (routing synthesis guard)
- selectAndApplyModel primary-model and fallback provider checks
- auto-start.ts dynamic-routing banner so the startup message matches
dispatch-time reality
Preferences:
- Add `flat_rate_providers?: string[]` to GSDPreferences and
KNOWN_PREFERENCE_KEYS in preferences-types.ts.
- Add a string-array validator in preferences-validation.ts that trims
whitespace and drops empty entries.
Tests:
- Extend flat-rate-routing-guard.test.ts with 13 new cases covering
externalCli auto-detection, userFlatRate preference matching
(case-insensitive), combined signals, buildFlatRateContext() behavior
(including registry-lookup-throws and non-canonical auth-mode
responses), plus regression cases for the built-in list.
- Add 5 validator cases in preferences.test.ts for the new
flat_rate_providers field (string-array accepted, whitespace trimmed,
non-array rejected, non-string elements rejected, known-key warning
check).
This commit is contained in:
parent
24f51fd76b
commit
9a93563a64
6 changed files with 309 additions and 13 deletions
|
|
@ -29,6 +29,10 @@ export function resolvePreferredModelConfig(
|
|||
/** When false, only return explicit per-phase model configs — do not
|
||||
* synthesize a routing ceiling from dynamic_routing.tier_models (#3962). */
|
||||
isAutoMode = true,
|
||||
/** Optional flat-rate context for the start model's provider. Used to
|
||||
* extend flat-rate detection beyond the built-in list (user
|
||||
* `flat_rate_providers` preference + externalCli auto-detection). */
|
||||
flatRateCtx?: FlatRateContext,
|
||||
) {
|
||||
const explicitConfig = resolveModelWithFallbacksForUnit(unitType);
|
||||
if (explicitConfig) return explicitConfig;
|
||||
|
|
@ -41,7 +45,7 @@ export function resolvePreferredModelConfig(
|
|||
if (!routingConfig.enabled || !routingConfig.tier_models) return undefined;
|
||||
|
||||
// Don't synthesize a routing config for flat-rate providers (#3453).
|
||||
if (autoModeStartModel && isFlatRateProvider(autoModeStartModel.provider)) return undefined;
|
||||
if (autoModeStartModel && isFlatRateProvider(autoModeStartModel.provider, flatRateCtx)) return undefined;
|
||||
|
||||
const ceilingModel = routingConfig.tier_models.heavy
|
||||
?? (autoModeStartModel ? `${autoModeStartModel.provider}/${autoModeStartModel.id}` : undefined);
|
||||
|
|
@ -79,9 +83,17 @@ export async function selectAndApplyModel(
|
|||
const effectiveSessionModelOverride = sessionModelOverride === undefined
|
||||
? getSessionModelOverride(ctx.sessionManager.getSessionId())
|
||||
: (sessionModelOverride ?? undefined);
|
||||
// Build a flat-rate context for the start model's provider up front so
|
||||
// routing synthesis and the dispatch-time guard see the same signals
|
||||
// (built-in list + user `flat_rate_providers` preference + externalCli
|
||||
// auto-detection). The dispatch-time primary-model check below builds
|
||||
// its own per-provider context when it has a resolved primary model.
|
||||
const startModelFlatRateCtx = autoModeStartModel
|
||||
? buildFlatRateContext(autoModeStartModel.provider, ctx, prefs)
|
||||
: undefined;
|
||||
const modelConfig = effectiveSessionModelOverride
|
||||
? undefined
|
||||
: resolvePreferredModelConfig(unitType, autoModeStartModel, isAutoMode);
|
||||
: resolvePreferredModelConfig(unitType, autoModeStartModel, isAutoMode, startModelFlatRateCtx);
|
||||
let routing: { tier: string; modelDowngraded: boolean } | null = null;
|
||||
let appliedModel: Model<Api> | null = null;
|
||||
|
||||
|
|
@ -107,12 +119,16 @@ export async function selectAndApplyModel(
|
|||
if (routingConfig.enabled) {
|
||||
const primaryModel = resolveModelId(modelConfig.primary, availableModels, ctx.model?.provider);
|
||||
if (primaryModel) {
|
||||
if (isFlatRateProvider(primaryModel.provider)) {
|
||||
const primaryFlatRateCtx = buildFlatRateContext(primaryModel.provider, ctx, prefs);
|
||||
if (isFlatRateProvider(primaryModel.provider, primaryFlatRateCtx)) {
|
||||
routingConfig.enabled = false;
|
||||
}
|
||||
} else if (
|
||||
(autoModeStartModel && isFlatRateProvider(autoModeStartModel.provider))
|
||||
|| (ctx.model?.provider && isFlatRateProvider(ctx.model.provider))
|
||||
(autoModeStartModel && isFlatRateProvider(autoModeStartModel.provider, startModelFlatRateCtx))
|
||||
|| (ctx.model?.provider && isFlatRateProvider(
|
||||
ctx.model.provider,
|
||||
buildFlatRateContext(ctx.model.provider, ctx, prefs),
|
||||
))
|
||||
) {
|
||||
// Primary model unresolvable but provider signals indicate flat-rate —
|
||||
// disable routing to prevent quality degradation.
|
||||
|
|
@ -416,8 +432,63 @@ export function resolveModelId<T extends { id: string; provider: string }>(
|
|||
* Uses case-insensitive matching with alias support to prevent fail-open on
|
||||
* provider naming variations (e.g. "copilot" vs "github-copilot").
|
||||
*/
|
||||
const FLAT_RATE_PROVIDERS = new Set(["github-copilot", "copilot", "claude-code"]);
|
||||
const BUILTIN_FLAT_RATE = new Set(["github-copilot", "copilot", "claude-code"]);
|
||||
|
||||
export function isFlatRateProvider(provider: string): boolean {
|
||||
return FLAT_RATE_PROVIDERS.has(provider.toLowerCase());
|
||||
/**
|
||||
* Optional context that lets callers extend flat-rate detection beyond the
|
||||
* hard-coded built-in list. Either signal on its own is enough to classify
|
||||
* a provider as flat-rate.
|
||||
*/
|
||||
export interface FlatRateContext {
|
||||
/**
|
||||
* Auth mode for the specific provider being checked, as returned by
|
||||
* `ctx.modelRegistry.getProviderAuthMode(provider)`. Any provider that
|
||||
* wraps a local CLI (externalCli) is, by definition, a flat-rate
|
||||
* subscription wrapper — every request costs the same regardless of
|
||||
* model, so dynamic routing only degrades quality.
|
||||
*/
|
||||
authMode?: "apiKey" | "oauth" | "externalCli" | "none";
|
||||
/**
|
||||
* Case-insensitive list of extra provider IDs the user has declared as
|
||||
* flat-rate via `preferences.flat_rate_providers`. Used for private
|
||||
* subscription-backed proxies and enterprise-gated deployments that the
|
||||
* built-in list doesn't know about.
|
||||
*/
|
||||
userFlatRate?: readonly string[];
|
||||
}
|
||||
|
||||
export function isFlatRateProvider(provider: string, opts?: FlatRateContext): boolean {
|
||||
const p = provider.toLowerCase();
|
||||
if (BUILTIN_FLAT_RATE.has(p)) return true;
|
||||
if (opts?.userFlatRate?.some(id => id.toLowerCase() === p)) return true;
|
||||
if (opts?.authMode === "externalCli") return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a FlatRateContext for a given provider from live runtime state.
|
||||
* Safe to call when ctx or prefs are undefined — missing pieces are
|
||||
* treated as "no signal".
|
||||
*/
|
||||
export function buildFlatRateContext(
|
||||
provider: string,
|
||||
ctx?: { modelRegistry?: { getProviderAuthMode?: (p: string) => string } },
|
||||
prefs?: { flat_rate_providers?: readonly string[] },
|
||||
): FlatRateContext {
|
||||
let authMode: FlatRateContext["authMode"];
|
||||
const getAuthMode = ctx?.modelRegistry?.getProviderAuthMode;
|
||||
if (typeof getAuthMode === "function") {
|
||||
try {
|
||||
const mode = getAuthMode(provider);
|
||||
if (mode === "apiKey" || mode === "oauth" || mode === "externalCli" || mode === "none") {
|
||||
authMode = mode;
|
||||
}
|
||||
} catch {
|
||||
// Registry lookup failure must never break flat-rate detection.
|
||||
}
|
||||
}
|
||||
return {
|
||||
authMode,
|
||||
userFlatRate: prefs?.flat_rate_providers,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -825,12 +825,19 @@ export async function bootstrapAutoSession(
|
|||
? `${s.autoModeStartModel.provider}/${s.autoModeStartModel.id}`
|
||||
: ctx.model ? `${ctx.model.provider}/${ctx.model.id}` : "default";
|
||||
|
||||
// Flat-rate providers (e.g. GitHub Copilot, claude-code) suppress routing
|
||||
// at dispatch time (#3453) — reflect that in the banner.
|
||||
const { isFlatRateProvider } = await import("./auto-model-selection.js");
|
||||
// Flat-rate providers (e.g. GitHub Copilot, claude-code, user-declared
|
||||
// subscription proxies, externalCli CLIs) suppress routing at dispatch
|
||||
// time (#3453) — reflect that in the banner. Thread the same
|
||||
// FlatRateContext used by selectAndApplyModel so user-declared
|
||||
// flat-rate providers and externalCli auto-detection are respected.
|
||||
const { isFlatRateProvider, buildFlatRateContext } = await import("./auto-model-selection.js");
|
||||
const bannerPrefs = loadEffectiveGSDPreferences()?.preferences;
|
||||
const effectiveProvider = s.autoModeStartModel?.provider ?? ctx.model?.provider;
|
||||
const effectivelyEnabled = routingConfig.enabled
|
||||
&& !(effectiveProvider && isFlatRateProvider(effectiveProvider));
|
||||
&& !(effectiveProvider && isFlatRateProvider(
|
||||
effectiveProvider,
|
||||
buildFlatRateContext(effectiveProvider, ctx, bannerPrefs),
|
||||
));
|
||||
|
||||
// The actual ceiling may come from tier_models.heavy, not the start model.
|
||||
const effectiveCeiling = (routingConfig.enabled && routingConfig.tier_models?.heavy)
|
||||
|
|
|
|||
|
|
@ -113,6 +113,7 @@ export const KNOWN_PREFERENCE_KEYS = new Set<string>([
|
|||
"discuss_preparation",
|
||||
"discuss_web_research",
|
||||
"discuss_depth",
|
||||
"flat_rate_providers",
|
||||
]);
|
||||
|
||||
/** Canonical list of all dispatch unit types. */
|
||||
|
|
@ -359,6 +360,17 @@ export interface GSDPreferences {
|
|||
* Default: "standard".
|
||||
*/
|
||||
discuss_depth?: "quick" | "standard" | "thorough";
|
||||
/**
|
||||
* Extra provider IDs to treat as flat-rate (no cost benefit from dynamic
|
||||
* routing). Dynamic routing is suppressed for any provider listed here,
|
||||
* in addition to the built-in list (github-copilot, copilot, claude-code)
|
||||
* and any provider auto-detected via `authMode: "externalCli"`.
|
||||
*
|
||||
* Intended for private subscription-backed proxies, enterprise-gated
|
||||
* deployments, and custom CLI wrappers where every request costs the
|
||||
* same regardless of model. Case-insensitive.
|
||||
*/
|
||||
flat_rate_providers?: string[];
|
||||
}
|
||||
|
||||
export interface LoadedGSDPreferences {
|
||||
|
|
|
|||
|
|
@ -180,6 +180,29 @@ export function validatePreferences(preferences: GSDPreferences): {
|
|||
}
|
||||
}
|
||||
|
||||
// ─── Flat-rate Providers ────────────────────────────────────────────
|
||||
// User-declared flat-rate providers for dynamic routing suppression.
|
||||
// Built-in providers (github-copilot, copilot, claude-code) and any
|
||||
// externalCli provider are already auto-detected; this list layers on
|
||||
// top for private subscription proxies and custom CLI wrappers.
|
||||
if (preferences.flat_rate_providers !== undefined) {
|
||||
if (Array.isArray(preferences.flat_rate_providers)) {
|
||||
const allStrings = preferences.flat_rate_providers.every(
|
||||
(item: unknown) => typeof item === "string",
|
||||
);
|
||||
if (allStrings) {
|
||||
// Strip empty/whitespace-only entries to avoid false matches.
|
||||
validated.flat_rate_providers = preferences.flat_rate_providers
|
||||
.map((s: string) => s.trim())
|
||||
.filter((s: string) => s.length > 0);
|
||||
} else {
|
||||
errors.push("flat_rate_providers must be an array of strings");
|
||||
}
|
||||
} else {
|
||||
errors.push("flat_rate_providers must be an array of strings");
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Phase Skip Preferences ─────────────────────────────────────────
|
||||
if (preferences.phases !== undefined) {
|
||||
if (typeof preferences.phases === "object" && preferences.phases !== null) {
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
|
||||
import { describe, test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { isFlatRateProvider, resolvePreferredModelConfig } from "../auto-model-selection.ts";
|
||||
import { buildFlatRateContext, isFlatRateProvider, resolvePreferredModelConfig } from "../auto-model-selection.ts";
|
||||
|
||||
describe("flat-rate provider routing guard (#3453)", () => {
|
||||
|
||||
|
|
@ -48,3 +48,139 @@ describe("flat-rate provider routing guard (#3453)", () => {
|
|||
assert.equal(result, undefined, "Should not create routing config for copilot");
|
||||
});
|
||||
});
|
||||
|
||||
describe("flat-rate provider extensibility (any/all/custom)", () => {
|
||||
test("regression: built-in providers still flat-rate with no context", () => {
|
||||
assert.equal(isFlatRateProvider("github-copilot"), true);
|
||||
assert.equal(isFlatRateProvider("copilot"), true);
|
||||
assert.equal(isFlatRateProvider("claude-code"), true);
|
||||
});
|
||||
|
||||
test("regression: non-flat-rate API providers return false with no context", () => {
|
||||
assert.equal(isFlatRateProvider("anthropic"), false);
|
||||
assert.equal(isFlatRateProvider("openai"), false);
|
||||
assert.equal(isFlatRateProvider("google-vertex"), false);
|
||||
});
|
||||
|
||||
test("auto-detection: externalCli auth mode marks provider flat-rate", () => {
|
||||
// Any provider registered with authMode: "externalCli" is a local
|
||||
// CLI wrapper around the user's subscription — every request costs
|
||||
// the same regardless of model, so dynamic routing provides no benefit.
|
||||
assert.equal(
|
||||
isFlatRateProvider("my-private-cli", { authMode: "externalCli" }),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test("auto-detection: non-externalCli auth modes do not mark provider flat-rate", () => {
|
||||
assert.equal(
|
||||
isFlatRateProvider("my-http-proxy", { authMode: "apiKey" }),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
isFlatRateProvider("my-http-proxy", { authMode: "oauth" }),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
isFlatRateProvider("my-http-proxy", { authMode: "none" }),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test("user preference: custom provider listed in userFlatRate is flat-rate", () => {
|
||||
assert.equal(
|
||||
isFlatRateProvider("my-ollama-proxy", { userFlatRate: ["my-ollama-proxy"] }),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test("user preference: case-insensitive match against userFlatRate list", () => {
|
||||
assert.equal(
|
||||
isFlatRateProvider("My-Proxy", { userFlatRate: ["my-proxy"] }),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
isFlatRateProvider("my-proxy", { userFlatRate: ["MY-PROXY"] }),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test("user preference: provider not in userFlatRate list is not flat-rate", () => {
|
||||
assert.equal(
|
||||
isFlatRateProvider("other-proxy", { userFlatRate: ["my-proxy"] }),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test("combined signals: built-in list wins even when context is empty", () => {
|
||||
assert.equal(
|
||||
isFlatRateProvider("claude-code", { authMode: "apiKey", userFlatRate: [] }),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test("combined signals: externalCli auto-detection wins alongside userFlatRate miss", () => {
|
||||
assert.equal(
|
||||
isFlatRateProvider("my-cli", {
|
||||
authMode: "externalCli",
|
||||
userFlatRate: ["a-different-cli"],
|
||||
}),
|
||||
true,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildFlatRateContext()", () => {
|
||||
test("builds a context from ctx.modelRegistry.getProviderAuthMode + prefs", () => {
|
||||
const ctx = {
|
||||
modelRegistry: {
|
||||
getProviderAuthMode: (p: string) =>
|
||||
p === "my-cli" ? "externalCli" : "apiKey",
|
||||
},
|
||||
};
|
||||
const prefs = { flat_rate_providers: ["my-proxy"] };
|
||||
|
||||
const ctxForCli = buildFlatRateContext("my-cli", ctx, prefs);
|
||||
assert.equal(ctxForCli.authMode, "externalCli");
|
||||
assert.deepEqual(ctxForCli.userFlatRate, ["my-proxy"]);
|
||||
assert.equal(isFlatRateProvider("my-cli", ctxForCli), true);
|
||||
|
||||
const ctxForProxy = buildFlatRateContext("my-proxy", ctx, prefs);
|
||||
assert.equal(ctxForProxy.authMode, "apiKey");
|
||||
assert.equal(isFlatRateProvider("my-proxy", ctxForProxy), true);
|
||||
|
||||
const ctxForOther = buildFlatRateContext("anthropic", ctx, prefs);
|
||||
assert.equal(ctxForOther.authMode, "apiKey");
|
||||
assert.equal(isFlatRateProvider("anthropic", ctxForOther), false);
|
||||
});
|
||||
|
||||
test("survives missing ctx and missing prefs", () => {
|
||||
const empty = buildFlatRateContext("anything");
|
||||
assert.equal(empty.authMode, undefined);
|
||||
assert.equal(empty.userFlatRate, undefined);
|
||||
assert.equal(isFlatRateProvider("anything", empty), false);
|
||||
});
|
||||
|
||||
test("survives a registry lookup that throws", () => {
|
||||
const ctx = {
|
||||
modelRegistry: {
|
||||
getProviderAuthMode: () => {
|
||||
throw new Error("registry boom");
|
||||
},
|
||||
},
|
||||
};
|
||||
const result = buildFlatRateContext("anything", ctx);
|
||||
// Error must be swallowed — authMode left undefined, function returns.
|
||||
assert.equal(result.authMode, undefined);
|
||||
});
|
||||
|
||||
test("registry returning a non-canonical auth mode is ignored", () => {
|
||||
const ctx = {
|
||||
modelRegistry: {
|
||||
getProviderAuthMode: () => "weird-mode",
|
||||
},
|
||||
};
|
||||
const result = buildFlatRateContext("anything", ctx);
|
||||
assert.equal(result.authMode, undefined);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -134,6 +134,53 @@ test("invalid value types produce errors and fall back to undefined", () => {
|
|||
}
|
||||
});
|
||||
|
||||
test("flat_rate_providers: accepts string array", () => {
|
||||
const { errors, preferences } = validatePreferences({
|
||||
flat_rate_providers: ["my-proxy", "private-cli"],
|
||||
});
|
||||
assert.equal(errors.length, 0);
|
||||
assert.deepEqual(preferences.flat_rate_providers, ["my-proxy", "private-cli"]);
|
||||
});
|
||||
|
||||
test("flat_rate_providers: trims whitespace and drops empty entries", () => {
|
||||
const { errors, preferences } = validatePreferences({
|
||||
flat_rate_providers: [" my-proxy ", "", " ", "private-cli"],
|
||||
});
|
||||
assert.equal(errors.length, 0);
|
||||
assert.deepEqual(preferences.flat_rate_providers, ["my-proxy", "private-cli"]);
|
||||
});
|
||||
|
||||
test("flat_rate_providers: non-array rejected", () => {
|
||||
const { errors } = validatePreferences({
|
||||
flat_rate_providers: "my-proxy" as any,
|
||||
});
|
||||
assert.ok(
|
||||
errors.some(e => e.includes("flat_rate_providers")),
|
||||
"should error on non-array value",
|
||||
);
|
||||
});
|
||||
|
||||
test("flat_rate_providers: non-string elements rejected", () => {
|
||||
const { errors } = validatePreferences({
|
||||
flat_rate_providers: ["ok", 123 as any, "also-ok"],
|
||||
});
|
||||
assert.ok(
|
||||
errors.some(e => e.includes("flat_rate_providers")),
|
||||
"should error when array contains non-strings",
|
||||
);
|
||||
});
|
||||
|
||||
test("flat_rate_providers is a recognized preference key (no warning)", () => {
|
||||
const { warnings } = validatePreferences({
|
||||
flat_rate_providers: ["my-proxy"],
|
||||
});
|
||||
assert.equal(
|
||||
warnings.filter(w => w.includes("flat_rate_providers")).length,
|
||||
0,
|
||||
"flat_rate_providers must be in KNOWN_PREFERENCE_KEYS",
|
||||
);
|
||||
});
|
||||
|
||||
test("valid values pass through correctly", () => {
|
||||
const { preferences: p1 } = validatePreferences({ budget_enforcement: "halt" });
|
||||
assert.equal(p1.budget_enforcement, "halt");
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue