feat(01-05): fire before_model_select hook, add verbose scoring output, load capability overrides
- Fire pi.emitBeforeModelSelect() in selectAndApplyModel before resolveModelForComplexity - Hook override bypasses capability scoring entirely with tier-only selectionMethod - Verbose output shows capability-scored breakdown: model scores sorted descending - Add loadCapabilityOverrides() to model-router.ts for deep-merge with built-in profiles - Extend resolveModelForComplexity signature with optional capabilityOverrides parameter - Pass capabilityOverrides through to scoreEligibleModels in STEP 2
This commit is contained in:
Jeremy
parent
6cc42bb504
commit
1645be072c
2 changed files with 104 additions and 20 deletions
|
|
@ -10,7 +10,7 @@ import type { GSDPreferences } from "./preferences.js";
|
|||
import { resolveModelWithFallbacksForUnit, resolveDynamicRoutingConfig } from "./preferences.js";
|
||||
import type { ComplexityTier } from "./complexity-classifier.js";
|
||||
import { classifyUnitComplexity, tierLabel } from "./complexity-classifier.js";
|
||||
import { resolveModelForComplexity, escalateTier } from "./model-router.js";
|
||||
import { resolveModelForComplexity, escalateTier, getEligibleModels, loadCapabilityOverrides } from "./model-router.js";
|
||||
import { getLedger, getProjectTotals } from "./metrics.js";
|
||||
import { unitPhaseLabel } from "./auto-dashboard.js";
|
||||
|
||||
|
|
@ -107,26 +107,89 @@ export async function selectAndApplyModel(
|
|||
}
|
||||
}
|
||||
|
||||
const routingResult = resolveModelForComplexity(
|
||||
classification,
|
||||
modelConfig,
|
||||
routingConfig,
|
||||
availableModelIds,
|
||||
unitType,
|
||||
classification.taskMetadata,
|
||||
// Load user capability overrides from preferences (D-17: deep-merged with built-in profiles)
|
||||
const capabilityOverrides = loadCapabilityOverrides(
|
||||
(prefs as { modelOverrides?: Record<string, { capabilities?: Record<string, number> }> } | undefined) ?? {},
|
||||
);
|
||||
|
||||
// Fire before_model_select hook (ADR-004, D-03)
|
||||
// Hook can override model selection entirely by returning { modelId }
|
||||
let hookOverride: string | undefined;
|
||||
if (routingConfig.hooks !== false) {
|
||||
const eligible = getEligibleModels(
|
||||
classification.tier,
|
||||
availableModelIds,
|
||||
routingConfig,
|
||||
);
|
||||
const hookResult = await pi.emitBeforeModelSelect({
|
||||
unitType,
|
||||
unitId,
|
||||
classification: {
|
||||
tier: classification.tier,
|
||||
reason: classification.reason,
|
||||
downgraded: classification.downgraded,
|
||||
},
|
||||
taskMetadata: classification.taskMetadata as Record<string, unknown> | undefined,
|
||||
eligibleModels: eligible,
|
||||
phaseConfig: modelConfig ? {
|
||||
primary: modelConfig.primary,
|
||||
fallbacks: modelConfig.fallbacks ?? [],
|
||||
} : undefined,
|
||||
});
|
||||
if (hookResult?.modelId) {
|
||||
hookOverride = hookResult.modelId;
|
||||
}
|
||||
}
|
||||
|
||||
let routingResult: ReturnType<typeof resolveModelForComplexity>;
|
||||
if (hookOverride) {
|
||||
// Hook override bypasses capability scoring entirely
|
||||
routingResult = {
|
||||
modelId: hookOverride,
|
||||
fallbacks: [
|
||||
...(modelConfig?.fallbacks ?? []).filter(f => f !== hookOverride),
|
||||
...(modelConfig?.primary && modelConfig.primary !== hookOverride ? [modelConfig.primary] : []),
|
||||
],
|
||||
tier: classification.tier,
|
||||
wasDowngraded: hookOverride !== modelConfig?.primary,
|
||||
reason: `hook override: ${hookOverride}`,
|
||||
selectionMethod: "tier-only",
|
||||
};
|
||||
} else {
|
||||
routingResult = resolveModelForComplexity(
|
||||
classification,
|
||||
modelConfig,
|
||||
routingConfig,
|
||||
availableModelIds,
|
||||
unitType,
|
||||
classification.taskMetadata,
|
||||
capabilityOverrides,
|
||||
);
|
||||
}
|
||||
|
||||
if (routingResult.wasDowngraded) {
|
||||
effectiveModelConfig = {
|
||||
primary: routingResult.modelId,
|
||||
fallbacks: routingResult.fallbacks,
|
||||
};
|
||||
if (verbose) {
|
||||
const method = routingResult.selectionMethod === "capability-scored" ? "capability-scored" : "tier-only";
|
||||
ctx.ui.notify(
|
||||
`Dynamic routing [${tierLabel(classification.tier)}]: ${routingResult.modelId} (${method} — ${classification.reason})`,
|
||||
"info",
|
||||
);
|
||||
if (routingResult.selectionMethod === "capability-scored" && routingResult.capabilityScores) {
|
||||
// Verbose scoring breakdown for capability-scored decisions (D-20)
|
||||
const tierLbl = tierLabel(classification.tier);
|
||||
const scores = Object.entries(routingResult.capabilityScores)
|
||||
.sort(([, a], [, b]) => b - a)
|
||||
.map(([id, score]) => `${id}: ${score.toFixed(1)}`)
|
||||
.join(", ");
|
||||
ctx.ui.notify(
|
||||
`Dynamic routing [${tierLbl}]: ${routingResult.modelId} (capability-scored) — ${scores}`,
|
||||
"info",
|
||||
);
|
||||
} else {
|
||||
ctx.ui.notify(
|
||||
`Dynamic routing [${tierLabel(classification.tier)}]: ${routingResult.modelId} (${classification.reason})`,
|
||||
"info",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
routingTierLabel = ` [${tierLabel(classification.tier)}]`;
|
||||
|
|
|
|||
|
|
@ -289,6 +289,25 @@ function buildFallbackChain(selectedModelId: string, phaseConfig: ResolvedModelC
|
|||
].filter(f => f !== selectedModelId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Load capability overrides from user preferences' modelOverrides section.
|
||||
* Returns a map of model ID → partial capability overrides to deep-merge with built-in profiles.
|
||||
*
|
||||
* Per D-17: partial capability overrides via models.json modelOverrides, deep-merged with defaults.
|
||||
*/
|
||||
export function loadCapabilityOverrides(
|
||||
prefs: { modelOverrides?: Record<string, { capabilities?: Partial<ModelCapabilities> }> },
|
||||
): Record<string, Partial<ModelCapabilities>> {
|
||||
const result: Record<string, Partial<ModelCapabilities>> = {};
|
||||
if (!prefs.modelOverrides) return result;
|
||||
for (const [modelId, overrideEntry] of Object.entries(prefs.modelOverrides)) {
|
||||
if (overrideEntry.capabilities) {
|
||||
result[modelId] = overrideEntry.capabilities;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the model to use for a given complexity tier.
|
||||
*
|
||||
|
|
@ -300,12 +319,13 @@ function buildFallbackChain(selectedModelId: string, phaseConfig: ResolvedModelC
|
|||
* when capability_routing is enabled and multiple eligible models exist.
|
||||
* STEP 3: Fallback chain assembly.
|
||||
*
|
||||
* @param classification The complexity classification result
|
||||
* @param phaseConfig The user's configured model for this phase (ceiling)
|
||||
* @param routingConfig Dynamic routing configuration
|
||||
* @param availableModelIds List of available model IDs (from registry)
|
||||
* @param unitType The unit type for capability requirement computation (optional)
|
||||
* @param taskMetadata Task metadata for refined requirement vectors (optional)
|
||||
* @param classification The complexity classification result
|
||||
* @param phaseConfig The user's configured model for this phase (ceiling)
|
||||
* @param routingConfig Dynamic routing configuration
|
||||
* @param availableModelIds List of available model IDs (from registry)
|
||||
* @param unitType The unit type for capability requirement computation (optional)
|
||||
* @param taskMetadata Task metadata for refined requirement vectors (optional)
|
||||
* @param capabilityOverrides User-provided capability overrides (deep-merged with built-in profiles, optional)
|
||||
*/
|
||||
export function resolveModelForComplexity(
|
||||
classification: ClassificationResult,
|
||||
|
|
@ -314,6 +334,7 @@ export function resolveModelForComplexity(
|
|||
availableModelIds: string[],
|
||||
unitType?: string,
|
||||
taskMetadata?: TaskMetadata,
|
||||
capabilityOverrides?: Record<string, Partial<ModelCapabilities>>,
|
||||
): RoutingDecision {
|
||||
// If no phase config or routing disabled, pass through
|
||||
if (!phaseConfig || !routingConfig.enabled) {
|
||||
|
|
@ -377,7 +398,7 @@ export function resolveModelForComplexity(
|
|||
// STEP 2: Capability scoring (when enabled and multiple eligible models exist)
|
||||
if (routingConfig.capability_routing !== false && eligible.length > 1 && unitType) {
|
||||
const requirements = computeTaskRequirements(unitType, taskMetadata);
|
||||
const scored = scoreEligibleModels(eligible, requirements);
|
||||
const scored = scoreEligibleModels(eligible, requirements, capabilityOverrides);
|
||||
const winner = scored[0];
|
||||
if (winner) {
|
||||
const capScores: Record<string, number> = {};
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue