feat(selection): wire before_model_select into FallbackResolver for outcome-aware fallback
When a model fails and FallbackResolver picks a replacement, it now: 1. Fires the before_model_select hook with reason='fallback' and the failing model's ID — the learning system records the failure outcome and returns the best Bayesian-blended replacement from llm_task_outcomes 2. Falls back to the existing heuristic sort (reasoning + context window) if the hook is unavailable or returns no override Changes: - BeforeModelSelectEvent: add optional currentModelId and reason fields - FallbackResolver: accept emitBeforeModelSelect in constructor; make _findAnyAvailableFallback async; fire hook before heuristic fallback - agent-session.ts: inject lazy emitBeforeModelSelect closure into resolver - register-hooks.js: record failure outcome when reason='fallback' before returning selectLearnedModel result Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
parent
fb1bd3e5fa
commit
009651e86f
4 changed files with 85 additions and 7 deletions
|
|
@ -402,6 +402,10 @@ export class AgentSession {
|
|||
this.settingsManager,
|
||||
this._modelRegistry.authStorage,
|
||||
this._modelRegistry,
|
||||
// Lazy closure — extension runner is set after construction
|
||||
(event) =>
|
||||
this._extensionRunnerRef?.current?.emitBeforeModelSelect(event) ??
|
||||
Promise.resolve(undefined),
|
||||
);
|
||||
this._extensionRunnerRef = config.extensionRunnerRef;
|
||||
this._initialActiveToolNames = config.initialActiveToolNames;
|
||||
|
|
|
|||
|
|
@ -700,6 +700,17 @@ export interface BeforeModelSelectEvent {
|
|||
taskMetadata?: Record<string, unknown>;
|
||||
eligibleModels: string[];
|
||||
phaseConfig?: { primary: string; fallbacks: string[] };
|
||||
/**
|
||||
* Set when fired by FallbackResolver after a model fails. Enables failure recording
|
||||
* in the learning system so the failed model is demoted in future selections.
|
||||
*/
|
||||
currentModelId?: string;
|
||||
/**
|
||||
* "fallback" = fired by FallbackResolver (model is down/rate-limited).
|
||||
* "auto" = fired by autonomous dispatch (normal unit scheduling).
|
||||
* Defaults to "auto" when absent.
|
||||
*/
|
||||
reason?: "fallback" | "auto";
|
||||
}
|
||||
|
||||
/** Result from before_model_select event handler. Return { modelId } to override selection. */
|
||||
|
|
|
|||
|
|
@ -10,6 +10,10 @@
|
|||
|
||||
import type { Api, Model } from "@singularity-forge/ai";
|
||||
import type { AuthStorage, UsageLimitErrorType } from "./auth-storage.js";
|
||||
import type {
|
||||
BeforeModelSelectEvent,
|
||||
BeforeModelSelectResult,
|
||||
} from "./extensions/types.js";
|
||||
import type { ModelRegistry } from "./model-registry.js";
|
||||
import type {
|
||||
FallbackChainEntry,
|
||||
|
|
@ -22,11 +26,18 @@ export interface FallbackResult {
|
|||
reason: string;
|
||||
}
|
||||
|
||||
type EmitBeforeModelSelect = (
|
||||
event: Omit<BeforeModelSelectEvent, "type">,
|
||||
) => Promise<BeforeModelSelectResult | undefined>;
|
||||
|
||||
export class FallbackResolver {
|
||||
constructor(
|
||||
private settingsManager: SettingsManager,
|
||||
private authStorage: AuthStorage,
|
||||
private modelRegistry: ModelRegistry,
|
||||
/** Optional hook emitter — when provided, fires before_model_select so the
|
||||
* learning system can influence which replacement model is chosen. */
|
||||
private emitBeforeModelSelect?: EmitBeforeModelSelect,
|
||||
) {}
|
||||
|
||||
/**
|
||||
|
|
@ -50,7 +61,7 @@ export class FallbackResolver {
|
|||
this.authStorage.markProviderExhausted(currentModel.provider, errorType);
|
||||
}
|
||||
|
||||
return this._findAnyAvailableFallback(currentModel);
|
||||
return this._findAnyAvailableFallback(currentModel, errorType);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -131,12 +142,14 @@ export class FallbackResolver {
|
|||
|
||||
/**
|
||||
* Free-selection fallback when no chain contains the current model.
|
||||
* Picks any available model from the registry with a different provider.
|
||||
* Prefers models with reasoning capability if the current model has it.
|
||||
* Fires before_model_select hook so the learning system can rank candidates
|
||||
* by outcome history and Bayesian benchmarks. Falls back to heuristic sort
|
||||
* (reasoning match + context window) if the hook is unavailable or returns nothing.
|
||||
*/
|
||||
private _findAnyAvailableFallback(
|
||||
private async _findAnyAvailableFallback(
|
||||
currentModel: Model<Api>,
|
||||
): FallbackResult | null {
|
||||
errorType?: UsageLimitErrorType,
|
||||
): Promise<FallbackResult | null> {
|
||||
const allModels = this.modelRegistry.getAvailable();
|
||||
const candidates = allModels.filter((m) => {
|
||||
// Exclude same provider — credential rotation was already tried
|
||||
|
|
@ -150,7 +163,46 @@ export class FallbackResolver {
|
|||
|
||||
if (candidates.length === 0) return null;
|
||||
|
||||
// Sort: prefer models with matching reasoning capability, then by context window
|
||||
// Fire before_model_select so the learning system can:
|
||||
// 1. Record the current model as failed (reason="fallback")
|
||||
// 2. Return the best outcome-weighted replacement
|
||||
if (this.emitBeforeModelSelect) {
|
||||
try {
|
||||
const result = await this.emitBeforeModelSelect({
|
||||
unitType: "execute-task",
|
||||
unitId: "",
|
||||
classification: {
|
||||
tier: "standard",
|
||||
reason: errorType ?? "unknown",
|
||||
downgraded: false,
|
||||
},
|
||||
eligibleModels: candidates.map((m) => `${m.provider}/${m.id}`),
|
||||
currentModelId: `${currentModel.provider}/${currentModel.id}`,
|
||||
reason: "fallback",
|
||||
});
|
||||
if (result?.modelId) {
|
||||
const slashIdx = result.modelId.indexOf("/");
|
||||
if (slashIdx > 0) {
|
||||
const provider = result.modelId.slice(0, slashIdx);
|
||||
const modelId = result.modelId.slice(slashIdx + 1);
|
||||
const preferred = candidates.find(
|
||||
(m) => m.provider === provider && m.id === modelId,
|
||||
);
|
||||
if (preferred) {
|
||||
return {
|
||||
model: preferred,
|
||||
chainName: "learned-fallback",
|
||||
reason: `learned routing selected ${result.modelId} (outcome-weighted)`,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Hook failure → fall through to heuristic sort
|
||||
}
|
||||
}
|
||||
|
||||
// Heuristic: prefer reasoning capability match, then larger context window
|
||||
candidates.sort((a, b) => {
|
||||
const aReasoningMatch = a.reasoning === currentModel.reasoning ? 1 : 0;
|
||||
const bReasoningMatch = b.reasoning === currentModel.reasoning ? 1 : 0;
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import {
|
|||
import { join, relative, resolve } from "node:path";
|
||||
import { isToolCallEventType } from "@singularity-forge/coding-agent";
|
||||
import { resetAskUserQuestionsCache } from "../../ask-user-questions.js";
|
||||
import { formatTokenCount } from "../../shared/format-utils.js";
|
||||
import { formatTokenCount } from "@singularity-forge/coding-agent";
|
||||
import { saveActivityLog } from "../activity-log.js";
|
||||
import {
|
||||
getAutoDashboardData,
|
||||
|
|
@ -1533,7 +1533,18 @@ export function registerHooks(pi, ecosystemHandlers = []) {
|
|||
// Capability-aware model routing hook (ADR-004)
|
||||
// Extensions can override model selection by returning { modelId: "..." }
|
||||
// Return undefined to let the built-in capability scoring proceed.
|
||||
// When reason="fallback", the current model just failed — record it as a failure
|
||||
// so the learning system demotes it in future selections.
|
||||
pi.on("before_model_select", async (event) => {
|
||||
if (event.reason === "fallback" && event.currentModelId) {
|
||||
recordLearnedOutcome({
|
||||
model_id: event.currentModelId,
|
||||
unit_type: event.unitType ?? "execute-task",
|
||||
unit_id: `fallback:${event.currentModelId}`,
|
||||
succeeded: false,
|
||||
recorded_at: Date.now(),
|
||||
});
|
||||
}
|
||||
return selectLearnedModel({
|
||||
unitType: event.unitType,
|
||||
eligibleModels: event.eligibleModels,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue