feat(selection): wire before_model_select into FallbackResolver for outcome-aware fallback

When a model fails and FallbackResolver picks a replacement, it now:
1. Fires the before_model_select hook with reason='fallback' and the
   failing model's ID — the learning system records the failure outcome
   and returns the best Bayesian-blended replacement from llm_task_outcomes
2. Falls back to the existing heuristic sort (reasoning + context window)
   if the hook is unavailable or returns no override

Changes:
- BeforeModelSelectEvent: add optional currentModelId and reason fields
- FallbackResolver: accept emitBeforeModelSelect in constructor; make
  _findAnyAvailableFallback async; fire hook before heuristic fallback
- agent-session.ts: inject lazy emitBeforeModelSelect closure into resolver
- register-hooks.js: record failure outcome when reason='fallback' before
  returning selectLearnedModel result

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Mikael Hugo 2026-05-10 23:05:33 +02:00
parent fb1bd3e5fa
commit 009651e86f
4 changed files with 85 additions and 7 deletions

View file

@ -402,6 +402,10 @@ export class AgentSession {
this.settingsManager,
this._modelRegistry.authStorage,
this._modelRegistry,
// Lazy closure — extension runner is set after construction
(event) =>
this._extensionRunnerRef?.current?.emitBeforeModelSelect(event) ??
Promise.resolve(undefined),
);
this._extensionRunnerRef = config.extensionRunnerRef;
this._initialActiveToolNames = config.initialActiveToolNames;

View file

@ -700,6 +700,17 @@ export interface BeforeModelSelectEvent {
taskMetadata?: Record<string, unknown>;
eligibleModels: string[];
phaseConfig?: { primary: string; fallbacks: string[] };
/**
* Set when fired by FallbackResolver after a model fails. Enables failure recording
* in the learning system so the failed model is demoted in future selections.
*/
currentModelId?: string;
/**
* "fallback" = fired by FallbackResolver (model is down/rate-limited).
* "auto" = fired by autonomous dispatch (normal unit scheduling).
* Defaults to "auto" when absent.
*/
reason?: "fallback" | "auto";
}
/** Result from before_model_select event handler. Return { modelId } to override selection. */

View file

@ -10,6 +10,10 @@
import type { Api, Model } from "@singularity-forge/ai";
import type { AuthStorage, UsageLimitErrorType } from "./auth-storage.js";
import type {
BeforeModelSelectEvent,
BeforeModelSelectResult,
} from "./extensions/types.js";
import type { ModelRegistry } from "./model-registry.js";
import type {
FallbackChainEntry,
@ -22,11 +26,18 @@ export interface FallbackResult {
reason: string;
}
type EmitBeforeModelSelect = (
event: Omit<BeforeModelSelectEvent, "type">,
) => Promise<BeforeModelSelectResult | undefined>;
export class FallbackResolver {
constructor(
private settingsManager: SettingsManager,
private authStorage: AuthStorage,
private modelRegistry: ModelRegistry,
/** Optional hook emitter when provided, fires before_model_select so the
* learning system can influence which replacement model is chosen. */
private emitBeforeModelSelect?: EmitBeforeModelSelect,
) {}
/**
@ -50,7 +61,7 @@ export class FallbackResolver {
this.authStorage.markProviderExhausted(currentModel.provider, errorType);
}
return this._findAnyAvailableFallback(currentModel);
return this._findAnyAvailableFallback(currentModel, errorType);
}
/**
@ -131,12 +142,14 @@ export class FallbackResolver {
/**
* Free-selection fallback when no chain contains the current model.
* Picks any available model from the registry with a different provider.
* Prefers models with reasoning capability if the current model has it.
* Fires before_model_select hook so the learning system can rank candidates
* by outcome history and Bayesian benchmarks. Falls back to heuristic sort
* (reasoning match + context window) if the hook is unavailable or returns nothing.
*/
private _findAnyAvailableFallback(
private async _findAnyAvailableFallback(
currentModel: Model<Api>,
): FallbackResult | null {
errorType?: UsageLimitErrorType,
): Promise<FallbackResult | null> {
const allModels = this.modelRegistry.getAvailable();
const candidates = allModels.filter((m) => {
// Exclude same provider — credential rotation was already tried
@ -150,7 +163,46 @@ export class FallbackResolver {
if (candidates.length === 0) return null;
// Sort: prefer models with matching reasoning capability, then by context window
// Fire before_model_select so the learning system can:
// 1. Record the current model as failed (reason="fallback")
// 2. Return the best outcome-weighted replacement
if (this.emitBeforeModelSelect) {
try {
const result = await this.emitBeforeModelSelect({
unitType: "execute-task",
unitId: "",
classification: {
tier: "standard",
reason: errorType ?? "unknown",
downgraded: false,
},
eligibleModels: candidates.map((m) => `${m.provider}/${m.id}`),
currentModelId: `${currentModel.provider}/${currentModel.id}`,
reason: "fallback",
});
if (result?.modelId) {
const slashIdx = result.modelId.indexOf("/");
if (slashIdx > 0) {
const provider = result.modelId.slice(0, slashIdx);
const modelId = result.modelId.slice(slashIdx + 1);
const preferred = candidates.find(
(m) => m.provider === provider && m.id === modelId,
);
if (preferred) {
return {
model: preferred,
chainName: "learned-fallback",
reason: `learned routing selected ${result.modelId} (outcome-weighted)`,
};
}
}
}
} catch {
// Hook failure → fall through to heuristic sort
}
}
// Heuristic: prefer reasoning capability match, then larger context window
candidates.sort((a, b) => {
const aReasoningMatch = a.reasoning === currentModel.reasoning ? 1 : 0;
const bReasoningMatch = b.reasoning === currentModel.reasoning ? 1 : 0;

View file

@ -10,7 +10,7 @@ import {
import { join, relative, resolve } from "node:path";
import { isToolCallEventType } from "@singularity-forge/coding-agent";
import { resetAskUserQuestionsCache } from "../../ask-user-questions.js";
import { formatTokenCount } from "../../shared/format-utils.js";
import { formatTokenCount } from "@singularity-forge/coding-agent";
import { saveActivityLog } from "../activity-log.js";
import {
getAutoDashboardData,
@ -1533,7 +1533,18 @@ export function registerHooks(pi, ecosystemHandlers = []) {
// Capability-aware model routing hook (ADR-004)
// Extensions can override model selection by returning { modelId: "..." }
// Return undefined to let the built-in capability scoring proceed.
// When reason="fallback", the current model just failed — record it as a failure
// so the learning system demotes it in future selections.
pi.on("before_model_select", async (event) => {
if (event.reason === "fallback" && event.currentModelId) {
recordLearnedOutcome({
model_id: event.currentModelId,
unit_type: event.unitType ?? "execute-task",
unit_id: `fallback:${event.currentModelId}`,
succeeded: false,
recorded_at: Date.now(),
});
}
return selectLearnedModel({
unitType: event.unitType,
eligibleModels: event.eligibleModels,