From 009651e86fd66afae21e294037f488be74e45adb Mon Sep 17 00:00:00 2001 From: Mikael Hugo Date: Sun, 10 May 2026 23:05:33 +0200 Subject: [PATCH] feat(selection): wire before_model_select into FallbackResolver for outcome-aware fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a model fails and FallbackResolver picks a replacement, it now: 1. Fires the before_model_select hook with reason='fallback' and the failing model's ID — the learning system records the failure outcome and returns the best Bayesian-blended replacement from llm_task_outcomes 2. Falls back to the existing heuristic sort (reasoning + context window) if the hook is unavailable or returns no override Changes: - BeforeModelSelectEvent: add optional currentModelId and reason fields - FallbackResolver: accept emitBeforeModelSelect in constructor; make _findAnyAvailableFallback async; fire hook before heuristic fallback - agent-session.ts: inject lazy emitBeforeModelSelect closure into resolver - register-hooks.js: record failure outcome when reason='fallback' before returning selectLearnedModel result Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../coding-agent/src/core/agent-session.ts | 4 ++ .../coding-agent/src/core/extensions/types.ts | 11 ++++ .../src/core/fallback-resolver.ts | 64 +++++++++++++++++-- .../extensions/sf/bootstrap/register-hooks.js | 13 +++- 4 files changed, 85 insertions(+), 7 deletions(-) diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index 6f19b76da..7567d9f61 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -402,6 +402,10 @@ export class AgentSession { this.settingsManager, this._modelRegistry.authStorage, this._modelRegistry, + // Lazy closure — extension runner is set after construction + (event) => + this._extensionRunnerRef?.current?.emitBeforeModelSelect(event) ?? + Promise.resolve(undefined), ); this._extensionRunnerRef = config.extensionRunnerRef; this._initialActiveToolNames = config.initialActiveToolNames; diff --git a/packages/coding-agent/src/core/extensions/types.ts b/packages/coding-agent/src/core/extensions/types.ts index b9d1e7844..259cc6134 100644 --- a/packages/coding-agent/src/core/extensions/types.ts +++ b/packages/coding-agent/src/core/extensions/types.ts @@ -700,6 +700,17 @@ export interface BeforeModelSelectEvent { taskMetadata?: Record; eligibleModels: string[]; phaseConfig?: { primary: string; fallbacks: string[] }; + /** + * Set when fired by FallbackResolver after a model fails. Enables failure recording + * in the learning system so the failed model is demoted in future selections. + */ + currentModelId?: string; + /** + * "fallback" = fired by FallbackResolver (model is down/rate-limited). + * "auto" = fired by autonomous dispatch (normal unit scheduling). + * Defaults to "auto" when absent. + */ + reason?: "fallback" | "auto"; } /** Result from before_model_select event handler. Return { modelId } to override selection. */ diff --git a/packages/coding-agent/src/core/fallback-resolver.ts b/packages/coding-agent/src/core/fallback-resolver.ts index 0924d6b93..f7c1994bd 100644 --- a/packages/coding-agent/src/core/fallback-resolver.ts +++ b/packages/coding-agent/src/core/fallback-resolver.ts @@ -10,6 +10,10 @@ import type { Api, Model } from "@singularity-forge/ai"; import type { AuthStorage, UsageLimitErrorType } from "./auth-storage.js"; +import type { + BeforeModelSelectEvent, + BeforeModelSelectResult, +} from "./extensions/types.js"; import type { ModelRegistry } from "./model-registry.js"; import type { FallbackChainEntry, @@ -22,11 +26,18 @@ export interface FallbackResult { reason: string; } +type EmitBeforeModelSelect = ( + event: Omit, +) => Promise; + export class FallbackResolver { constructor( private settingsManager: SettingsManager, private authStorage: AuthStorage, private modelRegistry: ModelRegistry, + /** Optional hook emitter — when provided, fires before_model_select so the + * learning system can influence which replacement model is chosen. */ + private emitBeforeModelSelect?: EmitBeforeModelSelect, ) {} /** @@ -50,7 +61,7 @@ export class FallbackResolver { this.authStorage.markProviderExhausted(currentModel.provider, errorType); } - return this._findAnyAvailableFallback(currentModel); + return this._findAnyAvailableFallback(currentModel, errorType); } /** @@ -131,12 +142,14 @@ export class FallbackResolver { /** * Free-selection fallback when no chain contains the current model. - * Picks any available model from the registry with a different provider. - * Prefers models with reasoning capability if the current model has it. + * Fires before_model_select hook so the learning system can rank candidates + * by outcome history and Bayesian benchmarks. Falls back to heuristic sort + * (reasoning match + context window) if the hook is unavailable or returns nothing. */ - private _findAnyAvailableFallback( + private async _findAnyAvailableFallback( currentModel: Model, - ): FallbackResult | null { + errorType?: UsageLimitErrorType, + ): Promise { const allModels = this.modelRegistry.getAvailable(); const candidates = allModels.filter((m) => { // Exclude same provider — credential rotation was already tried @@ -150,7 +163,46 @@ export class FallbackResolver { if (candidates.length === 0) return null; - // Sort: prefer models with matching reasoning capability, then by context window + // Fire before_model_select so the learning system can: + // 1. Record the current model as failed (reason="fallback") + // 2. Return the best outcome-weighted replacement + if (this.emitBeforeModelSelect) { + try { + const result = await this.emitBeforeModelSelect({ + unitType: "execute-task", + unitId: "", + classification: { + tier: "standard", + reason: errorType ?? "unknown", + downgraded: false, + }, + eligibleModels: candidates.map((m) => `${m.provider}/${m.id}`), + currentModelId: `${currentModel.provider}/${currentModel.id}`, + reason: "fallback", + }); + if (result?.modelId) { + const slashIdx = result.modelId.indexOf("/"); + if (slashIdx > 0) { + const provider = result.modelId.slice(0, slashIdx); + const modelId = result.modelId.slice(slashIdx + 1); + const preferred = candidates.find( + (m) => m.provider === provider && m.id === modelId, + ); + if (preferred) { + return { + model: preferred, + chainName: "learned-fallback", + reason: `learned routing selected ${result.modelId} (outcome-weighted)`, + }; + } + } + } + } catch { + // Hook failure → fall through to heuristic sort + } + } + + // Heuristic: prefer reasoning capability match, then larger context window candidates.sort((a, b) => { const aReasoningMatch = a.reasoning === currentModel.reasoning ? 1 : 0; const bReasoningMatch = b.reasoning === currentModel.reasoning ? 1 : 0; diff --git a/src/resources/extensions/sf/bootstrap/register-hooks.js b/src/resources/extensions/sf/bootstrap/register-hooks.js index aa8bc4579..1a2faf485 100644 --- a/src/resources/extensions/sf/bootstrap/register-hooks.js +++ b/src/resources/extensions/sf/bootstrap/register-hooks.js @@ -10,7 +10,7 @@ import { import { join, relative, resolve } from "node:path"; import { isToolCallEventType } from "@singularity-forge/coding-agent"; import { resetAskUserQuestionsCache } from "../../ask-user-questions.js"; -import { formatTokenCount } from "../../shared/format-utils.js"; +import { formatTokenCount } from "@singularity-forge/coding-agent"; import { saveActivityLog } from "../activity-log.js"; import { getAutoDashboardData, @@ -1533,7 +1533,18 @@ export function registerHooks(pi, ecosystemHandlers = []) { // Capability-aware model routing hook (ADR-004) // Extensions can override model selection by returning { modelId: "..." } // Return undefined to let the built-in capability scoring proceed. + // When reason="fallback", the current model just failed — record it as a failure + // so the learning system demotes it in future selections. pi.on("before_model_select", async (event) => { + if (event.reason === "fallback" && event.currentModelId) { + recordLearnedOutcome({ + model_id: event.currentModelId, + unit_type: event.unitType ?? "execute-task", + unit_id: `fallback:${event.currentModelId}`, + succeeded: false, + recorded_at: Date.now(), + }); + } return selectLearnedModel({ unitType: event.unitType, eligibleModels: event.eligibleModels,