singularity-forge/packages/coding-agent/src/core/fallback-resolver.test.ts

// SF Provider Fallback Resolver Tests
// Copyright (c) 2026 Jeremy McSpadden <jeremy@fluxlabs.net>

import assert from "node:assert/strict";
import type { Api, Model } from "@singularity-forge/ai";
import { describe, it, vi } from "vitest";
import type { AuthStorage } from "./auth-storage.js";
import { FallbackResolver } from "./fallback-resolver.js";
import type { ModelRegistry } from "./model-registry.js";
import type {
	FallbackChainEntry,
	SettingsManager,
} from "./settings-manager.js";

function createMockModel(provider: string, id: string): Model<Api> {
	return {
		id,
		name: id,
		api: "openai-completions" as Api,
		provider,
		baseUrl: `https://api.${provider}.com`,
		reasoning: false,
		input: ["text"],
		cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
		contextWindow: 128000,
		maxTokens: 16384,
	} as Model<Api>;
}

const zaiModel = createMockModel("zai", "glm-5");
const alibabaModel = createMockModel("alibaba", "glm-5");
const openaiModel = createMockModel("openai", "gpt-4.1");

const defaultChain: FallbackChainEntry[] = [
	{ provider: "zai", model: "glm-5", priority: 1 },
	{ provider: "alibaba", model: "glm-5", priority: 2 },
	{ provider: "openai", model: "gpt-4.1", priority: 3 },
];

function createResolver(overrides?: {
	enabled?: boolean;
	isProviderAvailable?: (provider: string) => boolean;
	hasAuth?: (provider: string) => boolean;
	isProviderRequestReady?: (provider: string) => boolean;
	find?: (provider: string, modelId: string) => Model<Api> | undefined;
	getAvailable?: () => Model<Api>[];
}) {
	const settingsManager = {
		getFallbackSettings: () => ({
			enabled: overrides?.enabled ?? true,
			chains: { coding: defaultChain },
		}),
	} as unknown as SettingsManager;

	const authStorage = {
		markProviderExhausted: vi.fn(),
		isProviderAvailable: overrides?.isProviderAvailable ?? (() => true),
		hasAuth: overrides?.hasAuth ?? (() => true),
	} as unknown as AuthStorage;

	const modelRegistry = {
		find:
			overrides?.find ??
			((provider: string, modelId: string) => {
				if (provider === "zai" && modelId === "glm-5") return zaiModel;
				if (provider === "alibaba" && modelId === "glm-5") return alibabaModel;
				if (provider === "openai" && modelId === "gpt-4.1") return openaiModel;
				return undefined;
			}),
		isProviderRequestReady:
			overrides?.isProviderRequestReady ?? overrides?.hasAuth ?? (() => true),
		getAvailable:
			overrides?.getAvailable ?? (() => [zaiModel, alibabaModel, openaiModel]),
	} as unknown as ModelRegistry;

	return {
		resolver: new FallbackResolver(settingsManager, authStorage, modelRegistry),
		authStorage,
	};
}

// ─── findFallback ────────────────────────────────────────────────────────────

describe("FallbackResolver — findFallback", () => {
	it("reselects from the current available models when current fails", async () => {
		const { resolver } = createResolver();
		const result = await resolver.findFallback(zaiModel, "quota_exhausted");

		assert.notEqual(result, null);
		assert.equal(result!.model.provider, "alibaba");
		assert.equal(result!.chainName, "fresh-selection");
	});

	it("marks current provider as exhausted for rate_limit errors", async () => {
		const { resolver, authStorage } = createResolver();
		await resolver.findFallback(zaiModel, "rate_limit");

		const fn = authStorage.markProviderExhausted as any;
		assert.equal(fn.mock.calls.length, 1);
		assert.equal(fn.mock.calls[0][0], "zai");
		assert.equal(fn.mock.calls[0][1], "rate_limit");
	});

	it("does NOT mark provider as exhausted for quota_exhausted (per-model quota)", async () => {
		const { resolver, authStorage } = createResolver();
		await resolver.findFallback(zaiModel, "quota_exhausted");

		const fn = authStorage.markProviderExhausted as any;
		assert.equal(
			fn.mock.calls.length,
			0,
			"quota_exhausted should not mark entire provider exhausted — other models may have quota",
		);
	});

	it("skips backed-off providers", async () => {
		const { resolver } = createResolver({
			isProviderAvailable: (provider: string) => provider !== "alibaba",
		});

		const result = await resolver.findFallback(zaiModel, "quota_exhausted");

		assert.notEqual(result, null);
		assert.equal(result!.model.provider, "openai");
		assert.equal(result!.model.id, "gpt-4.1");
	});

	it("returns null when all providers are backed off", async () => {
		const { resolver } = createResolver({
			isProviderAvailable: () => false,
			getAvailable: () => [zaiModel, alibabaModel, openaiModel],
		});

		const result = await resolver.findFallback(zaiModel, "quota_exhausted");
		assert.equal(result, null);
	});

	it("returns null when fallback is disabled", async () => {
		const { resolver } = createResolver({ enabled: false });
		const result = await resolver.findFallback(zaiModel, "quota_exhausted");
		assert.equal(result, null);
	});

	it("reselects from scratch when model is not in any chain", async () => {
		const { resolver } = createResolver();
		const unknownModel = createMockModel("unknown", "some-model");
		const result = await resolver.findFallback(unknownModel, "quota_exhausted");
		assert.notEqual(result, null);
		assert.equal(result!.chainName, "fresh-selection");
		// Should pick an available model with different provider
		assert.notEqual(result!.model.provider, "unknown");
	});

	it("free selection prefers models with matching reasoning capability", async () => {
		const reasoningModel = createMockModel("openai", "gpt-4.1");
		reasoningModel.reasoning = true;
		const nonReasoningModel = createMockModel("alibaba", "glm-5");
		nonReasoningModel.reasoning = false;

		const { resolver } = createResolver({
			getAvailable: () => [nonReasoningModel, reasoningModel],
		});

		const currentModel = createMockModel("unknown", "some-model");
		currentModel.reasoning = true;

		const result = await resolver.findFallback(currentModel, "quota_exhausted");
		assert.notEqual(result, null);
		assert.equal(result!.model.provider, "openai");
		assert.equal(result!.model.reasoning, true);
	});

	it("free selection excludes same provider", async () => {
		const sameProviderModel = createMockModel("zai", "glm-5-other");
		const differentProviderModel = createMockModel("alibaba", "glm-5");

		const { resolver } = createResolver({
			getAvailable: () => [sameProviderModel, differentProviderModel],
		});

		const result = await resolver.findFallback(zaiModel, "quota_exhausted");
		assert.notEqual(result, null);
		assert.equal(result!.model.provider, "alibaba");
	});

	it("skips providers that are not request-ready", async () => {
		const { resolver } = createResolver({
			isProviderRequestReady: (provider: string) => provider !== "alibaba",
		});

		const result = await resolver.findFallback(zaiModel, "quota_exhausted");

		assert.notEqual(result, null);
		assert.equal(result!.model.provider, "openai");
	});

	it("allows fallback to external-cli style providers without stored auth", async () => {
		const { resolver } = createResolver({
			hasAuth: () => false,
			isProviderRequestReady: (provider: string) => provider === "alibaba",
		});

		const result = await resolver.findFallback(zaiModel, "quota_exhausted");
		assert.notEqual(result, null);
		assert.equal(result!.model.provider, "alibaba");
	});

	it("skips providers with no model in registry", async () => {
		const { resolver } = createResolver({
			getAvailable: () => [openaiModel],
		});

		const result = await resolver.findFallback(zaiModel, "quota_exhausted");

		assert.notEqual(result, null);
		assert.equal(result!.model.provider, "openai");
	});
});

// ─── checkForRestoration ─────────────────────────────────────────────────────

describe("FallbackResolver — checkForRestoration", () => {
	it("returns null because restoration is disabled", async () => {
		const { resolver } = createResolver();
		const result = await resolver.checkForRestoration(alibabaModel);
		assert.equal(result, null);
	});
});

// ─── getBestAvailable ────────────────────────────────────────────────────────

describe("FallbackResolver — getBestAvailable", () => {
	it("returns highest-priority available provider", async () => {
		const { resolver } = createResolver();
		const result = await resolver.getBestAvailable("coding");

		assert.notEqual(result, null);
		assert.equal(result!.model.provider, "zai");
	});

	it("skips backed-off providers", async () => {
		const { resolver } = createResolver({
			isProviderAvailable: (provider: string) => provider !== "zai",
		});

		const result = await resolver.getBestAvailable("coding");

		assert.notEqual(result, null);
		assert.equal(result!.model.provider, "alibaba");
	});

	it("returns null for unknown chain", async () => {
		const { resolver } = createResolver();
		const result = await resolver.getBestAvailable("nonexistent");
		assert.equal(result, null);
	});
});

// ─── findChainsForModel ──────────────────────────────────────────────────────

describe("FallbackResolver — findChainsForModel", () => {
	it("finds chains containing a model", () => {
		const { resolver } = createResolver();
		const chains = resolver.findChainsForModel("zai", "glm-5");
		assert.deepEqual(chains, ["coding"]);
	});

	it("returns empty array for model not in any chain", () => {
		const { resolver } = createResolver();
		const chains = resolver.findChainsForModel("unknown", "model");
		assert.deepEqual(chains, []);
	});
});