diff --git a/src/resources/extensions/sf/tests/provider-catalog-discovery.test.mjs b/src/resources/extensions/sf/tests/provider-catalog-discovery.test.mjs index e51d695ad..ce59fd5b7 100644 --- a/src/resources/extensions/sf/tests/provider-catalog-discovery.test.mjs +++ b/src/resources/extensions/sf/tests/provider-catalog-discovery.test.mjs @@ -1,12 +1,20 @@ /** * provider-catalog-discovery.test.mjs * - * Regression tests for live-discovery entries in provider-catalog-config.js. - * Covers: opencode, opencode-go, minimax, kimi-coding, zai, ollama-cloud, - * and xiaomi baseline fields. + * Regression tests for live-discovery entries in provider-catalog-config.js + * and runtime behavior of model-catalog-cache.js. + * + * Sections: + * 1. Provider catalog config — baseUrl / modelsPath / auth fields + * 2. parseDiscoveredModels — OpenAI / Google format parsing + * 3. refreshSfManagedProviders — iteration, auth headers, SDK cache writes + * 4. DISCOVERABLE_PROVIDER_IDS completeness */ import assert from "node:assert/strict"; -import { describe, test } from "vitest"; +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, test, vi } from "vitest"; import { DISCOVERABLE_PROVIDER_IDS, getProviderCatalogConfig, @@ -215,4 +223,296 @@ describe("DISCOVERABLE_PROVIDER_IDS completeness", () => { assert.ok(ids.includes("opencode-go"), "missing opencode-go"); assert.ok(ids.includes("minimax"), "missing minimax"); }); + + test("every discoverable provider has non-null modelsPath and baseUrl", () => { + for (const id of DISCOVERABLE_PROVIDER_IDS) { + const cfg = getProviderCatalogConfig(id); + assert.ok(cfg, `no catalog config for ${id}`); + assert.ok(cfg.modelsPath, `${id}.modelsPath is null/empty — should not be in DISCOVERABLE_PROVIDER_IDS`); + assert.ok(cfg.baseUrl, `${id}.baseUrl is null/empty`); + } + }); +}); + +// ─── parseDiscoveredModels ──────────────────────────────────────────────────── + +const { parseDiscoveredModels } = await import("../model-catalog-cache.js"); + +describe("parseDiscoveredModels — OpenAI format", () => { + test("parses { data: [{id}] } response", () => { + const cfg = { type: "openai" }; + const json = { data: [{ id: "model-a" }, { id: "model-b" }] }; + const result = parseDiscoveredModels(cfg, json); + assert.deepEqual(result, [{ id: "model-a" }, { id: "model-b" }]); + }); + + test("parses { models: [{id}] } fallback format", () => { + const cfg = { type: "openai" }; + const json = { models: [{ id: "model-x" }] }; + const result = parseDiscoveredModels(cfg, json); + assert.deepEqual(result, [{ id: "model-x" }]); + }); + + test("returns null for empty/invalid response", () => { + const cfg = { type: "openai" }; + assert.equal(parseDiscoveredModels(cfg, null), null); + assert.equal(parseDiscoveredModels(cfg, {}), null); + assert.equal(parseDiscoveredModels(cfg, { items: [] }), null); + }); + + test("uses 'name' field as fallback when 'id' is absent", () => { + const cfg = { type: "openai" }; + // m.id ?? m.name — so { name: "fallback-name" } becomes id: "fallback-name" + const json = { data: [{ id: "explicit-id" }, { name: "fallback-name" }] }; + const result = parseDiscoveredModels(cfg, json); + assert.deepEqual(result, [{ id: "explicit-id" }, { id: "fallback-name" }]); + }); + + test("filters out entries with neither id nor name", () => { + const cfg = { type: "openai" }; + const json = { data: [{ id: "valid" }, {}] }; + const result = parseDiscoveredModels(cfg, json); + assert.deepEqual(result, [{ id: "valid" }]); + }); +}); + +describe("parseDiscoveredModels — Google format", () => { + test("parses { models: [{name: 'models/gemini-...'}] } and strips 'models/' prefix", () => { + const cfg = { type: "google" }; + const json = { + models: [ + { name: "models/gemini-2.0-flash" }, + { name: "models/gemini-1.5-pro" }, + ], + }; + const result = parseDiscoveredModels(cfg, json); + assert.deepEqual(result, [{ id: "gemini-2.0-flash" }, { id: "gemini-1.5-pro" }]); + }); + + test("filters out entries with empty name after stripping prefix", () => { + const cfg = { type: "google" }; + const json = { models: [{ name: "models/gemini-2.0-flash" }, { name: "" }, {}] }; + const result = parseDiscoveredModels(cfg, json); + assert.deepEqual(result, [{ id: "gemini-2.0-flash" }]); + }); + + test("returns null when models array is absent", () => { + const cfg = { type: "google" }; + assert.equal(parseDiscoveredModels(cfg, { data: [] }), null); + }); +}); + +describe("parseDiscoveredModels — OpenRouter pricing extraction", () => { + test("extracts cost object from pricing field", () => { + const cfg = { type: "openai", providerId: "openrouter" }; + const json = { + data: [{ + id: "openai/gpt-4o", + pricing: { prompt: "0.000005", completion: "0.000015", input_cache_read: "0.0000025", input_cache_write: "0.000005" }, + context_length: 128000, + }], + }; + const result = parseDiscoveredModels(cfg, json); + assert.equal(result.length, 1); + assert.equal(result[0].id, "openai/gpt-4o"); + assert.ok(result[0].cost, "should have cost field"); + assert.equal(result[0].cost.input, 0.000005); + assert.equal(result[0].cost.output, 0.000015); + assert.equal(result[0].contextWindow, 128000); + }); +}); + +// ─── refreshSfManagedProviders — auth headers and SDK cache writes ──────────── + +describe("refreshSfManagedProviders — auth header and iteration", () => { + let tmp; + let originalSfHome; + let originalFetch; + + /** Captured fetch calls: [{ url, headers }] */ + const fetchCalls = []; + + beforeEach(() => { + fetchCalls.length = 0; + // Temp dirs: basePath (runtime model-catalog) and SF_HOME (SDK discovery cache) + tmp = mkdtempSync(join(tmpdir(), "sf-discovery-test-")); + // Override SF_HOME so writeSdkDiscoveryCacheEntry writes to our temp dir. + originalSfHome = process.env.SF_HOME; + process.env.SF_HOME = tmp; + // Intercept fetch. + originalFetch = globalThis.fetch; + globalThis.fetch = async (url, options = {}) => { + fetchCalls.push({ url, headers: { ...(options.headers ?? {}) } }); + // Return a minimal valid model list for any URL. + return { + ok: true, + json: async () => ({ data: [{ id: "stub-model" }] }), + }; + }; + }); + + afterEach(() => { + if (tmp) rmSync(tmp, { recursive: true, force: true }); + process.env.SF_HOME = originalSfHome; + globalThis.fetch = originalFetch; + }); + + test("uses x-api-key header for xiaomi (not Bearer)", async () => { + const { refreshSfManagedProviders } = await import("../model-catalog-cache.js"); + + // Provide a key only for xiaomi so we don't accidentally test other providers. + const auth = { + getCredentialsForProvider(id) { + if (id === "xiaomi") return [{ type: "api_key", key: "test-key-xiaomi" }]; + return []; + }, + }; + + await refreshSfManagedProviders(tmp, auth); + + const xiaomiCall = fetchCalls.find((c) => c.url.includes("xiaomimimo.com")); + assert.ok(xiaomiCall, "should have fetched xiaomi models endpoint"); + assert.equal( + xiaomiCall.headers["x-api-key"], + "test-key-xiaomi", + "xiaomi must use x-api-key header", + ); + assert.equal( + xiaomiCall.headers["Authorization"], + undefined, + "xiaomi must NOT use Authorization/Bearer header", + ); + }); + + test("uses Bearer header for opencode (not x-api-key)", async () => { + const { refreshSfManagedProviders } = await import("../model-catalog-cache.js"); + + const auth = { + getCredentialsForProvider(id) { + if (id === "opencode") return [{ type: "api_key", key: "test-key-opencode" }]; + return []; + }, + }; + + await refreshSfManagedProviders(tmp, auth); + + const opencodeCall = fetchCalls.find((c) => c.url.includes("opencode.ai/zen") && !c.url.includes("/go")); + assert.ok(opencodeCall, "should have fetched opencode models endpoint"); + assert.equal( + opencodeCall.headers["Authorization"], + "Bearer test-key-opencode", + "opencode must use Bearer auth", + ); + assert.equal( + opencodeCall.headers["x-api-key"], + undefined, + "opencode must NOT use x-api-key header", + ); + }); + + test("skips providers with no API key", async () => { + const { refreshSfManagedProviders } = await import("../model-catalog-cache.js"); + + // No keys at all. + const auth = { getCredentialsForProvider: () => [] }; + + await refreshSfManagedProviders(tmp, auth); + + assert.equal(fetchCalls.length, 0, "should make no fetch calls when no keys are configured"); + }); + + test("writes to SDK discovery cache when runtime cache is absent", async () => { + const { refreshSfManagedProviders } = await import("../model-catalog-cache.js"); + + const auth = { + getCredentialsForProvider(id) { + if (id === "opencode") return [{ type: "api_key", key: "test-key-oc" }]; + return []; + }, + }; + + await refreshSfManagedProviders(tmp, auth); + + const cachePath = join(tmp, "agent", "discovery-cache.json"); + assert.ok(existsSync(cachePath), "discovery-cache.json should be created"); + + const cache = JSON.parse(readFileSync(cachePath, "utf-8")); + assert.equal(cache.version, 1); + assert.ok(cache.entries["opencode"], "opencode should have an entry in the SDK discovery cache"); + assert.ok( + Array.isArray(cache.entries["opencode"].models), + "opencode entry should have a models array", + ); + }); + + test("iterates all SF-managed providers that have keys (kimi-coding and opencode-go)", async () => { + const { refreshSfManagedProviders } = await import("../model-catalog-cache.js"); + + const auth = { + getCredentialsForProvider(id) { + if (id === "kimi-coding") return [{ type: "api_key", key: "key-kimi" }]; + if (id === "opencode-go") return [{ type: "api_key", key: "key-oc-go" }]; + return []; + }, + }; + + await refreshSfManagedProviders(tmp, auth); + + const kimiCall = fetchCalls.find((c) => c.url.includes("api.kimi.com")); + assert.ok(kimiCall, "should have fetched kimi-coding models endpoint"); + + const goCall = fetchCalls.find((c) => c.url.includes("opencode.ai/zen/go")); + assert.ok(goCall, "should have fetched opencode-go models endpoint"); + + // Both should be in the SDK discovery cache. + const cachePath = join(tmp, "agent", "discovery-cache.json"); + assert.ok(existsSync(cachePath)); + const cache = JSON.parse(readFileSync(cachePath, "utf-8")); + assert.ok(cache.entries["kimi-coding"], "kimi-coding should be in SDK discovery cache"); + assert.ok(cache.entries["opencode-go"], "opencode-go should be in SDK discovery cache"); + }); + + test("writes runtime-cached data to SDK discovery cache without re-fetching", async () => { + const { refreshSfManagedProviders } = await import("../model-catalog-cache.js"); + const { writeFileSync, mkdirSync } = await import("node:fs"); + + // Pre-populate the runtime cache for opencode to simulate a cache hit. + // sfRoot(tmp) → join(tmp, ".sf") when no .sf dir exists (probeSfRoot fallback). + // cacheDirPath(tmp) → join(sfRuntimeRoot(tmp), "model-catalog") + // → join(sfRoot(tmp), "model-catalog") (isRunningOnSelf=false for tmp) + // → join(tmp, ".sf", "model-catalog") + const cacheDir = join(tmp, ".sf", "model-catalog"); + mkdirSync(cacheDir, { recursive: true }); + writeFileSync( + join(cacheDir, "opencode.json"), + JSON.stringify({ + fetchedAt: new Date().toISOString(), + modelIds: [{ id: "cached-model-1" }, { id: "cached-model-2" }], + }), + "utf-8", + ); + + const auth = { + getCredentialsForProvider(id) { + if (id === "opencode") return [{ type: "api_key", key: "test-key-oc" }]; + return []; + }, + }; + + await refreshSfManagedProviders(tmp, auth); + + // Should NOT have made a network call for opencode (cache was fresh). + const opencodeCall = fetchCalls.find((c) => c.url.includes("opencode.ai/zen") && !c.url.includes("/go")); + assert.equal(opencodeCall, undefined, "should NOT re-fetch opencode when runtime cache is fresh"); + + // But should still have written to the SDK discovery cache from cached data. + const cachePath = join(tmp, "agent", "discovery-cache.json"); + assert.ok(existsSync(cachePath), "SDK discovery cache should be written even on cache hit"); + const cache = JSON.parse(readFileSync(cachePath, "utf-8")); + assert.ok(cache.entries["opencode"], "opencode should be in SDK discovery cache"); + assert.ok( + cache.entries["opencode"].models.length > 0, + "SDK discovery cache should include cached model data", + ); + }); });