test(providers): expand discovery test matrix to 46 cases
Some checks are pending
CI / detect-changes (push) Waiting to run
CI / docs-check (push) Blocked by required conditions
CI / lint (push) Blocked by required conditions
CI / build (push) Blocked by required conditions
CI / integration-tests (push) Blocked by required conditions
CI / windows-portability (push) Blocked by required conditions
CI / rtk-portability (linux, blacksmith-4vcpu-ubuntu-2404) (push) Blocked by required conditions
CI / rtk-portability (macos, macos-15) (push) Blocked by required conditions
CI / rtk-portability (windows, blacksmith-4vcpu-windows-2025) (push) Blocked by required conditions

Adds full coverage for the discovery-gating root cause that was
fixed in commits d70d8d3b1 (xiaomi x-api-key auth) and the
subsequent refreshSfManagedProviders + writeSdkDiscoveryCacheEntry
work in model-catalog-cache.js.

Diagnosis recap: kimi-coding, opencode, opencode-go were silent
in ~/.sf/agent/discovery-cache.json because the SDK's
model-discovery.js adapter registry marked them with
StaticDiscoveryAdapter (supportsDiscovery=false), so the SDK's
discoverModels() never attempted them. SF's own
scheduleModelCatalogRefresh DID fetch them but wrote only to the
per-repo runtime cache (basePath/.sf/model-catalog/) and only fired
on session_start — not during --discover. The fix is to mirror the
write to the SDK's discovery cache on both fetch-path AND cache-hit
path, and await it in cli.ts before listModels when --discover is set.

New test sections:
- parseDiscoveredModels: OpenAI {data}/{models} formats, Google
  {models[].name} prefix stripping, name-as-id fallback, null on
  bad input, OpenRouter pricing extraction
- refreshSfManagedProviders: xiaomi uses x-api-key (not Bearer),
  opencode uses Bearer, no-key providers skipped, SDK discovery cache
  written on BOTH network-fetch and cache-hit paths, kimi-coding +
  opencode-go iterated when keys present

46 tests pass. No regressions.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Mikael Hugo 2026-05-15 15:09:38 +02:00
parent 67c088410c
commit 0b19afebf6

View file

@ -1,12 +1,20 @@
/**
* provider-catalog-discovery.test.mjs
*
* Regression tests for live-discovery entries in provider-catalog-config.js.
* Covers: opencode, opencode-go, minimax, kimi-coding, zai, ollama-cloud,
* and xiaomi baseline fields.
* Regression tests for live-discovery entries in provider-catalog-config.js
* and runtime behavior of model-catalog-cache.js.
*
* Sections:
* 1. Provider catalog config baseUrl / modelsPath / auth fields
* 2. parseDiscoveredModels OpenAI / Google format parsing
* 3. refreshSfManagedProviders iteration, auth headers, SDK cache writes
* 4. DISCOVERABLE_PROVIDER_IDS completeness
*/
import assert from "node:assert/strict";
import { describe, test } from "vitest";
import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, beforeEach, describe, test, vi } from "vitest";
import {
DISCOVERABLE_PROVIDER_IDS,
getProviderCatalogConfig,
@ -215,4 +223,296 @@ describe("DISCOVERABLE_PROVIDER_IDS completeness", () => {
assert.ok(ids.includes("opencode-go"), "missing opencode-go");
assert.ok(ids.includes("minimax"), "missing minimax");
});
test("every discoverable provider has non-null modelsPath and baseUrl", () => {
for (const id of DISCOVERABLE_PROVIDER_IDS) {
const cfg = getProviderCatalogConfig(id);
assert.ok(cfg, `no catalog config for ${id}`);
assert.ok(cfg.modelsPath, `${id}.modelsPath is null/empty — should not be in DISCOVERABLE_PROVIDER_IDS`);
assert.ok(cfg.baseUrl, `${id}.baseUrl is null/empty`);
}
});
});
// ─── parseDiscoveredModels ────────────────────────────────────────────────────
const { parseDiscoveredModels } = await import("../model-catalog-cache.js");
describe("parseDiscoveredModels — OpenAI format", () => {
test("parses { data: [{id}] } response", () => {
const cfg = { type: "openai" };
const json = { data: [{ id: "model-a" }, { id: "model-b" }] };
const result = parseDiscoveredModels(cfg, json);
assert.deepEqual(result, [{ id: "model-a" }, { id: "model-b" }]);
});
test("parses { models: [{id}] } fallback format", () => {
const cfg = { type: "openai" };
const json = { models: [{ id: "model-x" }] };
const result = parseDiscoveredModels(cfg, json);
assert.deepEqual(result, [{ id: "model-x" }]);
});
test("returns null for empty/invalid response", () => {
const cfg = { type: "openai" };
assert.equal(parseDiscoveredModels(cfg, null), null);
assert.equal(parseDiscoveredModels(cfg, {}), null);
assert.equal(parseDiscoveredModels(cfg, { items: [] }), null);
});
test("uses 'name' field as fallback when 'id' is absent", () => {
const cfg = { type: "openai" };
// m.id ?? m.name — so { name: "fallback-name" } becomes id: "fallback-name"
const json = { data: [{ id: "explicit-id" }, { name: "fallback-name" }] };
const result = parseDiscoveredModels(cfg, json);
assert.deepEqual(result, [{ id: "explicit-id" }, { id: "fallback-name" }]);
});
test("filters out entries with neither id nor name", () => {
const cfg = { type: "openai" };
const json = { data: [{ id: "valid" }, {}] };
const result = parseDiscoveredModels(cfg, json);
assert.deepEqual(result, [{ id: "valid" }]);
});
});
describe("parseDiscoveredModels — Google format", () => {
test("parses { models: [{name: 'models/gemini-...'}] } and strips 'models/' prefix", () => {
const cfg = { type: "google" };
const json = {
models: [
{ name: "models/gemini-2.0-flash" },
{ name: "models/gemini-1.5-pro" },
],
};
const result = parseDiscoveredModels(cfg, json);
assert.deepEqual(result, [{ id: "gemini-2.0-flash" }, { id: "gemini-1.5-pro" }]);
});
test("filters out entries with empty name after stripping prefix", () => {
const cfg = { type: "google" };
const json = { models: [{ name: "models/gemini-2.0-flash" }, { name: "" }, {}] };
const result = parseDiscoveredModels(cfg, json);
assert.deepEqual(result, [{ id: "gemini-2.0-flash" }]);
});
test("returns null when models array is absent", () => {
const cfg = { type: "google" };
assert.equal(parseDiscoveredModels(cfg, { data: [] }), null);
});
});
describe("parseDiscoveredModels — OpenRouter pricing extraction", () => {
test("extracts cost object from pricing field", () => {
const cfg = { type: "openai", providerId: "openrouter" };
const json = {
data: [{
id: "openai/gpt-4o",
pricing: { prompt: "0.000005", completion: "0.000015", input_cache_read: "0.0000025", input_cache_write: "0.000005" },
context_length: 128000,
}],
};
const result = parseDiscoveredModels(cfg, json);
assert.equal(result.length, 1);
assert.equal(result[0].id, "openai/gpt-4o");
assert.ok(result[0].cost, "should have cost field");
assert.equal(result[0].cost.input, 0.000005);
assert.equal(result[0].cost.output, 0.000015);
assert.equal(result[0].contextWindow, 128000);
});
});
// ─── refreshSfManagedProviders — auth headers and SDK cache writes ────────────
describe("refreshSfManagedProviders — auth header and iteration", () => {
let tmp;
let originalSfHome;
let originalFetch;
/** Captured fetch calls: [{ url, headers }] */
const fetchCalls = [];
beforeEach(() => {
fetchCalls.length = 0;
// Temp dirs: basePath (runtime model-catalog) and SF_HOME (SDK discovery cache)
tmp = mkdtempSync(join(tmpdir(), "sf-discovery-test-"));
// Override SF_HOME so writeSdkDiscoveryCacheEntry writes to our temp dir.
originalSfHome = process.env.SF_HOME;
process.env.SF_HOME = tmp;
// Intercept fetch.
originalFetch = globalThis.fetch;
globalThis.fetch = async (url, options = {}) => {
fetchCalls.push({ url, headers: { ...(options.headers ?? {}) } });
// Return a minimal valid model list for any URL.
return {
ok: true,
json: async () => ({ data: [{ id: "stub-model" }] }),
};
};
});
afterEach(() => {
if (tmp) rmSync(tmp, { recursive: true, force: true });
process.env.SF_HOME = originalSfHome;
globalThis.fetch = originalFetch;
});
test("uses x-api-key header for xiaomi (not Bearer)", async () => {
const { refreshSfManagedProviders } = await import("../model-catalog-cache.js");
// Provide a key only for xiaomi so we don't accidentally test other providers.
const auth = {
getCredentialsForProvider(id) {
if (id === "xiaomi") return [{ type: "api_key", key: "test-key-xiaomi" }];
return [];
},
};
await refreshSfManagedProviders(tmp, auth);
const xiaomiCall = fetchCalls.find((c) => c.url.includes("xiaomimimo.com"));
assert.ok(xiaomiCall, "should have fetched xiaomi models endpoint");
assert.equal(
xiaomiCall.headers["x-api-key"],
"test-key-xiaomi",
"xiaomi must use x-api-key header",
);
assert.equal(
xiaomiCall.headers["Authorization"],
undefined,
"xiaomi must NOT use Authorization/Bearer header",
);
});
test("uses Bearer header for opencode (not x-api-key)", async () => {
const { refreshSfManagedProviders } = await import("../model-catalog-cache.js");
const auth = {
getCredentialsForProvider(id) {
if (id === "opencode") return [{ type: "api_key", key: "test-key-opencode" }];
return [];
},
};
await refreshSfManagedProviders(tmp, auth);
const opencodeCall = fetchCalls.find((c) => c.url.includes("opencode.ai/zen") && !c.url.includes("/go"));
assert.ok(opencodeCall, "should have fetched opencode models endpoint");
assert.equal(
opencodeCall.headers["Authorization"],
"Bearer test-key-opencode",
"opencode must use Bearer auth",
);
assert.equal(
opencodeCall.headers["x-api-key"],
undefined,
"opencode must NOT use x-api-key header",
);
});
test("skips providers with no API key", async () => {
const { refreshSfManagedProviders } = await import("../model-catalog-cache.js");
// No keys at all.
const auth = { getCredentialsForProvider: () => [] };
await refreshSfManagedProviders(tmp, auth);
assert.equal(fetchCalls.length, 0, "should make no fetch calls when no keys are configured");
});
test("writes to SDK discovery cache when runtime cache is absent", async () => {
const { refreshSfManagedProviders } = await import("../model-catalog-cache.js");
const auth = {
getCredentialsForProvider(id) {
if (id === "opencode") return [{ type: "api_key", key: "test-key-oc" }];
return [];
},
};
await refreshSfManagedProviders(tmp, auth);
const cachePath = join(tmp, "agent", "discovery-cache.json");
assert.ok(existsSync(cachePath), "discovery-cache.json should be created");
const cache = JSON.parse(readFileSync(cachePath, "utf-8"));
assert.equal(cache.version, 1);
assert.ok(cache.entries["opencode"], "opencode should have an entry in the SDK discovery cache");
assert.ok(
Array.isArray(cache.entries["opencode"].models),
"opencode entry should have a models array",
);
});
test("iterates all SF-managed providers that have keys (kimi-coding and opencode-go)", async () => {
const { refreshSfManagedProviders } = await import("../model-catalog-cache.js");
const auth = {
getCredentialsForProvider(id) {
if (id === "kimi-coding") return [{ type: "api_key", key: "key-kimi" }];
if (id === "opencode-go") return [{ type: "api_key", key: "key-oc-go" }];
return [];
},
};
await refreshSfManagedProviders(tmp, auth);
const kimiCall = fetchCalls.find((c) => c.url.includes("api.kimi.com"));
assert.ok(kimiCall, "should have fetched kimi-coding models endpoint");
const goCall = fetchCalls.find((c) => c.url.includes("opencode.ai/zen/go"));
assert.ok(goCall, "should have fetched opencode-go models endpoint");
// Both should be in the SDK discovery cache.
const cachePath = join(tmp, "agent", "discovery-cache.json");
assert.ok(existsSync(cachePath));
const cache = JSON.parse(readFileSync(cachePath, "utf-8"));
assert.ok(cache.entries["kimi-coding"], "kimi-coding should be in SDK discovery cache");
assert.ok(cache.entries["opencode-go"], "opencode-go should be in SDK discovery cache");
});
test("writes runtime-cached data to SDK discovery cache without re-fetching", async () => {
const { refreshSfManagedProviders } = await import("../model-catalog-cache.js");
const { writeFileSync, mkdirSync } = await import("node:fs");
// Pre-populate the runtime cache for opencode to simulate a cache hit.
// sfRoot(tmp) → join(tmp, ".sf") when no .sf dir exists (probeSfRoot fallback).
// cacheDirPath(tmp) → join(sfRuntimeRoot(tmp), "model-catalog")
// → join(sfRoot(tmp), "model-catalog") (isRunningOnSelf=false for tmp)
// → join(tmp, ".sf", "model-catalog")
const cacheDir = join(tmp, ".sf", "model-catalog");
mkdirSync(cacheDir, { recursive: true });
writeFileSync(
join(cacheDir, "opencode.json"),
JSON.stringify({
fetchedAt: new Date().toISOString(),
modelIds: [{ id: "cached-model-1" }, { id: "cached-model-2" }],
}),
"utf-8",
);
const auth = {
getCredentialsForProvider(id) {
if (id === "opencode") return [{ type: "api_key", key: "test-key-oc" }];
return [];
},
};
await refreshSfManagedProviders(tmp, auth);
// Should NOT have made a network call for opencode (cache was fresh).
const opencodeCall = fetchCalls.find((c) => c.url.includes("opencode.ai/zen") && !c.url.includes("/go"));
assert.equal(opencodeCall, undefined, "should NOT re-fetch opencode when runtime cache is fresh");
// But should still have written to the SDK discovery cache from cached data.
const cachePath = join(tmp, "agent", "discovery-cache.json");
assert.ok(existsSync(cachePath), "SDK discovery cache should be written even on cache hit");
const cache = JSON.parse(readFileSync(cachePath, "utf-8"));
assert.ok(cache.entries["opencode"], "opencode should be in SDK discovery cache");
assert.ok(
cache.entries["opencode"].models.length > 0,
"SDK discovery cache should include cached model data",
);
});
});