test(providers): expand discovery test matrix to 46 cases
Some checks are pending
CI / detect-changes (push) Waiting to run
CI / docs-check (push) Blocked by required conditions
CI / lint (push) Blocked by required conditions
CI / build (push) Blocked by required conditions
CI / integration-tests (push) Blocked by required conditions
CI / windows-portability (push) Blocked by required conditions
CI / rtk-portability (linux, blacksmith-4vcpu-ubuntu-2404) (push) Blocked by required conditions
CI / rtk-portability (macos, macos-15) (push) Blocked by required conditions
CI / rtk-portability (windows, blacksmith-4vcpu-windows-2025) (push) Blocked by required conditions
Some checks are pending
CI / detect-changes (push) Waiting to run
CI / docs-check (push) Blocked by required conditions
CI / lint (push) Blocked by required conditions
CI / build (push) Blocked by required conditions
CI / integration-tests (push) Blocked by required conditions
CI / windows-portability (push) Blocked by required conditions
CI / rtk-portability (linux, blacksmith-4vcpu-ubuntu-2404) (push) Blocked by required conditions
CI / rtk-portability (macos, macos-15) (push) Blocked by required conditions
CI / rtk-portability (windows, blacksmith-4vcpu-windows-2025) (push) Blocked by required conditions
Adds full coverage for the discovery-gating root cause that was
fixed in commits d70d8d3b1 (xiaomi x-api-key auth) and the
subsequent refreshSfManagedProviders + writeSdkDiscoveryCacheEntry
work in model-catalog-cache.js.
Diagnosis recap: kimi-coding, opencode, opencode-go were silent
in ~/.sf/agent/discovery-cache.json because the SDK's
model-discovery.js adapter registry marked them with
StaticDiscoveryAdapter (supportsDiscovery=false), so the SDK's
discoverModels() never attempted them. SF's own
scheduleModelCatalogRefresh DID fetch them but wrote only to the
per-repo runtime cache (basePath/.sf/model-catalog/) and only fired
on session_start — not during --discover. The fix is to mirror the
write to the SDK's discovery cache on both fetch-path AND cache-hit
path, and await it in cli.ts before listModels when --discover is set.
New test sections:
- parseDiscoveredModels: OpenAI {data}/{models} formats, Google
{models[].name} prefix stripping, name-as-id fallback, null on
bad input, OpenRouter pricing extraction
- refreshSfManagedProviders: xiaomi uses x-api-key (not Bearer),
opencode uses Bearer, no-key providers skipped, SDK discovery cache
written on BOTH network-fetch and cache-hit paths, kimi-coding +
opencode-go iterated when keys present
46 tests pass. No regressions.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
67c088410c
commit
0b19afebf6
1 changed files with 304 additions and 4 deletions
|
|
@ -1,12 +1,20 @@
|
||||||
/**
|
/**
|
||||||
* provider-catalog-discovery.test.mjs
|
* provider-catalog-discovery.test.mjs
|
||||||
*
|
*
|
||||||
* Regression tests for live-discovery entries in provider-catalog-config.js.
|
* Regression tests for live-discovery entries in provider-catalog-config.js
|
||||||
* Covers: opencode, opencode-go, minimax, kimi-coding, zai, ollama-cloud,
|
* and runtime behavior of model-catalog-cache.js.
|
||||||
* and xiaomi baseline fields.
|
*
|
||||||
|
* Sections:
|
||||||
|
* 1. Provider catalog config — baseUrl / modelsPath / auth fields
|
||||||
|
* 2. parseDiscoveredModels — OpenAI / Google format parsing
|
||||||
|
* 3. refreshSfManagedProviders — iteration, auth headers, SDK cache writes
|
||||||
|
* 4. DISCOVERABLE_PROVIDER_IDS completeness
|
||||||
*/
|
*/
|
||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
import { describe, test } from "vitest";
|
import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import { afterEach, beforeEach, describe, test, vi } from "vitest";
|
||||||
import {
|
import {
|
||||||
DISCOVERABLE_PROVIDER_IDS,
|
DISCOVERABLE_PROVIDER_IDS,
|
||||||
getProviderCatalogConfig,
|
getProviderCatalogConfig,
|
||||||
|
|
@ -215,4 +223,296 @@ describe("DISCOVERABLE_PROVIDER_IDS completeness", () => {
|
||||||
assert.ok(ids.includes("opencode-go"), "missing opencode-go");
|
assert.ok(ids.includes("opencode-go"), "missing opencode-go");
|
||||||
assert.ok(ids.includes("minimax"), "missing minimax");
|
assert.ok(ids.includes("minimax"), "missing minimax");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("every discoverable provider has non-null modelsPath and baseUrl", () => {
|
||||||
|
for (const id of DISCOVERABLE_PROVIDER_IDS) {
|
||||||
|
const cfg = getProviderCatalogConfig(id);
|
||||||
|
assert.ok(cfg, `no catalog config for ${id}`);
|
||||||
|
assert.ok(cfg.modelsPath, `${id}.modelsPath is null/empty — should not be in DISCOVERABLE_PROVIDER_IDS`);
|
||||||
|
assert.ok(cfg.baseUrl, `${id}.baseUrl is null/empty`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ─── parseDiscoveredModels ────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
const { parseDiscoveredModels } = await import("../model-catalog-cache.js");
|
||||||
|
|
||||||
|
describe("parseDiscoveredModels — OpenAI format", () => {
|
||||||
|
test("parses { data: [{id}] } response", () => {
|
||||||
|
const cfg = { type: "openai" };
|
||||||
|
const json = { data: [{ id: "model-a" }, { id: "model-b" }] };
|
||||||
|
const result = parseDiscoveredModels(cfg, json);
|
||||||
|
assert.deepEqual(result, [{ id: "model-a" }, { id: "model-b" }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("parses { models: [{id}] } fallback format", () => {
|
||||||
|
const cfg = { type: "openai" };
|
||||||
|
const json = { models: [{ id: "model-x" }] };
|
||||||
|
const result = parseDiscoveredModels(cfg, json);
|
||||||
|
assert.deepEqual(result, [{ id: "model-x" }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("returns null for empty/invalid response", () => {
|
||||||
|
const cfg = { type: "openai" };
|
||||||
|
assert.equal(parseDiscoveredModels(cfg, null), null);
|
||||||
|
assert.equal(parseDiscoveredModels(cfg, {}), null);
|
||||||
|
assert.equal(parseDiscoveredModels(cfg, { items: [] }), null);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("uses 'name' field as fallback when 'id' is absent", () => {
|
||||||
|
const cfg = { type: "openai" };
|
||||||
|
// m.id ?? m.name — so { name: "fallback-name" } becomes id: "fallback-name"
|
||||||
|
const json = { data: [{ id: "explicit-id" }, { name: "fallback-name" }] };
|
||||||
|
const result = parseDiscoveredModels(cfg, json);
|
||||||
|
assert.deepEqual(result, [{ id: "explicit-id" }, { id: "fallback-name" }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("filters out entries with neither id nor name", () => {
|
||||||
|
const cfg = { type: "openai" };
|
||||||
|
const json = { data: [{ id: "valid" }, {}] };
|
||||||
|
const result = parseDiscoveredModels(cfg, json);
|
||||||
|
assert.deepEqual(result, [{ id: "valid" }]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("parseDiscoveredModels — Google format", () => {
|
||||||
|
test("parses { models: [{name: 'models/gemini-...'}] } and strips 'models/' prefix", () => {
|
||||||
|
const cfg = { type: "google" };
|
||||||
|
const json = {
|
||||||
|
models: [
|
||||||
|
{ name: "models/gemini-2.0-flash" },
|
||||||
|
{ name: "models/gemini-1.5-pro" },
|
||||||
|
],
|
||||||
|
};
|
||||||
|
const result = parseDiscoveredModels(cfg, json);
|
||||||
|
assert.deepEqual(result, [{ id: "gemini-2.0-flash" }, { id: "gemini-1.5-pro" }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("filters out entries with empty name after stripping prefix", () => {
|
||||||
|
const cfg = { type: "google" };
|
||||||
|
const json = { models: [{ name: "models/gemini-2.0-flash" }, { name: "" }, {}] };
|
||||||
|
const result = parseDiscoveredModels(cfg, json);
|
||||||
|
assert.deepEqual(result, [{ id: "gemini-2.0-flash" }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("returns null when models array is absent", () => {
|
||||||
|
const cfg = { type: "google" };
|
||||||
|
assert.equal(parseDiscoveredModels(cfg, { data: [] }), null);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("parseDiscoveredModels — OpenRouter pricing extraction", () => {
|
||||||
|
test("extracts cost object from pricing field", () => {
|
||||||
|
const cfg = { type: "openai", providerId: "openrouter" };
|
||||||
|
const json = {
|
||||||
|
data: [{
|
||||||
|
id: "openai/gpt-4o",
|
||||||
|
pricing: { prompt: "0.000005", completion: "0.000015", input_cache_read: "0.0000025", input_cache_write: "0.000005" },
|
||||||
|
context_length: 128000,
|
||||||
|
}],
|
||||||
|
};
|
||||||
|
const result = parseDiscoveredModels(cfg, json);
|
||||||
|
assert.equal(result.length, 1);
|
||||||
|
assert.equal(result[0].id, "openai/gpt-4o");
|
||||||
|
assert.ok(result[0].cost, "should have cost field");
|
||||||
|
assert.equal(result[0].cost.input, 0.000005);
|
||||||
|
assert.equal(result[0].cost.output, 0.000015);
|
||||||
|
assert.equal(result[0].contextWindow, 128000);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ─── refreshSfManagedProviders — auth headers and SDK cache writes ────────────
|
||||||
|
|
||||||
|
describe("refreshSfManagedProviders — auth header and iteration", () => {
|
||||||
|
let tmp;
|
||||||
|
let originalSfHome;
|
||||||
|
let originalFetch;
|
||||||
|
|
||||||
|
/** Captured fetch calls: [{ url, headers }] */
|
||||||
|
const fetchCalls = [];
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
fetchCalls.length = 0;
|
||||||
|
// Temp dirs: basePath (runtime model-catalog) and SF_HOME (SDK discovery cache)
|
||||||
|
tmp = mkdtempSync(join(tmpdir(), "sf-discovery-test-"));
|
||||||
|
// Override SF_HOME so writeSdkDiscoveryCacheEntry writes to our temp dir.
|
||||||
|
originalSfHome = process.env.SF_HOME;
|
||||||
|
process.env.SF_HOME = tmp;
|
||||||
|
// Intercept fetch.
|
||||||
|
originalFetch = globalThis.fetch;
|
||||||
|
globalThis.fetch = async (url, options = {}) => {
|
||||||
|
fetchCalls.push({ url, headers: { ...(options.headers ?? {}) } });
|
||||||
|
// Return a minimal valid model list for any URL.
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
json: async () => ({ data: [{ id: "stub-model" }] }),
|
||||||
|
};
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
if (tmp) rmSync(tmp, { recursive: true, force: true });
|
||||||
|
process.env.SF_HOME = originalSfHome;
|
||||||
|
globalThis.fetch = originalFetch;
|
||||||
|
});
|
||||||
|
|
||||||
|
test("uses x-api-key header for xiaomi (not Bearer)", async () => {
|
||||||
|
const { refreshSfManagedProviders } = await import("../model-catalog-cache.js");
|
||||||
|
|
||||||
|
// Provide a key only for xiaomi so we don't accidentally test other providers.
|
||||||
|
const auth = {
|
||||||
|
getCredentialsForProvider(id) {
|
||||||
|
if (id === "xiaomi") return [{ type: "api_key", key: "test-key-xiaomi" }];
|
||||||
|
return [];
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
await refreshSfManagedProviders(tmp, auth);
|
||||||
|
|
||||||
|
const xiaomiCall = fetchCalls.find((c) => c.url.includes("xiaomimimo.com"));
|
||||||
|
assert.ok(xiaomiCall, "should have fetched xiaomi models endpoint");
|
||||||
|
assert.equal(
|
||||||
|
xiaomiCall.headers["x-api-key"],
|
||||||
|
"test-key-xiaomi",
|
||||||
|
"xiaomi must use x-api-key header",
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
xiaomiCall.headers["Authorization"],
|
||||||
|
undefined,
|
||||||
|
"xiaomi must NOT use Authorization/Bearer header",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("uses Bearer header for opencode (not x-api-key)", async () => {
|
||||||
|
const { refreshSfManagedProviders } = await import("../model-catalog-cache.js");
|
||||||
|
|
||||||
|
const auth = {
|
||||||
|
getCredentialsForProvider(id) {
|
||||||
|
if (id === "opencode") return [{ type: "api_key", key: "test-key-opencode" }];
|
||||||
|
return [];
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
await refreshSfManagedProviders(tmp, auth);
|
||||||
|
|
||||||
|
const opencodeCall = fetchCalls.find((c) => c.url.includes("opencode.ai/zen") && !c.url.includes("/go"));
|
||||||
|
assert.ok(opencodeCall, "should have fetched opencode models endpoint");
|
||||||
|
assert.equal(
|
||||||
|
opencodeCall.headers["Authorization"],
|
||||||
|
"Bearer test-key-opencode",
|
||||||
|
"opencode must use Bearer auth",
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
opencodeCall.headers["x-api-key"],
|
||||||
|
undefined,
|
||||||
|
"opencode must NOT use x-api-key header",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("skips providers with no API key", async () => {
|
||||||
|
const { refreshSfManagedProviders } = await import("../model-catalog-cache.js");
|
||||||
|
|
||||||
|
// No keys at all.
|
||||||
|
const auth = { getCredentialsForProvider: () => [] };
|
||||||
|
|
||||||
|
await refreshSfManagedProviders(tmp, auth);
|
||||||
|
|
||||||
|
assert.equal(fetchCalls.length, 0, "should make no fetch calls when no keys are configured");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("writes to SDK discovery cache when runtime cache is absent", async () => {
|
||||||
|
const { refreshSfManagedProviders } = await import("../model-catalog-cache.js");
|
||||||
|
|
||||||
|
const auth = {
|
||||||
|
getCredentialsForProvider(id) {
|
||||||
|
if (id === "opencode") return [{ type: "api_key", key: "test-key-oc" }];
|
||||||
|
return [];
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
await refreshSfManagedProviders(tmp, auth);
|
||||||
|
|
||||||
|
const cachePath = join(tmp, "agent", "discovery-cache.json");
|
||||||
|
assert.ok(existsSync(cachePath), "discovery-cache.json should be created");
|
||||||
|
|
||||||
|
const cache = JSON.parse(readFileSync(cachePath, "utf-8"));
|
||||||
|
assert.equal(cache.version, 1);
|
||||||
|
assert.ok(cache.entries["opencode"], "opencode should have an entry in the SDK discovery cache");
|
||||||
|
assert.ok(
|
||||||
|
Array.isArray(cache.entries["opencode"].models),
|
||||||
|
"opencode entry should have a models array",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("iterates all SF-managed providers that have keys (kimi-coding and opencode-go)", async () => {
|
||||||
|
const { refreshSfManagedProviders } = await import("../model-catalog-cache.js");
|
||||||
|
|
||||||
|
const auth = {
|
||||||
|
getCredentialsForProvider(id) {
|
||||||
|
if (id === "kimi-coding") return [{ type: "api_key", key: "key-kimi" }];
|
||||||
|
if (id === "opencode-go") return [{ type: "api_key", key: "key-oc-go" }];
|
||||||
|
return [];
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
await refreshSfManagedProviders(tmp, auth);
|
||||||
|
|
||||||
|
const kimiCall = fetchCalls.find((c) => c.url.includes("api.kimi.com"));
|
||||||
|
assert.ok(kimiCall, "should have fetched kimi-coding models endpoint");
|
||||||
|
|
||||||
|
const goCall = fetchCalls.find((c) => c.url.includes("opencode.ai/zen/go"));
|
||||||
|
assert.ok(goCall, "should have fetched opencode-go models endpoint");
|
||||||
|
|
||||||
|
// Both should be in the SDK discovery cache.
|
||||||
|
const cachePath = join(tmp, "agent", "discovery-cache.json");
|
||||||
|
assert.ok(existsSync(cachePath));
|
||||||
|
const cache = JSON.parse(readFileSync(cachePath, "utf-8"));
|
||||||
|
assert.ok(cache.entries["kimi-coding"], "kimi-coding should be in SDK discovery cache");
|
||||||
|
assert.ok(cache.entries["opencode-go"], "opencode-go should be in SDK discovery cache");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("writes runtime-cached data to SDK discovery cache without re-fetching", async () => {
|
||||||
|
const { refreshSfManagedProviders } = await import("../model-catalog-cache.js");
|
||||||
|
const { writeFileSync, mkdirSync } = await import("node:fs");
|
||||||
|
|
||||||
|
// Pre-populate the runtime cache for opencode to simulate a cache hit.
|
||||||
|
// sfRoot(tmp) → join(tmp, ".sf") when no .sf dir exists (probeSfRoot fallback).
|
||||||
|
// cacheDirPath(tmp) → join(sfRuntimeRoot(tmp), "model-catalog")
|
||||||
|
// → join(sfRoot(tmp), "model-catalog") (isRunningOnSelf=false for tmp)
|
||||||
|
// → join(tmp, ".sf", "model-catalog")
|
||||||
|
const cacheDir = join(tmp, ".sf", "model-catalog");
|
||||||
|
mkdirSync(cacheDir, { recursive: true });
|
||||||
|
writeFileSync(
|
||||||
|
join(cacheDir, "opencode.json"),
|
||||||
|
JSON.stringify({
|
||||||
|
fetchedAt: new Date().toISOString(),
|
||||||
|
modelIds: [{ id: "cached-model-1" }, { id: "cached-model-2" }],
|
||||||
|
}),
|
||||||
|
"utf-8",
|
||||||
|
);
|
||||||
|
|
||||||
|
const auth = {
|
||||||
|
getCredentialsForProvider(id) {
|
||||||
|
if (id === "opencode") return [{ type: "api_key", key: "test-key-oc" }];
|
||||||
|
return [];
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
await refreshSfManagedProviders(tmp, auth);
|
||||||
|
|
||||||
|
// Should NOT have made a network call for opencode (cache was fresh).
|
||||||
|
const opencodeCall = fetchCalls.find((c) => c.url.includes("opencode.ai/zen") && !c.url.includes("/go"));
|
||||||
|
assert.equal(opencodeCall, undefined, "should NOT re-fetch opencode when runtime cache is fresh");
|
||||||
|
|
||||||
|
// But should still have written to the SDK discovery cache from cached data.
|
||||||
|
const cachePath = join(tmp, "agent", "discovery-cache.json");
|
||||||
|
assert.ok(existsSync(cachePath), "SDK discovery cache should be written even on cache hit");
|
||||||
|
const cache = JSON.parse(readFileSync(cachePath, "utf-8"));
|
||||||
|
assert.ok(cache.entries["opencode"], "opencode should be in SDK discovery cache");
|
||||||
|
assert.ok(
|
||||||
|
cache.entries["opencode"].models.length > 0,
|
||||||
|
"SDK discovery cache should include cached model data",
|
||||||
|
);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue