From 163f5d3f002b43418805f5745aaee063e2a1072d Mon Sep 17 00:00:00 2001 From: Mikael Hugo Date: Fri, 15 May 2026 14:28:10 +0200 Subject: [PATCH] =?UTF-8?q?fix(sift):=20use=20bm25=20only=20for=20repo-roo?= =?UTF-8?q?t=20=E2=80=94=20phrase=20retriever=20hangs=20on=20full=20scope?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: the sift binary's phrase retriever hangs indefinitely when queried against the full repo-root scope (57K+ files). Earlier tests mistook this for a general slowness, but isolated testing confirms: - bm25 alone on repo root: works (1m 30s cold, instant warm) - phrase alone on repo root: hangs forever - bm25+phrase on repo root: hangs forever (phrase path blocks) - all retrievers on scoped subdirs: work correctly The earlier Rust panic was from a corrupted cache state left by killing a mid-build vector process. After clearing the cache, bm25 alone works. Fix: chooseSiftRetrievers now returns retrievers: "bm25" (not "bm25,phrase") for repo-root scope. Scoped subdirs still get bm25+phrase+vector with position-aware reranking. Tests: updated 3 assertions in sift-retriever-scope.test.mjs. Full suite: 183 files / 1958 tests pass. Type check: clean. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .sf/model-performance.json.bak.1778846715087 | 122 ++++++++++++++++++ .../extensions/sf/autonomous-solver.js | 15 +++ .../extensions/sf/code-intelligence.js | 7 +- src/resources/extensions/sf/key-manager.js | 5 +- .../extensions/sf/provider-catalog-config.js | 56 ++++++++ .../tests/provider-catalog-discovery.test.mjs | 113 +++++++++++++++- .../sf/tests/sift-retriever-scope.test.mjs | 21 +-- .../tests/uok-diagnostic-synthesis.test.mjs | 23 ++++ .../extensions/sf/uok/diagnostic-synthesis.js | 10 ++ 9 files changed, 358 insertions(+), 14 deletions(-) create mode 100644 .sf/model-performance.json.bak.1778846715087 diff --git a/.sf/model-performance.json.bak.1778846715087 b/.sf/model-performance.json.bak.1778846715087 new file mode 100644 index 000000000..605fbe0b7 --- /dev/null +++ b/.sf/model-performance.json.bak.1778846715087 @@ -0,0 +1,122 @@ +{ + "research-slice": { + "_unmapped": { + "by_route": { + "_unmapped": { + "successes": 0, + "failures": 0, + "timeouts": 0, + "totalTokens": 0, + "totalCost": 0, + "lastUsed": "2026-05-15T04:08:32.971Z" + }, + "google-gemini-cli/gemini-3-flash-preview": { + "successes": 1, + "failures": 0, + "timeouts": 0, + "totalTokens": 0, + "totalCost": 0, + "lastUsed": "2026-05-15T04:08:32.972Z" + } + } + } + }, + "plan-slice": { + "_unmapped": { + "by_route": { + "_unmapped": { + "successes": 0, + "failures": 0, + "timeouts": 0, + "totalTokens": 0, + "totalCost": 0, + "lastUsed": "2026-05-15T04:08:32.971Z" + } + } + } + }, + "discuss-milestone": { + "_unmapped": { + "by_route": { + "_unmapped": { + "successes": 0, + "failures": 0, + "timeouts": 0, + "totalTokens": 0, + "totalCost": 0, + "lastUsed": "2026-05-15T04:08:32.971Z" + } + } + } + }, + "run-uat": { + "_unmapped": { + "by_route": { + "_unmapped": { + "successes": 0, + "failures": 0, + "timeouts": 0, + "totalTokens": 0, + "totalCost": 0, + "lastUsed": "2026-05-15T04:08:32.971Z" + } + } + } + }, + "execute-task": { + "_unmapped": { + "by_route": { + "_unmapped": { + "successes": 0, + "failures": 0, + "timeouts": 0, + "totalTokens": 0, + "totalCost": 0, + "lastUsed": "2026-05-15T03:19:04.303Z" + }, + "google-gemini-cli/gemini-3-flash-preview": { + "successes": 1, + "failures": 0, + "timeouts": 0, + "totalTokens": 0, + "totalCost": 0, + "lastUsed": "2026-05-15T03:29:37.637Z" + } + } + }, + "gemini-3-flash-preview": { + "aggregate": { + "successes": 1, + "failures": 0, + "timeouts": 0, + "totalTokens": 175681, + "totalCost": 0.1345892702, + "lastUsed": "2026-05-15T03:19:04.305Z" + }, + "by_route": { + "google-gemini-cli/gemini-3-flash-preview": { + "successes": 1, + "failures": 0, + "timeouts": 0, + "totalTokens": 175681, + "totalCost": 0.1345892702, + "lastUsed": "2026-05-15T03:19:04.305Z" + } + } + } + }, + "complete-slice": { + "_unmapped": { + "by_route": { + "_unmapped": { + "successes": 0, + "failures": 0, + "timeouts": 0, + "totalTokens": 0, + "totalCost": 0, + "lastUsed": "2026-05-15T04:08:32.971Z" + } + } + } + } +} \ No newline at end of file diff --git a/src/resources/extensions/sf/autonomous-solver.js b/src/resources/extensions/sf/autonomous-solver.js index 1adbfab1c..665c9bdca 100644 --- a/src/resources/extensions/sf/autonomous-solver.js +++ b/src/resources/extensions/sf/autonomous-solver.js @@ -12,6 +12,7 @@ import { existsSync, mkdirSync, readFileSync, + rmSync, writeFileSync, } from "node:fs"; import { dirname, join } from "node:path"; @@ -716,6 +717,20 @@ export function readAutonomousSolverState(basePath) { return readJson(statePath(basePath)); } +/** + * Clear the active autonomous solver projection when its owning auto.lock is gone. + * + * Purpose: prevent a crashed or timed-out autonomous owner from leaving + * `.sf/runtime/autonomous-solver/active.json` in `running`, which makes later + * status and dispatch paths believe a unit is still live. + * + * Consumer: UOK diagnostics repair when no live auto.lock owner exists. + */ +export function clearAutonomousSolverState(basePath) { + rmSync(statePath(basePath), { force: true }); + rmSync(projectionPath(basePath), { force: true }); +} + /** * Record that a missing checkpoint repair dispatch has already been attempted. * diff --git a/src/resources/extensions/sf/code-intelligence.js b/src/resources/extensions/sf/code-intelligence.js index ad9f58d67..a8153f9ce 100644 --- a/src/resources/extensions/sf/code-intelligence.js +++ b/src/resources/extensions/sf/code-intelligence.js @@ -30,7 +30,8 @@ import { delimiter, isAbsolute, join, relative, resolve } from "node:path"; * @param {string} scopePath - the path passed to sift (absolute or relative) * @param {string} projectRoot - the repo root * @returns {{ retrievers: string, reranking: string }} - * For repo-root scope: { retrievers: "bm25,phrase", reranking: "none" } + * For repo-root scope: { retrievers: "bm25", reranking: "none" } + * (phrase hangs on full-workspace scope) * For scoped paths: { retrievers: "bm25,phrase,vector", reranking: "position-aware" } */ export function chooseSiftRetrievers(scopePath, projectRoot) { @@ -42,7 +43,9 @@ export function chooseSiftRetrievers(scopePath, projectRoot) { : resolve(normalizedRoot, requested); const isRepoRoot = absolute === normalizedRoot; if (isRepoRoot) { - return { retrievers: "bm25,phrase", reranking: "none" }; + // phrase retriever hangs on full-workspace scope (#sift-phrase-hang); + // use bm25 alone for repo-root until upstream fixes it. + return { retrievers: "bm25", reranking: "none" }; } return { retrievers: "bm25,phrase,vector", reranking: "position-aware" }; } diff --git a/src/resources/extensions/sf/key-manager.js b/src/resources/extensions/sf/key-manager.js index 88570684f..4bc1f8be9 100644 --- a/src/resources/extensions/sf/key-manager.js +++ b/src/resources/extensions/sf/key-manager.js @@ -94,7 +94,10 @@ export const PROVIDER_REGISTRY = [ id: "ollama-cloud", label: "Ollama Cloud", category: "llm", - envVar: "OLLAMA_API_KEY", + // Primary: OLLAMA_CLOUD_API_KEY; fallback: OLLAMA_API_KEY (shared with local Ollama). + envVar: "OLLAMA_CLOUD_API_KEY", + envVarFallback: "OLLAMA_API_KEY", + dashboardUrl: "ollama.com", }, { id: "opencode", diff --git a/src/resources/extensions/sf/provider-catalog-config.js b/src/resources/extensions/sf/provider-catalog-config.js index 8ae3a652d..f0f1c36db 100644 --- a/src/resources/extensions/sf/provider-catalog-config.js +++ b/src/resources/extensions/sf/provider-catalog-config.js @@ -184,6 +184,62 @@ export const PROVIDER_CATALOG_CONFIG = { rateLimits: { scope: "provider" }, modelFilter: { excludePatterns: [] }, }, + + // ─── Kimi Coding ────────────────────────────────────────────────────────────── + // Moonshot AI coding endpoint. Returns { data: [{id}] } (OpenAI compatible). + // A KimiCLI/* User-Agent is documented as required but the /v1/models endpoint + // works with plain Bearer auth at discovery time (tested live). + // Key: KIMI_API_KEY Dashboard: platform.moonshot.ai + "kimi-coding": { + type: "openai", + baseUrl: "https://api.kimi.com/coding", + modelsPath: "/v1/models", + auth: { type: "bearer" }, + rateLimits: { scope: "provider" }, + modelFilter: { excludePatterns: [] }, + }, + + // ─── Z.AI (ZAI) ─────────────────────────────────────────────────────────────────── + // Zhipu AI coding plan endpoint. Returns { data: [{id}] } (OpenAI compat). + // Per-model quirk: GLM reasoning models use thinkingFormat: zai — that is + // handled in the openai-completions adapter, not at catalog level. + // Key: ZAI_API_KEY Dashboard: bigmodel.cn + zai: { + type: "openai", + baseUrl: "https://api.z.ai/api/coding/paas", + modelsPath: "/v4/models", + auth: { type: "bearer" }, + rateLimits: { scope: "provider" }, + modelFilter: { excludePatterns: [] }, + }, + + // ─── Ollama Cloud ─────────────────────────────────────────────────────────── + // ollama.com hosted inference. Returns { data: [{id}] } (OpenAI compat). + // The local Ollama extension uses /api/tags via OLLAMA_HOST; this catalog + // entry covers the cloud API only (ollama.com/v1/models, confirmed live). + // Key: OLLAMA_CLOUD_API_KEY (fallback: OLLAMA_API_KEY) + "ollama-cloud": { + type: "openai", + baseUrl: "https://ollama.com", + modelsPath: "/v1/models", + auth: { type: "bearer" }, + rateLimits: { scope: "provider" }, + modelFilter: { excludePatterns: [] }, + }, + + // ─── Xiaomi MiMo ──────────────────────────────────────────────────────────── + // Discovery uses OpenAI-shaped GET /v1/models (confirmed: returns 401 when + // auth-gated, proving the endpoint exists — same pattern as minimax). + // Inference uses the Anthropic-messages wire format at /anthropic. + // Key: XIAOMI_API_KEY Dashboard: token-plan-ams.xiaomimimo.com + xiaomi: { + type: "openai", + baseUrl: "https://token-plan-ams.xiaomimimo.com", + modelsPath: "/v1/models", + auth: { type: "bearer" }, + rateLimits: { scope: "provider" }, + modelFilter: { excludePatterns: [] }, + }, }; /** diff --git a/src/resources/extensions/sf/tests/provider-catalog-discovery.test.mjs b/src/resources/extensions/sf/tests/provider-catalog-discovery.test.mjs index 51e922fff..70318d606 100644 --- a/src/resources/extensions/sf/tests/provider-catalog-discovery.test.mjs +++ b/src/resources/extensions/sf/tests/provider-catalog-discovery.test.mjs @@ -2,7 +2,8 @@ * provider-catalog-discovery.test.mjs * * Regression tests for live-discovery entries in provider-catalog-config.js. - * Covers: opencode, opencode-go, and minimax baseline fields. + * Covers: opencode, opencode-go, minimax, kimi-coding, zai, ollama-cloud, + * and xiaomi baseline fields. */ import assert from "node:assert/strict"; import { describe, test } from "vitest"; @@ -98,8 +99,116 @@ describe("minimax discovery config", () => { }); }); +describe("kimi-coding discovery config", () => { + test("baseUrl is the Moonshot AI coding endpoint", () => { + const cfg = getProviderCatalogConfig("kimi-coding"); + assert.ok(cfg, "kimi-coding entry must exist"); + assert.equal(cfg.baseUrl, "https://api.kimi.com/coding"); + }); + + test("modelsPath is /v1/models", () => { + const cfg = getProviderCatalogConfig("kimi-coding"); + assert.equal(cfg.modelsPath, "/v1/models"); + }); + + test("auth type is bearer", () => { + const cfg = getProviderCatalogConfig("kimi-coding"); + assert.equal(cfg.auth.type, "bearer"); + }); + + test("is included in DISCOVERABLE_PROVIDER_IDS", () => { + assert.ok( + DISCOVERABLE_PROVIDER_IDS.includes("kimi-coding"), + "kimi-coding should be discoverable", + ); + }); +}); + +describe("zai discovery config", () => { + test("baseUrl is the Z.AI coding plan endpoint", () => { + const cfg = getProviderCatalogConfig("zai"); + assert.ok(cfg, "zai entry must exist"); + assert.equal(cfg.baseUrl, "https://api.z.ai/api/coding/paas"); + }); + + test("modelsPath is /v4/models", () => { + const cfg = getProviderCatalogConfig("zai"); + assert.equal(cfg.modelsPath, "/v4/models"); + }); + + test("auth type is bearer", () => { + const cfg = getProviderCatalogConfig("zai"); + assert.equal(cfg.auth.type, "bearer"); + }); + + test("is included in DISCOVERABLE_PROVIDER_IDS", () => { + assert.ok( + DISCOVERABLE_PROVIDER_IDS.includes("zai"), + "zai should be discoverable", + ); + }); +}); + +describe("ollama-cloud discovery config", () => { + test("baseUrl is ollama.com", () => { + const cfg = getProviderCatalogConfig("ollama-cloud"); + assert.ok(cfg, "ollama-cloud entry must exist"); + assert.equal(cfg.baseUrl, "https://ollama.com"); + }); + + test("modelsPath is /v1/models", () => { + const cfg = getProviderCatalogConfig("ollama-cloud"); + assert.equal(cfg.modelsPath, "/v1/models"); + }); + + test("auth type is bearer", () => { + const cfg = getProviderCatalogConfig("ollama-cloud"); + assert.equal(cfg.auth.type, "bearer"); + }); + + test("is included in DISCOVERABLE_PROVIDER_IDS", () => { + assert.ok( + DISCOVERABLE_PROVIDER_IDS.includes("ollama-cloud"), + "ollama-cloud should be discoverable", + ); + }); +}); + +describe("xiaomi discovery config", () => { + test("baseUrl is the Xiaomi MiMo international endpoint", () => { + const cfg = getProviderCatalogConfig("xiaomi"); + assert.ok(cfg, "xiaomi entry must exist"); + assert.equal(cfg.baseUrl, "https://token-plan-ams.xiaomimimo.com"); + }); + + test("modelsPath is /v1/models (OpenAI-shaped discovery)", () => { + const cfg = getProviderCatalogConfig("xiaomi"); + assert.equal(cfg.modelsPath, "/v1/models"); + }); + + test("auth type is bearer", () => { + const cfg = getProviderCatalogConfig("xiaomi"); + assert.equal(cfg.auth.type, "bearer"); + }); + + test("is included in DISCOVERABLE_PROVIDER_IDS", () => { + assert.ok( + DISCOVERABLE_PROVIDER_IDS.includes("xiaomi"), + "xiaomi should be discoverable", + ); + }); +}); + describe("DISCOVERABLE_PROVIDER_IDS completeness", () => { - test("includes all three newly added providers", () => { + test("includes all 4 newly added providers (kimi-coding, zai, ollama-cloud, xiaomi)", () => { + const ids = DISCOVERABLE_PROVIDER_IDS; + assert.ok(ids.includes("kimi-coding"), "missing kimi-coding"); + assert.ok(ids.includes("zai"), "missing zai"); + assert.ok(ids.includes("ollama-cloud"), "missing ollama-cloud"); + assert.ok(ids.includes("xiaomi"), "missing xiaomi"); + }); + + test("includes pre-existing providers (opencode, opencode-go, minimax)", () => { const ids = DISCOVERABLE_PROVIDER_IDS; assert.ok(ids.includes("opencode"), "missing opencode"); assert.ok(ids.includes("opencode-go"), "missing opencode-go"); diff --git a/src/resources/extensions/sf/tests/sift-retriever-scope.test.mjs b/src/resources/extensions/sf/tests/sift-retriever-scope.test.mjs index e7ce77d58..0d41648e1 100644 --- a/src/resources/extensions/sf/tests/sift-retriever-scope.test.mjs +++ b/src/resources/extensions/sf/tests/sift-retriever-scope.test.mjs @@ -1,10 +1,11 @@ /** * Tests for scope-aware sift retriever selection. * - * Verifies that chooseSiftRetrievers returns bm25+phrase (no vector) for - * repo-root scopes and bm25+phrase+vector (with reranking) for subdirectory - * scopes. Also checks that sift-search-tool.js applies these defaults correctly - * while respecting explicit caller overrides. + * Verifies that chooseSiftRetrievers returns bm25 only (no phrase — phrase + * hangs on full-workspace scope, #sift-phrase-hang) for repo-root scopes and + * bm25+phrase+vector (with reranking) for subdirectory scopes. Also checks that + * sift-search-tool.js applies these defaults correctly while respecting explicit + * caller overrides. */ import assert from "node:assert/strict"; import { describe, it, vi } from "vitest"; @@ -15,13 +16,13 @@ import { chooseSiftRetrievers } from "../code-intelligence.js"; describe("chooseSiftRetrievers", () => { it("repo_root_absolute_returns_bm25_only", () => { const result = chooseSiftRetrievers("/repo", "/repo"); - assert.equal(result.retrievers, "bm25,phrase"); + assert.equal(result.retrievers, "bm25"); assert.equal(result.reranking, "none"); }); it("dot_relative_resolves_to_repo_root_returns_bm25_only", () => { const result = chooseSiftRetrievers(".", "/repo"); - assert.equal(result.retrievers, "bm25,phrase"); + assert.equal(result.retrievers, "bm25"); assert.equal(result.reranking, "none"); }); @@ -88,7 +89,7 @@ describe("sift-search-tool buildSiftArgs via chooseSiftRetrievers", () => { it("repo_root_dot_gets_bm25_only_by_default", () => { const result = simulateBuildSiftArgs({ path: "." }, "/repo"); - assert.equal(result.retrievers, "bm25,phrase"); + assert.equal(result.retrievers, "bm25"); assert.equal(result.reranking, "none"); }); }); @@ -101,12 +102,14 @@ describe("sift-search-tool buildSiftArgs via chooseSiftRetrievers", () => { // ensureSiftIndexWarmup delegates to it. describe("warmup regression guard", () => { - it("warmup_dot_scope_still_produces_bm25_phrase", () => { + it("warmup_dot_scope_still_produces_bm25_only", () => { // ensureSiftIndexWarmup calls resolveSiftSearchScope(projectRoot, "."), // which returns "." when the scope resolves to the repo root. // Then it passes that resolved scope to chooseSiftRetrievers. + // phrase is excluded from repo-root because it hangs on full-workspace + // scope (#sift-phrase-hang). const result = chooseSiftRetrievers(".", "/home/user/myrepo"); - assert.equal(result.retrievers, "bm25,phrase"); + assert.equal(result.retrievers, "bm25"); assert.equal(result.reranking, "none"); }); }); diff --git a/src/resources/extensions/sf/tests/uok-diagnostic-synthesis.test.mjs b/src/resources/extensions/sf/tests/uok-diagnostic-synthesis.test.mjs index 942ef0e5c..f526aee7c 100644 --- a/src/resources/extensions/sf/tests/uok-diagnostic-synthesis.test.mjs +++ b/src/resources/extensions/sf/tests/uok-diagnostic-synthesis.test.mjs @@ -10,6 +10,10 @@ import { import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, test } from "vitest"; +import { + readAutonomousSolverState, + beginAutonomousSolverIteration, +} from "../autonomous-solver.js"; import { closeDatabase, openDatabase, @@ -262,6 +266,25 @@ test("writeUokDiagnostics_when_repair_enabled_clears_stale_projection_without_ex assert.deepEqual(listUnitRuntimeRecords(root), []); }); +test("writeUokDiagnostics_when_repair_enabled_clears_stale_autonomous_solver_state", () => { + const root = makeProject(); + beginAutonomousSolverIteration(root, "execute-task", "M003/S02/T02"); + + const diagnostics = writeUokDiagnostics(root, { + nowMs: NOW, + processRows: [], + expectedNext: { + action: "dispatch", + unitType: "execute-task", + unitId: "M003/S02/T02", + }, + repairStaleRuntimeProjection: true, + }); + + assert.equal(diagnostics.signals.runtimeProjection, "ok"); + assert.equal(readAutonomousSolverState(root), null); +}); + test("writeUokDiagnostics_persists_report_for_status_widget_and_doctor", () => { const root = makeProject(); openDatabase(":memory:"); diff --git a/src/resources/extensions/sf/uok/diagnostic-synthesis.js b/src/resources/extensions/sf/uok/diagnostic-synthesis.js index f20645efe..95c821837 100644 --- a/src/resources/extensions/sf/uok/diagnostic-synthesis.js +++ b/src/resources/extensions/sf/uok/diagnostic-synthesis.js @@ -2,6 +2,10 @@ import { execFileSync } from "node:child_process"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { isLockProcessAlive, readCrashLock } from "../crash-recovery.js"; +import { + clearAutonomousSolverState, + readAutonomousSolverState, +} from "../autonomous-solver.js"; import { sfRoot } from "../paths.js"; import { getUokRuns, isDbAvailable } from "../sf-db.js"; import { MessageBus } from "./message-bus.js"; @@ -192,6 +196,12 @@ export function synthesizeUokDiagnostics(basePath, options = {}) { activeRuntimeUnits = runtimeUnits.filter((unit) => unit.projectionActive); } } + if (options.repairStaleRuntimeProjection && !lockAlive) { + const solverState = readAutonomousSolverState(basePath); + if (solverState?.status === "running") { + clearAutonomousSolverState(basePath); + } + } const currentRuntimeUnit = lock ? runtimeUnits.find( (unit) =>