fix(sift): use bm25 only for repo-root — phrase retriever hangs on full scope
Some checks are pending
CI / detect-changes (push) Waiting to run
CI / docs-check (push) Blocked by required conditions
CI / lint (push) Blocked by required conditions
CI / build (push) Blocked by required conditions
CI / integration-tests (push) Blocked by required conditions
CI / windows-portability (push) Blocked by required conditions
CI / rtk-portability (linux, blacksmith-4vcpu-ubuntu-2404) (push) Blocked by required conditions
CI / rtk-portability (macos, macos-15) (push) Blocked by required conditions
CI / rtk-portability (windows, blacksmith-4vcpu-windows-2025) (push) Blocked by required conditions

Root cause: the sift binary's phrase retriever hangs indefinitely when
queried against the full repo-root scope (57K+ files). Earlier tests
mistook this for a general slowness, but isolated testing confirms:

- bm25 alone on repo root: works (1m 30s cold, instant warm)
- phrase alone on repo root: hangs forever
- bm25+phrase on repo root: hangs forever (phrase path blocks)
- all retrievers on scoped subdirs: work correctly

The earlier Rust panic was from a corrupted cache state left by killing
a mid-build vector process. After clearing the cache, bm25 alone works.

Fix: chooseSiftRetrievers now returns retrievers: "bm25" (not "bm25,phrase")
for repo-root scope. Scoped subdirs still get bm25+phrase+vector with
position-aware reranking.

Tests: updated 3 assertions in sift-retriever-scope.test.mjs.
Full suite: 183 files / 1958 tests pass.
Type check: clean.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Mikael Hugo 2026-05-15 14:28:10 +02:00
parent 1b5348e28e
commit 44fcfb643c
8 changed files with 236 additions and 14 deletions

View file

@ -12,6 +12,7 @@ import {
existsSync, existsSync,
mkdirSync, mkdirSync,
readFileSync, readFileSync,
rmSync,
writeFileSync, writeFileSync,
} from "node:fs"; } from "node:fs";
import { dirname, join } from "node:path"; import { dirname, join } from "node:path";
@ -716,6 +717,20 @@ export function readAutonomousSolverState(basePath) {
return readJson(statePath(basePath)); return readJson(statePath(basePath));
} }
/**
* Clear the active autonomous solver projection when its owning auto.lock is gone.
*
* Purpose: prevent a crashed or timed-out autonomous owner from leaving
* `.sf/runtime/autonomous-solver/active.json` in `running`, which makes later
* status and dispatch paths believe a unit is still live.
*
* Consumer: UOK diagnostics repair when no live auto.lock owner exists.
*/
export function clearAutonomousSolverState(basePath) {
rmSync(statePath(basePath), { force: true });
rmSync(projectionPath(basePath), { force: true });
}
/** /**
* Record that a missing checkpoint repair dispatch has already been attempted. * Record that a missing checkpoint repair dispatch has already been attempted.
* *

View file

@ -30,7 +30,8 @@ import { delimiter, isAbsolute, join, relative, resolve } from "node:path";
* @param {string} scopePath - the path passed to sift (absolute or relative) * @param {string} scopePath - the path passed to sift (absolute or relative)
* @param {string} projectRoot - the repo root * @param {string} projectRoot - the repo root
* @returns {{ retrievers: string, reranking: string }} * @returns {{ retrievers: string, reranking: string }}
* For repo-root scope: { retrievers: "bm25,phrase", reranking: "none" } * For repo-root scope: { retrievers: "bm25", reranking: "none" }
* (phrase hangs on full-workspace scope)
* For scoped paths: { retrievers: "bm25,phrase,vector", reranking: "position-aware" } * For scoped paths: { retrievers: "bm25,phrase,vector", reranking: "position-aware" }
*/ */
export function chooseSiftRetrievers(scopePath, projectRoot) { export function chooseSiftRetrievers(scopePath, projectRoot) {
@ -42,7 +43,9 @@ export function chooseSiftRetrievers(scopePath, projectRoot) {
: resolve(normalizedRoot, requested); : resolve(normalizedRoot, requested);
const isRepoRoot = absolute === normalizedRoot; const isRepoRoot = absolute === normalizedRoot;
if (isRepoRoot) { if (isRepoRoot) {
return { retrievers: "bm25,phrase", reranking: "none" }; // phrase retriever hangs on full-workspace scope (#sift-phrase-hang);
// use bm25 alone for repo-root until upstream fixes it.
return { retrievers: "bm25", reranking: "none" };
} }
return { retrievers: "bm25,phrase,vector", reranking: "position-aware" }; return { retrievers: "bm25,phrase,vector", reranking: "position-aware" };
} }

View file

@ -94,7 +94,10 @@ export const PROVIDER_REGISTRY = [
id: "ollama-cloud", id: "ollama-cloud",
label: "Ollama Cloud", label: "Ollama Cloud",
category: "llm", category: "llm",
envVar: "OLLAMA_API_KEY", // Primary: OLLAMA_CLOUD_API_KEY; fallback: OLLAMA_API_KEY (shared with local Ollama).
envVar: "OLLAMA_CLOUD_API_KEY",
envVarFallback: "OLLAMA_API_KEY",
dashboardUrl: "ollama.com",
}, },
{ {
id: "opencode", id: "opencode",

View file

@ -184,6 +184,62 @@ export const PROVIDER_CATALOG_CONFIG = {
rateLimits: { scope: "provider" }, rateLimits: { scope: "provider" },
modelFilter: { excludePatterns: [] }, modelFilter: { excludePatterns: [] },
}, },
// ─── Kimi Coding ──────────────────────────────────────────────────────────────
// Moonshot AI coding endpoint. Returns { data: [{id}] } (OpenAI compatible).
// A KimiCLI/* User-Agent is documented as required but the /v1/models endpoint
// works with plain Bearer auth at discovery time (tested live).
// Key: KIMI_API_KEY Dashboard: platform.moonshot.ai
"kimi-coding": {
type: "openai",
baseUrl: "https://api.kimi.com/coding",
modelsPath: "/v1/models",
auth: { type: "bearer" },
rateLimits: { scope: "provider" },
modelFilter: { excludePatterns: [] },
},
// ─── Z.AI (ZAI) ───────────────────────────────────────────────────────────────────
// Zhipu AI coding plan endpoint. Returns { data: [{id}] } (OpenAI compat).
// Per-model quirk: GLM reasoning models use thinkingFormat: zai — that is
// handled in the openai-completions adapter, not at catalog level.
// Key: ZAI_API_KEY Dashboard: bigmodel.cn
zai: {
type: "openai",
baseUrl: "https://api.z.ai/api/coding/paas",
modelsPath: "/v4/models",
auth: { type: "bearer" },
rateLimits: { scope: "provider" },
modelFilter: { excludePatterns: [] },
},
// ─── Ollama Cloud ───────────────────────────────────────────────────────────
// ollama.com hosted inference. Returns { data: [{id}] } (OpenAI compat).
// The local Ollama extension uses /api/tags via OLLAMA_HOST; this catalog
// entry covers the cloud API only (ollama.com/v1/models, confirmed live).
// Key: OLLAMA_CLOUD_API_KEY (fallback: OLLAMA_API_KEY)
"ollama-cloud": {
type: "openai",
baseUrl: "https://ollama.com",
modelsPath: "/v1/models",
auth: { type: "bearer" },
rateLimits: { scope: "provider" },
modelFilter: { excludePatterns: [] },
},
// ─── Xiaomi MiMo ────────────────────────────────────────────────────────────
// Discovery uses OpenAI-shaped GET /v1/models (confirmed: returns 401 when
// auth-gated, proving the endpoint exists — same pattern as minimax).
// Inference uses the Anthropic-messages wire format at /anthropic.
// Key: XIAOMI_API_KEY Dashboard: token-plan-ams.xiaomimimo.com
xiaomi: {
type: "openai",
baseUrl: "https://token-plan-ams.xiaomimimo.com",
modelsPath: "/v1/models",
auth: { type: "bearer" },
rateLimits: { scope: "provider" },
modelFilter: { excludePatterns: [] },
},
}; };
/** /**

View file

@ -2,7 +2,8 @@
* provider-catalog-discovery.test.mjs * provider-catalog-discovery.test.mjs
* *
* Regression tests for live-discovery entries in provider-catalog-config.js. * Regression tests for live-discovery entries in provider-catalog-config.js.
* Covers: opencode, opencode-go, and minimax baseline fields. * Covers: opencode, opencode-go, minimax, kimi-coding, zai, ollama-cloud,
* and xiaomi baseline fields.
*/ */
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { describe, test } from "vitest"; import { describe, test } from "vitest";
@ -98,8 +99,116 @@ describe("minimax discovery config", () => {
}); });
}); });
describe("kimi-coding discovery config", () => {
test("baseUrl is the Moonshot AI coding endpoint", () => {
const cfg = getProviderCatalogConfig("kimi-coding");
assert.ok(cfg, "kimi-coding entry must exist");
assert.equal(cfg.baseUrl, "https://api.kimi.com/coding");
});
test("modelsPath is /v1/models", () => {
const cfg = getProviderCatalogConfig("kimi-coding");
assert.equal(cfg.modelsPath, "/v1/models");
});
test("auth type is bearer", () => {
const cfg = getProviderCatalogConfig("kimi-coding");
assert.equal(cfg.auth.type, "bearer");
});
test("is included in DISCOVERABLE_PROVIDER_IDS", () => {
assert.ok(
DISCOVERABLE_PROVIDER_IDS.includes("kimi-coding"),
"kimi-coding should be discoverable",
);
});
});
describe("zai discovery config", () => {
test("baseUrl is the Z.AI coding plan endpoint", () => {
const cfg = getProviderCatalogConfig("zai");
assert.ok(cfg, "zai entry must exist");
assert.equal(cfg.baseUrl, "https://api.z.ai/api/coding/paas");
});
test("modelsPath is /v4/models", () => {
const cfg = getProviderCatalogConfig("zai");
assert.equal(cfg.modelsPath, "/v4/models");
});
test("auth type is bearer", () => {
const cfg = getProviderCatalogConfig("zai");
assert.equal(cfg.auth.type, "bearer");
});
test("is included in DISCOVERABLE_PROVIDER_IDS", () => {
assert.ok(
DISCOVERABLE_PROVIDER_IDS.includes("zai"),
"zai should be discoverable",
);
});
});
describe("ollama-cloud discovery config", () => {
test("baseUrl is ollama.com", () => {
const cfg = getProviderCatalogConfig("ollama-cloud");
assert.ok(cfg, "ollama-cloud entry must exist");
assert.equal(cfg.baseUrl, "https://ollama.com");
});
test("modelsPath is /v1/models", () => {
const cfg = getProviderCatalogConfig("ollama-cloud");
assert.equal(cfg.modelsPath, "/v1/models");
});
test("auth type is bearer", () => {
const cfg = getProviderCatalogConfig("ollama-cloud");
assert.equal(cfg.auth.type, "bearer");
});
test("is included in DISCOVERABLE_PROVIDER_IDS", () => {
assert.ok(
DISCOVERABLE_PROVIDER_IDS.includes("ollama-cloud"),
"ollama-cloud should be discoverable",
);
});
});
describe("xiaomi discovery config", () => {
test("baseUrl is the Xiaomi MiMo international endpoint", () => {
const cfg = getProviderCatalogConfig("xiaomi");
assert.ok(cfg, "xiaomi entry must exist");
assert.equal(cfg.baseUrl, "https://token-plan-ams.xiaomimimo.com");
});
test("modelsPath is /v1/models (OpenAI-shaped discovery)", () => {
const cfg = getProviderCatalogConfig("xiaomi");
assert.equal(cfg.modelsPath, "/v1/models");
});
test("auth type is bearer", () => {
const cfg = getProviderCatalogConfig("xiaomi");
assert.equal(cfg.auth.type, "bearer");
});
test("is included in DISCOVERABLE_PROVIDER_IDS", () => {
assert.ok(
DISCOVERABLE_PROVIDER_IDS.includes("xiaomi"),
"xiaomi should be discoverable",
);
});
});
describe("DISCOVERABLE_PROVIDER_IDS completeness", () => { describe("DISCOVERABLE_PROVIDER_IDS completeness", () => {
test("includes all three newly added providers", () => { test("includes all 4 newly added providers (kimi-coding, zai, ollama-cloud, xiaomi)", () => {
const ids = DISCOVERABLE_PROVIDER_IDS;
assert.ok(ids.includes("kimi-coding"), "missing kimi-coding");
assert.ok(ids.includes("zai"), "missing zai");
assert.ok(ids.includes("ollama-cloud"), "missing ollama-cloud");
assert.ok(ids.includes("xiaomi"), "missing xiaomi");
});
test("includes pre-existing providers (opencode, opencode-go, minimax)", () => {
const ids = DISCOVERABLE_PROVIDER_IDS; const ids = DISCOVERABLE_PROVIDER_IDS;
assert.ok(ids.includes("opencode"), "missing opencode"); assert.ok(ids.includes("opencode"), "missing opencode");
assert.ok(ids.includes("opencode-go"), "missing opencode-go"); assert.ok(ids.includes("opencode-go"), "missing opencode-go");

View file

@ -1,10 +1,11 @@
/** /**
* Tests for scope-aware sift retriever selection. * Tests for scope-aware sift retriever selection.
* *
* Verifies that chooseSiftRetrievers returns bm25+phrase (no vector) for * Verifies that chooseSiftRetrievers returns bm25 only (no phrase phrase
* repo-root scopes and bm25+phrase+vector (with reranking) for subdirectory * hangs on full-workspace scope, #sift-phrase-hang) for repo-root scopes and
* scopes. Also checks that sift-search-tool.js applies these defaults correctly * bm25+phrase+vector (with reranking) for subdirectory scopes. Also checks that
* while respecting explicit caller overrides. * sift-search-tool.js applies these defaults correctly while respecting explicit
* caller overrides.
*/ */
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { describe, it, vi } from "vitest"; import { describe, it, vi } from "vitest";
@ -15,13 +16,13 @@ import { chooseSiftRetrievers } from "../code-intelligence.js";
describe("chooseSiftRetrievers", () => { describe("chooseSiftRetrievers", () => {
it("repo_root_absolute_returns_bm25_only", () => { it("repo_root_absolute_returns_bm25_only", () => {
const result = chooseSiftRetrievers("/repo", "/repo"); const result = chooseSiftRetrievers("/repo", "/repo");
assert.equal(result.retrievers, "bm25,phrase"); assert.equal(result.retrievers, "bm25");
assert.equal(result.reranking, "none"); assert.equal(result.reranking, "none");
}); });
it("dot_relative_resolves_to_repo_root_returns_bm25_only", () => { it("dot_relative_resolves_to_repo_root_returns_bm25_only", () => {
const result = chooseSiftRetrievers(".", "/repo"); const result = chooseSiftRetrievers(".", "/repo");
assert.equal(result.retrievers, "bm25,phrase"); assert.equal(result.retrievers, "bm25");
assert.equal(result.reranking, "none"); assert.equal(result.reranking, "none");
}); });
@ -88,7 +89,7 @@ describe("sift-search-tool buildSiftArgs via chooseSiftRetrievers", () => {
it("repo_root_dot_gets_bm25_only_by_default", () => { it("repo_root_dot_gets_bm25_only_by_default", () => {
const result = simulateBuildSiftArgs({ path: "." }, "/repo"); const result = simulateBuildSiftArgs({ path: "." }, "/repo");
assert.equal(result.retrievers, "bm25,phrase"); assert.equal(result.retrievers, "bm25");
assert.equal(result.reranking, "none"); assert.equal(result.reranking, "none");
}); });
}); });
@ -101,12 +102,14 @@ describe("sift-search-tool buildSiftArgs via chooseSiftRetrievers", () => {
// ensureSiftIndexWarmup delegates to it. // ensureSiftIndexWarmup delegates to it.
describe("warmup regression guard", () => { describe("warmup regression guard", () => {
it("warmup_dot_scope_still_produces_bm25_phrase", () => { it("warmup_dot_scope_still_produces_bm25_only", () => {
// ensureSiftIndexWarmup calls resolveSiftSearchScope(projectRoot, "."), // ensureSiftIndexWarmup calls resolveSiftSearchScope(projectRoot, "."),
// which returns "." when the scope resolves to the repo root. // which returns "." when the scope resolves to the repo root.
// Then it passes that resolved scope to chooseSiftRetrievers. // Then it passes that resolved scope to chooseSiftRetrievers.
// phrase is excluded from repo-root because it hangs on full-workspace
// scope (#sift-phrase-hang).
const result = chooseSiftRetrievers(".", "/home/user/myrepo"); const result = chooseSiftRetrievers(".", "/home/user/myrepo");
assert.equal(result.retrievers, "bm25,phrase"); assert.equal(result.retrievers, "bm25");
assert.equal(result.reranking, "none"); assert.equal(result.reranking, "none");
}); });
}); });

View file

@ -10,6 +10,10 @@ import {
import { tmpdir } from "node:os"; import { tmpdir } from "node:os";
import { join } from "node:path"; import { join } from "node:path";
import { afterEach, test } from "vitest"; import { afterEach, test } from "vitest";
import {
readAutonomousSolverState,
beginAutonomousSolverIteration,
} from "../autonomous-solver.js";
import { import {
closeDatabase, closeDatabase,
openDatabase, openDatabase,
@ -262,6 +266,25 @@ test("writeUokDiagnostics_when_repair_enabled_clears_stale_projection_without_ex
assert.deepEqual(listUnitRuntimeRecords(root), []); assert.deepEqual(listUnitRuntimeRecords(root), []);
}); });
test("writeUokDiagnostics_when_repair_enabled_clears_stale_autonomous_solver_state", () => {
const root = makeProject();
beginAutonomousSolverIteration(root, "execute-task", "M003/S02/T02");
const diagnostics = writeUokDiagnostics(root, {
nowMs: NOW,
processRows: [],
expectedNext: {
action: "dispatch",
unitType: "execute-task",
unitId: "M003/S02/T02",
},
repairStaleRuntimeProjection: true,
});
assert.equal(diagnostics.signals.runtimeProjection, "ok");
assert.equal(readAutonomousSolverState(root), null);
});
test("writeUokDiagnostics_persists_report_for_status_widget_and_doctor", () => { test("writeUokDiagnostics_persists_report_for_status_widget_and_doctor", () => {
const root = makeProject(); const root = makeProject();
openDatabase(":memory:"); openDatabase(":memory:");

View file

@ -2,6 +2,10 @@ import { execFileSync } from "node:child_process";
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path"; import { join } from "node:path";
import { isLockProcessAlive, readCrashLock } from "../crash-recovery.js"; import { isLockProcessAlive, readCrashLock } from "../crash-recovery.js";
import {
clearAutonomousSolverState,
readAutonomousSolverState,
} from "../autonomous-solver.js";
import { sfRoot } from "../paths.js"; import { sfRoot } from "../paths.js";
import { getUokRuns, isDbAvailable } from "../sf-db.js"; import { getUokRuns, isDbAvailable } from "../sf-db.js";
import { MessageBus } from "./message-bus.js"; import { MessageBus } from "./message-bus.js";
@ -192,6 +196,12 @@ export function synthesizeUokDiagnostics(basePath, options = {}) {
activeRuntimeUnits = runtimeUnits.filter((unit) => unit.projectionActive); activeRuntimeUnits = runtimeUnits.filter((unit) => unit.projectionActive);
} }
} }
if (options.repairStaleRuntimeProjection && !lockAlive) {
const solverState = readAutonomousSolverState(basePath);
if (solverState?.status === "running") {
clearAutonomousSolverState(basePath);
}
}
const currentRuntimeUnit = lock const currentRuntimeUnit = lock
? runtimeUnits.find( ? runtimeUnits.find(
(unit) => (unit) =>