diff --git a/packages/pi-coding-agent/src/modes/rpc/rpc-client.ts b/packages/pi-coding-agent/src/modes/rpc/rpc-client.ts index 9e6fccc37..bec5e63f5 100644 --- a/packages/pi-coding-agent/src/modes/rpc/rpc-client.ts +++ b/packages/pi-coding-agent/src/modes/rpc/rpc-client.ts @@ -58,6 +58,10 @@ function isTypeScriptEntrypoint(cliPath: string): boolean { return cliPath.endsWith(".ts") || cliPath.endsWith(".tsx"); } +function isJavaScriptEntrypoint(cliPath: string): boolean { + return cliPath.endsWith(".js") || cliPath.endsWith(".mjs") || cliPath.endsWith(".cjs"); +} + function findResolveTsLoader(cliPath: string): string | null { let currentDir = resolve(dirname(cliPath)); while (true) { @@ -74,13 +78,20 @@ function findResolveTsLoader(cliPath: string): string | null { } export function buildRpcLaunchSpec(cliPath: string): RpcLaunchSpec { - if (!isTypeScriptEntrypoint(cliPath)) { + if (isJavaScriptEntrypoint(cliPath)) { return { command: "node", args: [cliPath], }; } + if (!isTypeScriptEntrypoint(cliPath)) { + return { + command: cliPath, + args: [], + }; + } + const resolveTsLoader = findResolveTsLoader(cliPath); if (!resolveTsLoader) { throw new Error(`Could not find resolve-ts.mjs for TypeScript CLI path: ${cliPath}`); diff --git a/packages/pi-coding-agent/src/modes/rpc/rpc-protocol-v2.test.ts b/packages/pi-coding-agent/src/modes/rpc/rpc-protocol-v2.test.ts index e05913660..9d538d3e5 100644 --- a/packages/pi-coding-agent/src/modes/rpc/rpc-protocol-v2.test.ts +++ b/packages/pi-coding-agent/src/modes/rpc/rpc-protocol-v2.test.ts @@ -524,6 +524,12 @@ describe("RpcClient command serialization", () => { assert.equal(launchSpec.command, "node"); assert.deepEqual(launchSpec.args, ["/tmp/dist/cli.js"]); }); + + it("non-js executable shims launch directly", () => { + const launchSpec = buildRpcLaunchSpec("/tmp/bin/sf-from-source"); + assert.equal(launchSpec.command, "/tmp/bin/sf-from-source"); + assert.deepEqual(launchSpec.args, []); + }); }); // ============================================================================ diff --git a/packages/rpc-client/src/rpc-client.test.ts b/packages/rpc-client/src/rpc-client.test.ts index 229acbe64..d99aeb4de 100644 --- a/packages/rpc-client/src/rpc-client.test.ts +++ b/packages/rpc-client/src/rpc-client.test.ts @@ -278,6 +278,12 @@ describe("buildRpcLaunchSpec", () => { assert.deepEqual(launchSpec.args, ["/tmp/dist/cli.js"]); }); + it("executes non-js executable shims directly", () => { + const launchSpec = buildRpcLaunchSpec("/tmp/bin/sf-from-source"); + assert.equal(launchSpec.command, "/tmp/bin/sf-from-source"); + assert.deepEqual(launchSpec.args, []); + }); + it("wraps typescript entrypoints with resolve-ts loader", () => { const repoRoot = new URL("../../..", import.meta.url).pathname; const cliPath = `${repoRoot}src/loader.ts`; diff --git a/packages/rpc-client/src/rpc-client.ts b/packages/rpc-client/src/rpc-client.ts index afc242dfe..557c7624a 100644 --- a/packages/rpc-client/src/rpc-client.ts +++ b/packages/rpc-client/src/rpc-client.ts @@ -66,6 +66,10 @@ function isTypeScriptEntrypoint(cliPath: string): boolean { return cliPath.endsWith(".ts") || cliPath.endsWith(".tsx"); } +function isJavaScriptEntrypoint(cliPath: string): boolean { + return cliPath.endsWith(".js") || cliPath.endsWith(".mjs") || cliPath.endsWith(".cjs"); +} + function findResolveTsLoader(cliPath: string): string | null { let currentDir = resolve(dirname(cliPath)); while (true) { @@ -82,13 +86,20 @@ function findResolveTsLoader(cliPath: string): string | null { } export function buildRpcLaunchSpec(cliPath: string): RpcLaunchSpec { - if (!isTypeScriptEntrypoint(cliPath)) { + if (isJavaScriptEntrypoint(cliPath)) { return { command: "node", args: [cliPath], }; } + if (!isTypeScriptEntrypoint(cliPath)) { + return { + command: cliPath, + args: [], + }; + } + const resolveTsLoader = findResolveTsLoader(cliPath); if (!resolveTsLoader) { throw new Error(`Could not find resolve-ts.mjs for TypeScript CLI path: ${cliPath}`); diff --git a/scripts/model-smoke-benchmark.mjs b/scripts/model-smoke-benchmark.mjs new file mode 100644 index 000000000..f209fc8e3 --- /dev/null +++ b/scripts/model-smoke-benchmark.mjs @@ -0,0 +1,217 @@ +#!/usr/bin/env node + +import { readFileSync, writeFileSync, mkdirSync } from "node:fs"; +import { homedir } from "node:os"; +import { dirname, resolve } from "node:path"; +import { spawnSync } from "node:child_process"; +import { performance } from "node:perf_hooks"; + +const repoRoot = resolve(import.meta.dirname, ".."); +const defaultOutputPath = resolve(repoRoot, ".sf", "model-benchmarks", `${new Date().toISOString().replace(/[:.]/g, "-")}.json`); + +const args = parseArgs(process.argv.slice(2)); +const modelsArg = args.models ?? args.model; +const outputPath = resolve(args.output ?? defaultOutputPath); +const maxModels = Number.parseInt(args.maxModels ?? args["max-models"] ?? "8", 10); +const maxTokens = Number.parseInt(args.maxTokens ?? args["max-tokens"] ?? "420", 10); + +await loadSfScopedEnv(); + +const { getModel, streamSimpleOpenAICompletions } = await import("../packages/pi-ai/src/index.ts"); + +const modelIds = modelsArg + ? modelsArg.split(",").map((s) => s.trim()).filter(Boolean) + : [ + "kimi-coding/kimi-for-coding", + "minimax/MiniMax-M2.7-highspeed", + "zai/glm-4.6", + "mistral/devstral-latest", + "alibaba-coding-plan/qwen3-coder-plus", + "xiaomi/mimo-v2-pro", + "opencode-go/minimax-m2.7", + "openrouter/inclusionai/ling-2.6-1t:free", + ]; + +const tasks = [ + { + id: "json-repair", + maxTokens: Math.min(maxTokens, 280), + prompt: `Return ONLY valid JSON matching { "bug": string, "fix": string, "tests": string[] }. +Broken payload: {"bug":"path traversal\\n- accepts ../foo","fix":123,"tests":"none"}. +Normalize it semantically; no markdown.`, + check: (text) => { + try { + const parsed = JSON.parse(text); + return typeof parsed.bug === "string" && typeof parsed.fix === "string" && Array.isArray(parsed.tests); + } catch { + return false; + } + }, + }, + { + id: "path-debug", + maxTokens, + prompt: `Find the bug and propose the minimal patch. Code: +function isSafe(base, target) { + const resolved = path.resolve(base, target) + return resolved.startsWith(base) +} +Explain why it is unsafe in <= 8 bullets, then provide a corrected JS function.`, + check: (text) => /startsWith|prefix/i.test(text) && /path\.sep|relative|normalize|resolve/i.test(text), + }, + { + id: "routing-plan", + maxTokens, + prompt: `Produce a concise implementation plan with risks and verification for migrating an LLM routing table from alias k2p5 to semantic ids kimi-k2.5 and kimi-k2.6.`, + check: (text) => /kimi-k2\.5/.test(text) && /kimi-k2\.6/.test(text) && /test|verify|validation/i.test(text), + }, +]; + +const selectedModels = modelIds.slice(0, Number.isFinite(maxModels) ? maxModels : modelIds.length); +const results = []; + +for (const fullId of selectedModels) { + const slash = fullId.indexOf("/"); + if (slash === -1) { + results.push({ model: fullId, ok: false, error: "expected provider/model id" }); + continue; + } + const provider = fullId.slice(0, slash); + const modelId = fullId.slice(slash + 1); + const model = getModel(provider, modelId); + if (!model) { + results.push({ model: fullId, ok: false, error: "model not found in registry" }); + continue; + } + + for (const task of tasks) { + const started = performance.now(); + let text = ""; + let result; + try { + const stream = streamSimpleOpenAICompletions( + model, + { + systemPrompt: "You are a precise software engineering benchmark model. Follow requested output formats exactly.", + messages: [{ role: "user", content: task.prompt, timestamp: Date.now() }], + }, + { temperature: 0, maxTokens: task.maxTokens }, + ); + for await (const event of stream) { + if (event.type === "text_delta") text += event.delta; + } + result = await stream.result(); + } catch (error) { + results.push({ + model: fullId, + task: task.id, + ok: false, + elapsedMs: Math.round(performance.now() - started), + error: error instanceof Error ? error.message : String(error), + }); + continue; + } + + const elapsedMs = Math.round(performance.now() - started); + const passed = result.stopReason !== "error" && task.check(text); + results.push({ + model: fullId, + task: task.id, + ok: passed, + stopReason: result.stopReason, + errorMessage: result.errorMessage, + elapsedMs, + chars: text.length, + usage: result.usage, + sample: text.slice(0, 700), + }); + console.log(`${passed ? "PASS" : "FAIL"} ${fullId} ${task.id} ${elapsedMs}ms ${result.stopReason}`); + } +} + +const report = { + createdAt: new Date().toISOString(), + models: selectedModels, + tasks: tasks.map((t) => t.id), + results, +}; + +mkdirSync(dirname(outputPath), { recursive: true }); +writeFileSync(outputPath, `${JSON.stringify(report, null, 2)}\n`); +console.log(`wrote ${outputPath}`); + +function parseArgs(argv) { + const parsed = {}; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (!arg.startsWith("--")) continue; + const key = arg.slice(2); + const next = argv[i + 1]; + if (!next || next.startsWith("--")) { + parsed[key] = "true"; + } else { + parsed[key] = next; + i++; + } + } + return parsed; +} + +async function loadSfScopedEnv() { + const secretsFile = `${homedir()}/.dotfiles/secrets/api-keys.yaml`; + const sopsConfig = `${homedir()}/.dotfiles/.sops.yaml`; + const wrapperPath = `${homedir()}/.local/bin/sf`; + const envNames = readSfScopedEnvNames(wrapperPath); + for (const name of envNames) delete process.env[name]; + + const decrypted = spawnSync("sops", ["--config", sopsConfig, "-d", secretsFile], { + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }); + if (decrypted.status !== 0 || !decrypted.stdout) return; + + const extracted = spawnSync("yq", [ + "-r", + `( + (.sf // {} | to_entries[] + | select((.value | type) == "string" or (.value | type) == "number" or (.value | type) == "boolean") + | select(.value != null and .value != "") + | "\\(.key)=\\(.value)"), + (.sf.env // {} | to_entries[] + | select(.value != null and .value != "") + | "\\(.key)=\\(.value)"), + (.sf.providers // {} | to_entries[] + | (.value.env // {}) + | to_entries[] + | select(.value != null and .value != "") + | "\\(.key)=\\(.value)") + )`, + ], { + input: decrypted.stdout, + encoding: "utf8", + stdio: ["pipe", "pipe", "ignore"], + }); + if (extracted.status !== 0 || !extracted.stdout) return; + + for (const line of extracted.stdout.split(/\r?\n/)) { + const idx = line.indexOf("="); + if (idx <= 0) continue; + const key = line.slice(0, idx); + const value = line.slice(idx + 1); + if (/^[A-Za-z_][A-Za-z0-9_]*$/.test(key) && value) process.env[key] = value; + } +} + +function readSfScopedEnvNames(wrapperPath) { + try { + const source = readFileSync(wrapperPath, "utf8"); + const match = source.match(/sf_scoped_env=\(\n([\s\S]*?)\n\)/); + if (!match) return []; + return match[1] + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => /^[A-Z0-9_]+$/.test(line)); + } catch { + return []; + } +} diff --git a/src/headless-context.ts b/src/headless-context.ts index 7fb4ffb29..fbbdd86b1 100644 --- a/src/headless-context.ts +++ b/src/headless-context.ts @@ -6,6 +6,7 @@ */ import { + type Dirent, existsSync, mkdirSync, readdirSync, @@ -19,6 +20,8 @@ import { ensurePreferences, untrackRuntimeFiles, } from "./resources/extensions/sf/gitignore.js"; +import { ensureAgenticDocsScaffold } from "./resources/extensions/sf/agentic-docs-scaffold.js"; +import { ensureSiftIndexWarmup } from "./resources/extensions/sf/code-intelligence.js"; import { nativeInit, nativeIsRepo, @@ -170,6 +173,8 @@ export function buildAutoBootstrapContext(basePath: string): string { "Use explorer-style subagents or equivalent high-context research passes before planning when the runtime supports them.", "Recommended explorer passes: docs/purpose/vision; source architecture and dependency map; tests/gates/tooling; risks/backlog/eval candidates.", "Merge explorer findings into one repo map with cited file paths before creating milestones.", + "Follow harness-engineering principles: keep AGENTS.md short as a table of contents, make docs/ the system of record, create versioned plans/evals, prefer mechanically enforced architecture/taste rules, and add cleanup/gardening work when repo knowledge is stale.", + "Optimize for agent legibility: every milestone should improve the next agent's ability to understand, validate, and safely modify the repo.", "Create actionable milestones and slices from the repo's docs and source tree rather than asking the user to restate them.", "", ]; @@ -272,9 +277,9 @@ function walkFiles( ): string[] { const found: string[] = []; const visit = (dir: string) => { - let entries: ReturnType; + let entries: Dirent[]; try { - entries = readdirSync(dir, { withFileTypes: true }); + entries = readdirSync(dir, { withFileTypes: true }) as Dirent[]; } catch { return; } @@ -325,5 +330,7 @@ export function bootstrapProject(basePath: string): void { ensureGitignore(basePath); ensurePreferences(basePath); + ensureAgenticDocsScaffold(basePath); + ensureSiftIndexWarmup(basePath); untrackRuntimeFiles(basePath); } diff --git a/src/headless-ui.ts b/src/headless-ui.ts index 4710c75ec..b61aa2f55 100644 --- a/src/headless-ui.ts +++ b/src/headless-ui.ts @@ -397,6 +397,9 @@ export function formatProgress( return line; } + case "extension_error": + return `${c.red}${tag("error")}${formatExtensionError(event)}${c.reset}`; + case "extension_ui_request": { const method = String(event.method ?? ""); @@ -443,6 +446,19 @@ export function formatProgress( } } +function formatExtensionError(event: Record): string { + const extensionPath = String(event.extensionPath ?? "unknown extension"); + const eventName = String(event.event ?? "unknown event"); + const error = event.error as Record | string | undefined; + const errorText = + typeof error === "string" + ? error + : error && typeof error === "object" + ? String(error.message ?? error.name ?? JSON.stringify(error)) + : "unknown error"; + return `Extension error in ${extensionPath} during ${eventName}: ${errorText}`; +} + /** * Format a thinking preview line from accumulated LLM text deltas. * Used as a fallback when streaming is not enabled — shows a truncated one-liner. diff --git a/src/headless.ts b/src/headless.ts index c1ec507ca..8a1e04472 100644 --- a/src/headless.ts +++ b/src/headless.ts @@ -812,7 +812,9 @@ async function runHeadlessOnce( ? String(event.toolName ?? "") : type === "extension_ui_request" ? `${event.method}: ${event.title ?? event.message ?? ""}` - : undefined; + : type === "extension_error" + ? `${event.extensionPath ?? "unknown extension"} ${event.event ?? "unknown event"}` + : undefined; recentEvents.push({ type, timestamp: Date.now(), detail }); if (recentEvents.length > 20) recentEvents.shift(); @@ -1171,6 +1173,13 @@ async function runHeadlessOnce( } } + if (eventType === "extension_error" && !completed) { + exitCode = EXIT_ERROR; + completed = true; + resolveCompletion(); + return; + } + // Handle execution_complete (v2 structured completion) // Skip for multi-turn commands (auto, next) — their completion is detected via // isTerminalNotification("Auto-mode stopped..."/"Step-mode stopped..."), not per-turn events diff --git a/src/resources/extensions/sf/agentic-docs-scaffold.ts b/src/resources/extensions/sf/agentic-docs-scaffold.ts new file mode 100644 index 000000000..621ce52c2 --- /dev/null +++ b/src/resources/extensions/sf/agentic-docs-scaffold.ts @@ -0,0 +1,306 @@ +import { existsSync, mkdirSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; + +interface ScaffoldFile { + path: string; + content: string; +} + +const SCAFFOLD_FILES: ScaffoldFile[] = [ + { + path: "AGENTS.md", + content: `# Agent Map + +Keep this file short. Use it as a table of contents for agents and humans. + +- Read \`ARCHITECTURE.md\` first for the system map and invariants. +- Read \`docs/PLANS.md\` and \`docs/exec-plans/active/\` for current work. +- Read \`docs/QUALITY_SCORE.md\`, \`docs/RELIABILITY.md\`, and \`docs/SECURITY.md\` before changing production behavior. +- Put durable product decisions in \`docs/product-specs/\`. +- Put durable design and architecture decisions in \`docs/design-docs/\`. +- Put generated reference material in \`docs/generated/\`. +- Use \`docs/RECORDS_KEEPER.md\` as the repo-order checklist after meaningful changes. +- Use the \`records-keeper\` skill when repo docs, plans, or architecture records need triage. +- Follow deeper \`AGENTS.md\` files when present. The closest one to the changed file wins. + +Before implementation, inspect the relevant docs and source files, state observed facts before inferred facts, and define the command or eval that proves the change. +`, + }, + { + path: "src/AGENTS.md", + content: `# Source Agent Notes + +- Start by mapping the owning module and its tests. +- Preserve existing public contracts unless the active plan explicitly changes them. +- Prefer typed/domain helpers over ad hoc parsing or duplicated logic. +- Keep edits scoped to the smallest module boundary that satisfies the plan. +- Update \`ARCHITECTURE.md\` when a source change creates a new subsystem or invariant. +`, + }, + { + path: "tests/AGENTS.md", + content: `# Test Agent Notes + +- Treat tests as executable specs, not coverage decoration. +- Add regression tests for changed behavior and failure modes. +- Prefer focused tests that name the behavior under test. +- Include the exact verification command in the plan or completion summary. +`, + }, + { + path: "ARCHITECTURE.md", + content: `# Architecture + +This file is the short map of the codebase. Keep it current and compact. + +## Purpose + +Describe the product, its users, and the job this repository exists to do. + +## Codemap + +- \`src/\`: primary implementation. +- \`tests/\`: behavior and regression coverage. +- \`docs/\`: durable product, design, plan, reliability, and security context. + +## Invariants + +- Prefer small, named modules with clear ownership. +- Behavior changes need tests or an explicit eval. +- Keep generated artifacts out of hand-written design docs. +- Update this map when new top-level concepts or directories become important. +`, + }, + { + path: "docs/design-docs/index.md", + content: `# Design Docs + +Durable design decisions live here. Link active proposals, completed decisions, and rejected alternatives. +`, + }, + { + path: "docs/AGENTS.md", + content: `# Docs Agent Notes + +- Docs are the durable project memory. Keep them concise, navigable, and current. +- Put stable decisions here; keep transient execution state in active plans. +- Prefer links to source paths, commands, and eval artifacts over broad prose. +- When docs and code disagree, inspect the code and update the stale document. +- Run the records keeper checklist in \`RECORDS_KEEPER.md\` after meaningful code, product, or architecture changes. +`, + }, + { + path: "docs/records/AGENTS.md", + content: `# Records Agent Notes + +- Keep repository memory ordered, current, and easy to inspect. +- Prefer moving durable facts to the narrowest canonical document over duplicating them. +- Preserve historical decisions; mark superseded records instead of deleting useful context. +- Escalate conflicts between docs and source by citing the exact files that disagree. +`, + }, + { + path: "docs/records/index.md", + content: `# Records + +This folder holds repo-memory audits, decision ledgers, context-gardening notes, and records-keeper outputs. +`, + }, + { + path: "docs/RECORDS_KEEPER.md", + content: `# Records Keeper + +The records keeper keeps repo memory ordered after meaningful changes. Run this checklist at milestone close, after architecture changes, after product behavior changes, and whenever docs/source disagree. + +Use the \`records-keeper\` skill for this workflow when SF skills are available. Use \`context-doctor\` instead when stale state lives under \`.sf/\` or the memory store. + +## Canonical Homes + +- Root \`AGENTS.md\`: short routing map for agents. +- \`ARCHITECTURE.md\`: short system map, boundaries, invariants, critical flows, and verification. +- \`docs/product-specs/\`: durable user-facing behavior and product decisions. +- \`docs/design-docs/\`: durable design and architecture decisions. +- \`docs/exec-plans/\`: active/completed work plans and technical debt. +- \`docs/generated/\`: generated references only. +- \`docs/records/\`: audits, ledgers, and context-gardening outputs. + +## Checklist + +- Root map is current: \`AGENTS.md\` points to the right canonical docs and local \`AGENTS.md\` files. +- Architecture is current: new subsystems, boundaries, invariants, data/state, or critical flows are reflected in \`ARCHITECTURE.md\`. +- Product specs are current: user-visible behavior changes are reflected in \`docs/product-specs/\`. +- Execution plans are filed: active work is in \`docs/exec-plans/active/\`; completed summaries and evidence are in \`docs/exec-plans/completed/\`. +- Debt is visible: discovered cleanup is listed in \`docs/exec-plans/tech-debt-tracker.md\`. +- Generated docs are marked: generated material stays under \`docs/generated/\` or clearly says how to regenerate it. +- Contradictions are resolved: stale docs are updated or marked superseded with links to the source of truth. +- Verification is recorded: changed checks, evals, and commands are listed in the relevant plan or quality document. + +## Output + +When records work is non-trivial, write a dated note under \`docs/records/\` with: + +- What changed. +- What canonical docs were updated. +- What contradictions were found. +- What remains unresolved. +`, + }, + { + path: "docs/design-docs/AGENTS.md", + content: `# Design Doc Agent Notes + +- Capture problem, context, options, decision, consequences, and validation. +- Separate observed facts from inferred product or architecture intent. +- Record rejected alternatives when they would prevent repeated debate. +`, + }, + { + path: "docs/design-docs/core-beliefs.md", + content: `# Core Beliefs + +- The repo should explain itself to humans and agents. +- Plans should carry acceptance criteria, falsifiers, and verification commands. +- Architecture should be mechanically checkable where possible. +`, + }, + { + path: "docs/exec-plans/active/index.md", + content: `# Active Execution Plans + +Link active plans here. Each plan should state purpose, scope, tasks, acceptance criteria, and verification. +`, + }, + { + path: "docs/exec-plans/AGENTS.md", + content: `# Execution Plan Agent Notes + +- Every plan needs purpose, scope, tasks, acceptance criteria, falsifier, and verification. +- Active plans live in \`active/\`; completed evidence summaries live in \`completed/\`. +- Add discovered cleanup to \`tech-debt-tracker.md\` instead of hiding it in chat. +`, + }, + { + path: "docs/exec-plans/completed/index.md", + content: `# Completed Execution Plans + +Move finished plan summaries here with evidence links and follow-up debt. +`, + }, + { + path: "docs/exec-plans/tech-debt-tracker.md", + content: `# Tech Debt Tracker + +Track cleanup discovered during implementation. Include owner, impact, proposed fix, and verification. +`, + }, + { + path: "docs/generated/db-schema.md", + content: `# Database Schema + +Generated or refreshed schema notes belong here. Do not hand-maintain stale schema copies. +`, + }, + { + path: "docs/product-specs/index.md", + content: `# Product Specs + +Durable user-facing behavior, workflows, and product decisions live here. +`, + }, + { + path: "docs/product-specs/AGENTS.md", + content: `# Product Spec Agent Notes + +- Describe the user, job-to-be-done, workflow, edge cases, and non-goals. +- Keep implementation details out unless they are product-visible constraints. +- Update specs when behavior changes, especially onboarding, permissions, billing, or destructive actions. +`, + }, + { + path: "docs/product-specs/new-user-onboarding.md", + content: `# New User Onboarding + +Describe the first-run experience, success criteria, and failure states when this product has an onboarding flow. +`, + }, + { + path: "docs/references/design-system-reference-llms.txt", + content: `Reference slot for design-system guidance intended for LLM consumption. +`, + }, + { + path: "docs/references/nixpacks-llms.txt", + content: `Reference slot for Nixpacks deployment/build guidance intended for LLM consumption. +`, + }, + { + path: "docs/references/uv-llms.txt", + content: `Reference slot for uv/Python tooling guidance intended for LLM consumption. +`, + }, + { + path: "docs/DESIGN.md", + content: `# Design + +Record interaction patterns, visual constraints, and design-system usage here. +`, + }, + { + path: "docs/FRONTEND.md", + content: `# Frontend + +Record frontend architecture, component ownership, accessibility constraints, and browser support here. +`, + }, + { + path: "docs/PLANS.md", + content: `# Plans + +Use this as the index for current and upcoming work. Link detailed plans in \`docs/exec-plans/\`. +`, + }, + { + path: "docs/PRODUCT_SENSE.md", + content: `# Product Sense + +Capture user goals, non-goals, tradeoffs, and examples of good product judgment for this repo. +`, + }, + { + path: "docs/QUALITY_SCORE.md", + content: `# Quality Score + +Define what good looks like for this repo. Include fast checks, slow checks, evals, and known blind spots. + +Use these principles: + +- Make code legible to agents with semantic names and explicit boundaries. +- Prefer small, testable modules over files that require broad context to edit. +- Enforce style, architecture, and reliability rules mechanically where possible. +- Keep a cleanup loop for stale docs, generated artifacts, and accumulated implementation debt. +`, + }, + { + path: "docs/RELIABILITY.md", + content: `# Reliability + +Document expected failure modes, recovery paths, observability, and release checks here. +`, + }, + { + path: "docs/SECURITY.md", + content: `# Security + +Document trust boundaries, secrets handling, dependency risk, and security review requirements here. +`, + }, +]; + +export function ensureAgenticDocsScaffold(basePath: string): void { + for (const file of SCAFFOLD_FILES) { + const target = join(basePath, file.path); + if (existsSync(target)) continue; + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, file.content, "utf-8"); + } +} diff --git a/src/resources/extensions/sf/auto-bootstrap-context.ts b/src/resources/extensions/sf/auto-bootstrap-context.ts new file mode 100644 index 000000000..8b0c39b0e --- /dev/null +++ b/src/resources/extensions/sf/auto-bootstrap-context.ts @@ -0,0 +1,219 @@ +import { + existsSync, + readdirSync, + readFileSync, + statSync, + type Dirent, +} from "node:fs"; +import { join, relative } from "node:path"; + +const AUTO_BOOTSTRAP_MAX_BYTES = 180_000; +const AUTO_BOOTSTRAP_MAX_FILE_BYTES = 40_000; +const AUTO_BOOTSTRAP_ROOT_FILES = [ + "TODO.md", + "SPEC.md", + "VISION.md", + "PURPOSE.md", + "MISSION.md", + "ROADMAP.md", + "ARCHITECTURE.md", + "BUILD_PLAN.md", + "README.md", + "AGENTS.md", + "CLAUDE.md", + "CONTRIBUTING.md", +]; +const AUTO_BOOTSTRAP_SOURCE_EXTENSIONS = new Set([ + ".go", + ".ts", + ".tsx", + ".js", + ".jsx", + ".mjs", + ".cjs", + ".py", + ".rs", + ".java", + ".kt", + ".kts", + ".rb", + ".php", + ".cs", + ".c", + ".cc", + ".cpp", + ".h", + ".hpp", + ".swift", + ".scala", + ".sh", + ".bash", + ".zsh", + ".fish", + ".sql", + ".yaml", + ".yml", + ".toml", + ".json", + ".jsonc", + ".xml", + ".html", + ".css", + ".scss", + ".sass", + ".vue", + ".svelte", + ".lua", + ".ex", + ".exs", + ".erl", + ".hrl", + ".clj", + ".cljs", + ".nix", + ".proto", +]); +const AUTO_BOOTSTRAP_EXCLUDED_DIRS = new Set([ + ".git", + ".sf", + ".gsd", + "node_modules", + "vendor", + "dist", + "build", + "target", + ".next", + ".cache", +]); + +export function buildAutoBootstrapContext(basePath: string): string { + const selectedFiles = collectAutoBootstrapFiles(basePath); + const sourceFiles = collectSourceFiles(basePath); + const chunks: string[] = [ + "# Autonomous Repo Bootstrap", + "", + "SF headless auto found no milestones. Use the repository files below as the seed context.", + "Research every relevant markdown document and every source file path before creating the initial milestone plan.", + "Use tool-based repository inspection for source contents; do not assume the seed excerpt is complete.", + "Extract the project purpose, vision, architecture, constraints, current TODOs, risks, eval/gate ideas, and implementation backlog.", + "Apply the ACE spec-first TDD shape when planning: purpose and consumer first, behavior contract before implementation, tests as specs, evidence after gates.", + "For each proposed slice, capture Observed/Inferred/Proposed facts, a falsifier, acceptance criteria, and the verification command or eval that proves it.", + "Use explorer-style subagents or equivalent high-context research passes before planning when the runtime supports them.", + "Recommended explorer passes: docs/purpose/vision; source architecture and dependency map; tests/gates/tooling; risks/backlog/eval candidates.", + "Merge explorer findings into one repo map with cited file paths before creating milestones.", + "Follow harness-engineering principles: keep AGENTS.md short as a table of contents, make docs/ the system of record, create versioned plans/evals, prefer mechanically enforced architecture/taste rules, and add cleanup/gardening work when repo knowledge is stale.", + "Optimize for agent legibility: every milestone should improve the next agent's ability to understand, validate, and safely modify the repo.", + "Create actionable milestones and slices from the repo's docs and source tree rather than asking the user to restate them.", + "", + ]; + + let used = chunks.join("\n").length; + for (const filePath of selectedFiles) { + let content: string; + try { + content = readFileSync(filePath, "utf-8"); + } catch { + continue; + } + if (content.length > AUTO_BOOTSTRAP_MAX_FILE_BYTES) { + content = + content.slice(0, AUTO_BOOTSTRAP_MAX_FILE_BYTES) + + "\n\n[truncated by SF headless auto bootstrap]\n"; + } + + const relPath = relative(basePath, filePath); + const block = `\n\n## ${relPath}\n\n${content.trim()}\n`; + if (used + block.length > AUTO_BOOTSTRAP_MAX_BYTES) break; + chunks.push(block); + used += block.length; + } + + if (sourceFiles.length > 0) { + const inventoryLines = [ + "\n\n## Source File Inventory\n", + "Inspect these source/config/test files during repo research before finalizing the plan.\n", + ...sourceFiles.map((filePath) => `- ${relative(basePath, filePath)}`), + "", + ]; + const block = inventoryLines.join("\n"); + if (used + block.length <= AUTO_BOOTSTRAP_MAX_BYTES) { + chunks.push(block); + } else { + const remaining = AUTO_BOOTSTRAP_MAX_BYTES - used; + if (remaining > 1000) chunks.push(block.slice(0, remaining)); + } + } + + if (selectedFiles.length === 0) { + chunks.push( + "No markdown docs were found. Inspect the repository directly and create an initial milestone from source layout, package metadata, tests, and git status.", + ); + } + + return chunks.join("\n").trim() + "\n"; +} + +function collectAutoBootstrapFiles(basePath: string): string[] { + const seen = new Set(); + const files: string[] = []; + + for (const name of AUTO_BOOTSTRAP_ROOT_FILES) { + const path = join(basePath, name); + if (existsMarkdownFile(path)) { + seen.add(path); + files.push(path); + } + } + + for (const path of walkMarkdownFiles(basePath)) { + if (seen.has(path)) continue; + seen.add(path); + files.push(path); + } + + return files; +} + +function existsMarkdownFile(path: string): boolean { + try { + const stat = statSync(path); + return stat.isFile() && path.toLowerCase().endsWith(".md"); + } catch { + return false; + } +} + +function collectSourceFiles(basePath: string): string[] { + return walkFiles(basePath, (path) => { + const lower = path.toLowerCase(); + if (lower.endsWith(".md")) return false; + const dot = lower.lastIndexOf("."); + return dot !== -1 && AUTO_BOOTSTRAP_SOURCE_EXTENSIONS.has(lower.slice(dot)); + }); +} + +function walkMarkdownFiles(root: string): string[] { + return walkFiles(root, (path) => path.toLowerCase().endsWith(".md")); +} + +function walkFiles(root: string, includeFile: (path: string) => boolean): string[] { + const found: string[] = []; + const visit = (dir: string) => { + let entries: Dirent[]; + try { + entries = readdirSync(dir, { withFileTypes: true }) as Dirent[]; + } catch { + return; + } + for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { + const path = join(dir, entry.name); + if (entry.isDirectory()) { + if (!AUTO_BOOTSTRAP_EXCLUDED_DIRS.has(entry.name)) visit(path); + continue; + } + if (entry.isFile() && includeFile(path)) found.push(path); + } + }; + visit(root); + return found; +} diff --git a/src/resources/extensions/sf/auto-start.ts b/src/resources/extensions/sf/auto-start.ts index 5a41b3b37..978c8e37f 100644 --- a/src/resources/extensions/sf/auto-start.ts +++ b/src/resources/extensions/sf/auto-start.ts @@ -25,6 +25,8 @@ import type { import { collectSecretsFromManifest } from "../get-secrets-from-user.js"; import type { AutoSession } from "./auto/session.js"; import { hideFooter } from "./auto-dashboard.js"; +import { ensureAgenticDocsScaffold } from "./agentic-docs-scaffold.js"; +import { ensureSiftIndexWarmup } from "./code-intelligence.js"; import { cleanStaleRuntimeUnits, getAutoWorktreePath, @@ -487,9 +489,14 @@ export async function bootstrapAutoSession( // ensureGitignore checks for git-tracked .sf/ files and skips the // ".sf" pattern if the project intentionally tracks .sf/ in git. const gitPrefs = loadEffectiveSFPreferences()?.preferences?.git; - const manageGitignore = gitPrefs?.manage_gitignore; - ensureGitignore(base, { manageGitignore }); - if (manageGitignore !== false) untrackRuntimeFiles(base); + const manageGitignore = gitPrefs?.manage_gitignore; + ensureGitignore(base, { manageGitignore }); + ensureAgenticDocsScaffold(base); + ensureSiftIndexWarmup( + base, + loadEffectiveSFPreferences()?.preferences?.codebase, + ); + if (manageGitignore !== false) untrackRuntimeFiles(base); // Bootstrap milestones/ if it doesn't exist. // Check milestones/ directly — ensureGsdSymlink above already created .sf/, @@ -691,18 +698,22 @@ export async function bootstrapAutoSession( // Auto mode: autonomously map the codebase and create milestones // without waiting for user answers. Uses discuss-headless prompt. ctx.ui.notify( - "No milestones found. Bootstrapping from codebase analysis.", + "No milestones found. Bootstrapping from repo docs and source inventory.", "info", ); + const { buildAutoBootstrapContext } = await import( + "./auto-bootstrap-context.js" + ); const { bootstrapNewMilestone, dispatchNewMilestoneDiscuss, injectTodoContext, } = await import("./guided-flow.js"); + const bootstrapContext = buildAutoBootstrapContext(base); const nextId = bootstrapNewMilestone(base); await dispatchNewMilestoneDiscuss(ctx, pi, base, nextId, { auto: true, - preamble: injectTodoContext(base, "This is an autonomous session."), + preamble: injectTodoContext(base, bootstrapContext), }); invalidateAllCaches(); @@ -719,6 +730,8 @@ export async function bootstrapAutoSession( [ `This is an autonomous roadmap bootstrap repair for ${nextId}.`, "The previous bootstrap turn ended without writing CONTEXT, CONTEXT-DRAFT, or ROADMAP artifacts.", + "Use the repo-doc/source bootstrap context below as the source of truth.", + bootstrapContext, "Start the roadmap planning session now: build project knowledge, run the planning meeting, and persist artifacts.", "Do not stop after reflection. At minimum write CONTEXT-DRAFT with evidence and open questions.", "If confidence is high enough, write CONTEXT and call sf_plan_milestone so auto-mode can continue.", @@ -758,11 +771,13 @@ export async function bootstrapAutoSession( preamble: injectTodoContext( base, [ - `This is an autonomous roadmap bootstrap repair for existing milestone ${repairId}.`, - "The previous bootstrap created a milestone shell but did not write CONTEXT.md, CONTEXT-DRAFT.md, or ROADMAP.md.", - "Reuse this milestone ID. Do not create a new milestone for the same bootstrap work.", - "Run the roadmap planning session now and persist CONTEXT or CONTEXT-DRAFT at minimum.", - "If confidence is high enough, write CONTEXT and call sf_plan_milestone so auto-mode can continue.", + `This is an autonomous roadmap bootstrap repair for existing milestone ${repairId}.`, + "The previous bootstrap created a milestone shell but did not write CONTEXT.md, CONTEXT-DRAFT.md, or ROADMAP.md.", + "Use the repo-doc/source bootstrap context below as the source of truth.", + bootstrapContext, + "Reuse this milestone ID. Do not create a new milestone for the same bootstrap work.", + "Run the roadmap planning session now and persist CONTEXT or CONTEXT-DRAFT at minimum.", + "If confidence is high enough, write CONTEXT and call sf_plan_milestone so auto-mode can continue.", ].join("\n"), ), }); @@ -827,11 +842,15 @@ export async function bootstrapAutoSession( const hasContext = !!(contextFile && (await loadFile(contextFile))); if (!hasContext) { ctx.ui.notify( - `Milestone ${mid} has no context. Bootstrapping from codebase analysis.`, + `Milestone ${mid} has no context. Bootstrapping from repo docs and source inventory.`, "info", ); + const { buildAutoBootstrapContext } = await import( + "./auto-bootstrap-context.js" + ); const { dispatchNewMilestoneDiscuss, injectTodoContext } = await import("./guided-flow.js"); + const bootstrapContext = buildAutoBootstrapContext(base); await dispatchNewMilestoneDiscuss(ctx, pi, base, mid, { auto: true, preamble: injectTodoContext( @@ -839,6 +858,8 @@ export async function bootstrapAutoSession( [ `This is an autonomous roadmap bootstrap repair for existing milestone ${mid}.`, "The milestone exists but has no CONTEXT.md yet.", + "Use the repo-doc/source bootstrap context below as the source of truth.", + bootstrapContext, "Reuse this milestone ID. Do not create a new milestone for the same bootstrap work.", "Build project knowledge, run the planning meeting, and persist CONTEXT or CONTEXT-DRAFT.", ].join("\n"), diff --git a/src/resources/extensions/sf/auto.ts b/src/resources/extensions/sf/auto.ts index 686ebf6ef..6ead72628 100644 --- a/src/resources/extensions/sf/auto.ts +++ b/src/resources/extensions/sf/auto.ts @@ -1643,10 +1643,15 @@ export async function startAuto( return; } + // Preserve the paused session path for recovery synthesis before clearing + // mutable resume state. The file can be unlinked from runtime metadata, but + // the provider JSONL must remain available for synthesizeCrashRecovery(). + const resumeSessionFile = s.pausedSessionFile; + // Lock acquired — now safe to delete the pause file - if (s.pausedSessionFile) { + if (resumeSessionFile) { try { - unlinkSync(s.pausedSessionFile); + unlinkSync(resumeSessionFile); } catch (err) { if ((err as NodeJS.ErrnoException).code !== "ENOENT") { logWarning( @@ -1750,13 +1755,13 @@ export async function startAuto( } invalidateAllCaches(); - if (s.pausedSessionFile) { + if (resumeSessionFile) { const activityDir = join(sfRoot(s.basePath), "activity"); const recovery = synthesizeCrashRecovery( s.basePath, s.currentUnit?.type ?? s.pausedUnitType ?? "unknown", s.currentUnit?.id ?? s.pausedUnitId ?? "unknown", - s.pausedSessionFile ?? undefined, + resumeSessionFile ?? undefined, activityDir, ); if (recovery && recovery.trace.toolCallCount > 0) { @@ -1766,7 +1771,6 @@ export async function startAuto( "info", ); } - s.pausedSessionFile = null; } updateSessionLock( diff --git a/src/resources/extensions/sf/code-intelligence.ts b/src/resources/extensions/sf/code-intelligence.ts index 40c7e195e..8452912bc 100644 --- a/src/resources/extensions/sf/code-intelligence.ts +++ b/src/resources/extensions/sf/code-intelligence.ts @@ -5,8 +5,14 @@ * accelerators for local code retrieval. */ -import { spawnSync } from "node:child_process"; -import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { spawn, spawnSync } from "node:child_process"; +import { + existsSync, + mkdirSync, + readFileSync, + statSync, + writeFileSync, +} from "node:fs"; import { delimiter, join, resolve } from "node:path"; import type { @@ -110,6 +116,31 @@ export interface ProjectRagBuildResult { stderr: string; } +export interface SiftIndexWarmupResult { + status: "started" | "skipped" | "unavailable" | "error"; + reason: string; + command?: string; + args?: string[]; + markerPath?: string; +} + +interface SiftIndexWarmupOptions { + env?: NodeJS.ProcessEnv; + force?: boolean; + ttlMs?: number; + query?: string; + limit?: number; + retrieverTimeoutMs?: number; + spawnFn?: typeof spawn; + now?: number; +} + +const DEFAULT_SIFT_WARMUP_TTL_MS = 6 * 60 * 60 * 1000; +const DEFAULT_SIFT_WARMUP_QUERY = + "repo architecture source tests entrypoints configuration"; +const DEFAULT_SIFT_WARMUP_LIMIT = 1; +const DEFAULT_SIFT_WARMUP_RETRIEVER_TIMEOUT_MS = 30_000; + function readJsonConfig(configPath: string): McpConfigFile { if (!existsSync(configPath)) return {}; const raw = readFileSync(configPath, "utf-8"); @@ -305,6 +336,118 @@ export function detectSift( }; } +function isFreshMarker( + markerPath: string, + now: number, + ttlMs: number, +): boolean { + try { + const stat = statSync(markerPath); + return now - stat.mtimeMs < ttlMs; + } catch { + return false; + } +} + +export function ensureSiftIndexWarmup( + projectRoot: string, + prefs?: CodebaseMapPreferences, + options: SiftIndexWarmupOptions = {}, +): SiftIndexWarmupResult { + const env = options.env ?? process.env; + const backendName = resolveEffectiveCodebaseIndexerBackendName( + projectRoot, + prefs, + env, + ); + if (backendName !== "sift") { + return { + status: "skipped", + reason: `effective codebase indexer is ${backendName}`, + }; + } + + const detection = detectSift(projectRoot, prefs, env); + if (detection.status !== "configured" || !detection.binaryPath) { + return { + status: "unavailable", + reason: detection.reason, + }; + } + + const markerPath = join( + projectRoot, + ".sf", + "runtime", + "sift-index-warmup.json", + ); + const now = options.now ?? Date.now(); + const ttlMs = options.ttlMs ?? DEFAULT_SIFT_WARMUP_TTL_MS; + if (!options.force && isFreshMarker(markerPath, now, ttlMs)) { + return { + status: "skipped", + reason: "recent sift warmup marker exists", + markerPath, + }; + } + + const args = [ + "search", + "--json", + "--strategy", + "page-index-hybrid", + "--limit", + String(options.limit ?? DEFAULT_SIFT_WARMUP_LIMIT), + "--retriever-timeout-ms", + String( + options.retrieverTimeoutMs ?? + DEFAULT_SIFT_WARMUP_RETRIEVER_TIMEOUT_MS, + ), + projectRoot, + options.query ?? DEFAULT_SIFT_WARMUP_QUERY, + ]; + + try { + mkdirSync(join(projectRoot, ".sf", "runtime"), { recursive: true }); + writeFileSync( + markerPath, + `${JSON.stringify( + { + startedAt: new Date(now).toISOString(), + command: detection.binaryPath, + args, + }, + null, + 2, + )}\n`, + "utf-8", + ); + + const child = (options.spawnFn ?? spawn)(detection.binaryPath, args, { + cwd: projectRoot, + env, + stdio: "ignore", + detached: true, + }); + child.unref(); + return { + status: "started", + reason: "sift page-index-hybrid warmup started", + command: detection.binaryPath, + args, + markerPath, + }; + } catch (err) { + return { + status: "error", + reason: err instanceof Error ? err.message : String(err), + command: detection.binaryPath, + args, + markerPath, + }; + } +} + function projectRagBinaryFromSource(sourceDir: string): string | null { const candidate = join( sourceDir, @@ -591,6 +734,17 @@ export function resolveCodebaseIndexerBackendName( return prefs?.indexer_backend ?? "projectRag"; } +export function resolveEffectiveCodebaseIndexerBackendName( + projectRoot: string, + prefs?: CodebaseMapPreferences, + env: NodeJS.ProcessEnv = process.env, +): CodebaseIndexerBackendName { + if (prefs?.indexer_backend) return prefs.indexer_backend; + const sift = detectSift(projectRoot, prefs, env); + if (sift.status === "configured") return "sift"; + return "projectRag"; +} + export function getCodebaseIndexerBackend( prefsOrName?: CodebaseMapPreferences | CodebaseIndexerBackendName, ): CodebaseIndexerBackend { @@ -606,7 +760,12 @@ export function detectCodebaseIndexer( prefs?: CodebaseMapPreferences, env: NodeJS.ProcessEnv = process.env, ): CodebaseIndexerDetection { - return getCodebaseIndexerBackend(prefs).detect(projectRoot, prefs, env); + const backendName = resolveEffectiveCodebaseIndexerBackendName( + projectRoot, + prefs, + env, + ); + return getCodebaseIndexerBackend(backendName).detect(projectRoot, prefs, env); } export function formatCodebaseIndexerStatus( @@ -614,7 +773,16 @@ export function formatCodebaseIndexerStatus( prefs?: CodebaseMapPreferences, env: NodeJS.ProcessEnv = process.env, ): string { - return getCodebaseIndexerBackend(prefs).formatStatus(projectRoot, prefs, env); + const backendName = resolveEffectiveCodebaseIndexerBackendName( + projectRoot, + prefs, + env, + ); + return getCodebaseIndexerBackend(backendName).formatStatus( + projectRoot, + prefs, + env, + ); } export function buildCodeIntelligenceContextBlock( @@ -622,11 +790,16 @@ export function buildCodeIntelligenceContextBlock( prefs?: CodebaseMapPreferences, env: NodeJS.ProcessEnv = process.env, ): string { + const backendName = resolveEffectiveCodebaseIndexerBackendName( + projectRoot, + prefs, + env, + ); const lines = [ "[PROJECT CODE INTELLIGENCE]", "", "- Durable baseline: use `.sf/CODEBASE.md` for structural orientation and persistent project knowledge.", - ...getCodebaseIndexerBackend(prefs).buildContextLines( + ...getCodebaseIndexerBackend(backendName).buildContextLines( projectRoot, prefs, env, diff --git a/src/resources/extensions/sf/commands-codebase.ts b/src/resources/extensions/sf/commands-codebase.ts index 6b5229fd2..881470785 100644 --- a/src/resources/extensions/sf/commands-codebase.ts +++ b/src/resources/extensions/sf/commands-codebase.ts @@ -39,7 +39,7 @@ const USAGE = ' exclude_patterns: ["docs/", "fixtures/"]\n' + " max_files: 1000\n" + " collapse_threshold: 15\n" + - " indexer_backend: projectRag # projectRag | sift | none\n" + + " indexer_backend: sift # projectRag | sift | none; omit for auto-detect\n" + " project_rag: auto # auto | off | required\n" + " project_rag_auto_index: true"; diff --git a/src/resources/extensions/sf/docs/preferences-reference.md b/src/resources/extensions/sf/docs/preferences-reference.md index 56c5cdf38..6fe9c7068 100644 --- a/src/resources/extensions/sf/docs/preferences-reference.md +++ b/src/resources/extensions/sf/docs/preferences-reference.md @@ -175,7 +175,7 @@ Setting `prefer_skills: []` does **not** disable skill discovery — it just mea - `exclude_patterns`: string[] — extra file or directory patterns to omit from CODEBASE.md. - `max_files`: number — maximum files to include in CODEBASE.md. Default: `500`. - `collapse_threshold`: number — files-per-directory threshold before collapsing a directory summary. Default: `20`. - - `indexer_backend`: `"projectRag"`, `"sift"`, or `"none"` — codebase-indexer backend used for prompt guidance and `/sf codebase indexer status`. Default: `"projectRag"`. + - `indexer_backend`: `"projectRag"`, `"sift"`, or `"none"` — codebase-indexer backend used for prompt guidance and `/sf codebase indexer status`. Default: use Sift when it is on `PATH`; set `projectRag` explicitly to use the MCP RAG backend. - `project_rag`: `"auto"`, `"off"`, or `"required"` — use Brainwires/project-rag MCP search when configured. Default: `"auto"`. - `project_rag_server`: string — explicit MCP server name when the server cannot be detected from command or args. - `project_rag_auto_index`: boolean — whether agents should prefer indexing before querying a configured Project RAG backend. Default: `true`. diff --git a/src/resources/extensions/sf/guided-flow.ts b/src/resources/extensions/sf/guided-flow.ts index 9eda14413..a9b89dce7 100644 --- a/src/resources/extensions/sf/guided-flow.ts +++ b/src/resources/extensions/sf/guided-flow.ts @@ -22,9 +22,11 @@ import type { } from "@singularity-forge/pi-coding-agent"; import { showConfirm, showNextAction } from "../shared/tui.js"; import { resolveExpectedArtifactPath, startAutoDetached } from "./auto.js"; +import { ensureAgenticDocsScaffold } from "./agentic-docs-scaffold.js"; import { selectAndApplyModel } from "./auto-model-selection.js"; import { buildSkillActivationBlock } from "./auto-prompts.js"; import { invalidateAllCaches } from "./cache.js"; +import { ensureSiftIndexWarmup } from "./code-intelligence.js"; import { DISCUSS_TOOLS_ALLOWLIST } from "./constants.js"; import { clearLock } from "./crash-recovery.js"; import { debugLog } from "./debug-logger.js"; @@ -742,6 +744,8 @@ export function bootstrapProject(basePath: string): void { ensureGitignore(basePath); ensurePreferences(basePath); + ensureAgenticDocsScaffold(basePath); + ensureSiftIndexWarmup(basePath, loadEffectiveSFPreferences()?.preferences?.codebase); untrackRuntimeFiles(basePath); } diff --git a/src/resources/extensions/sf/init-wizard.ts b/src/resources/extensions/sf/init-wizard.ts index cc9ea6d36..92d0fe580 100644 --- a/src/resources/extensions/sf/init-wizard.ts +++ b/src/resources/extensions/sf/init-wizard.ts @@ -13,7 +13,9 @@ import type { ExtensionCommandContext, } from "@singularity-forge/pi-coding-agent"; import { showNextAction } from "../shared/tui.js"; +import { ensureAgenticDocsScaffold } from "./agentic-docs-scaffold.js"; import { generateCodebaseMap, writeCodebaseMap } from "./codebase-generator.js"; +import { ensureSiftIndexWarmup } from "./code-intelligence.js"; import type { ProjectDetection, ProjectSignals } from "./detection.js"; import { ensureGitignore, untrackRuntimeFiles } from "./gitignore.js"; import { nativeInit } from "./native-git-bridge.js"; @@ -591,6 +593,9 @@ function bootstrapGsdDirectory( if (contextContent) { writeFileSync(join(sf, "CONTEXT.md"), contextContent, "utf-8"); } + + ensureAgenticDocsScaffold(basePath); + ensureSiftIndexWarmup(basePath); } function buildPreferencesFile(prefs: ProjectPreferences): string { diff --git a/src/resources/extensions/sf/preferences-types.ts b/src/resources/extensions/sf/preferences-types.ts index 093e15522..2417afddc 100644 --- a/src/resources/extensions/sf/preferences-types.ts +++ b/src/resources/extensions/sf/preferences-types.ts @@ -355,7 +355,7 @@ export interface CodebaseMapPreferences { max_files?: number; /** Files-per-directory threshold before collapsing to a summary line. Default: 20. */ collapse_threshold?: number; - /** Optional codebase-indexer backend. Default: "projectRag" for backward compatibility. */ + /** Optional codebase-indexer backend. Default: sift when available; set projectRag explicitly to use MCP RAG. */ indexer_backend?: CodebaseIndexerBackendName; /** Optional Brainwires/project-rag MCP backend. Default: "auto" (use if configured, never block if missing). */ project_rag?: "auto" | "off" | "required"; diff --git a/src/resources/extensions/sf/session-forensics.ts b/src/resources/extensions/sf/session-forensics.ts index 177b28e13..732dbd28c 100644 --- a/src/resources/extensions/sf/session-forensics.ts +++ b/src/resources/extensions/sf/session-forensics.ts @@ -442,10 +442,12 @@ function formatRecoveryPrompt( sections.push( "", "### Resume Instructions", - "1. Check the task plan for remaining work", - "2. Verify files listed above exist and look correct on disk", - "3. Continue from where the previous session left off", - "4. Do NOT re-read files or re-run commands that already succeeded above", + "Recovery budget is limited. Do not restart discovery.", + "1. Inspect the current diff and the files listed above first.", + "2. Do not re-read files or re-run commands that already succeeded above.", + "3. If files were already edited, finish the smallest verifiable change from that diff before any new exploration.", + "4. Run the narrowest verification command that proves the resumed work.", + "5. Complete the unit, or explicitly report the blocker with the exact remaining file/command.", ); return sections.join("\n"); diff --git a/src/resources/extensions/sf/skills/records-keeper/SKILL.md b/src/resources/extensions/sf/skills/records-keeper/SKILL.md new file mode 100644 index 000000000..97ff38529 --- /dev/null +++ b/src/resources/extensions/sf/skills/records-keeper/SKILL.md @@ -0,0 +1,67 @@ +--- +name: records-keeper +description: Keep repository records ordered after meaningful code, product, architecture, or planning changes. Use when docs may be stale, when milestone work closes, when AGENTS.md or ARCHITECTURE.md may need updates, or when durable decisions need to be filed in the right repo document. Complements context-doctor, which repairs SF's .sf persistent context. +--- + +# Records Keeper + +This skill keeps repo-local memory ordered. It updates canonical project documents, resolves stale or contradictory docs, and leaves a clear audit trail. It does not manage `.sf/` runtime state; use `context-doctor` for `.sf` persistent-context decay. + +## When To Run + +- After a milestone, slice, or large implementation finishes. +- After architecture, boundary, data/state, permission, or product behavior changes. +- When root `AGENTS.md`, local `AGENTS.md`, `ARCHITECTURE.md`, or docs disagree with source. +- When a user dumps planning notes that need to become durable repo knowledge. +- Before handing off a repo to autonomous agents. + +## Canonical Homes + +- `AGENTS.md`: short routing map and high-level agent instructions. +- Local `AGENTS.md`: directory-specific instructions; closest file wins. +- `ARCHITECTURE.md`: short operational map, boundaries, invariants, critical flows, data/state, and verification. +- `docs/product-specs/`: durable user-visible behavior. +- `docs/design-docs/`: durable design and architecture decisions. +- `docs/exec-plans/`: active/completed execution plans and tech debt. +- `docs/generated/`: generated references only. +- `docs/records/`: records audits, contradiction notes, and context-gardening outputs. +- `docs/RECORDS_KEEPER.md`: repo-specific records checklist. + +## Procedure + +1. Build a quick map: + - Read root `AGENTS.md`, `ARCHITECTURE.md`, and `docs/RECORDS_KEEPER.md` if present. + - Use Sift, `rg`, or file search to find docs and source paths touched by the change. + - Separate observed facts from inferred intent. + +2. Triage records: + - New durable behavior goes to product specs. + - New subsystem, boundary, invariant, critical flow, or state model goes to `ARCHITECTURE.md`. + - Work status/evidence goes to `docs/exec-plans/`. + - Cleanup found during work goes to the tech-debt tracker. + - Temporary notes either become a canonical record or are removed from active flow. + +3. Repair contradictions: + - Prefer source and tests for implemented behavior. + - Prefer product/design docs for intended behavior not yet implemented. + - Mark superseded decisions instead of deleting useful history. + - Cite exact files when ambiguity remains. + +4. Verify: + - Read each touched doc after editing. + - Check links and referenced paths. + - Ensure no two files claim canonical ownership of the same fact. + - Record verification commands or evals in the relevant plan/quality doc. + +5. Leave an audit note when non-trivial: + - Write a dated note under `docs/records/`. + - Include what changed, docs updated, contradictions found, and unresolved follow-up. + +## Guardrails + +- Keep root `AGENTS.md` short. Route; do not duplicate detailed doctrine. +- Do not rewrite large docs for style. Make minimal records repairs. +- Do not edit protected human-authored strategy docs destructively without a clear reason. +- Do not move generated material into hand-maintained docs. +- If `.sf/CODEBASE.md`, `.sf/KNOWLEDGE.md`, memories, or `.sf/DECISIONS.md` are stale, switch to `context-doctor`. + diff --git a/src/resources/extensions/sf/tests/agentic-docs-scaffold.test.ts b/src/resources/extensions/sf/tests/agentic-docs-scaffold.test.ts new file mode 100644 index 000000000..31114c943 --- /dev/null +++ b/src/resources/extensions/sf/tests/agentic-docs-scaffold.test.ts @@ -0,0 +1,52 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; + +import { ensureAgenticDocsScaffold } from "../agentic-docs-scaffold.ts"; + +test("ensureAgenticDocsScaffold creates repo docs and nested AGENTS files without overwriting", () => { + const dir = mkdtempSync(join(tmpdir(), "sf-agentic-docs-")); + try { + writeFileSync(join(dir, "AGENTS.md"), "# Existing\n", "utf-8"); + + ensureAgenticDocsScaffold(dir); + + assert.equal(readFileSync(join(dir, "AGENTS.md"), "utf-8"), "# Existing\n"); + assert.match( + readFileSync(join(dir, "ARCHITECTURE.md"), "utf-8"), + /short map of the codebase/, + ); + assert.match( + readFileSync(join(dir, "src", "AGENTS.md"), "utf-8"), + /Source Agent Notes/, + ); + assert.match( + readFileSync(join(dir, "tests", "AGENTS.md"), "utf-8"), + /Test Agent Notes/, + ); + assert.match( + readFileSync(join(dir, "docs", "AGENTS.md"), "utf-8"), + /records keeper checklist/, + ); + assert.match( + readFileSync(join(dir, "docs", "records", "AGENTS.md"), "utf-8"), + /Records Agent Notes/, + ); + assert.match( + readFileSync(join(dir, "docs", "RECORDS_KEEPER.md"), "utf-8"), + /Canonical Homes/, + ); + assert.match( + readFileSync(join(dir, "docs", "exec-plans", "AGENTS.md"), "utf-8"), + /Execution Plan Agent Notes/, + ); + assert.match( + readFileSync(join(dir, "docs", "QUALITY_SCORE.md"), "utf-8"), + /Make code legible to agents/, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/src/resources/extensions/sf/tests/auto-paused-session-validation.test.ts b/src/resources/extensions/sf/tests/auto-paused-session-validation.test.ts index 8804e33c4..ee4d5340f 100644 --- a/src/resources/extensions/sf/tests/auto-paused-session-validation.test.ts +++ b/src/resources/extensions/sf/tests/auto-paused-session-validation.test.ts @@ -35,20 +35,37 @@ test("auto.ts validates milestone before restoring paused session (#1664)", () = // The resume block must check for a SUMMARY file to detect completed milestones assert.ok( - source.includes('resolveMilestoneFile(base, meta.milestoneId, "SUMMARY")'), + /resolveMilestoneFile\(\s*base,\s*meta\.milestoneId,\s*"SUMMARY",?\s*\)/.test( + source, + ), "auto.ts must check for SUMMARY file to detect completed milestones", ); // Resume path must sanitize paused session file metadata before unlink/recovery. assert.ok( - source.includes("normalizeSessionFilePath(meta.sessionFile ?? null)"), + /normalizeSessionFilePath\(\s*meta\.sessionFile\s*\?\?\s*null,?\s*\)/.test( + source, + ), "auto.ts must sanitize paused-session metadata sessionFile before using it", ); + // Resume path must preserve the sanitized provider session path long enough + // to synthesize a recovery briefing. A regression nulled s.pausedSessionFile + // before synthesizeCrashRecovery(), causing resumed units to restart blind. + assert.ok( + source.includes("const resumeSessionFile = s.pausedSessionFile"), + "auto.ts must capture pausedSessionFile before cleanup", + ); + assert.ok( + source.includes("if (resumeSessionFile)") && + source.includes("resumeSessionFile ?? undefined"), + "auto.ts must pass the captured paused session file into synthesizeCrashRecovery", + ); + // Pause path must sanitize live session file path before persisting metadata. assert.ok( - source.includes( - "normalizeSessionFilePath(ctx?.sessionManager?.getSessionFile() ?? null)", + /normalizeSessionFilePath\(\s*ctx\?\.sessionManager\?\.getSessionFile\(\)\s*\?\?\s*null,?\s*\)/.test( + source, ), "auto.ts must sanitize sessionManager getSessionFile output before persisting", ); diff --git a/src/resources/extensions/sf/tests/code-intelligence.test.ts b/src/resources/extensions/sf/tests/code-intelligence.test.ts index 1b6a79082..525739738 100644 --- a/src/resources/extensions/sf/tests/code-intelligence.test.ts +++ b/src/resources/extensions/sf/tests/code-intelligence.test.ts @@ -15,10 +15,12 @@ import { detectProjectRag, detectSift, ensureProjectRagMcpConfig, + ensureSiftIndexWarmup, findProjectRagSourceDir, formatProjectRagStatus, formatSiftStatus, PROJECT_RAG_MCP_SERVER_NAME, + resolveEffectiveCodebaseIndexerBackendName, resolveProjectRagBinaryForProject, resolveProjectRagBuildJobs, resolveSiftBinary, @@ -270,7 +272,9 @@ test("buildCodeIntelligenceContextBlock injects project-rag usage guidance when "utf-8", ); - const block = buildCodeIntelligenceContextBlock(projectRoot); + const block = buildCodeIntelligenceContextBlock(projectRoot, { + indexer_backend: "projectRag", + }); assert.match(block, /PROJECT CODE INTELLIGENCE/); assert.match(block, /Project RAG: configured/); assert.match(block, /query_codebase/); @@ -281,6 +285,167 @@ test("buildCodeIntelligenceContextBlock injects project-rag usage guidance when } }); +test("effective codebase indexer uses project-rag only when explicitly selected", () => { + const projectRoot = makeProject(); + try { + writeFakeSiftBinary(projectRoot); + writeFileSync( + join(projectRoot, ".mcp.json"), + `${JSON.stringify( + { + mcpServers: { + "project-rag": { + command: join(projectRoot, "bin", "sift"), + }, + }, + }, + null, + 2, + )}\n`, + "utf-8", + ); + + assert.equal( + resolveEffectiveCodebaseIndexerBackendName( + projectRoot, + { indexer_backend: "projectRag" }, + { PATH: join(projectRoot, "bin") }, + ), + "projectRag", + ); + assert.equal( + resolveEffectiveCodebaseIndexerBackendName( + projectRoot, + undefined, + { PATH: join(projectRoot, "bin") }, + ), + "sift", + ); + } finally { + cleanup(projectRoot); + } +}); + +test("effective codebase indexer defaults to sift when project-rag is absent", () => { + const projectRoot = makeProject(); + try { + writeFakeSiftBinary(projectRoot); + + assert.equal( + resolveEffectiveCodebaseIndexerBackendName( + projectRoot, + undefined, + { PATH: join(projectRoot, "bin") }, + ), + "sift", + ); + assert.match( + buildCodeIntelligenceContextBlock(projectRoot, undefined, { + PATH: join(projectRoot, "bin"), + }), + /Sift: configured/, + ); + } finally { + cleanup(projectRoot); + } +}); + +test("ensureSiftIndexWarmup starts page-index-hybrid warmup and writes marker", () => { + const projectRoot = makeProject(); + try { + const fakeSift = writeFakeSiftBinary(projectRoot); + const calls: Array<{ command: string; args: string[] }> = []; + const fakeSpawn = ((command: string, args: string[]) => { + calls.push({ command, args }); + return { unref() {} }; + }) as unknown as typeof import("node:child_process").spawn; + + const result = ensureSiftIndexWarmup(projectRoot, undefined, { + env: { PATH: join(projectRoot, "bin") }, + spawnFn: fakeSpawn, + now: Date.parse("2026-04-30T12:00:00.000Z"), + }); + + assert.equal(result.status, "started"); + assert.equal(calls.length, 1); + assert.equal(calls[0].command, fakeSift); + assert.deepEqual(calls[0].args.slice(0, 7), [ + "search", + "--json", + "--strategy", + "page-index-hybrid", + "--limit", + "1", + "--retriever-timeout-ms", + ]); + assert.equal(calls[0].args.at(-2), projectRoot); + assert.match(calls[0].args.at(-1) ?? "", /repo architecture/); + assert.match( + readFileSync( + join(projectRoot, ".sf", "runtime", "sift-index-warmup.json"), + "utf-8", + ), + /page-index-hybrid/, + ); + } finally { + cleanup(projectRoot); + } +}); + +test("ensureSiftIndexWarmup skips recent marker and explicit non-sift backends", () => { + const projectRoot = makeProject(); + try { + writeFakeSiftBinary(projectRoot); + let spawnCount = 0; + const fakeSpawn = (() => { + spawnCount += 1; + return { unref() {} }; + }) as unknown as typeof import("node:child_process").spawn; + + const first = ensureSiftIndexWarmup(projectRoot, undefined, { + env: { PATH: join(projectRoot, "bin") }, + spawnFn: fakeSpawn, + now: Date.parse("2026-04-30T12:00:00.000Z"), + }); + assert.equal(first.status, "started"); + + const second = ensureSiftIndexWarmup(projectRoot, undefined, { + env: { PATH: join(projectRoot, "bin") }, + spawnFn: fakeSpawn, + now: Date.parse("2026-04-30T12:01:00.000Z"), + }); + assert.equal(second.status, "skipped"); + assert.equal(spawnCount, 1); + + const explicitProjectRag = ensureSiftIndexWarmup( + projectRoot, + { indexer_backend: "projectRag" }, + { + env: { PATH: join(projectRoot, "bin") }, + force: true, + spawnFn: fakeSpawn, + }, + ); + assert.equal(explicitProjectRag.status, "skipped"); + assert.match(explicitProjectRag.reason, /projectRag/); + + const disabled = ensureSiftIndexWarmup( + projectRoot, + { indexer_backend: "none" }, + { + env: { PATH: join(projectRoot, "bin") }, + force: true, + spawnFn: fakeSpawn, + }, + ); + assert.equal(disabled.status, "skipped"); + assert.match(disabled.reason, /none/); + assert.equal(spawnCount, 1); + } finally { + cleanup(projectRoot); + } +}); + test("resolveSiftBinary honors SIFT_PATH before PATH lookup", () => { const projectRoot = makeProject(); try { diff --git a/src/resources/extensions/sf/tests/records-keeper-skill.test.ts b/src/resources/extensions/sf/tests/records-keeper-skill.test.ts new file mode 100644 index 000000000..b45881ca5 --- /dev/null +++ b/src/resources/extensions/sf/tests/records-keeper-skill.test.ts @@ -0,0 +1,26 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { test } from "node:test"; + +const skillPath = join( + process.cwd(), + "src", + "resources", + "extensions", + "sf", + "skills", + "records-keeper", + "SKILL.md", +); + +test("records-keeper skill defines repo records workflow and context-doctor boundary", () => { + const skill = readFileSync(skillPath, "utf-8"); + assert.match(skill, /name: records-keeper/); + assert.match(skill, /description: Keep repository records ordered/); + assert.match(skill, /Canonical Homes/); + assert.match(skill, /ARCHITECTURE\.md/); + assert.match(skill, /docs\/RECORDS_KEEPER\.md/); + assert.match(skill, /context-doctor/); +}); + diff --git a/src/tests/headless-context-autobootstrap.test.ts b/src/tests/headless-context-autobootstrap.test.ts index 7171af0df..ead0e7f67 100644 --- a/src/tests/headless-context-autobootstrap.test.ts +++ b/src/tests/headless-context-autobootstrap.test.ts @@ -25,6 +25,7 @@ test("buildAutoBootstrapContext includes purpose docs and source inventory", () assert.match(context, /purpose, vision, architecture/); assert.match(context, /ACE spec-first TDD/); assert.match(context, /explorer-style subagents/); + assert.match(context, /harness-engineering principles/); assert.match(context, /## VISION\.md/); assert.match(context, /## TODO\.md/); assert.match(context, /## docs\/architecture\.md/); diff --git a/src/tests/headless-progress.test.ts b/src/tests/headless-progress.test.ts index 6eb2421d5..6245e620a 100644 --- a/src/tests/headless-progress.test.ts +++ b/src/tests/headless-progress.test.ts @@ -180,6 +180,25 @@ describe("formatProgress", () => { }); }); + describe("extension_error", () => { + it("shows extension path, event, and error message", () => { + const result = formatProgress( + { + type: "extension_error", + extensionPath: "/tmp/extensions/sf/index.js", + event: "startup", + error: { message: "boom" }, + }, + ctx(), + ); + assert.ok(result); + assert.ok(result.includes("Extension error")); + assert.ok(result.includes("/tmp/extensions/sf/index.js")); + assert.ok(result.includes("startup")); + assert.ok(result.includes("boom")); + }); + }); + describe("extension_ui_request", () => { it("shows notify with message", () => { const result = formatProgress(