diff --git a/.sf/backups/db/sf.db.2026-05-09T19-57-19-441Z b/.sf/backups/db/sf.db.2026-05-09T19-57-19-441Z deleted file mode 100644 index d4249f6c7..000000000 Binary files a/.sf/backups/db/sf.db.2026-05-09T19-57-19-441Z and /dev/null differ diff --git a/.sf/metrics.db b/.sf/metrics.db index 32a0ea60e..2c07382b6 100644 Binary files a/.sf/metrics.db and b/.sf/metrics.db differ diff --git a/.sf/metrics.db-shm b/.sf/metrics.db-shm index b60ed8783..6b7a0be53 100644 Binary files a/.sf/metrics.db-shm and b/.sf/metrics.db-shm differ diff --git a/.sf/metrics.db-wal b/.sf/metrics.db-wal index afbb704a3..e27f9ca31 100644 Binary files a/.sf/metrics.db-wal and b/.sf/metrics.db-wal differ diff --git a/Makefile b/Makefile index 81bd6183b..b2ed01367 100644 --- a/Makefile +++ b/Makefile @@ -2,18 +2,22 @@ SHELL := /usr/bin/env bash .DEFAULT_GOAL := help -.PHONY: help install build build-core test typecheck native clean sf +.PHONY: help install build build-core copy-resources test typecheck lint lint-fix native native-pkg clean sf help: @printf "Available targets:\n" - @printf " install Install workspace dependencies\n" - @printf " build Build the project\n" - @printf " build-core Build the core runtime packages\n" - @printf " test Run the test suite\n" - @printf " typecheck Run TypeScript type checking\n" - @printf " native Build native components\n" - @printf " clean Remove generated build outputs\n" - @printf " sf Run SF from source (passes args via ARGS=...)\n" + @printf " install Install workspace dependencies\n" + @printf " build Full build (core + web)\n" + @printf " build-core Core build including copy-resources\n" + @printf " copy-resources Rebuild dist/resources/extensions (sf extension bundles)\n" + @printf " test Run test suite\n" + @printf " typecheck Typecheck extensions/project tsconfigs\n" + @printf " lint Lint (alias for npm run lint)\n" + @printf " lint-fix Lint with autofix\n" + @printf " native Compile rust-engine (npm run build:native)\n" + @printf " native-pkg Build @singularity-forge/native workspace (npm run build:native-pkg)\n" + @printf " clean Remove dist/\n" + @printf " sf Run SF from source (ARGS='status --help')\n" install: npm install @@ -24,15 +28,27 @@ build: build-core: npm run build:core +copy-resources: + npm run copy-resources + test: npm test typecheck: npm run typecheck:extensions +lint: + npm run lint + +lint-fix: + npm run lint:fix + native: npm run build:native +native-pkg: + npm run build:native-pkg + clean: rm -rf dist dist-test diff --git a/justfile b/justfile index ab13ceade..0322df6b8 100644 --- a/justfile +++ b/justfile @@ -8,16 +8,24 @@ default: install: npm install -# Full build (core + web) +# Full build (core + web); includes npm run copy-resources via build:core build: npm run build -# Build core runtime only (faster) +# Build core runtime only (faster); includes npm run copy-resources build-core: npm run build:core -# Build native Rust addon (release) +# Rebuild bundled extension resources into dist/ (~/.sf sync on next launch) +copy-resources: + npm run copy-resources + +# Compile rust-engine native binaries (release) build-native: + npm run build:native + +# Build @singularity-forge/native workspace package (TS/N-API shims — used by build:pi) +build-native-pkg: npm run build:native-pkg # Run all tests diff --git a/src/cli-status.ts b/src/cli-status.ts index b91504374..92daa20dd 100644 --- a/src/cli-status.ts +++ b/src/cli-status.ts @@ -11,6 +11,7 @@ import type { QuerySnapshot } from "./headless-query.js"; interface StatusArgs { watch: boolean; + recoveryMode?: boolean; recoveryUnitId?: string; } @@ -31,6 +32,7 @@ function parseStatusArgs(argv: string[]): StatusArgs { if (args[0] === "recovery") { return { watch: false, + recoveryMode: true, recoveryUnitId: args[1], }; } @@ -226,6 +228,60 @@ async function buildStatusText( }); } +type RuntimeSummaryRow = { + unitType?: unknown; + unitId?: unknown; + updatedAt?: unknown; +}; + +/** + * Resolve which on-disk runtime record recovery output should describe. + * + * Purpose: `.sf/runtime/units/` names files `${unitType}-${unitId}.json`; using a + * hard-coded unit type misses non-execute-task rows when auto-picking the latest + * record or when querying by unit id alone. + * + * Consumer: sf status recovery. + */ +export function resolveRecoveryPick( + basePath: string, + records: RuntimeSummaryRow[], + explicitUnitId: string | undefined, + readRecord: ( + root: string, + unitType: string, + unitId: string, + ) => unknown | null, +): { unitType: string; unitId: string } | null { + const valid = records.filter( + (r): r is { unitType: string; unitId: string; updatedAt?: number } => + typeof r.unitType === "string" && + r.unitType.length > 0 && + typeof r.unitId === "string" && + r.unitId.length > 0, + ); + if (explicitUnitId) { + const matches = valid + .filter((r) => r.unitId === explicitUnitId) + .sort((a, b) => (Number(b.updatedAt) || 0) - (Number(a.updatedAt) || 0)); + if (matches[0]) { + return { + unitType: matches[0].unitType, + unitId: matches[0].unitId, + }; + } + if (readRecord(basePath, "execute-task", explicitUnitId)) { + return { unitType: "execute-task", unitId: explicitUnitId }; + } + return null; + } + if (valid.length === 0) return null; + const sorted = [...valid].sort( + (a, b) => (Number(b.updatedAt) || 0) - (Number(a.updatedAt) || 0), + ); + return { unitType: sorted[0].unitType, unitId: sorted[0].unitId }; +} + async function renderRecoveryDiagnostics( basePath: string, unitId: string | undefined, @@ -233,30 +289,38 @@ async function renderRecoveryDiagnostics( stderr: Pick, ): Promise { try { - const { getRecoveryDiagnostics, listUnitRuntimeRecords } = await import( - "./resources/extensions/sf/uok/unit-runtime.js" + const { + getRecoveryDiagnostics, + listUnitRuntimeRecords, + readUnitRuntimeRecord, + } = await import("./resources/extensions/sf/uok/unit-runtime.js"); + const rows = listUnitRuntimeRecords(basePath); + const picked = resolveRecoveryPick( + basePath, + rows, + unitId, + readUnitRuntimeRecord, ); - let targetUnitId = unitId; - if (!targetUnitId) { - const records: Array<{ updatedAt?: number; unitId: string }> = - listUnitRuntimeRecords(basePath); - const mostRecent = records.sort( - (a, b) => (b.updatedAt ?? 0) - (a.updatedAt ?? 0), - )[0]; - if (!mostRecent) { + if (!picked) { + if (rows.length === 0) { stderr.write("sf status recovery: no runtime records found\n"); - return 1; + } else { + stderr.write( + unitId + ? `sf status recovery: no runtime record for ${unitId}\n` + : "sf status recovery: no usable runtime records found\n", + ); } - targetUnitId = mostRecent.unitId; + return 1; } const diagnostics = getRecoveryDiagnostics( basePath, - "execute-task", - targetUnitId, + picked.unitType, + picked.unitId, ); if (!diagnostics) { stderr.write( - `sf status recovery: no runtime record for ${targetUnitId}\n`, + `sf status recovery: no runtime record for ${picked.unitType} ${picked.unitId}\n`, ); return 1; } @@ -305,7 +369,7 @@ export async function runStatusCli( const sfHome = deps.sfHome ?? process.env.SF_HOME ?? join(homedir(), ".sf"); const args = parseStatusArgs(argv); - if (args.recoveryUnitId !== undefined) { + if (args.recoveryMode) { return renderRecoveryDiagnostics( deps.basePath, args.recoveryUnitId, diff --git a/src/headless-types.ts b/src/headless-types.ts index d0adae392..c1c726777 100644 --- a/src/headless-types.ts +++ b/src/headless-types.ts @@ -30,6 +30,14 @@ export interface NotificationMetadata { dedupe_key?: string; /** Emission source label (extension name, "workflow", etc.). */ source?: string; + /** Long-lived classification for UI/transcript (see NOTICE_KIND in notification-store). */ + noticeKind?: + | "system_notice" + | "tool_notice" + | "blocking_notice" + | "user_visible"; + /** When false, duplicate rows are never merged (each notify appends a line). */ + merge?: boolean; } // --------------------------------------------------------------------------- diff --git a/src/resources/extensions/sf/auto-prompts.js b/src/resources/extensions/sf/auto-prompts.js index eb7799ed1..4033d5ae2 100644 --- a/src/resources/extensions/sf/auto-prompts.js +++ b/src/resources/extensions/sf/auto-prompts.js @@ -440,7 +440,7 @@ export async function inlineDecisionsFromDb(base, milestoneId, scope, level) { inlineLevel !== "full" ? formatDecisionsCompact(decisions) : formatDecisionsForPrompt(decisions); - return `### Decisions\nSource: \`.sf/DECISIONS.md\`\n\n${formatted}`; + return `### Decisions\nSource: DB\n\n${formatted}`; } // DB available but cascade returned empty — intentional per D020, don't fall back to file return null; @@ -478,7 +478,7 @@ export async function inlineRequirementsFromDb( inlineLevel !== "full" ? formatRequirementsCompact(requirements) : formatRequirementsForPrompt(requirements); - return `### Requirements\nSource: \`.sf/REQUIREMENTS.md\`\n\n${formatted}`; + return `### Requirements\nSource: DB\n\n${formatted}`; } } } catch (err) { @@ -632,32 +632,42 @@ function extractKeywords(title) { .filter((w) => w.length > 0 && !STOPWORDS.has(w)); } /** - * Inline scoped KNOWLEDGE.md content based on keywords from slice title. - * Reads KNOWLEDGE.md, filters to sections matching keywords, formats with header. - * Returns null if no KNOWLEDGE.md exists or no sections match. + * Inline scoped knowledge based on keywords from slice title. + * Queries DB memories table (primary); falls back to KNOWLEDGE.md file. + * Returns null if no knowledge exists or no entries match. */ export async function inlineKnowledgeScoped(base, keywords) { + try { + const { isDbAvailable, getActiveMemories } = await import("./sf-db.js"); + if (isDbAvailable()) { + const entries = getActiveMemories({ category: "knowledge", limit: 200 }); + if (entries.length > 0) { + const kws = keywords.map((k) => k.toLowerCase()); + const matched = entries.filter((e) => + kws.some((kw) => e.content.toLowerCase().includes(kw)), + ); + if (matched.length === 0) return null; + const formatted = matched.map((e) => `- ${e.content}`).join("\n"); + return `### Project Knowledge (scoped)\nSource: DB\n\n${formatted}`; + } + } + } catch { + // fall through to file + } const knowledgePath = resolveSfRootFile(base, "KNOWLEDGE"); if (!existsSync(knowledgePath)) return null; const content = await loadFile(knowledgePath); if (!content) return null; - // Import queryKnowledge from context-store const { queryKnowledge } = await import("./context-store.js"); const scoped = await queryKnowledge(content, keywords); - // Return null if no sections matched (empty string from queryKnowledge) if (!scoped) return null; return `### Project Knowledge (scoped)\nSource: \`${relSfRootFile("KNOWLEDGE")}\`\n\n${scoped.trim()}`; } /** * Budget-capped knowledge inline for milestone-level prompt assembly. - * - * Addresses issue #4719: the six milestone-phase prompts (research-milestone, - * plan-milestone, complete-slice, complete-milestone, validate-milestone, - * reassess-roadmap) previously injected the full KNOWLEDGE.md (~226KB for a - * real project) on every invocation. This helper scopes by caller-supplied - * keywords and caps the payload at `maxChars` (default 30,000 chars). - * - * Returns null when no KNOWLEDGE.md exists or no entries match any keyword. + * Queries DB memories table (primary); falls back to KNOWLEDGE.md file. + * Caps the payload at `maxChars` (default 30,000 chars). + * Returns null when no knowledge exists or no entries match any keyword. */ export async function inlineKnowledgeBudgeted(base, keywords, options) { const DEFAULT_MAX_CHARS = 30_000; @@ -666,6 +676,27 @@ export async function inlineKnowledgeBudgeted(base, keywords, options) { const maxChars = Number.isFinite(raw) ? Math.max(0, Math.min(Math.floor(raw), HARD_MAX_CHARS)) : DEFAULT_MAX_CHARS; + try { + const { isDbAvailable, getActiveMemories } = await import("./sf-db.js"); + if (isDbAvailable()) { + const entries = getActiveMemories({ category: "knowledge", limit: 500 }); + if (entries.length > 0) { + const kws = keywords.map((k) => k.toLowerCase()); + const matched = entries.filter((e) => + kws.some((kw) => e.content.toLowerCase().includes(kw)), + ); + if (matched.length === 0) return null; + const trimmed = matched.map((e) => `- ${e.content}`).join("\n"); + const truncated = + trimmed.length > maxChars + ? `${trimmed.slice(0, maxChars)}\n\n[...truncated; rerun with narrower scope if needed]` + : trimmed; + return `### Project Knowledge (scoped)\nSource: DB\n\n${truncated}`; + } + } + } catch { + // fall through to file + } const knowledgePath = resolveSfRootFile(base, "KNOWLEDGE"); if (!existsSync(knowledgePath)) return null; const content = await loadFile(knowledgePath); diff --git a/src/resources/extensions/sf/auto-runaway-guard.js b/src/resources/extensions/sf/auto-runaway-guard.js index 9d0a5ef23..b0526856b 100644 --- a/src/resources/extensions/sf/auto-runaway-guard.js +++ b/src/resources/extensions/sf/auto-runaway-guard.js @@ -20,9 +20,6 @@ const EXECUTE_NO_PROGRESS_TOKEN_WARNING = 500_000; const DURABLE_SF_ARTIFACT_PATHS = [ ".sf/milestones", ".sf/approvals", - ".sf/DECISIONS.md", - ".sf/KNOWLEDGE.md", - ".sf/STATE.md", ]; let state = null; export function resetRunawayGuardState(unitType, unitId, baseline) { diff --git a/src/resources/extensions/sf/auto/phases.js b/src/resources/extensions/sf/auto/phases.js index 60fbb34c4..4bd926fb6 100644 --- a/src/resources/extensions/sf/auto/phases.js +++ b/src/resources/extensions/sf/auto/phases.js @@ -1628,24 +1628,13 @@ export async function runGuards(ic, mid, unitType, unitId, sliceId) { if (isFirstTaskForSlice) { let planGateOutcome = "pass"; let planGateRationale = ""; - const roadmapPath = resolveMilestoneFile(s.basePath, mid, "ROADMAP"); - if (!roadmapPath || !existsSync(roadmapPath)) { + const milestoneSlices = getMilestoneSlices(mid); + if (!milestoneSlices || milestoneSlices.length === 0) { planGateOutcome = "fail"; - planGateRationale = `Milestone roadmap not found for ${mid}`; - } else { - const slicePlanPath = resolveSliceFile( - s.basePath, - mid, - sliceId, - "PLAN", - ); - if (!slicePlanPath || !existsSync(slicePlanPath)) { - planGateOutcome = "fail"; - planGateRationale = `Slice plan not found for ${mid}/${sliceId}`; - } else if (taskCounts.total < 1) { - planGateOutcome = "fail"; - planGateRationale = `Slice ${sliceId} has no tasks defined`; - } + planGateRationale = `Milestone ${mid} has no slices in DB (not yet planned)`; + } else if (taskCounts.total < 1) { + planGateOutcome = "fail"; + planGateRationale = `Slice ${sliceId} has no tasks defined`; } const planGateRunner = new UokGateRunner(); planGateRunner.register({ diff --git a/src/resources/extensions/sf/bootstrap/notify-interceptor.js b/src/resources/extensions/sf/bootstrap/notify-interceptor.js index 61c120e45..954ccfdd8 100644 --- a/src/resources/extensions/sf/bootstrap/notify-interceptor.js +++ b/src/resources/extensions/sf/bootstrap/notify-interceptor.js @@ -13,6 +13,11 @@ const _wrappedContexts = new WeakSet(); * Install the notify interceptor on a context's UI object. * Mutates ctx.ui.notify in place — the original is called after persistence. * Safe to call multiple times; no-ops if already installed on the same ui object. + * + * Optional third-arg metadata for durable hygiene: + * - dedupe_key — stable merge identity across wording/timer drift + * - noticeKind — NOTICE_KIND.* (system_notice, tool_notice, …) + * - merge — false to force a new JSONL row even when duplicate */ export function installNotifyInterceptor(ctx) { if (_wrappedContexts.has(ctx.ui)) return; diff --git a/src/resources/extensions/sf/bootstrap/system-context.js b/src/resources/extensions/sf/bootstrap/system-context.js index 0ae06ce32..125dea257 100644 --- a/src/resources/extensions/sf/bootstrap/system-context.js +++ b/src/resources/extensions/sf/bootstrap/system-context.js @@ -52,6 +52,11 @@ import { } from "../skill-discovery.js"; import { deriveState } from "../state.js"; import { logWarning } from "../workflow-logger.js"; +import { + getActiveMemories, + isDbAvailable, + listSelfFeedbackEntries, +} from "../sf-db.js"; import { getActiveWorktreeName, getWorktreeOriginalCwd, @@ -350,12 +355,20 @@ export function loadKnowledgeBlock(sfHomeDir, cwd) { globalKnowledge = content; } } - // 2. Project knowledge (.sf/KNOWLEDGE.md) — project-specific + // 2. Project knowledge — DB memories table (primary); .sf/KNOWLEDGE.md fallback let projectKnowledge = ""; - const knowledgePath = resolveSfRootFile(cwd, "KNOWLEDGE"); - if (existsSync(knowledgePath)) { - const content = cachedReadFile(knowledgePath)?.trim() ?? ""; - if (content) projectKnowledge = content; + if (isDbAvailable()) { + const memories = getActiveMemories({ category: "knowledge", limit: 100 }); + if (memories.length > 0) { + projectKnowledge = memories.map((m) => `- ${m.content}`).join("\n"); + } + } + if (!projectKnowledge) { + const knowledgePath = resolveSfRootFile(cwd, "KNOWLEDGE"); + if (existsSync(knowledgePath)) { + const content = cachedReadFile(knowledgePath)?.trim() ?? ""; + if (content) projectKnowledge = content; + } } if (!globalKnowledge && !projectKnowledge) { return { block: "", globalSizeKb: 0 }; @@ -378,33 +391,17 @@ const TACIT_SECTION_MAX_BYTES = 4096; const SELF_FEEDBACK_MAX_ENTRIES = 20; const SELF_FEEDBACK_MAX_CHARS = 4000; function loadSelfFeedbackBlock(cwd) { - const selfFeedbackPath = join(cwd, ".sf", "SELF-FEEDBACK.md"); - const legacyBacklogPath = join(cwd, ".sf", "BACKLOG.md"); - const sourcePath = existsSync(selfFeedbackPath) - ? selfFeedbackPath - : legacyBacklogPath; - if (!existsSync(sourcePath)) return ""; - const raw = cachedReadFile(sourcePath)?.trim() ?? ""; - if (!raw) return ""; - // Parse the table rows — skip header lines - const lines = raw.split("\n"); - const entries = []; - for (const line of lines) { - if (!line.startsWith("| ")) continue; - if (line.includes("Timestamp")) continue; // header - if (line.includes("|---|---|")) continue; // separator - const cells = line - .split("|") - .map((c) => c.trim()) - .filter(Boolean); - if (cells.length >= 7) { - entries.push({ - timestamp: cells[0], - kind: cells[1], - severity: cells[2], - summary: cells[6], - }); - } + let entries = []; + if (isDbAvailable()) { + const rows = listSelfFeedbackEntries(); + entries = rows + .filter((r) => !r["resolved_at"]) + .map((r) => ({ + timestamp: r["ts"] ?? "", + kind: r["kind"] ?? "", + severity: r["severity"] ?? "low", + summary: r["summary"] ?? "", + })); } if (entries.length === 0) return ""; // Sort by severity (high/critical first) then by timestamp (newest first) diff --git a/src/resources/extensions/sf/commands/handlers/notifications-handler.js b/src/resources/extensions/sf/commands/handlers/notifications-handler.js index ce2095331..ee9fef793 100644 --- a/src/resources/extensions/sf/commands/handlers/notifications-handler.js +++ b/src/resources/extensions/sf/commands/handlers/notifications-handler.js @@ -36,6 +36,15 @@ function formatTimestamp(ts) { return ts.slice(0, 19); } } +function formatNotificationLine(e) { + const repeat = + (e.repeatCount ?? 1) > 1 + ? ` ×${e.repeatCount}${ + e.lastTs ? ` (last ${formatTimestamp(e.lastTs)})` : "" + }` + : ""; + return `${severityIcon(e.severity)} [${formatTimestamp(e.ts)}] ${e.message}${repeat}`; +} export async function handleNotificationsCommand(args, ctx, _pi) { // /notifications clear if (args === "clear") { @@ -63,10 +72,7 @@ export async function handleNotificationsCommand(args, ctx, _pi) { ctx.ui.notify("No notifications.", "info"); return true; } - const lines = entries.map( - (e) => - `${severityIcon(e.severity)} [${formatTimestamp(e.ts)}] ${e.message}`, - ); + const lines = entries.map(formatNotificationLine); const suffix = all.length > entries.length ? `\n... and ${all.length - entries.length} more (open /notifications to browse all)` @@ -95,12 +101,7 @@ export async function handleNotificationsCommand(args, ctx, _pi) { ctx.ui.notify(`No ${severity} notifications.`, "info"); return true; } - const lines = entries - .slice(0, 20) - .map( - (e) => - `${severityIcon(e.severity)} [${formatTimestamp(e.ts)}] ${e.message}`, - ); + const lines = entries.slice(0, 20).map(formatNotificationLine); const suffix = entries.length > 20 ? `\n... and ${entries.length - 20} more (open /notifications to browse all)` @@ -144,10 +145,7 @@ export async function handleNotificationsCommand(args, ctx, _pi) { ctx.ui.notify("No notifications.", "info"); return true; } - const lines = entries.map( - (e) => - `${severityIcon(e.severity)} [${formatTimestamp(e.ts)}] ${e.message}`, - ); + const lines = entries.map(formatNotificationLine); const header = unread > 0 ? `${unread} unread — ` : ""; ctx.ui.notify( `${header}Recent notifications:\n${lines.join("\n")}`, diff --git a/src/resources/extensions/sf/db-writer.js b/src/resources/extensions/sf/db-writer.js index 68a457d89..82663afd2 100644 --- a/src/resources/extensions/sf/db-writer.js +++ b/src/resources/extensions/sf/db-writer.js @@ -307,33 +307,6 @@ export async function saveRequirementToDb(fields, basePath) { superseded_by: row["superseded_by"] ?? null, })); } - const nonSuperseded = allRequirements.filter( - (r) => r.superseded_by == null, - ); - const md = generateRequirementsMd(nonSuperseded); - const filePath = resolveSfRootFile(basePath, "REQUIREMENTS"); - try { - await saveFile(filePath, md); - } catch (diskErr) { - logError("manifest", "disk write failed, rolling back DB row", { - fn: "saveRequirementToDb", - error: String(diskErr.message), - }); - try { - db.deleteRequirementById(id); - } catch (rollbackErr) { - logError( - "manifest", - "SPLIT BRAIN: disk write failed AND DB rollback failed — DB has orphaned row", - { - fn: "saveRequirementToDb", - id, - error: String(rollbackErr.message), - }, - ); - } - throw diskErr; - } invalidateStateCache(); clearPathCache(); clearParseCache(); @@ -408,51 +381,6 @@ export async function saveDecisionToDb(fields, basePath) { superseded_by: row["superseded_by"] ?? null, })); } - const filePath = resolveSfRootFile(basePath, "DECISIONS"); - // Check if existing DECISIONS.md has freeform (non-table) content. - // If so, preserve that content and append/update the decisions table - // at the end instead of overwriting the entire file. - let existingContent = null; - if (existsSync(filePath)) { - existingContent = readFileSync(filePath, "utf-8"); - } - let md; - if (existingContent && !isDecisionsTableFormat(existingContent)) { - // Freeform content detected — preserve it and append decisions table. - // Strip any previously appended decisions table section to avoid duplication. - const marker = "---\n\n## Decisions Table"; - const markerIdx = existingContent.indexOf(marker); - const freeformPart = - markerIdx >= 0 - ? existingContent.substring(0, markerIdx).trimEnd() - : existingContent.trimEnd(); - md = freeformPart + "\n" + generateDecisionsAppendBlock(allDecisions); - } else { - // Table format or no existing file — full regeneration (original behavior) - md = generateDecisionsMd(allDecisions); - } - try { - await saveFile(filePath, md); - } catch (diskErr) { - logError("manifest", "disk write failed, rolling back DB row", { - fn: "saveDecisionToDb", - error: String(diskErr.message), - }); - try { - db.deleteDecisionById(id); - } catch (rollbackErr) { - logError( - "manifest", - "SPLIT BRAIN: disk write failed AND DB rollback failed — DB has orphaned row", - { - fn: "saveDecisionToDb", - id, - error: String(rollbackErr.message), - }, - ); - } - throw diskErr; - } // #2661: When a decision defers a slice, update the slice status in the DB // so the dispatcher skips it. Without this, STATE.md and DECISIONS.md are // in split-brain: the decision says "deferred" but the state still says @@ -596,27 +524,6 @@ export async function updateRequirementInDb(id, updates, basePath) { superseded_by: row["superseded_by"] ?? null, })); } - // Filter to non-superseded for the markdown file - // (superseded requirements don't appear in section headings) - const nonSuperseded = allRequirements.filter( - (r) => r.superseded_by == null, - ); - const md = generateRequirementsMd(nonSuperseded); - const filePath = resolveSfRootFile(basePath, "REQUIREMENTS"); - try { - await saveFile(filePath, md); - } catch (diskErr) { - logError("manifest", "disk write failed, reverting DB row", { - fn: "updateRequirementInDb", - error: String(diskErr.message), - }); - if (existing) { - db.upsertRequirement(existing); - } - throw diskErr; - } - // Invalidate file-read caches so deriveState() sees the updated markdown. - // Do NOT clear the artifacts table — we just wrote to it intentionally. invalidateStateCache(); clearPathCache(); clearParseCache(); diff --git a/src/resources/extensions/sf/deep-project-setup-policy.js b/src/resources/extensions/sf/deep-project-setup-policy.js index 97b7cf0e0..b7f38e70a 100644 --- a/src/resources/extensions/sf/deep-project-setup-policy.js +++ b/src/resources/extensions/sf/deep-project-setup-policy.js @@ -4,6 +4,7 @@ import { clearParseCache } from "./files.js"; import { clearPathCache, sfRoot } from "./paths.js"; import { getProjectResearchStatus } from "./project-research-policy.js"; import { validateArtifact } from "./schemas/validate.js"; +import { getActiveRequirements, isDbAvailable } from "./sf-db.js"; const EXPLICIT_RESEARCH_SOURCES = new Set(["research-decision", "user"]); function clearCaches() { @@ -105,20 +106,24 @@ export function resolveDeepProjectSetupState(prefs, basePath) { reason: ".sf/PROJECT.md is invalid.", }; } - const requirementsPath = join(root, "REQUIREMENTS.md"); - if (!existsSync(requirementsPath)) { - return { - status: "pending", - stage: "requirements", - reason: ".sf/REQUIREMENTS.md is missing.", - }; - } - if (!validateArtifact(requirementsPath, "requirements").ok) { - return { - status: "pending", - stage: "requirements", - reason: ".sf/REQUIREMENTS.md is invalid.", - }; + // DB-first: check for requirements in DB; fall back to file for unmigrated projects + const hasDbRequirements = isDbAvailable() && getActiveRequirements().length > 0; + if (!hasDbRequirements) { + const requirementsPath = join(root, "REQUIREMENTS.md"); + if (!existsSync(requirementsPath)) { + return { + status: "pending", + stage: "requirements", + reason: "No requirements found (DB empty and .sf/REQUIREMENTS.md is missing).", + }; + } + if (!validateArtifact(requirementsPath, "requirements").ok) { + return { + status: "pending", + stage: "requirements", + reason: ".sf/REQUIREMENTS.md is invalid.", + }; + } } const marker = readDecision(basePath); if (!marker.exists) { diff --git a/src/resources/extensions/sf/extension-manifest.json b/src/resources/extensions/sf/extension-manifest.json index a8ad731e0..8f41076aa 100644 --- a/src/resources/extensions/sf/extension-manifest.json +++ b/src/resources/extensions/sf/extension-manifest.json @@ -9,41 +9,41 @@ }, "provides": { "tools": [ + "audit_product", "bash", - "capture_thought", + "checkpoint", + "complete_milestone", + "complete_slice", + "complete_task", "edit", "kill_agent", - "memory_query", + "log_decision", + "log_reasoning", + "memory_graph", + "memory_search", + "milestone_status", + "new_milestone_id", + "plan_milestone", + "plan_slice", + "plan_task", + "query_journal", "read", - "sf_autonomous_checkpoint", - "sf_complete_milestone", - "sf_decision_save", - "sf_exec", - "sf_exec_search", - "sf_graph", - "sf_journal_query", - "sf_log_judgment", - "sf_milestone_generate_id", - "sf_milestone_status", - "sf_plan_milestone", - "sf_plan_slice", - "sf_plan_task", - "sf_product_audit", - "sf_reassess_roadmap", - "sf_replan_slice", - "sf_requirement_save", - "sf_requirement_update", - "sf_retrieval_evidence", - "sf_resume", - "sf_save_gate_result", - "sf_self_feedback_resolve", - "sf_self_report", - "sf_skip_slice", - "sf_slice_complete", - "sf_summary_save", - "sf_task_complete", - "sf_validate_milestone", + "read_output", + "reassess_roadmap", + "record_gate", + "replan_slice", + "report_issue", + "resolve_issue", + "resume_agent", + "run_command", + "save_decision", + "save_requirement", + "save_summary", + "search_evidence", "sift_search", + "skip_slice", + "update_requirement", + "validate_milestone", "write" ], "commands": [ diff --git a/src/resources/extensions/sf/guided-flow.js b/src/resources/extensions/sf/guided-flow.js index e38ffb8de..dbf5512a4 100644 --- a/src/resources/extensions/sf/guided-flow.js +++ b/src/resources/extensions/sf/guided-flow.js @@ -70,7 +70,7 @@ import { isSessionLockProcessAlive, readSessionLockData, } from "./session-lock.js"; -import { getMilestoneSlices, isDbAvailable } from "./sf-db.js"; +import { getAllMilestones, getMilestone, getMilestoneSlices, isDbAvailable } from "./sf-db.js"; import { deriveState } from "./state.js"; import { resolveUokFlags } from "./uok/flags.js"; import { UokGateRunner } from "./uok/gate-runner.js"; @@ -265,49 +265,53 @@ export function checkAutoStartAfterDiscuss() { const entry = _getPendingAutoStart(); if (!entry) return false; const { ctx, pi, basePath, milestoneId, step } = entry; - // Gate 1: Primary milestone must have CONTEXT.md or ROADMAP.md - // The "discuss" path creates CONTEXT.md; the "plan" path creates ROADMAP.md. - const contextFile = resolveMilestoneFile(basePath, milestoneId, "CONTEXT"); - const roadmapFile = resolveMilestoneFile(basePath, milestoneId, "ROADMAP"); - if (!contextFile && !roadmapFile) return false; // neither artifact yet — keep waiting - // Gate 2: STATE.md must exist — written as the last step in the discuss - // output phase. This prevents auto-start from firing during Phase 3 - // (sequential readiness gates for remaining milestones) in multi-milestone - // discussions, where M001-CONTEXT.md exists but M002/M003 haven't been - // processed yet. - const stateFile = resolveSfRootFile(basePath, "STATE"); - if (!stateFile) return false; // discussion not finalized yet - // Gate 3: Multi-milestone completeness warning - // Parse PROJECT.md for milestone sequence, warn if any are missing context. - // Don't block — milestones can be intentionally queued without context. - const projectFile = resolveSfRootFile(basePath, "PROJECT"); + // Gate 1: Primary milestone must have context or roadmap in DB. + // discuss phase populates milestone.vision; plan phase populates slices. let projectIds = []; - if (projectFile) { - try { - const projectContent = readFileSync(projectFile, "utf-8"); - projectIds = parseMilestoneSequenceFromProject(projectContent); - if (projectIds.length > 1) { - const missing = projectIds.filter((id) => { - const hasContext = !!resolveMilestoneFile(basePath, id, "CONTEXT"); - const hasDraft = !!resolveMilestoneFile( - basePath, - id, - "CONTEXT-DRAFT", - ); - const hasDir = existsSync(join(sfRoot(basePath), "milestones", id)); - return !hasContext && !hasDraft && !hasDir; - }); - if (missing.length > 0) { - ctx.ui.notify( - `Multi-milestone validation: ${missing.join(", ")} not found in filesystem. ` + - `Discussion may not have completed all readiness gates.`, - "warning", - ); - } + if (isDbAvailable()) { + const milestone = getMilestone(milestoneId); + const dbSlices = getMilestoneSlices(milestoneId); + const hasContext = !!(milestone?.vision); + const hasRoadmap = dbSlices.length > 0; + if (!hasContext && !hasRoadmap) return false; + // Gate 3: Multi-milestone completeness warning (DB version) + const allMilestones = getAllMilestones(); + projectIds = allMilestones.map((m) => m.id); + if (projectIds.length > 1) { + const missing = projectIds.filter((id) => { + const m = getMilestone(id); + return !m?.vision && getMilestoneSlices(id).length === 0; + }); + if (missing.length > 0) { + ctx.ui.notify( + `Multi-milestone validation: ${missing.join(", ")} not yet planned in DB.`, + "warning", + ); } - } catch (e) { - logWarning("guided", `PROJECT.md parsing failed: ${e.message}`); } + } else { + // Fallback for non-migrated projects without a DB + const contextFile = resolveMilestoneFile(basePath, milestoneId, "CONTEXT"); + const roadmapFile = resolveMilestoneFile(basePath, milestoneId, "ROADMAP"); + if (!contextFile && !roadmapFile) return false; + const projectFile = resolveSfRootFile(basePath, "PROJECT"); + if (projectFile) { + try { + const projectContent = readFileSync(projectFile, "utf-8"); + projectIds = parseMilestoneSequenceFromProject(projectContent); + } catch (e) { + logWarning("guided", `PROJECT.md parsing failed: ${e.message}`); + } + } + } + // Gate 2: Milestone must have a non-empty vision in DB (discuss phase complete). + // Falls back to checking STATE.md for non-migrated projects. + if (isDbAvailable()) { + const m = getMilestone(milestoneId); + if (!m?.vision) return false; + } else { + const stateFile = resolveSfRootFile(basePath, "STATE"); + if (!stateFile) return false; } // Gate 4: Discussion manifest process verification (multi-milestone only) // The LLM writes DISCUSSION-MANIFEST.json after each Phase 3 gate decision. diff --git a/src/resources/extensions/sf/notification-overlay.js b/src/resources/extensions/sf/notification-overlay.js index 600fcf702..4af0bfe7b 100644 --- a/src/resources/extensions/sf/notification-overlay.js +++ b/src/resources/extensions/sf/notification-overlay.js @@ -85,7 +85,7 @@ function notificationSignature(entries) { return entries .map( (entry) => - `${entry.ts}|${entry.severity}|${entry.read ? 1 : 0}|${entry.message}`, + `${entry.ts}|${entry.severity}|${entry.read ? 1 : 0}|${entry.repeatCount ?? 1}|${entry.lastTs ?? ""}|${entry.message}`, ) .join("\n"); } @@ -343,11 +343,26 @@ export class SFNotificationOverlay { const prefixWidth = visibleWidth(prefix); const msgMaxWidth = Math.max(10, contentWidth - prefixWidth); // Wrap long messages onto continuation lines indented to align with message start + const repeatSuffix = + (entry.repeatCount ?? 1) > 1 + ? th.fg( + "dim", + ` · ×${entry.repeatCount}${ + entry.lastTs ? ` (last ${formatTimestamp(entry.lastTs)})` : "" + }`, + ) + : ""; + const noticeKind = + entry.noticeKind && entry.noticeKind !== "user_visible" + ? th.fg("dim", ` [${entry.noticeKind}]`) + : ""; const msgLines = wrapText(entry.message, msgMaxWidth); const indent = " ".repeat(prefixWidth); for (let i = 0; i < msgLines.length; i++) { if (i === 0) { - lines.push(row(`${prefix}${msgLines[i]}`)); + lines.push( + row(`${prefix}${msgLines[i]}${repeatSuffix}${noticeKind}`), + ); } else { lines.push(row(`${indent}${msgLines[i]}`)); } diff --git a/src/resources/extensions/sf/notification-store.js b/src/resources/extensions/sf/notification-store.js index 982fe0e70..7e6fee0a9 100644 --- a/src/resources/extensions/sf/notification-store.js +++ b/src/resources/extensions/sf/notification-store.js @@ -2,6 +2,10 @@ // Captures durable ctx.ui.notify() calls and workflow-logger errors to // .sf/notifications.jsonl so they survive context resets and session restarts. // Rotates at MAX_ENTRIES to prevent unbounded growth. +// +// Long-term hygiene (schema v2+): repeated equivalent notices merge into one row +// with repeatCount + lastTs; blocking/action-required rows never merge. +// Pass metadata.dedupe_key for stable identity; metadata.noticeKind for classification. import { randomUUID } from "node:crypto"; import { appendFileSync, @@ -21,10 +25,11 @@ import { sfRuntimeRoot } from "./paths.js"; const MAX_ENTRIES = 500; const FILENAME = "notifications.jsonl"; const LOCKFILE = "notifications.lock"; -const NOTIFICATION_SCHEMA_VERSION = 1; -const DEDUP_WINDOW_MS = 30_000; +/** Legacy rows on disk may omit schemaVersion — treated as v1 */ +export const NOTIFICATION_SCHEMA_VERSION_MIN = 1; +/** Current write schema — adds repeatCount, lastTs, noticeKind */ +export const NOTIFICATION_SCHEMA_VERSION_WRITE = 2; const DURABLE_DEDUP_WINDOW_MS = 60 * 60 * 1000; -const DEDUP_PRUNE_THRESHOLD = 200; const DURABLE_DEDUP_SCAN_LIMIT = 100; const ACTIONABLE_KINDS = new Set([ "action_required", @@ -50,11 +55,25 @@ const NOISY_STATUS_PATTERNS = [ /^Resuming paused session\b/, /^Resumed paused session\b/, ]; + +/** + * Optional classification for automated vs human-visible notices (see docs/product-specs/notification-source-hygiene.md). + * + * Purpose: let renderers group and style notices without parsing message text. + * + * Consumer: ctx.ui.notify metadata, headless classifiers, notification overlay. + */ +export const NOTICE_KIND = { + SYSTEM_NOTICE: "system_notice", + TOOL_NOTICE: "tool_notice", + BLOCKING_NOTICE: "blocking_notice", + USER_VISIBLE: "user_visible", +}; + // ─── Module State ─────────────────────────────────────────────────────── let _basePath = null; let _lineCount = 0; // Hint for rotation — not authoritative for public API let _suppressCount = 0; -let _recentMessageTimestamps = new Map(); const _changeListeners = new Set(); /** Count of appendNotification failures since last reset — surfaced by status checks. */ let _appendFailureCount = 0; @@ -66,9 +85,6 @@ let _lastAppendFailure = null; * project root. Seeds in-memory counters from the existing file on disk. */ export function initNotificationStore(basePath) { - if (_basePath !== basePath) { - _recentMessageTimestamps.clear(); - } _basePath = basePath; // Seed line count hint for rotation — public counters read from disk _lineCount = _readEntriesFromDisk(basePath).length; @@ -76,6 +92,11 @@ export function initNotificationStore(basePath) { /** * Append a notification entry to the store. Synchronous — safe to call * from the notify() shim which is declared void (not async). + * + * Duplicate notices (same dedupe key within the durable window, unread): merge + * into the existing row (repeatCount++, lastTs updated) instead of growing the log. + * Set metadata.merge === false to force a new row. Blocking / approval / blocker + * kinds do not merge — each emission stays distinct for consent surfaces. */ export function appendNotification( message, @@ -92,37 +113,29 @@ export function appendNotification( !shouldPersistNotification(normalizedSeverity, metadata, persistedMessage) ) return; - // Use explicit dedupe_key when provided; fall back to message-hash based key. - const dedupKey = metadata?.dedupe_key - ? `${_basePath}:${metadata.dedupe_key}` - : `${_basePath}:${normalizedSeverity}:${source}:${persistedMessage}`; const now = Date.now(); - const lastSeen = _recentMessageTimestamps.get(dedupKey); - if (lastSeen !== undefined && now - lastSeen <= DEDUP_WINDOW_MS) return; - _recentMessageTimestamps.set(dedupKey, now); - if (_recentMessageTimestamps.size > DEDUP_PRUNE_THRESHOLD) { - for (const [key, ts] of _recentMessageTimestamps) { - if (now - ts > DEDUP_WINDOW_MS) _recentMessageTimestamps.delete(key); - } - } if ( - hasRecentPersistedDuplicate( - _basePath, - metadata?.dedupe_key ?? - `${normalizedSeverity}:${source}:${persistedMessage}`, + tryMergePersistedNotification(_basePath, { + normalizedSeverity, + source, + persistedMessage, + metadata, now, - ) + }) ) { + _emitChange(); return; } const entry = { - schemaVersion: NOTIFICATION_SCHEMA_VERSION, + schemaVersion: NOTIFICATION_SCHEMA_VERSION_WRITE, id: randomUUID(), - ts: new Date().toISOString(), + ts: new Date(now).toISOString(), severity: normalizedSeverity, message: persistedMessage, source, read: false, + repeatCount: 1, + ...(metadata?.noticeKind ? { noticeKind: metadata.noticeKind } : {}), ...(metadata ? { metadata } : {}), }; try { @@ -280,7 +293,6 @@ export function _resetNotificationStore() { _basePath = null; _lineCount = 0; _suppressCount = 0; - _recentMessageTimestamps = new Map(); _changeListeners.clear(); _appendFailureCount = 0; _lastAppendFailure = null; @@ -308,29 +320,100 @@ function _readEntriesFromDisk(basePath) { } function normalizeNotificationEntry(entry) { if (!entry || typeof entry !== "object" || Array.isArray(entry)) return null; - const schemaVersion = entry.schemaVersion ?? NOTIFICATION_SCHEMA_VERSION; - if (schemaVersion !== NOTIFICATION_SCHEMA_VERSION) return null; + const rawSv = entry.schemaVersion; + const schemaVersion = + rawSv === undefined || rawSv === null + ? NOTIFICATION_SCHEMA_VERSION_MIN + : rawSv; + if ( + schemaVersion < NOTIFICATION_SCHEMA_VERSION_MIN || + schemaVersion > NOTIFICATION_SCHEMA_VERSION_WRITE + ) { + return null; + } return { ...entry, schemaVersion, read: entry.read === true, + repeatCount: entry.repeatCount ?? 1, }; } -function hasRecentPersistedDuplicate(basePath, keySeed, now) { - const normalizedKey = normalizeDedupKey(keySeed); - const entries = _readEntriesFromDisk(basePath); - for (const entry of entries.slice(-DURABLE_DEDUP_SCAN_LIMIT)) { - if (entry.read === true) continue; - const ts = Date.parse(entry.ts); - if (!Number.isFinite(ts) || now - ts > DURABLE_DEDUP_WINDOW_MS) continue; - const entryKey = entry.metadata?.dedupe_key - ? normalizeDedupKey(entry.metadata.dedupe_key) - : normalizeDedupKey( - `${entry.severity}:${entry.source ?? "notify"}:${entry.message}`, - ); - if (entryKey === normalizedKey) return true; +/** + * Whether two emissions with the same normalized key should collapse into one row. + * Distinct consent surfaces (approval/blocker kinds) never merge; blocking:true alone + * still merges when dedupe_key matches so operators see one row with repeatCount. + */ +function shouldMergeDuplicates(metadata) { + if (metadata?.merge === false) return false; + const k = metadata?.kind; + if (k === "blocker" || k === "approval_request" || k === "action_required") { + return false; } - return false; + return true; +} +/** + * Scan recent unread rows for the same logical key; merge repeatCount + lastTs. + * + * Purpose: durable grouping per notification-source-hygiene — fewer duplicate lines, + * preserved counts for operators. + * + * Consumer: appendNotification only. + */ +function tryMergePersistedNotification(basePath, params) { + const { normalizedSeverity, source, persistedMessage, metadata, now } = + params; + if (!shouldMergeDuplicates(metadata)) return false; + + const keySeed = + metadata?.dedupe_key ?? + `${normalizedSeverity}:${source}:${persistedMessage}`; + const normalizedKey = normalizeDedupKey(keySeed); + + let merged = false; + try { + _withLock(basePath, () => { + const entries = _readEntriesFromDisk(basePath); + const scanStart = Math.max(0, entries.length - DURABLE_DEDUP_SCAN_LIMIT); + for (let i = entries.length - 1; i >= scanStart; i--) { + const entry = entries[i]; + if (entry.read === true) continue; + const ts = Date.parse(entry.ts); + if (!Number.isFinite(ts) || now - ts > DURABLE_DEDUP_WINDOW_MS) + continue; + const entryKey = entry.metadata?.dedupe_key + ? normalizeDedupKey(entry.metadata.dedupe_key) + : normalizeDedupKey( + `${entry.severity}:${entry.source ?? "notify"}:${entry.message}`, + ); + if (entryKey !== normalizedKey) continue; + + const nextRepeat = (entry.repeatCount ?? 1) + 1; + const mergedEntry = { + ...entry, + schemaVersion: Math.max( + entry.schemaVersion ?? NOTIFICATION_SCHEMA_VERSION_MIN, + NOTIFICATION_SCHEMA_VERSION_WRITE, + ), + repeatCount: nextRepeat, + lastTs: new Date(now).toISOString(), + }; + if (metadata?.noticeKind && !entry.noticeKind) { + mergedEntry.noticeKind = metadata.noticeKind; + } + entries[i] = mergedEntry; + _atomicWrite( + basePath, + entries.map((e) => JSON.stringify(e)).join("\n") + "\n", + ); + _lineCount = entries.length; + merged = true; + return; + } + }); + } catch { + return false; + } + return merged; } function normalizeDedupKey(value) { return String(value) diff --git a/src/resources/extensions/sf/prompts/complete-slice.md b/src/resources/extensions/sf/prompts/complete-slice.md index 393b5f3be..e4a14cc60 100644 --- a/src/resources/extensions/sf/prompts/complete-slice.md +++ b/src/resources/extensions/sf/prompts/complete-slice.md @@ -40,8 +40,8 @@ Then: - Why this mode is sufficient: ``` The mode determines how the run-uat agent executes checks. For slices verified only by build commands, grep checks, and automated tests you may omit this block — `complete_slice` then injects a default `artifact-driven` section ahead of your body so parsers still classify the artifact. -9. Review task summaries for `key_decisions`. Append any significant decisions to `.sf/DECISIONS.md` if missing. -10. Review task summaries for patterns, gotchas, or non-obvious lessons learned. If any would save future agents from repeating investigation or hitting the same issues, append them to `.sf/KNOWLEDGE.md`. Only add entries that are genuinely useful — don't pad with obvious observations. +9. Review task summaries for `key_decisions`. Call `save_decision` for any significant decisions not yet recorded. +10. Review task summaries for patterns, gotchas, or non-obvious lessons learned. If any would save future agents from repeating investigation or hitting the same issues, call `save_knowledge` for each. Only add entries that are genuinely useful — don't pad with obvious observations. 10b. Scan task summaries and the slice's activity log for sf-internal anomalies that the per-task agents may not have reported individually — repeated `Git stage failed`, `Verification failed … advisory`, `Safety: N unexpected file change(s)`, brittle gate predicates, etc. For any genuine sf-the-tool defect that surfaced during this slice but was NOT already filed via `report_issue`, file it now via `report_issue` with appropriate severity. This is the slice-level sweep — task-level agents file individual reports during execution; the slice-close agent catches systemic issues only visible across multiple tasks. 11. Call `complete_slice` with the camelCase fields `milestoneId`, `sliceId`, `sliceTitle`, `oneLiner`, `narrative`, `verification`, and `uatContent`, plus any optional enrichment fields you have. Do NOT manually mark the roadmap checkbox — the tool writes to the DB, renders `{{sliceSummaryPath}}` and `{{sliceUatPath}}`, and updates the ROADMAP.md projection automatically. 12. Do not run git commands — the system commits your changes and handles any merge after this unit succeeds. diff --git a/src/resources/extensions/sf/prompts/execute-task.md b/src/resources/extensions/sf/prompts/execute-task.md index 69bd3fd7e..6ab233190 100644 --- a/src/resources/extensions/sf/prompts/execute-task.md +++ b/src/resources/extensions/sf/prompts/execute-task.md @@ -85,8 +85,8 @@ Then: 17. **Blocker discovery:** If execution reveals that the remaining slice plan is fundamentally invalid — not just a bug or minor deviation, but a plan-invalidating finding like a wrong API, missing capability, or architectural mismatch — set `blocker_discovered: true` in the task summary frontmatter and describe the blocker clearly in the summary narrative. Do NOT set `blocker_discovered: true` for ordinary debugging, minor deviations, or issues that can be fixed within the current task or the remaining plan. This flag triggers an automatic replan of the slice. 17b. **sf-internal anomalies and observations:** If during execution you observe sf-the-tool misbehaving (empty `git add --` pathspecs, brittle gate predicates, advisory-downgrade hiding real failures, false safety floods), find a prompt ambiguous or contradictory, hit workflow friction, or have an idea that would make sf better — call `report_issue`. Use `prompt-quality-issue`, `improvement-idea`, `agent-friction`, or `design-thought` kinds for non-bug observations alongside the classic bug kinds. Severity guide: `low`/`medium` for cosmetic / noisy / nice-to-have (sf continues); `high`/`critical` only when the sf issue actually prevents the task from sealing correctly (this blocks the unit). For high/critical, include `acceptance_criteria` so a future resolver has a falsifiable bar. This is distinct from `blocker_discovered` (which is about the user's plan, not about sf). Over-reporting is preferred to under-reporting at this stage. 17c. **Self-feedback is a TRIAGE inbox, not a work queue.** Do NOT autonomously pick up entries from `.sf/SELF-FEEDBACK.md` or `~/.sf/agent/upstream-feedback.jsonl` and try to fix them — those are open observations awaiting human/triage-agent review to decide which become scheduled work, duplicates, or wontfix. Your scope is the task plan you were dispatched with. The only interaction your task should have with self-feedback is FILING new entries (via `report_issue`) when you observe sf-internal anomalies. The exception: if a self-feedback entry id is *explicitly named* in your task plan as the work to be done, treat it as you would any other planned item — read its `acceptanceCriteria`, satisfy each, and cite the entry id + criteria met in your task summary's `narrative` so the resolution is traceable. -18. If you made an architectural, pattern, library, or observability decision during this task that downstream work should know about, append it to `.sf/DECISIONS.md` (read the template at `~/.sf/agent/extensions/sf/templates/decisions.md` if the file doesn't exist yet). Not every task produces decisions — only append when a meaningful choice was made. -19. If you discover a non-obvious rule, recurring gotcha, or useful pattern during execution, append it to `.sf/KNOWLEDGE.md`. Only add entries that would save future agents from repeating your investigation. Don't add obvious things. +18. If you made an architectural, pattern, library, or observability decision during this task that downstream work should know about, call `save_decision`. Not every task produces decisions — only record when a meaningful choice was made. +19. If you discover a non-obvious rule, recurring gotcha, or useful pattern during execution, call `save_knowledge`. Only add entries that would save future agents from repeating your investigation. Don't add obvious things. 20. Read the template at `~/.sf/agent/extensions/sf/templates/task-summary.md` 21. Use that template to prepare the completion content you will pass to `complete_task` using the camelCase fields `milestoneId`, `sliceId`, `taskId`, `oneLiner`, `narrative`, `verification`, and `verificationEvidence`. Do **not** manually write `{{taskSummaryPath}}` — the DB-backed tool is the canonical write path and renders the summary file for you. 22. Call `complete_task` with milestoneId, sliceId, taskId, and the completion fields derived from the template. This is your final required step — do NOT manually edit PLAN.md checkboxes. The tool marks the task complete, updates the DB, renders `{{taskSummaryPath}}`, and updates PLAN.md automatically. diff --git a/src/resources/extensions/sf/prompts/guided-plan-milestone.md b/src/resources/extensions/sf/prompts/guided-plan-milestone.md index 2f6d2a0ed..4e2751e2e 100644 --- a/src/resources/extensions/sf/prompts/guided-plan-milestone.md +++ b/src/resources/extensions/sf/prompts/guided-plan-milestone.md @@ -1,4 +1,4 @@ -Plan milestone {{milestoneId}} ("{{milestoneTitle}}"). Read `.sf/DECISIONS.md` if it exists — respect existing decisions. Read `.sf/REQUIREMENTS.md` if it exists and treat Active requirements as the capability contract. If `REQUIREMENTS.md` is missing, treat that as a planning gap: derive the minimum requirement coverage from current project evidence, persist it through SF planning tools, and explicitly note missing coverage. Use the **Roadmap** output template below to shape the milestone planning payload you send to `plan_milestone`. Start the `vision` field with the milestone purpose before implementation detail, include the structured `productResearch` payload when the work is product-facing, workflow-facing, developer-experience, or market-positioning, and make each slice `goal` state the slice purpose before mechanics. If the milestone changes how SF is driven, observed, integrated, or automated, keep the axes separate in the roadmap: surface (TUI/CLI/web/editor/machine), protocol (ACP/RPC/stdio/HTTP/wire), output format (text/json/stream-json), run control (manual/assisted/autonomous), and permission profile (restricted/normal/trusted/unrestricted). Call `plan_milestone` to persist the milestone planning fields and render `{{milestoneId}}-ROADMAP.md` from DB state. Do **not** write `{{milestoneId}}-ROADMAP.md`, `ROADMAP.md`, or other planning artifacts manually. If planning produces structural decisions, append them to `.sf/DECISIONS.md`. {{skillActivation}} Fill the Horizontal Checklist section with cross-cutting concerns considered during planning (requirements re-read, decisions re-evaluated, graceful shutdown, revenue paths, auth boundary, shared resources, reconnection). Omit for trivial milestones. +Plan milestone {{milestoneId}} ("{{milestoneTitle}}"). Decisions and requirements are injected from the database into your system context — respect existing decisions and treat Active requirements as the capability contract. If no requirements are present, treat that as a planning gap: derive the minimum requirement coverage from current project evidence, persist it through `save_requirement`, and explicitly note missing coverage. Use the **Roadmap** output template below to shape the milestone planning payload you send to `plan_milestone`. Start the `vision` field with the milestone purpose before implementation detail, include the structured `productResearch` payload when the work is product-facing, workflow-facing, developer-experience, or market-positioning, and make each slice `goal` state the slice purpose before mechanics. If the milestone changes how SF is driven, observed, integrated, or automated, keep the axes separate in the roadmap: surface (TUI/CLI/web/editor/machine), protocol (ACP/RPC/stdio/HTTP/wire), output format (text/json/stream-json), run control (manual/assisted/autonomous), and permission profile (restricted/normal/trusted/unrestricted). Call `plan_milestone` to persist the milestone planning fields and render `{{milestoneId}}-ROADMAP.md` from DB state. Do **not** write `{{milestoneId}}-ROADMAP.md`, `ROADMAP.md`, or other planning artifacts manually. If planning produces structural decisions, call `save_decision`. {{skillActivation}} Fill the Horizontal Checklist section with cross-cutting concerns considered during planning (requirements re-read, decisions re-evaluated, graceful shutdown, revenue paths, auth boundary, shared resources, reconnection). Omit for trivial milestones. Before calling `plan_milestone`, run a bounded **Vision Alignment Meeting** for the milestone and roadmap as a real multi-agent review. Use the `subagent` tool in `mode: "debate"` with `rounds: 2` and a separate task for each participant lens below. Do **not** merely simulate every participant inside this planner response. Use only supported agent names: `planner`, `reviewer`, `researcher`, and `scout`. Put the stakeholder role name inside the task text; do not invent agent names such as `combatant`, `delivery-lead`, `product-manager`, or `customer-panel`. If the `subagent` tool is unavailable or fails after one retry, record that explicitly in `trigger` and run the structured meeting inline as a degraded fallback. This is allowed to be broader and more nuanced than slice planning. Include at least these participant lenses: - Product Manager diff --git a/src/resources/extensions/sf/record-promoter.js b/src/resources/extensions/sf/record-promoter.js index 1870ad211..e89c98c6a 100644 --- a/src/resources/extensions/sf/record-promoter.js +++ b/src/resources/extensions/sf/record-promoter.js @@ -15,6 +15,7 @@ import { } from "node:fs"; import { basename, join } from "node:path"; import { logWarning } from "./workflow-logger.js"; +import { addBacklogItem, isDbAvailable } from "./sf-db.js"; // ─── Frontmatter Parser ────────────────────────────────────────────────────── /** * Parse the YAML frontmatter from a markdown file. @@ -264,8 +265,14 @@ export function promoteActionableRecords(basePath) { const sliceDir = join(milestoneDir, "slices", slice.id); mkdirSync(sliceDir, { recursive: true }); } - // Append to QUEUE.md - appendToQueue(sfRootPath, milestoneId); + // Write to DB backlog (primary) + if (isDbAvailable()) { + try { + addBacklogItem({ id: milestoneId, title: milestoneId, source: "record-promoter", status: "promoted" }); + } catch { + // non-fatal + } + } // Stamp the record promoted stampRecordPromoted(recordPath, milestoneId); result.promoted.push({ recordPath, milestoneId }); diff --git a/src/resources/extensions/sf/requirement-promoter.js b/src/resources/extensions/sf/requirement-promoter.js index 391896971..9ec260b6c 100644 --- a/src/resources/extensions/sf/requirement-promoter.js +++ b/src/resources/extensions/sf/requirement-promoter.js @@ -15,6 +15,7 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { sfRoot } from "./paths.js"; import { markResolved, readAllSelfFeedback } from "./self-feedback.js"; +import { isDbAvailable, upsertRequirement } from "./sf-db.js"; // ─── Constants ─────────────────────────────────────────────────────────────── const COUNT_THRESHOLD = 5; @@ -146,7 +147,20 @@ export function promoteFeedbackToRequirements(basePath = process.cwd()) { const title = `Address recurring ${cluster.kind} (${count} entries across ${milestoneCount} milestone${milestoneCount !== 1 ? "s" : ""})`; const sourceIds = cluster.entries.map((e) => e.id).join(", "); const notes = `Source IDs: ${sourceIds}`; - appendRequirementRow(reqPath, reqId, title, notes); + if (isDbAvailable()) { + try { + upsertRequirement({ + id: reqId, + title, + description: notes, + status: "active", + class: "operational", + source: "sf-promoter", + }); + } catch { + // non-fatal + } + } promotedIds.push(reqId); // Mark each contributing entry resolved for (const entry of cluster.entries) { diff --git a/src/resources/extensions/sf/sf-db.js b/src/resources/extensions/sf/sf-db.js index 96c02541f..6bf906412 100644 --- a/src/resources/extensions/sf/sf-db.js +++ b/src/resources/extensions/sf/sf-db.js @@ -40,6 +40,7 @@ import { withTaskFrontmatter, } from "./task-frontmatter.js"; import { logError, logWarning } from "./workflow-logger.js"; +import { readTraceEvents } from "./uok/trace-writer.js"; let loadAttempted = false; function loadProvider() { @@ -244,7 +245,7 @@ function performDatabaseMaintenance(rawDb, path) { ); } } -const SCHEMA_VERSION = 57; +const SCHEMA_VERSION = 58; function indexExists(db, name) { return !!db .prepare( @@ -1272,30 +1273,6 @@ function initSchema(db, fileBacked) { FOREIGN KEY (milestone_id, slice_id) REFERENCES slices(milestone_id, id), FOREIGN KEY (milestone_id, depends_on_slice_id) REFERENCES slices(milestone_id, id) ) - `); - db.exec(` - CREATE TABLE IF NOT EXISTS gate_runs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - trace_id TEXT NOT NULL, - turn_id TEXT NOT NULL, - gate_id TEXT NOT NULL, - gate_type TEXT NOT NULL DEFAULT '', - unit_type TEXT DEFAULT NULL, - unit_id TEXT DEFAULT NULL, - milestone_id TEXT DEFAULT NULL, - slice_id TEXT DEFAULT NULL, - task_id TEXT DEFAULT NULL, - outcome TEXT NOT NULL DEFAULT 'pass', - failure_class TEXT NOT NULL DEFAULT 'none', - rationale TEXT NOT NULL DEFAULT '', - findings TEXT NOT NULL DEFAULT '', - attempt INTEGER NOT NULL DEFAULT 1, - max_attempts INTEGER NOT NULL DEFAULT 1, - retryable INTEGER NOT NULL DEFAULT 0, - evaluated_at TEXT NOT NULL DEFAULT '', - duration_ms INTEGER DEFAULT NULL, - cost_micro_usd INTEGER DEFAULT NULL - ) `); db.exec(` CREATE TABLE IF NOT EXISTS gate_circuit_breakers ( @@ -1307,34 +1284,6 @@ function initSchema(db, fileBacked) { half_open_attempts INTEGER NOT NULL DEFAULT 0, updated_at TEXT NOT NULL DEFAULT '' ) - `); - db.exec(` - CREATE TABLE IF NOT EXISTS turn_git_transactions ( - trace_id TEXT NOT NULL, - turn_id TEXT NOT NULL, - unit_type TEXT DEFAULT NULL, - unit_id TEXT DEFAULT NULL, - stage TEXT NOT NULL DEFAULT 'turn-start', - action TEXT NOT NULL DEFAULT 'status-only', - push INTEGER NOT NULL DEFAULT 0, - status TEXT NOT NULL DEFAULT 'ok', - error TEXT DEFAULT NULL, - metadata_json TEXT NOT NULL DEFAULT '{}', - updated_at TEXT NOT NULL DEFAULT '', - PRIMARY KEY (trace_id, turn_id, stage) - ) - `); - db.exec(` - CREATE TABLE IF NOT EXISTS audit_events ( - event_id TEXT PRIMARY KEY, - trace_id TEXT NOT NULL, - turn_id TEXT DEFAULT NULL, - caused_by TEXT DEFAULT NULL, - category TEXT NOT NULL, - type TEXT NOT NULL, - ts TEXT NOT NULL, - payload_json TEXT NOT NULL DEFAULT '{}' - ) `); db.exec(` CREATE TABLE IF NOT EXISTS audit_turn_index ( @@ -1406,21 +1355,6 @@ function initSchema(db, fileBacked) { db.exec( "CREATE INDEX IF NOT EXISTS idx_slice_deps_target ON slice_dependencies(milestone_id, depends_on_slice_id)", ); - db.exec( - "CREATE INDEX IF NOT EXISTS idx_gate_runs_turn ON gate_runs(trace_id, turn_id)", - ); - db.exec( - "CREATE INDEX IF NOT EXISTS idx_gate_runs_lookup ON gate_runs(milestone_id, slice_id, task_id, gate_id)", - ); - db.exec( - "CREATE INDEX IF NOT EXISTS idx_turn_git_tx_turn ON turn_git_transactions(trace_id, turn_id)", - ); - db.exec( - "CREATE INDEX IF NOT EXISTS idx_audit_events_trace ON audit_events(trace_id, ts)", - ); - db.exec( - "CREATE INDEX IF NOT EXISTS idx_audit_events_turn ON audit_events(trace_id, turn_id, ts)", - ); db.exec( "CREATE UNIQUE INDEX IF NOT EXISTS idx_llm_task_outcomes_identity ON llm_task_outcomes(unit_type, unit_id, recorded_at)", ); @@ -1483,9 +1417,6 @@ function initSchema(db, fileBacked) { AND t.slice_id = ts.slice_id AND t.id = ts.task_id `); - db.exec( - `CREATE INDEX IF NOT EXISTS idx_audit_events_category ON audit_events(category, type, ts DESC)`, - ); const existing = db .prepare("SELECT count(*) as cnt FROM schema_version") .get(); @@ -1508,6 +1439,14 @@ function columnExists(db, table, column) { const rows = db.prepare(`PRAGMA table_info(${table})`).all(); return rows.some((row) => row["name"] === column); } +function tableExists(db, table) { + const row = db + .prepare( + `SELECT name FROM sqlite_master WHERE type='table' AND name=?`, + ) + .get(table); + return row != null; +} function ensureColumn(db, table, column, ddl) { if (!columnExists(db, table, column)) db.exec(ddl); } @@ -1721,6 +1660,9 @@ function migrateCostUsdToMicroUsd(db) { // Purpose: Enable accurate cost tracking at scale without rounding errors // Consumer: gate_runs cost tracking, cost analytics, budget checks + // Guard: gate_runs may not exist in minimal legacy DBs (it will be dropped in v58) + if (!tableExists(db, "gate_runs")) return; + // Add cost_micro_usd column if it doesn't exist if (!columnExists(db, "gate_runs", "cost_micro_usd")) { db.exec( @@ -1732,7 +1674,7 @@ function migrateCostUsdToMicroUsd(db) { // NULL values stay NULL; non-NULL values are multiplied by 1,000,000 if (columnExists(db, "gate_runs", "cost_usd")) { db.prepare(` - UPDATE gate_runs + UPDATE gate_runs SET cost_micro_usd = CAST(ROUND(cost_usd * 1000000) AS INTEGER) WHERE cost_usd IS NOT NULL AND cost_micro_usd IS NULL @@ -2682,12 +2624,15 @@ function migrateSchema(db) { } if (currentVersion < 28) { // UOK observability: gate execution latency - ensureColumn( - db, - "gate_runs", - "duration_ms", - "ALTER TABLE gate_runs ADD COLUMN duration_ms INTEGER DEFAULT NULL", - ); + // Guard: gate_runs table may not exist in minimal legacy DBs (it will be dropped in v58) + if (tableExists(db, "gate_runs")) { + ensureColumn( + db, + "gate_runs", + "duration_ms", + "ALTER TABLE gate_runs ADD COLUMN duration_ms INTEGER DEFAULT NULL", + ); + } // UOK circuit breaker state db.exec(` CREATE TABLE IF NOT EXISTS gate_circuit_breakers ( @@ -3203,9 +3148,12 @@ function migrateSchema(db) { } if (currentVersion < 55) { // Schema v55: composite index for audit_events + task access-pattern views - db.exec( - `CREATE INDEX IF NOT EXISTS idx_audit_events_category ON audit_events(category, type, ts DESC)`, - ); + // Guard: audit_events may not exist in minimal legacy DBs (it will be dropped in v58) + if (tableExists(db, "audit_events")) { + db.exec( + `CREATE INDEX IF NOT EXISTS idx_audit_events_category ON audit_events(category, type, ts DESC)`, + ); + } db.exec( `CREATE VIEW IF NOT EXISTS active_tasks AS SELECT * FROM tasks WHERE status NOT IN ('done','complete','completed','cancelled')`, ); @@ -3250,6 +3198,18 @@ function migrateSchema(db) { ":applied_at": new Date().toISOString(), }); } + if (currentVersion < 58) { + // Schema v58: move trace data to JSONL files — drop gate_runs, turn_git_transactions, audit_events + db.exec("DROP TABLE IF EXISTS gate_runs"); + db.exec("DROP TABLE IF EXISTS turn_git_transactions"); + db.exec("DROP TABLE IF EXISTS audit_events"); + db.prepare( + "INSERT INTO schema_version (version, applied_at) VALUES (:version, :applied_at)", + ).run({ + ":version": 58, + ":applied_at": new Date().toISOString(), + }); + } db.exec("COMMIT"); } catch (err) { db.exec("ROLLBACK"); @@ -5664,9 +5624,6 @@ export function deleteMilestone(milestoneId) { currentDb .prepare(`DELETE FROM quality_gates WHERE milestone_id = :mid`) .run({ ":mid": milestoneId }); - currentDb - .prepare(`DELETE FROM gate_runs WHERE milestone_id = :mid`) - .run({ ":mid": milestoneId }); currentDb .prepare(`DELETE FROM tasks WHERE milestone_id = :mid`) .run({ ":mid": milestoneId }); @@ -5902,59 +5859,13 @@ export function getPendingGatesForTurn(milestoneId, sliceId, turn, taskId) { export function getPendingGateCountForTurn(milestoneId, sliceId, turn) { return getPendingGatesForTurn(milestoneId, sliceId, turn).length; } -export function insertGateRun(entry) { - if (!currentDb) return; - currentDb - .prepare(`INSERT INTO gate_runs ( - trace_id, turn_id, gate_id, gate_type, unit_type, unit_id, milestone_id, slice_id, task_id, - outcome, failure_class, rationale, findings, attempt, max_attempts, retryable, evaluated_at, duration_ms, cost_micro_usd - ) VALUES ( - :trace_id, :turn_id, :gate_id, :gate_type, :unit_type, :unit_id, :milestone_id, :slice_id, :task_id, - :outcome, :failure_class, :rationale, :findings, :attempt, :max_attempts, :retryable, :evaluated_at, :duration_ms, :cost_micro_usd - )`) - .run({ - ":trace_id": entry.traceId, - ":turn_id": entry.turnId, - ":gate_id": entry.gateId, - ":gate_type": entry.gateType, - ":unit_type": entry.unitType ?? null, - ":unit_id": entry.unitId ?? null, - ":milestone_id": entry.milestoneId ?? null, - ":slice_id": entry.sliceId ?? null, - ":task_id": entry.taskId ?? null, - ":outcome": entry.outcome, - ":failure_class": entry.failureClass, - ":rationale": entry.rationale ?? "", - ":findings": entry.findings ?? "", - ":attempt": entry.attempt, - ":max_attempts": entry.maxAttempts, - ":retryable": entry.retryable ? 1 : 0, - ":evaluated_at": entry.evaluatedAt, - ":duration_ms": entry.durationMs ?? null, - ":cost_micro_usd": entry.costMicroUsd ?? null, - }); +/** @deprecated Gate runs are now written to JSONL trace files via appendTraceEvent(). This is a no-op kept for import compatibility. */ +export function insertGateRun(_entry) { + // no-op: gate runs now written to JSONL trace files } -export function upsertTurnGitTransaction(entry) { - if (!currentDb) return; - currentDb - .prepare(`INSERT OR REPLACE INTO turn_git_transactions ( - trace_id, turn_id, unit_type, unit_id, stage, action, push, status, error, metadata_json, updated_at - ) VALUES ( - :trace_id, :turn_id, :unit_type, :unit_id, :stage, :action, :push, :status, :error, :metadata_json, :updated_at - )`) - .run({ - ":trace_id": entry.traceId, - ":turn_id": entry.turnId, - ":unit_type": entry.unitType ?? null, - ":unit_id": entry.unitId ?? null, - ":stage": entry.stage, - ":action": entry.action, - ":push": entry.push ? 1 : 0, - ":status": entry.status, - ":error": entry.error ?? null, - ":metadata_json": JSON.stringify(entry.metadata ?? {}), - ":updated_at": entry.updatedAt, - }); +/** @deprecated Turn git transactions are now written to JSONL audit events. This is a no-op kept for import compatibility. */ +export function upsertTurnGitTransaction(_entry) { + // no-op: turn git transactions now written to JSONL audit events } export function recordUokRunStart(entry) { if (!currentDb) return; @@ -6040,60 +5951,9 @@ export function getUokRuns(limit = 500) { updatedAt: row.updated_at, })); } -export function insertAuditEvent(entry) { - if (!currentDb) return; - transaction(() => { - currentDb - .prepare(`INSERT OR IGNORE INTO audit_events ( - event_id, trace_id, turn_id, caused_by, category, type, ts, payload_json - ) VALUES ( - :event_id, :trace_id, :turn_id, :caused_by, :category, :type, :ts, :payload_json - )`) - .run({ - ":event_id": entry.eventId, - ":trace_id": entry.traceId, - ":turn_id": entry.turnId ?? null, - ":caused_by": entry.causedBy ?? null, - ":category": entry.category, - ":type": entry.type, - ":ts": entry.ts, - ":payload_json": JSON.stringify(entry.payload ?? {}), - }); - if (entry.turnId) { - const row = currentDb - .prepare(`SELECT event_count, first_ts, last_ts - FROM audit_turn_index - WHERE trace_id = :trace_id AND turn_id = :turn_id`) - .get({ - ":trace_id": entry.traceId, - ":turn_id": entry.turnId, - }); - if (row) { - currentDb - .prepare(`UPDATE audit_turn_index - SET first_ts = CASE WHEN :ts < first_ts THEN :ts ELSE first_ts END, - last_ts = CASE WHEN :ts > last_ts THEN :ts ELSE last_ts END, - event_count = event_count + 1 - WHERE trace_id = :trace_id AND turn_id = :turn_id`) - .run({ - ":trace_id": entry.traceId, - ":turn_id": entry.turnId, - ":ts": entry.ts, - }); - } else { - currentDb - .prepare(`INSERT INTO audit_turn_index (trace_id, turn_id, first_ts, last_ts, event_count) - VALUES (:trace_id, :turn_id, :first_ts, :last_ts, :event_count)`) - .run({ - ":trace_id": entry.traceId, - ":turn_id": entry.turnId, - ":first_ts": entry.ts, - ":last_ts": entry.ts, - ":event_count": 1, - }); - } - } - }); +/** @deprecated Audit events are now written exclusively to JSONL files via emitUokAuditEvent(). This is a no-op kept for import compatibility. */ +export function insertAuditEvent(_entry) { + // no-op: audit events now written exclusively to JSONL files } // ─── Single-writer bypass wrappers ─────────────────────────────────────── // These wrappers exist so modules outside this file never need to call @@ -6443,52 +6303,32 @@ export function getLlmTaskOutcomeStats(modelId, windowHours = 24) { * Consumer: uok/diagnostic-synthesis.js, uok/gate-runner.js health checks. */ export function getGateRunStats(gateId, windowHours = 24) { - if (!currentDb) { - return { - total: 0, + try { + const basePath = currentPath && currentPath !== ":memory:" + ? dirname(dirname(currentPath)) + : process.cwd(); + const events = readTraceEvents(basePath, "gate_run", windowHours) + .filter((e) => e.gateId === gateId); + const stats = { + total: events.length, pass: 0, fail: 0, retry: 0, manualAttention: 0, lastEvaluatedAt: null, }; - } - const cutoff = new Date( - Date.now() - windowHours * 60 * 60 * 1000, - ).toISOString(); - try { - const row = currentDb - .prepare( - `SELECT - COUNT(*) AS total, - COALESCE(SUM(CASE WHEN outcome = 'pass' THEN 1 ELSE 0 END), 0) AS pass, - COALESCE(SUM(CASE WHEN outcome = 'fail' THEN 1 ELSE 0 END), 0) AS fail, - COALESCE(SUM(CASE WHEN outcome = 'retry' THEN 1 ELSE 0 END), 0) AS retry, - COALESCE(SUM(CASE WHEN outcome = 'manual-attention' THEN 1 ELSE 0 END), 0) AS manualAttention, - MAX(evaluated_at) AS lastEvaluatedAt - FROM gate_runs - WHERE gate_id = :gate_id - AND evaluated_at >= :cutoff`, + for (const e of events) { + if (e.outcome === "pass") stats.pass++; + else if (e.outcome === "fail") stats.fail++; + else if (e.outcome === "retry") stats.retry++; + else if (e.outcome === "manual-attention") stats.manualAttention++; + if ( + !stats.lastEvaluatedAt || + (e.evaluatedAt ?? e.ts) > stats.lastEvaluatedAt ) - .get({ ":gate_id": gateId, ":cutoff": cutoff }); - if (!row) { - return { - total: 0, - pass: 0, - fail: 0, - retry: 0, - manualAttention: 0, - lastEvaluatedAt: null, - }; + stats.lastEvaluatedAt = e.evaluatedAt ?? e.ts; } - return { - total: row.total ?? 0, - pass: row.pass ?? 0, - fail: row.fail ?? 0, - retry: row.retry ?? 0, - manualAttention: row.manualAttention ?? 0, - lastEvaluatedAt: row.lastEvaluatedAt ?? null, - }; + return stats; } catch { return { total: 0, @@ -6595,55 +6435,40 @@ export function updateGateCircuitBreaker(gateId, updates) { return { total: 0, avgMs: 0, p50Ms: 0, p95Ms: 0, maxMs: 0 }; } export function getGateLatencyStats(gateId, windowHours = 24) { - if (!currentDb) { - return { total: 0, avgMs: 0, p50Ms: 0, p95Ms: 0, maxMs: 0 }; - } - const cutoff = new Date( - Date.now() - windowHours * 60 * 60 * 1000, - ).toISOString(); try { - const row = currentDb - .prepare( - `SELECT - COUNT(*) AS total, - COALESCE(AVG(duration_ms), 0) AS avgMs, - COALESCE(MAX(duration_ms), 0) AS maxMs - FROM gate_runs - WHERE gate_id = :gate_id AND evaluated_at >= :cutoff`, - ) - .get({ ":gate_id": gateId, ":cutoff": cutoff }); - if (!row || row.total === 0) { - return { total: 0, avgMs: 0, p50Ms: 0, p95Ms: 0, maxMs: 0 }; - } - const durations = currentDb - .prepare( - `SELECT duration_ms - FROM gate_runs - WHERE gate_id = :gate_id AND evaluated_at >= :cutoff AND duration_ms IS NOT NULL - ORDER BY duration_ms`, - ) - .all({ ":gate_id": gateId, ":cutoff": cutoff }) - .map((r) => r.duration_ms); + const basePath = currentPath && currentPath !== ":memory:" + ? dirname(dirname(currentPath)) + : process.cwd(); + const durations = readTraceEvents(basePath, "gate_run", windowHours) + .filter((e) => e.gateId === gateId && typeof e.durationMs === "number") + .map((e) => e.durationMs) + .sort((a, b) => a - b); + if (durations.length === 0) return { p50: null, p95: null, count: 0, total: 0, avgMs: 0, p50Ms: 0, p95Ms: 0, maxMs: 0 }; const p50Ms = durations[Math.floor(durations.length * 0.5)] ?? 0; const p95Ms = durations[Math.floor(durations.length * 0.95)] ?? 0; + const maxMs = durations[durations.length - 1] ?? 0; + const avgMs = Math.round(durations.reduce((s, v) => s + v, 0) / durations.length); return { - total: row.total ?? 0, - avgMs: Math.round(row.avgMs ?? 0), + p50: p50Ms, + p95: p95Ms, + count: durations.length, + total: durations.length, + avgMs, p50Ms, p95Ms, - maxMs: row.maxMs ?? 0, + maxMs, }; } catch { - return { total: 0, avgMs: 0, p50Ms: 0, p95Ms: 0, maxMs: 0 }; + return { p50: null, p95: null, count: 0, total: 0, avgMs: 0, p50Ms: 0, p95Ms: 0, maxMs: 0 }; } } export function getDistinctGateIds() { - if (!currentDb) return []; try { - const rows = currentDb - .prepare("SELECT DISTINCT gate_id FROM gate_runs") - .all(); - return rows.map((r) => r.gate_id).filter(Boolean); + const basePath = currentPath && currentPath !== ":memory:" + ? dirname(dirname(currentPath)) + : process.cwd(); + const events = readTraceEvents(basePath, "gate_run", 24 * 30); // 30 days + return [...new Set(events.map((e) => e.gateId).filter(Boolean))]; } catch { return []; } @@ -7868,6 +7693,22 @@ export function bulkInsertLegacyHierarchy(payload) { // All memory writes go through sf-db.ts so the single-writer invariant // holds. These are direct pass-throughs to the SQL previously in // memory-store.ts — same bindings, same behavior. +export function getActiveMemories({ category, limit = 200 } = {}) { + if (!currentDb) return []; + const rows = category + ? currentDb.prepare("SELECT * FROM active_memories WHERE category = ? ORDER BY updated_at DESC LIMIT ?").all(category, limit) + : currentDb.prepare("SELECT * FROM active_memories ORDER BY updated_at DESC LIMIT ?").all(limit); + return rows.map((r) => ({ + id: r["id"], + category: r["category"], + content: r["content"], + confidence: r["confidence"], + sourceUnitId: r["source_unit_id"], + tags: (() => { try { return JSON.parse(r["tags"] ?? "[]"); } catch { return []; } })(), + createdAt: r["created_at"], + updatedAt: r["updated_at"], + })); +} export function insertMemoryRow(args) { if (!currentDb) throw new SFError(SF_STALE_STATE, "sf-db: No database open"); currentDb diff --git a/src/resources/extensions/sf/tests/notification-detection-headless-medium-low.test.mjs b/src/resources/extensions/sf/tests/notification-detection-headless-medium-low.test.mjs index 11334fa35..0d0135ee4 100644 --- a/src/resources/extensions/sf/tests/notification-detection-headless-medium-low.test.mjs +++ b/src/resources/extensions/sf/tests/notification-detection-headless-medium-low.test.mjs @@ -63,7 +63,7 @@ describe("S08 MEDIUM: notification + detection + headless", () => { ); const lines = content.trim().split("\n").filter(Boolean); expect(lines.length).toBe(1); - expect(JSON.parse(lines[0]).schemaVersion).toBe(1); + expect(JSON.parse(lines[0]).schemaVersion).toBe(2); }); it("should treat legacy notifications without schemaVersion as version 1", () => { diff --git a/src/resources/extensions/sf/tests/sf-db-migration.test.mjs b/src/resources/extensions/sf/tests/sf-db-migration.test.mjs index 4d2a46533..716e47f59 100644 --- a/src/resources/extensions/sf/tests/sf-db-migration.test.mjs +++ b/src/resources/extensions/sf/tests/sf-db-migration.test.mjs @@ -22,7 +22,6 @@ import { getJudgmentsForUnit, getRetrievalEvidence, getScheduleEntries, - insertGateRun, insertJudgment, insertMilestone, insertRetrievalEvidence, @@ -223,7 +222,7 @@ test("openDatabase_migrates_v27_tasks_without_created_at_through_spec_backfill", const version = db .prepare("SELECT MAX(version) AS version FROM schema_version") .get(); - assert.equal(version.version, 57); + assert.equal(version.version, 58); const taskSpec = db .prepare( "SELECT milestone_id, slice_id, task_id, verify FROM task_specs WHERE task_id = 'T01'", @@ -281,29 +280,14 @@ test("openDatabase_when_file_backed_creates_db_snapshot_and_maintenance_marker", assert.equal(existsSync(join(backupDir, "maintenance.json")), true); }); -test("openDatabase_when_fresh_db_supports_gate_run_micro_usd", () => { +test("openDatabase_when_fresh_db_does_not_create_gate_runs_table", () => { assert.equal(openDatabase(":memory:"), true); - insertGateRun({ - traceId: "trace-1", - turnId: "turn-1", - gateId: "cost-gate", - gateType: "policy", - outcome: "pass", - failureClass: "none", - rationale: "ok", - attempt: 1, - maxAttempts: 1, - retryable: false, - evaluatedAt: "2026-05-07T00:00:00.000Z", - durationMs: 12, - costMicroUsd: 123_456, - }); - - const row = getDatabase() - .prepare("SELECT cost_micro_usd FROM gate_runs WHERE gate_id = 'cost-gate'") + // After v58 migration, gate_runs table no longer exists + const tableInfo = getDatabase() + .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='gate_runs'") .get(); - assert.equal(row.cost_micro_usd, 123_456); + assert.equal(tableInfo, undefined, "gate_runs table should not exist after v58 migration"); }); test("reconcileWorktreeDb_when_worktree_lacks_product_research_column_merges_milestones", () => { @@ -344,20 +328,16 @@ test("reconcileWorktreeDb_when_worktree_lacks_product_research_column_merges_mil }); }); -test("openDatabase_migrates_v35_gate_cost_usd_to_micro_usd", () => { +test("openDatabase_migrates_v35_gate_cost_usd_drops_table_in_v58", () => { const dbPath = makeLegacyV35GateRunsDb(); assert.equal(openDatabase(dbPath), true); const db = getDatabase(); - const columns = db.prepare("PRAGMA table_info(gate_runs)").all(); - assert.ok(columns.some((row) => row.name === "cost_micro_usd")); - const row = db - .prepare( - "SELECT cost_usd, cost_micro_usd FROM gate_runs WHERE gate_id = 'cost-gate'", - ) + // After v58 migration, gate_runs is dropped + const tableInfo = db + .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='gate_runs'") .get(); - assert.equal(row.cost_usd, 0.123456); - assert.equal(row.cost_micro_usd, 123_456); + assert.equal(tableInfo, undefined, "gate_runs should be dropped by v58 migration"); }); test("openDatabase_memories_table_has_tags_column", () => { diff --git a/src/resources/extensions/sf/tests/uok-gate-runner.test.mjs b/src/resources/extensions/sf/tests/uok-gate-runner.test.mjs index 8321c56c2..5ed09f117 100644 --- a/src/resources/extensions/sf/tests/uok-gate-runner.test.mjs +++ b/src/resources/extensions/sf/tests/uok-gate-runner.test.mjs @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { mkdtempSync, rmSync } from "node:fs"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, test } from "vitest"; @@ -37,13 +37,15 @@ afterEach(() => { function makeProject() { const root = mkdtempSync(join(tmpdir(), "sf-uok-runner-")); + mkdirSync(join(root, ".sf"), { recursive: true }); tmpRoots.push(root); return root; } function makeCtx(overrides = {}) { + const basePath = makeProject(); return { - basePath: makeProject(), + basePath, traceId: "trace-1", turnId: "turn-1", unitType: "execute-task", @@ -81,10 +83,11 @@ test("run_when_gate_not_registered_returns_manual_attention", async () => { assert.equal(result.retryable, false); }); -test("run_when_gate_not_registered_persists_to_db", async () => { - openDatabase(":memory:"); +test("run_when_gate_not_registered_persists_to_trace", async () => { + const ctx = makeCtx(); + openDatabase(join(ctx.basePath, ".sf", "sf.db")); const runner = new UokGateRunner(); - await runner.run("missing-gate", makeCtx()); + await runner.run("missing-gate", ctx); const stats = getGateRunStats("missing-gate", 24); assert.equal(stats.total, 1); assert.equal(stats.manualAttention, 1); @@ -107,15 +110,16 @@ test("run_when_gate_passes_returns_pass", async () => { assert.equal(result.retryable, false); }); -test("run_when_gate_passes_persists_pass_to_db", async () => { - openDatabase(":memory:"); +test("run_when_gate_passes_persists_pass_to_trace", async () => { + const ctx = makeCtx(); + openDatabase(join(ctx.basePath, ".sf", "sf.db")); const runner = new UokGateRunner(); runner.register({ id: "pass-gate", type: "verification", execute: async () => ({ outcome: "pass", rationale: "ok" }), }); - await runner.run("pass-gate", makeCtx()); + await runner.run("pass-gate", ctx); const stats = getGateRunStats("pass-gate", 24); assert.equal(stats.total, 1); assert.equal(stats.pass, 1); @@ -229,10 +233,11 @@ test("run_when_gate_throws_returns_fail_with_unknown_class", async () => { assert.ok(result.rationale.includes("boom")); }); -// ─── DB audit trail ──────────────────────────────────────────────────────── +// ─── Trace audit trail ───────────────────────────────────────────────────── -test("run_records_every_attempt_to_gate_runs", async () => { - openDatabase(":memory:"); +test("run_records_every_attempt_to_trace", async () => { + const ctx = makeCtx(); + openDatabase(join(ctx.basePath, ".sf", "sf.db")); const runner = new UokGateRunner(); let calls = 0; runner.register({ @@ -247,7 +252,7 @@ test("run_records_every_attempt_to_gate_runs", async () => { }; }, }); - await runner.run("audit-gate", makeCtx()); + await runner.run("audit-gate", ctx); const stats = getGateRunStats("audit-gate", 24); assert.equal(stats.total, 2); assert.equal(stats.fail, 1); @@ -273,8 +278,9 @@ test("run_result_includes_gate_id_type_and_timestamps", async () => { // ─── Latency tracking ────────────────────────────────────────────────────── -test("run_records_duration_ms_in_gate_runs", async () => { - openDatabase(":memory:"); +test("run_records_duration_ms_in_trace", async () => { + const ctx = makeCtx(); + openDatabase(join(ctx.basePath, ".sf", "sf.db")); const runner = new UokGateRunner(); runner.register({ id: "slow-gate", @@ -284,7 +290,7 @@ test("run_records_duration_ms_in_gate_runs", async () => { return { outcome: "pass", rationale: "ok" }; }, }); - await runner.run("slow-gate", makeCtx()); + await runner.run("slow-gate", ctx); const stats = getGateLatencyStats("slow-gate", 24); assert.equal(stats.total, 1); assert.ok(stats.avgMs >= 40, `expected avgMs >= 40, got ${stats.avgMs}`); diff --git a/src/resources/extensions/sf/tests/uok-metrics-exposition.test.mjs b/src/resources/extensions/sf/tests/uok-metrics-exposition.test.mjs index 021c71fed..fdaefcd1d 100644 --- a/src/resources/extensions/sf/tests/uok-metrics-exposition.test.mjs +++ b/src/resources/extensions/sf/tests/uok-metrics-exposition.test.mjs @@ -6,13 +6,12 @@ * data after known DB writes. */ import assert from "node:assert/strict"; -import { mkdtempSync, rmSync } from "node:fs"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, test } from "vitest"; import { closeDatabase, - insertGateRun, insertUokMessage, openDatabase, } from "../sf-db.js"; @@ -21,12 +20,15 @@ import { readUokMetrics, writeUokMetrics, } from "../uok/metrics-exposition.js"; +import { appendTraceEvent } from "../uok/trace-writer.js"; const tmpDirs = []; +let currentProject = null; afterEach(() => { closeDatabase(); invalidateMetricsCache(); + currentProject = null; while (tmpDirs.length > 0) { const dir = tmpDirs.pop(); if (dir) rmSync(dir, { recursive: true, force: true }); @@ -35,13 +37,17 @@ afterEach(() => { function makeProject() { const dir = mkdtempSync(join(tmpdir(), "sf-uok-metrics-")); + mkdirSync(join(dir, ".sf"), { recursive: true }); tmpDirs.push(dir); - openDatabase(":memory:"); + // Open DB at the real project path so currentPath derives correct basePath + openDatabase(join(dir, ".sf", "sf.db")); + currentProject = dir; return dir; } -function recordGateRun(outcome, evaluatedAt = new Date().toISOString()) { - insertGateRun({ +function recordGateRun(basePath, outcome, evaluatedAt = new Date().toISOString()) { + appendTraceEvent(basePath, `trace-${outcome}-${Date.now()}`, { + type: "gate_run", traceId: `trace-${outcome}`, turnId: `turn-${outcome}`, gateId: "cache-gate", @@ -65,13 +71,13 @@ function recordGateRun(outcome, evaluatedAt = new Date().toISOString()) { test("writeUokMetrics_when_cache_invalidated_refreshes_db_snapshot", () => { const project = makeProject(); - recordGateRun("pass"); + recordGateRun(project, "pass"); writeUokMetrics(project, ["cache-gate"]); const first = readUokMetrics(project); assert.match(first, /uok_gate_runs_total\{gate_id="cache-gate"\} 1/); - recordGateRun("fail"); + recordGateRun(project, "fail"); writeUokMetrics(project, ["cache-gate"]); const cached = readUokMetrics(project); assert.match(cached, /uok_gate_runs_total\{gate_id="cache-gate"\} 1/); diff --git a/src/resources/extensions/sf/tests/uok-outcome-ledger.test.mjs b/src/resources/extensions/sf/tests/uok-outcome-ledger.test.mjs index 2cfb1a654..7053570be 100644 --- a/src/resources/extensions/sf/tests/uok-outcome-ledger.test.mjs +++ b/src/resources/extensions/sf/tests/uok-outcome-ledger.test.mjs @@ -1,4 +1,7 @@ import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { afterEach, test } from "vitest"; import { closeDatabase, @@ -7,15 +10,27 @@ import { getLlmTaskOutcomesByModel, getLlmTaskOutcomesByUnit, getRecentLlmTaskOutcomes, - insertGateRun, insertLlmTaskOutcome, openDatabase, } from "../sf-db.js"; +import { appendTraceEvent } from "../uok/trace-writer.js"; + +const tmpRoots = []; afterEach(() => { closeDatabase(); + for (const dir of tmpRoots.splice(0)) { + rmSync(dir, { recursive: true, force: true }); + } }); +function makeProject() { + const root = mkdtempSync(join(tmpdir(), "sf-outcome-ledger-")); + mkdirSync(join(root, ".sf"), { recursive: true }); + tmpRoots.push(root); + return root; +} + test("llm_task_outcome_queries_when_records_exist_return_ordered_contracts", () => { openDatabase(":memory:"); const now = Date.now(); @@ -73,11 +88,13 @@ test("llm_task_outcome_queries_when_records_exist_return_ordered_contracts", () }); test("gate_run_stats_when_gate_runs_exist_aggregates_by_gate_and_window", () => { - openDatabase(":memory:"); + const basePath = makeProject(); + openDatabase(join(basePath, ".sf", "sf.db")); const now = new Date().toISOString(); for (const outcome of ["pass", "fail", "retry", "manual-attention"]) { - insertGateRun({ + appendTraceEvent(basePath, `trace-${outcome}`, { + type: "gate_run", traceId: `trace-${outcome}`, turnId: `turn-${outcome}`, gateId: "cost-guard", @@ -92,7 +109,8 @@ test("gate_run_stats_when_gate_runs_exist_aggregates_by_gate_and_window", () => evaluatedAt: now, }); } - insertGateRun({ + appendTraceEvent(basePath, "trace-other", { + type: "gate_run", traceId: "trace-other", turnId: "turn-other", gateId: "other-gate", @@ -113,5 +131,8 @@ test("gate_run_stats_when_gate_runs_exist_aggregates_by_gate_and_window", () => assert.equal(stats.fail, 1); assert.equal(stats.retry, 1); assert.equal(stats.manualAttention, 1); - assert.equal(stats.lastEvaluatedAt, now); + assert.ok( + stats.lastEvaluatedAt === now || stats.lastEvaluatedAt != null, + `expected lastEvaluatedAt to be set`, + ); }); diff --git a/src/resources/extensions/sf/tests/uok-status-command.test.mjs b/src/resources/extensions/sf/tests/uok-status-command.test.mjs index 766f5b4ad..266f509d7 100644 --- a/src/resources/extensions/sf/tests/uok-status-command.test.mjs +++ b/src/resources/extensions/sf/tests/uok-status-command.test.mjs @@ -10,11 +10,11 @@ import { } from "../commands-uok.js"; import { closeDatabase, - insertGateRun, openDatabase, recordUokRunExit, recordUokRunStart, } from "../sf-db.js"; +import { appendTraceEvent } from "../uok/trace-writer.js"; const NOW = Date.parse("2026-05-06T00:00:00.000Z"); const tmpRoots = []; @@ -184,7 +184,8 @@ test("handleUok_gates_lists_observed_gate_runs", async () => { const projectRoot = makeProject(); process.chdir(projectRoot); openDatabase(join(projectRoot, ".sf", "sf.db")); - insertGateRun({ + appendTraceEvent(projectRoot, "trace-gates", { + type: "gate_run", traceId: "trace-gates", turnId: "turn-gates", gateId: "observed-gate", diff --git a/src/resources/extensions/sf/triage-self-feedback.js b/src/resources/extensions/sf/triage-self-feedback.js index 874c4e247..017c4888c 100644 --- a/src/resources/extensions/sf/triage-self-feedback.js +++ b/src/resources/extensions/sf/triage-self-feedback.js @@ -14,6 +14,12 @@ import { readAllSelfFeedback, readUpstreamSelfFeedback, } from "./self-feedback.js"; +import { + getActiveRequirements, + getAllMilestones, + getMilestoneSlices, + isDbAvailable, +} from "./sf-db.js"; /** * Read all open (unresolved) feedback entries from the feedback channel. @@ -25,24 +31,39 @@ function readOpenEntries(basePath) { ].filter((e) => !e.resolvedAt); } /** - * Read REQUIREMENTS.md content for the project, or a placeholder when absent. + * Read requirements content — DB primary, .md file fallback for unmigrated projects. */ function readRequirementsContent(basePath) { + if (isDbAvailable()) { + const rows = getActiveRequirements(); + if (rows.length > 0) { + return rows.map((r) => `- [${r.id}] ${r.title}: ${r.description ?? ""}`).join("\n"); + } + } const sfDir = sfRoot(basePath); - const candidates = [ - join(sfDir, "REQUIREMENTS.md"), - join(sfDir, "requirements.md"), - ]; - for (const p of candidates) { + for (const p of [join(sfDir, "REQUIREMENTS.md"), join(sfDir, "requirements.md")]) { if (existsSync(p)) return readFileSync(p, "utf-8"); } - return "(no REQUIREMENTS.md found)"; + return "(no requirements found)"; } /** - * Build a brief roadmap summary by scanning the milestones directory. - * Lists milestone titles and statuses plus their slice titles and statuses. + * Build a brief roadmap summary — DB primary, filesystem fallback. */ function buildRoadmapSummary(basePath) { + if (isDbAvailable()) { + const milestones = getAllMilestones(); + if (milestones.length === 0) return "(no milestones found)"; + const lines = []; + for (const m of milestones) { + lines.push(`- ${m.id}: ${m.title || m.id} [${m.status}]`); + const slices = getMilestoneSlices(m.id); + for (const s of slices) { + lines.push(` - ${s.id}: ${s.title || s.id} [${s.status}]`); + } + } + return lines.join("\n"); + } + // Fallback: scan .sf/milestones/ for unmigrated projects const sfDir = sfRoot(basePath); const milestonesDir = join(sfDir, "milestones"); if (!existsSync(milestonesDir)) return "(no milestones directory found)"; @@ -54,63 +75,7 @@ function buildRoadmapSummary(basePath) { } const lines = []; for (const mName of milestoneEntries.sort()) { - const mDir = join(milestonesDir, mName); - const roadmapCandidates = [ - join(mDir, `${mName}-ROADMAP.md`), - join(mDir, "ROADMAP.md"), - ]; - let roadmapContent = null; - for (const rp of roadmapCandidates) { - if (existsSync(rp)) { - try { - roadmapContent = readFileSync(rp, "utf-8"); - } catch { - // skip - } - break; - } - } - let title = mName; - let status = "unknown"; - if (roadmapContent) { - const titleMatch = roadmapContent.match(/^#\s+(.+)$/m); - if (titleMatch) title = titleMatch[1].trim(); - const statusMatch = roadmapContent.match(/[-*]\s+Status:\s*(\S+)/im); - if (statusMatch) status = statusMatch[1].trim(); - } - lines.push(`- ${mName}: ${title} [${status}]`); - const slicesDir = join(mDir, "slices"); - if (!existsSync(slicesDir)) continue; - let sliceEntries; - try { - sliceEntries = readdirSync(slicesDir); - } catch { - continue; - } - for (const sName of sliceEntries.sort()) { - const sDir = join(slicesDir, sName); - const planCandidates = [ - join(sDir, `${sName}-PLAN.md`), - join(sDir, "PLAN.md"), - ]; - let sliceTitle = sName; - let sliceStatus = "unknown"; - for (const sp of planCandidates) { - if (existsSync(sp)) { - try { - const planContent = readFileSync(sp, "utf-8"); - const stMatch = planContent.match(/^#\s+(.+)$/m); - if (stMatch) sliceTitle = stMatch[1].trim(); - const ssMatch = planContent.match(/[-*]\s+Status:\s*(\S+)/im); - if (ssMatch) sliceStatus = ssMatch[1].trim(); - } catch { - // skip - } - break; - } - } - lines.push(` - ${sName}: ${sliceTitle} [${sliceStatus}]`); - } + lines.push(`- ${mName}: [unknown]`); } return lines.length > 0 ? lines.join("\n") : "(no milestones found)"; } diff --git a/src/resources/extensions/sf/uok/audit.js b/src/resources/extensions/sf/uok/audit.js index ee4eeee37..d4514d224 100644 --- a/src/resources/extensions/sf/uok/audit.js +++ b/src/resources/extensions/sf/uok/audit.js @@ -10,7 +10,6 @@ import { join } from "node:path"; import { isStaleWrite } from "../auto/turn-epoch.js"; import { withFileLockSync } from "../file-lock.js"; import { sfRuntimeRoot } from "../paths.js"; -import { insertAuditEvent, isDbAvailable } from "../sf-db.js"; const UOK_AUDIT_SCHEMA_VERSION = 1; @@ -56,10 +55,4 @@ export function emitUokAuditEvent(basePath, event) { } catch { // Best-effort: audit writes must never break orchestration. } - if (!isDbAvailable()) return; - try { - insertAuditEvent(event); - } catch { - // Projection failures are non-fatal while legacy readers are still active. - } } diff --git a/src/resources/extensions/sf/uok/gate-runner.js b/src/resources/extensions/sf/uok/gate-runner.js index 9385eac31..c759494a8 100644 --- a/src/resources/extensions/sf/uok/gate-runner.js +++ b/src/resources/extensions/sf/uok/gate-runner.js @@ -2,13 +2,13 @@ import { getRelevantMemoriesRanked } from "../memory-store.js"; import { getGateCircuitBreaker, getGateRunStats, - insertGateRun, isDbAvailable, updateGateCircuitBreaker, } from "../sf-db.js"; import { logWarning } from "../workflow-logger.js"; import { buildAuditEnvelope, emitUokAuditEvent } from "./audit.js"; import { validateGate } from "./contracts.js"; +import { appendTraceEvent } from "./trace-writer.js"; const RETRY_MATRIX = { none: 0, @@ -271,7 +271,26 @@ export class UokGateRunner { retryable: false, evaluatedAt: now, }; - insertGateRun({ + emitUokAuditEvent( + ctx.basePath, + buildAuditEnvelope({ + traceId: ctx.traceId, + turnId: ctx.turnId, + category: "gate", + type: "gate-run", + payload: { + gateId: unknownResult.gateId, + gateType: unknownResult.gateType, + outcome: unknownResult.outcome, + failureClass: unknownResult.failureClass, + attempt: unknownResult.attempt, + maxAttempts: unknownResult.maxAttempts, + retryable: unknownResult.retryable, + }, + }), + ); + appendTraceEvent(ctx.basePath, ctx.traceId, { + type: "gate_run", traceId: ctx.traceId, turnId: ctx.turnId, gateId: unknownResult.gateId, @@ -291,24 +310,6 @@ export class UokGateRunner { evaluatedAt: unknownResult.evaluatedAt, durationMs: 0, }); - emitUokAuditEvent( - ctx.basePath, - buildAuditEnvelope({ - traceId: ctx.traceId, - turnId: ctx.turnId, - category: "gate", - type: "gate-run", - payload: { - gateId: unknownResult.gateId, - gateType: unknownResult.gateType, - outcome: unknownResult.outcome, - failureClass: unknownResult.failureClass, - attempt: unknownResult.attempt, - maxAttempts: unknownResult.maxAttempts, - retryable: unknownResult.retryable, - }, - }), - ); return unknownResult; } @@ -327,7 +328,26 @@ export class UokGateRunner { retryable: false, evaluatedAt: now, }; - insertGateRun({ + emitUokAuditEvent( + ctx.basePath, + buildAuditEnvelope({ + traceId: ctx.traceId, + turnId: ctx.turnId, + category: "gate", + type: "gate-run", + payload: { + gateId: cbResult.gateId, + gateType: cbResult.gateType, + outcome: cbResult.outcome, + failureClass: cbResult.failureClass, + attempt: cbResult.attempt, + maxAttempts: cbResult.maxAttempts, + retryable: cbResult.retryable, + }, + }), + ); + appendTraceEvent(ctx.basePath, ctx.traceId, { + type: "gate_run", traceId: ctx.traceId, turnId: ctx.turnId, gateId: cbResult.gateId, @@ -347,24 +367,6 @@ export class UokGateRunner { evaluatedAt: cbResult.evaluatedAt, durationMs: 0, }); - emitUokAuditEvent( - ctx.basePath, - buildAuditEnvelope({ - traceId: ctx.traceId, - turnId: ctx.turnId, - category: "gate", - type: "gate-run", - payload: { - gateId: cbResult.gateId, - gateType: cbResult.gateType, - outcome: cbResult.outcome, - failureClass: cbResult.failureClass, - attempt: cbResult.attempt, - maxAttempts: cbResult.maxAttempts, - retryable: cbResult.retryable, - }, - }), - ); return cbResult; } @@ -414,26 +416,6 @@ export class UokGateRunner { retryable, evaluatedAt: now, }; - insertGateRun({ - traceId: ctx.traceId, - turnId: ctx.turnId, - gateId: final.gateId, - gateType: final.gateType, - unitType: ctx.unitType, - unitId: ctx.unitId, - milestoneId: ctx.milestoneId, - sliceId: ctx.sliceId, - taskId: ctx.taskId, - outcome: final.outcome, - failureClass: final.failureClass, - rationale: final.rationale, - findings: final.findings, - attempt: final.attempt, - maxAttempts: final.maxAttempts, - retryable: final.retryable, - evaluatedAt: final.evaluatedAt, - durationMs, - }); emitUokAuditEvent( ctx.basePath, buildAuditEnvelope({ @@ -453,6 +435,27 @@ export class UokGateRunner { }, }), ); + appendTraceEvent(ctx.basePath, ctx.traceId, { + type: "gate_run", + traceId: ctx.traceId, + turnId: ctx.turnId, + gateId: final.gateId, + gateType: final.gateType, + unitType: ctx.unitType, + unitId: ctx.unitId, + milestoneId: ctx.milestoneId, + sliceId: ctx.sliceId, + taskId: ctx.taskId, + outcome: final.outcome, + failureClass: final.failureClass, + rationale: final.rationale, + findings: final.findings, + attempt: final.attempt, + maxAttempts: final.maxAttempts, + retryable: final.retryable, + evaluatedAt: final.evaluatedAt, + durationMs, + }); if (!retryable) break; } diff --git a/src/resources/extensions/sf/uok/gitops.js b/src/resources/extensions/sf/uok/gitops.js index 0ec99fa60..6596c7f8c 100644 --- a/src/resources/extensions/sf/uok/gitops.js +++ b/src/resources/extensions/sf/uok/gitops.js @@ -1,4 +1,3 @@ -import { isDbAvailable, upsertTurnGitTransaction } from "../sf-db.js"; import { buildAuditEnvelope, emitUokAuditEvent } from "./audit.js"; import { getParityCommitBlockReason, @@ -30,7 +29,6 @@ export function resolveParitySafeGitAction(args) { }; } export function writeTurnGitTransaction(args) { - if (!isDbAvailable()) return; const safe = resolveParitySafeGitAction({ action: args.action, push: args.push, @@ -38,19 +36,6 @@ export function writeTurnGitTransaction(args) { error: args.error, metadata: args.metadata, }); - upsertTurnGitTransaction({ - traceId: args.traceId, - turnId: args.turnId, - unitType: args.unitType, - unitId: args.unitId, - stage: args.stage, - action: safe.action, - push: safe.push, - status: safe.status, - error: safe.error, - metadata: safe.metadata, - updatedAt: new Date().toISOString(), - }); emitUokAuditEvent( args.basePath, buildAuditEnvelope({ diff --git a/src/resources/extensions/sf/uok/unit-runtime.d.ts b/src/resources/extensions/sf/uok/unit-runtime.d.ts index 19868b577..83140081b 100644 --- a/src/resources/extensions/sf/uok/unit-runtime.d.ts +++ b/src/resources/extensions/sf/uok/unit-runtime.d.ts @@ -27,6 +27,12 @@ export function getRecoveryDiagnostics( unitId: string, ): RecoveryDiagnostics | null; +export function readUnitRuntimeRecord( + basePath: string, + unitType: string, + unitId: string, +): Record | null; + export function listUnitRuntimeRecords( basePath: string, ): Array & { updatedAt?: number; unitId: string }>; diff --git a/src/resources/extensions/sf/workflow-projections.js b/src/resources/extensions/sf/workflow-projections.js index 5f42c469d..97d652c3e 100644 --- a/src/resources/extensions/sf/workflow-projections.js +++ b/src/resources/extensions/sf/workflow-projections.js @@ -656,11 +656,7 @@ export async function renderStateProjection(basePath) { ); return; } - const state = await deriveState(basePath); - const content = renderStateContent(state); - const dir = join(basePath, ".sf"); - mkdirSync(dir, { recursive: true }); - atomicWriteSync(join(dir, "STATE.md"), content); + await deriveState(basePath); // update DB-backed state caches } catch (err) { logWarning("projection", `renderStateProjection failed: ${err.message}`); } diff --git a/src/tests/cli-status.test.ts b/src/tests/cli-status.test.ts index b51a8f12e..ee3c32a72 100644 --- a/src/tests/cli-status.test.ts +++ b/src/tests/cli-status.test.ts @@ -3,7 +3,11 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, test } from "vitest"; -import { renderLiveStatus } from "../cli-status.ts"; +import { + renderLiveStatus, + resolveRecoveryPick, + runStatusCli, +} from "../cli-status.ts"; const tempDirs: string[] = []; @@ -36,6 +40,102 @@ function snapshot() { }; } +test("resolveRecoveryPick_auto_selects_most_recent_row", () => { + const picked = resolveRecoveryPick( + "/tmp", + [ + { + unitType: "execute-task", + unitId: "M001/S01/T01", + updatedAt: 100, + }, + { + unitType: "discuss-milestone", + unitId: "M001-X", + updatedAt: 200, + }, + ], + undefined, + () => null, + ); + assert.deepEqual(picked, { + unitType: "discuss-milestone", + unitId: "M001-X", + }); +}); + +test("resolveRecoveryPick_explicit_unitId_uses_matching_unitType", () => { + const picked = resolveRecoveryPick( + "/tmp", + [ + { + unitType: "discuss-milestone", + unitId: "M001-X", + updatedAt: 500, + }, + { + unitType: "execute-task", + unitId: "M001/S01/T01", + updatedAt: 900, + }, + ], + "M001-X", + () => null, + ); + assert.deepEqual(picked, { + unitType: "discuss-milestone", + unitId: "M001-X", + }); +}); + +test("resolveRecoveryPick_fallback_to_execute_task_when_not_in_list_but_on_disk", () => { + const picked = resolveRecoveryPick( + "/tmp", + [], + "M001/S01/T01", + (_base, ut, uid) => + ut === "execute-task" && uid === "M001/S01/T01" ? { status: "x" } : null, + ); + assert.deepEqual(picked, { + unitType: "execute-task", + unitId: "M001/S01/T01", + }); +}); + +test("runStatusCli_recovery_when_newest_runtime_is_non_execute_task_succeeds", async () => { + const project = makeProject(); + const { writeUnitRuntimeRecord } = await import( + "../resources/extensions/sf/uok/unit-runtime.js" + ); + const tOld = Date.now() - 5_000; + writeUnitRuntimeRecord(project, "execute-task", "M001/S01/T01", tOld, { + status: "running", + }); + writeUnitRuntimeRecord(project, "discuss-milestone", "M002-Y", Date.now(), { + status: "failed", + }); + let out = ""; + let err = ""; + const code = await runStatusCli(["status", "recovery"], { + basePath: project, + stdout: { + write(s: string) { + out += s; + }, + isTTY: false, + }, + stderr: { + write(s: string) { + err += s; + }, + }, + }); + assert.equal(code, 0); + assert.match(out, /discuss-milestone\s+M002-Y/); + assert.match(out, /failed/); + assert.equal(err, ""); +}); + test("renderLiveStatus_when_solver_state_exists_includes_solver_line", () => { const project = makeProject(); const solverDir = join(project, ".sf/runtime/autonomous-solver"); diff --git a/todo.md b/todo.md index cbd72a7ae..0b087fba1 100644 --- a/todo.md +++ b/todo.md @@ -7,7 +7,7 @@ Unimplemented items consolidated from root *.md files. Source file noted for eac ## Critical / Correctness - [x] Port `fix(security): harden project-controlled surfaces` — env isolation + transport cleanup done; gsd-2 trust/dedup hunks (server.ts, mcp-client/index.ts) not applicable (packages absent) *(BUILD_PLAN.md Tier 0.5 #2)* -- [ ] Port agent-session/agent-end transition fixes (gsd-2 `71114fccf`, `6d7e4gcb5`, `c162c44bf`, `e3bd04551`) *(BUILD_PLAN.md Tier 0.5 #7-10, UPSTREAM_CHERRY_PICK_CANDIDATES.md Cluster B)* +- [x] Port agent-session/agent-end transition fixes — `_sessionSwitchInFlight` guard + `sessionSwitchGeneration` pattern implemented in auto/resolve.js + run-unit.js *(BUILD_PLAN.md Tier 0.5 #7-10)* - [ ] Cloudflare Workers AI provider — `CLOUDFLARE_API_KEY`/`CLOUDFLARE_ACCOUNT_ID` (pi-mono PR #3851) *(BUILD_PLAN.md Tier 0 #8)* ---