feat: gap audit + upstream bridge + backlog prompt injection
- gap-audit.ts: automatic detection of orphaned prompts, handlers, native modules, and advertised commands. Deduped by content hash, runs at session_start. - upstream-bridge.ts: rolls up recurring upstream anomalies into forge-local backlog when threshold crossed (≥3 entries, ≥2 repos, 30d window). Severity capped at medium. - system-context.ts: injects top-5 backlog entries into system prompt, sorted by severity then recency. Capped at 2K chars. - register-hooks.ts: wires both gap audit and upstream bridge into session_start drain. - Tests: 13 upstream-bridge tests covering thresholds, idempotency, resolution, severity capping, and multi-kind handling.
This commit is contained in:
Mikael Hugo
parent
1990d2a2ee
commit
f9116f5514
6 changed files with 985 additions and 1 deletions
|
|
@ -235,6 +235,32 @@ export function registerHooks(
|
|||
} catch {
|
||||
/* non-fatal — self-feedback drain must never block session start */
|
||||
}
|
||||
// Run gap audit to detect orphaned prompts, handlers, native modules, commands
|
||||
try {
|
||||
const { runGapAudit } = await import("../gap-audit.js");
|
||||
const filed = runGapAudit(process.cwd());
|
||||
if (filed > 0) {
|
||||
ctx.ui?.notify?.(
|
||||
`Gap audit filed ${filed} new finding${filed === 1 ? "" : "s"} in .sf/BACKLOG.md`,
|
||||
"info",
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
/* non-fatal — gap audit must never block session start */
|
||||
}
|
||||
// Bridge upstream feedback into forge-local backlog
|
||||
try {
|
||||
const { bridgeUpstreamFeedback } = await import("../upstream-bridge.js");
|
||||
const filed = bridgeUpstreamFeedback(process.cwd());
|
||||
if (filed > 0) {
|
||||
ctx.ui?.notify?.(
|
||||
`Upstream bridge filed ${filed} rollup${filed === 1 ? "" : "s"} in .sf/BACKLOG.md`,
|
||||
"info",
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
/* non-fatal — upstream bridge must never block session start */
|
||||
}
|
||||
});
|
||||
|
||||
pi.on("session_switch", async (_event, ctx) => {
|
||||
|
|
|
|||
|
|
@ -364,7 +364,9 @@ export async function buildBeforeAgentStartResult(
|
|||
? `\n\n[JUDGMENT LOG — autonomous mode]\nWhen you make a judgment call between alternatives at an ambiguous point, call sf_log_judgment with: decision, alternatives, reasoning, confidence. This lets the user review your reasoning at milestone close. It does NOT delay or block the work.`
|
||||
: "";
|
||||
|
||||
const fullSystem = `${event.systemPrompt}\n\n[SYSTEM CONTEXT — SF]\n\n${escalationPolicyBlock}${systemContent}${preferenceBlock}${knowledgeBlock}${architectureBlock}${tacitKnowledgeBlock}${codebaseBlock}${codeIntelligenceBlock}${memoryBlock}${newSkillsBlock}${worktreeBlock}${repositoryVcsBlock}${modelIdentityBlock}${subagentModelBlock}${judgmentLogBlock}`;
|
||||
const backlogBlock = loadBacklogBlock(process.cwd());
|
||||
|
||||
const fullSystem = `${event.systemPrompt}\n\n[SYSTEM CONTEXT — SF]\n\n${escalationPolicyBlock}${systemContent}${preferenceBlock}${knowledgeBlock}${architectureBlock}${tacitKnowledgeBlock}${codebaseBlock}${codeIntelligenceBlock}${memoryBlock}${newSkillsBlock}${backlogBlock}${worktreeBlock}${repositoryVcsBlock}${modelIdentityBlock}${subagentModelBlock}${judgmentLogBlock}`;
|
||||
|
||||
stopContextTimer({
|
||||
systemPromptSize: fullSystem.length,
|
||||
|
|
@ -433,6 +435,52 @@ export function loadKnowledgeBlock(
|
|||
}
|
||||
|
||||
const TACIT_SECTION_MAX_BYTES = 4096;
|
||||
const BACKLOG_MAX_ENTRIES = 5;
|
||||
const BACKLOG_MAX_CHARS = 2000;
|
||||
|
||||
function loadBacklogBlock(cwd: string): string {
|
||||
const backlogPath = join(cwd, ".sf", "BACKLOG.md");
|
||||
if (!existsSync(backlogPath)) return "";
|
||||
const raw = cachedReadFile(backlogPath)?.trim() ?? "";
|
||||
if (!raw) return "";
|
||||
|
||||
// Parse the table rows — skip header lines
|
||||
const lines = raw.split("\n");
|
||||
const entries: Array<{ timestamp: string; kind: string; severity: string; summary: string }> = [];
|
||||
for (const line of lines) {
|
||||
if (!line.startsWith("| ")) continue;
|
||||
if (line.includes("Timestamp")) continue; // header
|
||||
if (line.includes("|---|---|")) continue; // separator
|
||||
const cells = line.split("|").map((c) => c.trim()).filter(Boolean);
|
||||
if (cells.length >= 7) {
|
||||
entries.push({
|
||||
timestamp: cells[0],
|
||||
kind: cells[1],
|
||||
severity: cells[2],
|
||||
summary: cells[6],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (entries.length === 0) return "";
|
||||
|
||||
// Sort by severity (high/critical first) then by timestamp (newest first)
|
||||
const severityOrder: Record<string, number> = { critical: 0, high: 1, medium: 2, low: 3 };
|
||||
entries.sort((a, b) => {
|
||||
const sa = severityOrder[a.severity] ?? 99;
|
||||
const sb = severityOrder[b.severity] ?? 99;
|
||||
if (sa !== sb) return sa - sb;
|
||||
return b.timestamp.localeCompare(a.timestamp);
|
||||
});
|
||||
|
||||
const top = entries.slice(0, BACKLOG_MAX_ENTRIES);
|
||||
const rows = top.map((e) => `- **${e.severity}** \`${e.kind}\` — ${e.summary}`).join("\n");
|
||||
const block = `## Recent Self-Feedback Entries (from .sf/BACKLOG.md)\n\n${rows}`;
|
||||
if (block.length > BACKLOG_MAX_CHARS) {
|
||||
return block.slice(0, BACKLOG_MAX_CHARS) + "\n\n*(truncated — see .sf/BACKLOG.md for full backlog)*";
|
||||
}
|
||||
return `\n\n[BACKLOG — Recent sf-internal anomalies]\n\n${block}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load tacit knowledge files (.sf/PRINCIPLES.md, .sf/TASTE.md, .sf/ANTI-GOALS.md)
|
||||
|
|
|
|||
255
src/resources/extensions/sf/gap-audit.ts
Normal file
255
src/resources/extensions/sf/gap-audit.ts
Normal file
|
|
@ -0,0 +1,255 @@
|
|||
/**
|
||||
* Gap Audit — detect orphaned/unwired artifacts in the SF extension.
|
||||
*
|
||||
* Purpose: automatically find dead code, unreferenced prompts, undispatched
|
||||
* command handlers, and shipped-but-unimported native modules. Results are
|
||||
* written to self-feedback so they surface in BACKLOG.md and can be triaged.
|
||||
*
|
||||
* Consumer: session_start drain hook in register-hooks.ts.
|
||||
*/
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
|
||||
import { join, relative } from "node:path";
|
||||
import { recordSelfFeedback } from "./self-feedback.js";
|
||||
|
||||
const EXTENSION_SRC = import.meta.dirname;
|
||||
const PROMPTS_DIR = join(EXTENSION_SRC, "prompts");
|
||||
const COMMANDS_DIR = join(EXTENSION_SRC, "commands");
|
||||
const HANDLERS_DIR = join(COMMANDS_DIR, "handlers");
|
||||
const NATIVE_PKG = join(EXTENSION_SRC, "..", "..", "..", "native");
|
||||
|
||||
interface GapFinding {
|
||||
kind: "orphan-prompt" | "orphan-handler" | "orphan-native" | "orphan-command";
|
||||
name: string;
|
||||
path: string;
|
||||
detail: string;
|
||||
}
|
||||
|
||||
function hashFindings(findings: GapFinding[]): string {
|
||||
const data = findings
|
||||
.map((f) => `${f.kind}:${f.name}:${f.path}`)
|
||||
.sort()
|
||||
.join("\n");
|
||||
return createHash("sha256").update(data).digest("hex").slice(0, 16);
|
||||
}
|
||||
|
||||
function readFileLines(path: string): string[] {
|
||||
try {
|
||||
return readFileSync(path, "utf-8").split("\n");
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function grepImports(sourceDir: string, symbol: string): boolean {
|
||||
try {
|
||||
const files = readdirSync(sourceDir, { recursive: true }) as string[];
|
||||
for (const file of files) {
|
||||
if (!file.endsWith(".ts")) continue;
|
||||
const content = readFileSync(join(sourceDir, file), "utf-8");
|
||||
if (content.includes(symbol)) return true;
|
||||
}
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function findOrphanPrompts(): GapFinding[] {
|
||||
const findings: GapFinding[] = [];
|
||||
try {
|
||||
const files = readdirSync(PROMPTS_DIR).filter((f) => f.endsWith(".md"));
|
||||
for (const file of files) {
|
||||
const name = file.slice(0, -3);
|
||||
// Skip templates that are loaded by convention (guided-* variants)
|
||||
if (name.startsWith("guided-")) continue;
|
||||
const loaded = grepImports(EXTENSION_SRC, `loadPrompt("${name}"`)
|
||||
|| grepImports(EXTENSION_SRC, `loadPrompt('${name}'`);
|
||||
if (!loaded) {
|
||||
findings.push({
|
||||
kind: "orphan-prompt",
|
||||
name,
|
||||
path: relative(EXTENSION_SRC, join(PROMPTS_DIR, file)),
|
||||
detail: `Prompt "${name}" exists but no loadPrompt("${name}") call found in extension source`,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
/* prompts dir may not exist in test env */
|
||||
}
|
||||
return findings;
|
||||
}
|
||||
|
||||
function findOrphanHandlers(): GapFinding[] {
|
||||
const findings: GapFinding[] = [];
|
||||
try {
|
||||
const files = readdirSync(HANDLERS_DIR).filter((f) => f.endsWith(".ts"));
|
||||
for (const file of files) {
|
||||
const path = join(HANDLERS_DIR, file);
|
||||
const lines = readFileLines(path);
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
// Look for exported handle* functions
|
||||
const match = line.match(/export\s+(?:async\s+)?function\s+(handle\w+)/);
|
||||
if (!match) continue;
|
||||
const handlerName = match[1];
|
||||
// Check if dispatched from ops.ts, workflow.ts, core.ts, auto.ts
|
||||
const dispatched = grepImports(COMMANDS_DIR, handlerName);
|
||||
if (!dispatched) {
|
||||
findings.push({
|
||||
kind: "orphan-handler",
|
||||
name: handlerName,
|
||||
path: relative(EXTENSION_SRC, path),
|
||||
detail: `${handlerName} exported from ${file} but never dispatched from commands/handlers/*.ts`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
/* handlers dir may not exist */
|
||||
}
|
||||
return findings;
|
||||
}
|
||||
|
||||
function findOrphanNative(): GapFinding[] {
|
||||
const findings: GapFinding[] = [];
|
||||
const nativeEditDir = join(NATIVE_PKG, "src", "edit");
|
||||
try {
|
||||
if (!existsSync(nativeEditDir)) return findings;
|
||||
const indexPath = join(nativeEditDir, "index.ts");
|
||||
if (!existsSync(indexPath)) return findings;
|
||||
const lines = readFileLines(indexPath);
|
||||
for (const line of lines) {
|
||||
const match = line.match(/export\s+(?:async\s+)?function\s+(\w+)/);
|
||||
if (!match) continue;
|
||||
const symbol = match[1];
|
||||
const imported = grepImports(EXTENSION_SRC, symbol);
|
||||
if (!imported) {
|
||||
findings.push({
|
||||
kind: "orphan-native",
|
||||
name: symbol,
|
||||
path: relative(EXTENSION_SRC, indexPath),
|
||||
detail: `Native edit function ${symbol} exported but never imported from SF extension`,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
/* native pkg may not exist */
|
||||
}
|
||||
return findings;
|
||||
}
|
||||
|
||||
function findOrphanCommands(): GapFinding[] {
|
||||
const findings: GapFinding[] = [];
|
||||
const catalogPath = join(COMMANDS_DIR, "catalog.ts");
|
||||
if (!existsSync(catalogPath)) return findings;
|
||||
|
||||
const catalogLines = readFileLines(catalogPath);
|
||||
const advertisedCommands: string[] = [];
|
||||
for (const line of catalogLines) {
|
||||
// Match { cmd: "rate", desc: "..." } patterns
|
||||
const match = line.match(/cmd:\s*["'](\w+)["']/);
|
||||
if (match) advertisedCommands.push(match[1]);
|
||||
}
|
||||
|
||||
// Check which are dispatched from ops.ts / workflow.ts / core.ts
|
||||
const dispatchFiles = ["ops.ts", "workflow.ts", "core.ts", "auto.ts"]
|
||||
.map((f) => join(HANDLERS_DIR, f))
|
||||
.filter(existsSync);
|
||||
|
||||
for (const cmd of advertisedCommands) {
|
||||
let dispatched = false;
|
||||
for (const path of dispatchFiles) {
|
||||
const content = readFileSync(path, "utf-8");
|
||||
// Look for startsWith("cmd ") or includes("cmd ") patterns
|
||||
if (content.includes(`"${cmd} "`) || content.includes(`'${cmd} '`)) {
|
||||
dispatched = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!dispatched) {
|
||||
findings.push({
|
||||
kind: "orphan-command",
|
||||
name: cmd,
|
||||
path: relative(EXTENSION_SRC, catalogPath),
|
||||
detail: `/sf ${cmd} advertised in catalog but no dispatch branch found in handlers`,
|
||||
});
|
||||
}
|
||||
}
|
||||
return findings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the gap audit and file self-feedback entries for any findings.
|
||||
* Deduped by content hash so repeat runs don't multiply entries.
|
||||
*
|
||||
* @returns number of new findings filed (0 if all were already reported)
|
||||
*/
|
||||
export function runGapAudit(basePath: string = process.cwd()): number {
|
||||
const findings: GapFinding[] = [
|
||||
...findOrphanPrompts(),
|
||||
...findOrphanHandlers(),
|
||||
...findOrphanNative(),
|
||||
...findOrphanCommands(),
|
||||
];
|
||||
|
||||
if (findings.length === 0) return 0;
|
||||
|
||||
const hash = hashFindings(findings);
|
||||
const hashPath = join(basePath, ".sf", "runtime", ".gap-audit-hash");
|
||||
|
||||
// Check if we've already reported this exact set
|
||||
try {
|
||||
if (existsSync(hashPath)) {
|
||||
const prior = readFileSync(hashPath, "utf-8").trim();
|
||||
if (prior === hash) return 0;
|
||||
}
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
|
||||
// File one self-feedback entry per finding kind, grouped
|
||||
const byKind = new Map<string, GapFinding[]>();
|
||||
for (const f of findings) {
|
||||
const list = byKind.get(f.kind) ?? [];
|
||||
list.push(f);
|
||||
byKind.set(f.kind, list);
|
||||
}
|
||||
|
||||
let filed = 0;
|
||||
for (const [kind, items] of byKind) {
|
||||
const severity = kind === "orphan-native" ? "high" : "medium";
|
||||
const summary = items.map((i) => i.name).join(", ");
|
||||
const evidence = items.map((i) => `- ${i.name}: ${i.detail}`).join("\n");
|
||||
const result = recordSelfFeedback(
|
||||
{
|
||||
kind: `gap-audit-${kind}`,
|
||||
severity: severity as "high" | "medium" | "low" | "critical",
|
||||
summary: `${kind.replace("-", " ")}: ${summary}`,
|
||||
evidence,
|
||||
suggestedFix:
|
||||
kind === "orphan-prompt"
|
||||
? "Remove unused prompt or wire it into a loadPrompt call"
|
||||
: kind === "orphan-handler"
|
||||
? "Add dispatch branch in ops.ts/workflow.ts or remove dead export"
|
||||
: kind === "orphan-native"
|
||||
? "Wire native function into SF extension or remove from native package"
|
||||
: "Add dispatch branch for advertised command or remove from catalog",
|
||||
source: "agent" as const,
|
||||
},
|
||||
basePath,
|
||||
);
|
||||
if (result) filed++;
|
||||
}
|
||||
|
||||
// Write hash to prevent re-filing
|
||||
try {
|
||||
mkdirSync(join(basePath, ".sf", "runtime"), { recursive: true });
|
||||
writeFileSync(hashPath, hash, "utf-8");
|
||||
} catch {
|
||||
/* non-fatal */
|
||||
}
|
||||
|
||||
return filed;
|
||||
}
|
||||
209
src/resources/extensions/sf/requirement-promoter.ts
Normal file
209
src/resources/extensions/sf/requirement-promoter.ts
Normal file
|
|
@ -0,0 +1,209 @@
|
|||
/**
|
||||
* Requirement Promoter — threshold-to-requirement promotion sweeper.
|
||||
*
|
||||
* When feedback entries cluster (e.g., 5 instances of `git-empty-pathspec`,
|
||||
* or `runaway-guard-hard-pause` recurring across 3 milestones), this module
|
||||
* auto-promotes to a row in `.sf/REQUIREMENTS.md`.
|
||||
*
|
||||
* Requirements flow into prompt context via the existing planning pipeline,
|
||||
* so promotion turns "noise that piles up in BACKLOG.md" into "something
|
||||
* the next planning round naturally addresses."
|
||||
*
|
||||
* Consumer: session_start drain hook in register-hooks.ts (wired separately).
|
||||
*/
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
type PersistedSelfFeedbackEntry,
|
||||
markResolved,
|
||||
readAllSelfFeedback,
|
||||
} from "./self-feedback.js";
|
||||
import { sfRoot } from "./paths.js";
|
||||
|
||||
// ─── Constants ───────────────────────────────────────────────────────────────
|
||||
|
||||
const COUNT_THRESHOLD = 5;
|
||||
const MILESTONE_THRESHOLD = 3;
|
||||
const LOOKBACK_DAYS = 90;
|
||||
|
||||
const REQUIREMENTS_HEADER =
|
||||
"# Requirements\n\n" +
|
||||
"This file is the explicit capability and coverage contract for the project.\n\n" +
|
||||
"## Active\n\n";
|
||||
|
||||
// ─── Forge detection (local — isForgeRepo is not exported) ───────────────────
|
||||
|
||||
function isForgeRepo(basePath: string): boolean {
|
||||
try {
|
||||
const pkgPath = join(basePath, "package.json");
|
||||
if (!existsSync(pkgPath)) return false;
|
||||
const pkg = JSON.parse(readFileSync(pkgPath, "utf-8"));
|
||||
return pkg?.name === "singularity-forge";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// ─── REQUIREMENTS.md helpers ─────────────────────────────────────────────────
|
||||
|
||||
function requirementsPath(basePath: string): string {
|
||||
return join(sfRoot(basePath), "REQUIREMENTS.md");
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the highest R-number present in REQUIREMENTS.md.
|
||||
* Returns 0 if the file is absent or contains no R-IDs.
|
||||
*/
|
||||
function readHighestRNumber(filePath: string): number {
|
||||
try {
|
||||
if (!existsSync(filePath)) return 0;
|
||||
const content = readFileSync(filePath, "utf-8");
|
||||
const matches = content.matchAll(/\bR(\d+)\b/g);
|
||||
let max = 0;
|
||||
for (const m of matches) {
|
||||
const n = parseInt(m[1], 10);
|
||||
if (n > max) max = n;
|
||||
}
|
||||
return max;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Append one requirement block to REQUIREMENTS.md.
|
||||
* Creates the file with header if it does not exist.
|
||||
* Appends into the ## Active section (or at end if already structured).
|
||||
*/
|
||||
function appendRequirementRow(
|
||||
filePath: string,
|
||||
id: string,
|
||||
title: string,
|
||||
notes: string,
|
||||
): void {
|
||||
const dir = join(filePath, "..");
|
||||
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
||||
|
||||
const block =
|
||||
`### ${id} — ${title}\n` +
|
||||
`- Class: operational\n` +
|
||||
`- Status: active\n` +
|
||||
`- Description: ${title}\n` +
|
||||
`- Source: sf-promoter\n` +
|
||||
`- Notes: ${notes}\n\n`;
|
||||
|
||||
if (!existsSync(filePath)) {
|
||||
writeFileSync(filePath, REQUIREMENTS_HEADER + block, "utf-8");
|
||||
} else {
|
||||
// Append before any ## Traceability or ## Coverage Summary section if
|
||||
// present; otherwise just append at the end.
|
||||
const content = readFileSync(filePath, "utf-8");
|
||||
const insertionMarker = content.match(/\n## (?:Traceability|Coverage Summary)/);
|
||||
if (insertionMarker && insertionMarker.index !== undefined) {
|
||||
const before = content.slice(0, insertionMarker.index);
|
||||
const after = content.slice(insertionMarker.index);
|
||||
writeFileSync(filePath, before + "\n" + block + after, "utf-8");
|
||||
} else {
|
||||
const appended = content.endsWith("\n") ? content + block : content + "\n" + block;
|
||||
writeFileSync(filePath, appended, "utf-8");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Cluster & promote ───────────────────────────────────────────────────────
|
||||
|
||||
interface Cluster {
|
||||
kind: string;
|
||||
entries: PersistedSelfFeedbackEntry[];
|
||||
distinctMilestones: Set<string | undefined>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Promote feedback entries to REQUIREMENTS.md when they cross threshold.
|
||||
*
|
||||
* Only runs when basePath is the forge repo itself. Bails silently otherwise.
|
||||
* Never throws — returns { promoted: 0, requirementIds: [] } on failure.
|
||||
*
|
||||
* @returns { promoted: number; requirementIds: string[] }
|
||||
*/
|
||||
export function promoteFeedbackToRequirements(
|
||||
basePath: string = process.cwd(),
|
||||
): { promoted: number; requirementIds: string[] } {
|
||||
const empty = { promoted: 0, requirementIds: [] as string[] };
|
||||
|
||||
try {
|
||||
// Gate: only runs on singularity-forge itself
|
||||
if (!isForgeRepo(basePath)) return empty;
|
||||
|
||||
const cutoff = new Date(Date.now() - LOOKBACK_DAYS * 24 * 60 * 60 * 1000);
|
||||
|
||||
// Read all entries, filter to open forge entries within the lookback window
|
||||
const eligible = readAllSelfFeedback(basePath).filter(
|
||||
(e) =>
|
||||
!e.resolvedAt &&
|
||||
e.repoIdentity === "forge" &&
|
||||
new Date(e.ts) >= cutoff,
|
||||
);
|
||||
|
||||
if (eligible.length === 0) return empty;
|
||||
|
||||
// Cluster by kind
|
||||
const clusters = new Map<string, Cluster>();
|
||||
for (const e of eligible) {
|
||||
const existing = clusters.get(e.kind);
|
||||
if (existing) {
|
||||
existing.entries.push(e);
|
||||
existing.distinctMilestones.add(e.occurredIn?.milestone);
|
||||
} else {
|
||||
clusters.set(e.kind, {
|
||||
kind: e.kind,
|
||||
entries: [e],
|
||||
distinctMilestones: new Set([e.occurredIn?.milestone]),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Determine which clusters cross the promotion threshold
|
||||
const promotable = [...clusters.values()].filter(
|
||||
(c) =>
|
||||
c.entries.length >= COUNT_THRESHOLD ||
|
||||
c.distinctMilestones.size >= MILESTONE_THRESHOLD,
|
||||
);
|
||||
|
||||
if (promotable.length === 0) return empty;
|
||||
|
||||
const reqPath = requirementsPath(basePath);
|
||||
const promotedIds: string[] = [];
|
||||
|
||||
for (const cluster of promotable) {
|
||||
const nextNum = readHighestRNumber(reqPath) + 1;
|
||||
const reqId = `R${String(nextNum).padStart(3, "0")}`;
|
||||
|
||||
const count = cluster.entries.length;
|
||||
const milestoneCount = cluster.distinctMilestones.size;
|
||||
const title = `Address recurring ${cluster.kind} (${count} entries across ${milestoneCount} milestone${milestoneCount !== 1 ? "s" : ""})`;
|
||||
const sourceIds = cluster.entries.map((e) => e.id).join(", ");
|
||||
const notes = `Source IDs: ${sourceIds}`;
|
||||
|
||||
appendRequirementRow(reqPath, reqId, title, notes);
|
||||
promotedIds.push(reqId);
|
||||
|
||||
// Mark each contributing entry resolved
|
||||
for (const entry of cluster.entries) {
|
||||
markResolved(
|
||||
entry.id,
|
||||
{
|
||||
reason: `Promoted to requirement ${reqId} by threshold-promotion sweeper`,
|
||||
evidence: { kind: "promoted-to-requirement", requirementId: reqId },
|
||||
},
|
||||
basePath,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return { promoted: promotedIds.length, requirementIds: promotedIds };
|
||||
} catch {
|
||||
return empty;
|
||||
}
|
||||
}
|
||||
282
src/resources/extensions/sf/tests/upstream-bridge.test.ts
Normal file
282
src/resources/extensions/sf/tests/upstream-bridge.test.ts
Normal file
|
|
@ -0,0 +1,282 @@
|
|||
import assert from "node:assert/strict";
|
||||
import {
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, beforeEach, test } from "vitest";
|
||||
import {
|
||||
type PersistedSelfFeedbackEntry,
|
||||
markResolved,
|
||||
readAllSelfFeedback,
|
||||
} from "../self-feedback.ts";
|
||||
import { bridgeUpstreamFeedback } from "../upstream-bridge.ts";
|
||||
|
||||
// ─── Test scaffolding ─────────────────────────────────────────────────────────
|
||||
|
||||
let tmpBase: string;
|
||||
let forgeDir: string;
|
||||
let sfHomeDir: string;
|
||||
let upstreamLog: string;
|
||||
let originalSfHome: string | undefined;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpBase = mkdtempSync(join(tmpdir(), "sf-upstream-bridge-"));
|
||||
forgeDir = join(tmpBase, "forge");
|
||||
sfHomeDir = join(tmpBase, "sf-home");
|
||||
upstreamLog = join(sfHomeDir, "agent", "upstream-feedback.jsonl");
|
||||
|
||||
mkdirSync(join(forgeDir, ".sf"), { recursive: true });
|
||||
mkdirSync(join(sfHomeDir, "agent"), { recursive: true });
|
||||
|
||||
// Fake forge repo — name matches but no loader.ts (so sfRuntimeRoot → basePath/.sf)
|
||||
writeFileSync(
|
||||
join(forgeDir, "package.json"),
|
||||
JSON.stringify({ name: "singularity-forge", version: "0.0.1" }),
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
originalSfHome = process.env.SF_HOME;
|
||||
process.env.SF_HOME = sfHomeDir;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (originalSfHome === undefined) delete process.env.SF_HOME;
|
||||
else process.env.SF_HOME = originalSfHome;
|
||||
rmSync(tmpBase, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// ─── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
function makeEntry(
|
||||
overrides: Partial<PersistedSelfFeedbackEntry> = {},
|
||||
): PersistedSelfFeedbackEntry {
|
||||
return {
|
||||
id: `sf-${Math.random().toString(36).slice(2, 10)}`,
|
||||
kind: "runaway-guard-hard-pause",
|
||||
severity: "medium",
|
||||
summary: "Runaway guard triggered",
|
||||
source: "detector",
|
||||
ts: new Date().toISOString(),
|
||||
basePath: "/some/external/repo",
|
||||
repoIdentity: "external",
|
||||
sfVersion: "1.0.0",
|
||||
blocking: false,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function writeUpstream(entries: PersistedSelfFeedbackEntry[]): void {
|
||||
const lines = entries.map((e) => JSON.stringify(e)).join("\n") + "\n";
|
||||
writeFileSync(upstreamLog, lines, "utf-8");
|
||||
}
|
||||
|
||||
function readForgeJsonl(): PersistedSelfFeedbackEntry[] {
|
||||
return readAllSelfFeedback(forgeDir);
|
||||
}
|
||||
|
||||
// ─── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
test("bails silently when basePath is not forge — no package.json", () => {
|
||||
const nonForge = join(tmpBase, "other");
|
||||
mkdirSync(nonForge, { recursive: true });
|
||||
|
||||
writeUpstream([
|
||||
makeEntry({ basePath: "/repo/a" }),
|
||||
makeEntry({ basePath: "/repo/b" }),
|
||||
makeEntry({ basePath: "/repo/c" }),
|
||||
]);
|
||||
|
||||
const count = bridgeUpstreamFeedback(nonForge);
|
||||
assert.equal(count, 0);
|
||||
// No forge-local jsonl created
|
||||
assert.ok(!existsSync(join(nonForge, ".sf", "self-feedback.jsonl")));
|
||||
});
|
||||
|
||||
test("bails silently when basePath has wrong package name", () => {
|
||||
const nonForge = join(tmpBase, "other2");
|
||||
mkdirSync(nonForge, { recursive: true });
|
||||
writeFileSync(
|
||||
join(nonForge, "package.json"),
|
||||
JSON.stringify({ name: "some-other-tool" }),
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
writeUpstream([
|
||||
makeEntry({ basePath: "/repo/a" }),
|
||||
makeEntry({ basePath: "/repo/b" }),
|
||||
makeEntry({ basePath: "/repo/c" }),
|
||||
]);
|
||||
|
||||
assert.equal(bridgeUpstreamFeedback(nonForge), 0);
|
||||
});
|
||||
|
||||
test("files a rollup when ≥3 entries of same kind from ≥2 distinct repos", () => {
|
||||
writeUpstream([
|
||||
makeEntry({ basePath: "/repo/a", kind: "runaway-guard-hard-pause" }),
|
||||
makeEntry({ basePath: "/repo/b", kind: "runaway-guard-hard-pause" }),
|
||||
makeEntry({ basePath: "/repo/c", kind: "runaway-guard-hard-pause" }),
|
||||
]);
|
||||
|
||||
const count = bridgeUpstreamFeedback(forgeDir);
|
||||
assert.equal(count, 1);
|
||||
|
||||
const entries = readForgeJsonl();
|
||||
assert.equal(entries.length, 1);
|
||||
const rollup = entries[0];
|
||||
assert.equal(rollup.kind, "upstream-rollup:runaway-guard-hard-pause");
|
||||
assert.equal(rollup.source, "detector");
|
||||
assert.match(rollup.summary, /3 external-repo entries/);
|
||||
assert.match(rollup.summary, /3 repos/);
|
||||
|
||||
// Rollup appears in BACKLOG.md
|
||||
const backlog = readFileSync(join(forgeDir, ".sf", "BACKLOG.md"), "utf-8");
|
||||
assert.match(backlog, /upstream-rollup:runaway-guard-hard-pause/);
|
||||
});
|
||||
|
||||
test("idempotent — re-running with same upstream state files 0 new entries", () => {
|
||||
writeUpstream([
|
||||
makeEntry({ basePath: "/repo/a" }),
|
||||
makeEntry({ basePath: "/repo/b" }),
|
||||
makeEntry({ basePath: "/repo/c" }),
|
||||
]);
|
||||
|
||||
const first = bridgeUpstreamFeedback(forgeDir);
|
||||
assert.equal(first, 1);
|
||||
|
||||
const second = bridgeUpstreamFeedback(forgeDir);
|
||||
assert.equal(second, 0);
|
||||
|
||||
assert.equal(readForgeJsonl().length, 1);
|
||||
});
|
||||
|
||||
test("skips a kind when an open rollup for it already exists", () => {
|
||||
// Pre-populate forge backlog with an open rollup of the same kind
|
||||
writeUpstream([
|
||||
makeEntry({ basePath: "/repo/a" }),
|
||||
makeEntry({ basePath: "/repo/b" }),
|
||||
makeEntry({ basePath: "/repo/c" }),
|
||||
]);
|
||||
bridgeUpstreamFeedback(forgeDir); // file initial rollup
|
||||
|
||||
// Add more upstream entries
|
||||
const existing = JSON.parse(
|
||||
readFileSync(upstreamLog, "utf-8").trim().split("\n")[0],
|
||||
);
|
||||
writeUpstream([
|
||||
existing as PersistedSelfFeedbackEntry,
|
||||
makeEntry({ basePath: "/repo/d" }),
|
||||
makeEntry({ basePath: "/repo/e" }),
|
||||
makeEntry({ basePath: "/repo/f" }),
|
||||
]);
|
||||
|
||||
const count = bridgeUpstreamFeedback(forgeDir);
|
||||
assert.equal(count, 0, "should not re-file while open rollup exists");
|
||||
assert.equal(readForgeJsonl().length, 1);
|
||||
});
|
||||
|
||||
test("re-fires after the previous rollup is resolved and new upstream entries arrive", () => {
|
||||
writeUpstream([
|
||||
makeEntry({ basePath: "/repo/a" }),
|
||||
makeEntry({ basePath: "/repo/b" }),
|
||||
makeEntry({ basePath: "/repo/c" }),
|
||||
]);
|
||||
bridgeUpstreamFeedback(forgeDir);
|
||||
|
||||
// Resolve the open rollup
|
||||
const [rollup] = readForgeJsonl();
|
||||
const resolved = markResolved(
|
||||
rollup.id,
|
||||
{ reason: "fixed", evidence: { kind: "human-clear" } },
|
||||
forgeDir,
|
||||
);
|
||||
assert.ok(resolved);
|
||||
|
||||
// New upstream entries still present → should re-file
|
||||
const count = bridgeUpstreamFeedback(forgeDir);
|
||||
assert.equal(count, 1);
|
||||
const entries = readForgeJsonl();
|
||||
assert.equal(entries.filter((e) => !e.resolvedAt).length, 1);
|
||||
});
|
||||
|
||||
test("filters out entries older than 30 days", () => {
|
||||
const oldTs = new Date(Date.now() - 31 * 24 * 60 * 60 * 1000).toISOString();
|
||||
writeUpstream([
|
||||
makeEntry({ basePath: "/repo/a", ts: oldTs }),
|
||||
makeEntry({ basePath: "/repo/b", ts: oldTs }),
|
||||
makeEntry({ basePath: "/repo/c", ts: oldTs }),
|
||||
]);
|
||||
|
||||
assert.equal(bridgeUpstreamFeedback(forgeDir), 0);
|
||||
assert.equal(readForgeJsonl().length, 0);
|
||||
});
|
||||
|
||||
test("caps severity at medium even when contributing entries are higher", () => {
|
||||
writeUpstream([
|
||||
makeEntry({ basePath: "/repo/a", severity: "critical" }),
|
||||
makeEntry({ basePath: "/repo/b", severity: "high" }),
|
||||
makeEntry({ basePath: "/repo/c", severity: "critical" }),
|
||||
]);
|
||||
|
||||
bridgeUpstreamFeedback(forgeDir);
|
||||
const [rollup] = readForgeJsonl();
|
||||
assert.equal(rollup.severity, "medium");
|
||||
assert.equal(rollup.blocking, false);
|
||||
});
|
||||
|
||||
test("does not file when count < 3", () => {
|
||||
writeUpstream([
|
||||
makeEntry({ basePath: "/repo/a" }),
|
||||
makeEntry({ basePath: "/repo/b" }),
|
||||
]);
|
||||
|
||||
assert.equal(bridgeUpstreamFeedback(forgeDir), 0);
|
||||
});
|
||||
|
||||
test("does not file when distinct repos < 2 (all from same repo)", () => {
|
||||
writeUpstream([
|
||||
makeEntry({ basePath: "/repo/a" }),
|
||||
makeEntry({ basePath: "/repo/a" }),
|
||||
makeEntry({ basePath: "/repo/a" }),
|
||||
]);
|
||||
|
||||
assert.equal(bridgeUpstreamFeedback(forgeDir), 0);
|
||||
});
|
||||
|
||||
test("skips resolved upstream entries", () => {
|
||||
writeUpstream([
|
||||
makeEntry({ basePath: "/repo/a", resolvedAt: new Date().toISOString() }),
|
||||
makeEntry({ basePath: "/repo/b", resolvedAt: new Date().toISOString() }),
|
||||
makeEntry({ basePath: "/repo/c", resolvedAt: new Date().toISOString() }),
|
||||
]);
|
||||
|
||||
assert.equal(bridgeUpstreamFeedback(forgeDir), 0);
|
||||
});
|
||||
|
||||
test("handles multiple distinct kinds independently", () => {
|
||||
writeUpstream([
|
||||
makeEntry({ basePath: "/repo/a", kind: "kind-alpha" }),
|
||||
makeEntry({ basePath: "/repo/b", kind: "kind-alpha" }),
|
||||
makeEntry({ basePath: "/repo/c", kind: "kind-alpha" }),
|
||||
makeEntry({ basePath: "/repo/a", kind: "kind-beta" }),
|
||||
makeEntry({ basePath: "/repo/b", kind: "kind-beta" }),
|
||||
makeEntry({ basePath: "/repo/c", kind: "kind-beta" }),
|
||||
]);
|
||||
|
||||
const count = bridgeUpstreamFeedback(forgeDir);
|
||||
assert.equal(count, 2);
|
||||
const entries = readForgeJsonl();
|
||||
const kinds = new Set(entries.map((e) => e.kind));
|
||||
assert.ok(kinds.has("upstream-rollup:kind-alpha"));
|
||||
assert.ok(kinds.has("upstream-rollup:kind-beta"));
|
||||
});
|
||||
|
||||
test("returns 0 when upstream log does not exist", () => {
|
||||
// Don't write any upstream log
|
||||
assert.equal(bridgeUpstreamFeedback(forgeDir), 0);
|
||||
});
|
||||
164
src/resources/extensions/sf/upstream-bridge.ts
Normal file
164
src/resources/extensions/sf/upstream-bridge.ts
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
/**
|
||||
* Upstream-feedback → forge-backlog bridge.
|
||||
*
|
||||
* Rolls up recurring upstream anomalies (observed while sf runs on external
|
||||
* repos) into the forge-local self-feedback backlog so they can be triaged and
|
||||
* addressed as forge-side fixes.
|
||||
*
|
||||
* Called from register-hooks.ts session_start drain (wired externally).
|
||||
* Never throws — any I/O failure returns 0.
|
||||
*/
|
||||
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
type PersistedSelfFeedbackEntry,
|
||||
type SelfFeedbackSeverity,
|
||||
readAllSelfFeedback,
|
||||
recordSelfFeedback,
|
||||
} from "./self-feedback.js";
|
||||
|
||||
// ─── Constants ────────────────────────────────────────────────────────────────
|
||||
|
||||
const SEVERITY_ORDER: SelfFeedbackSeverity[] = ["low", "medium", "high", "critical"];
|
||||
const ROLLUP_CAP: SelfFeedbackSeverity = "medium";
|
||||
const THRESHOLD_COUNT = 3;
|
||||
const THRESHOLD_REPOS = 2;
|
||||
const THIRTY_DAYS_MS = 30 * 24 * 60 * 60 * 1000;
|
||||
|
||||
// ─── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
function getUpstreamLogPath(): string {
|
||||
const sfHome = process.env.SF_HOME || join(homedir(), ".sf");
|
||||
return join(sfHome, "agent", "upstream-feedback.jsonl");
|
||||
}
|
||||
|
||||
function isForgeRepo(basePath: string): boolean {
|
||||
try {
|
||||
const pkgPath = join(basePath, "package.json");
|
||||
if (!existsSync(pkgPath)) return false;
|
||||
const pkg = JSON.parse(readFileSync(pkgPath, "utf-8"));
|
||||
return pkg?.name === "singularity-forge";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function readUpstreamEntries(): PersistedSelfFeedbackEntry[] {
|
||||
const path = getUpstreamLogPath();
|
||||
try {
|
||||
if (!existsSync(path)) return [];
|
||||
const out: PersistedSelfFeedbackEntry[] = [];
|
||||
for (const line of readFileSync(path, "utf-8").split("\n")) {
|
||||
if (!line.trim()) continue;
|
||||
try {
|
||||
out.push(JSON.parse(line) as PersistedSelfFeedbackEntry);
|
||||
} catch {
|
||||
/* skip malformed lines */
|
||||
}
|
||||
}
|
||||
return out;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function capSeverity(sev: SelfFeedbackSeverity): SelfFeedbackSeverity {
|
||||
const idx = SEVERITY_ORDER.indexOf(sev);
|
||||
const capIdx = SEVERITY_ORDER.indexOf(ROLLUP_CAP);
|
||||
return SEVERITY_ORDER[Math.min(idx, capIdx)];
|
||||
}
|
||||
|
||||
function maxSeverity(entries: PersistedSelfFeedbackEntry[]): SelfFeedbackSeverity {
|
||||
let max = 0;
|
||||
for (const e of entries) {
|
||||
const idx = SEVERITY_ORDER.indexOf(e.severity);
|
||||
if (idx > max) max = idx;
|
||||
}
|
||||
return SEVERITY_ORDER[max];
|
||||
}
|
||||
|
||||
// ─── Public API ───────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Roll up upstream feedback entries into the forge-local backlog.
|
||||
* Only runs when basePath is the singularity-forge repo itself.
|
||||
*
|
||||
* @returns count of new rollup entries filed (0 on bail/failure)
|
||||
*/
|
||||
export function bridgeUpstreamFeedback(basePath: string = process.cwd()): number {
|
||||
try {
|
||||
if (!isForgeRepo(basePath)) return 0;
|
||||
|
||||
const cutoff = Date.now() - THIRTY_DAYS_MS;
|
||||
const upstream = readUpstreamEntries().filter(
|
||||
(e) =>
|
||||
!e.resolvedAt &&
|
||||
e.repoIdentity === "external" &&
|
||||
new Date(e.ts).getTime() >= cutoff,
|
||||
);
|
||||
if (upstream.length === 0) return 0;
|
||||
|
||||
// Group by kind, then compute distinct basePaths per group
|
||||
const byKind = new Map<string, PersistedSelfFeedbackEntry[]>();
|
||||
for (const e of upstream) {
|
||||
const list = byKind.get(e.kind) ?? [];
|
||||
list.push(e);
|
||||
byKind.set(e.kind, list);
|
||||
}
|
||||
|
||||
// Read existing forge-local entries once for idempotency checks
|
||||
const existing = readAllSelfFeedback(basePath);
|
||||
const openRollupKinds = new Set(
|
||||
existing
|
||||
.filter((e) => !e.resolvedAt && e.kind.startsWith("upstream-rollup:"))
|
||||
.map((e) => e.kind),
|
||||
);
|
||||
|
||||
let filed = 0;
|
||||
for (const [kind, entries] of byKind) {
|
||||
if (entries.length < THRESHOLD_COUNT) continue;
|
||||
const distinctRepos = new Set(entries.map((e) => e.basePath)).size;
|
||||
if (distinctRepos < THRESHOLD_REPOS) continue;
|
||||
|
||||
const rollupKind = `upstream-rollup:${kind}`;
|
||||
if (openRollupKinds.has(rollupKind)) continue;
|
||||
|
||||
// Derive severity, capped at medium
|
||||
const severity = capSeverity(maxSeverity(entries));
|
||||
|
||||
// Build evidence block: up to 5 samples + full id list
|
||||
const samples = entries.slice(0, 5);
|
||||
const sampleLines = samples
|
||||
.map((e) => ` [${e.id}] ${e.basePath} — ${e.summary}`)
|
||||
.join("\n");
|
||||
const allIds = entries.map((e) => e.id).join(", ");
|
||||
const evidence =
|
||||
`Samples (${samples.length} of ${entries.length}):\n${sampleLines}\n\n` +
|
||||
`All upstream ids: ${allIds}`;
|
||||
|
||||
const result = recordSelfFeedback(
|
||||
{
|
||||
kind: rollupKind,
|
||||
severity,
|
||||
summary: `${entries.length} external-repo entries of kind '${kind}' across ${distinctRepos} repos`,
|
||||
evidence,
|
||||
suggestedFix:
|
||||
"Triage the cluster — common kinds suggest a forge-side fix (threshold tweak, gate fix, prompt clarification). Mark resolved with kind: 'agent-fix' citing the commit, or 'human-clear' with reason.",
|
||||
source: "detector",
|
||||
occurredIn: undefined,
|
||||
},
|
||||
basePath,
|
||||
);
|
||||
if (result) {
|
||||
filed++;
|
||||
openRollupKinds.add(rollupKind); // prevent double-filing in same run
|
||||
}
|
||||
}
|
||||
|
||||
return filed;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue