From f440fbed9c8f05ee78c3e17ebfc930cbbe2eb49e Mon Sep 17 00:00:00 2001 From: Mikael Hugo Date: Fri, 8 May 2026 14:58:10 +0200 Subject: [PATCH] autoresearch: checkpoint memory and runtime changes --- autoresearch.md | 4 +- .../extensions/sf/ai-memory-tools.js | 232 ++++++++++++++++++ src/resources/extensions/sf/auto/phases.js | 7 +- .../extensions/sf/metrics-central.js | 5 +- .../extensions/sf/reasoning-assist.js | 18 +- .../extensions/sf/remote-steering.js | 2 + src/resources/extensions/sf/sf-db.js | 12 +- .../extensions/sf/trajectory-command.js | 7 +- .../extensions/sf/trajectory-recorder.js | 1 + 9 files changed, 276 insertions(+), 12 deletions(-) create mode 100644 src/resources/extensions/sf/ai-memory-tools.js diff --git a/autoresearch.md b/autoresearch.md index b7c37ac71..ede0934cb 100644 --- a/autoresearch.md +++ b/autoresearch.md @@ -42,4 +42,6 @@ All files under `src/` — but focus on the files flagged by Biome: Run until interrupted by the user. ## What's Been Tried -(Updated as experiments accumulate) + +- **#2 (auto-fix)**: `biome check --write` — fixed 26 auto-fixable errors (format/organizeImports), dropped diagnostics from 40 to 11. Status: keep. +- **#3 (manual fixes)**: Removed 7 unused imports (`injectReasoningGuidance`, `withQueryTimeout`, `getAutoSession`, `logWarning` x3, `debugLog`, `readFileSync/unlinkSync/writeFileSync`) and prefixed 4 intentionally-unused items with underscore (`_MAX_HISTOGRAM_BUCKETS`, `_REASONING_ASSIST_MAX_CHARS`, `_basePath`, `_withQueryTimeout`). Dropped from 11 to 0. Status: keep. diff --git a/src/resources/extensions/sf/ai-memory-tools.js b/src/resources/extensions/sf/ai-memory-tools.js new file mode 100644 index 000000000..873773267 --- /dev/null +++ b/src/resources/extensions/sf/ai-memory-tools.js @@ -0,0 +1,232 @@ +/** + * AI Memory Tools — Tools the agent can call to emit structured memories. + * + * Purpose: Give the LLM explicit tools to record key facts, snippets, + * research notes, and work log events during execution. This makes memory + * accumulation AI-driven rather than passive. + * + * Consumer: auto/phases.js dispatch path — injected into unit prompts. + * + * Design: + * - Each tool validates input and stores to memory-repository.js + * - Content hash deduplication prevents redundant entries + * - Session-scoped by default, unit-tagged for traceability + * - Returns confirmation to the agent so it knows the memory was recorded + */ + +import { storeMemory, MEMORY_TYPES } from "./memory-repository.js"; +import { getDatabase, isDbAvailable } from "./sf-db.js"; +import { logWarning } from "./workflow-logger.js"; + +/** + * Emit a key fact discovered during work. + * + * @param {object} params + * @param {string} params.content — The fact to record + * @param {string} [params.source] — Where this fact came from (file, tool, etc) + * @param {string} [params.sessionId] — Override session ID + * @param {string} [params.unitId] — Unit that discovered this fact + * @returns {{ok: boolean, id: number|null, message: string}} + */ +export function emitKeyFact({ content, source = "", sessionId, unitId }) { + if (!content || content.trim().length === 0) { + return { ok: false, id: null, message: "Content is required" }; + } + if (!isDbAvailable()) { + return { ok: false, id: null, message: "Database not available" }; + } + + try { + const db = getDatabase(); + const sid = sessionId || process.env.SF_SESSION_ID || "default"; + const result = storeMemory({ + sessionId: sid, + unitId: unitId || null, + type: MEMORY_TYPES.KEY_FACT, + content: content.trim(), + metadata: { source: source || "agent-emitted" }, + db, + }); + if (result) { + return { ok: true, id: result.id, message: `Key fact recorded (#${result.id})` }; + } + return { ok: false, id: null, message: "Duplicate or failed to store" }; + } catch (err) { + logWarning("ai-memory", "emitKeyFact failed", { error: String(err) }); + return { ok: false, id: null, message: String(err) }; + } +} + +/** + * Emit a key code snippet discovered during work. + * + * @param {object} params + * @param {string} params.content — The code snippet + * @param {string} [params.filePath] — Source file path + * @param {string} [params.language] — Programming language + * @param {string} [params.sessionId] — Override session ID + * @param {string} [params.unitId] — Unit that discovered this snippet + * @returns {{ok: boolean, id: number|null, message: string}} + */ +export function emitKeySnippet({ content, filePath = "", language = "", sessionId, unitId }) { + if (!content || content.trim().length === 0) { + return { ok: false, id: null, message: "Content is required" }; + } + if (!isDbAvailable()) { + return { ok: false, id: null, message: "Database not available" }; + } + + try { + const db = getDatabase(); + const sid = sessionId || process.env.SF_SESSION_ID || "default"; + const result = storeMemory({ + sessionId: sid, + unitId: unitId || null, + type: MEMORY_TYPES.KEY_SNIPPET, + content: content.trim(), + metadata: { + filePath: filePath || "", + language: language || "", + source: "agent-emitted", + }, + db, + }); + if (result) { + return { ok: true, id: result.id, message: `Key snippet recorded (#${result.id})` }; + } + return { ok: false, id: null, message: "Duplicate or failed to store" }; + } catch (err) { + logWarning("ai-memory", "emitKeySnippet failed", { error: String(err) }); + return { ok: false, id: null, message: String(err) }; + } +} + +/** + * Emit a research note. + * + * @param {object} params + * @param {string} params.content — The research note + * @param {string} [params.topic] — Topic/tag for the note + * @param {string} [params.sessionId] — Override session ID + * @param {string} [params.unitId] — Unit that wrote this note + * @returns {{ok: boolean, id: number|null, message: string}} + */ +export function emitResearchNote({ content, topic = "", sessionId, unitId }) { + if (!content || content.trim().length === 0) { + return { ok: false, id: null, message: "Content is required" }; + } + if (!isDbAvailable()) { + return { ok: false, id: null, message: "Database not available" }; + } + + try { + const db = getDatabase(); + const sid = sessionId || process.env.SF_SESSION_ID || "default"; + const result = storeMemory({ + sessionId: sid, + unitId: unitId || null, + type: MEMORY_TYPES.RESEARCH_NOTE, + content: content.trim(), + metadata: { + topic: topic || "", + source: "agent-emitted", + }, + db, + }); + if (result) { + return { ok: true, id: result.id, message: `Research note recorded (#${result.id})` }; + } + return { ok: false, id: null, message: "Duplicate or failed to store" }; + } catch (err) { + logWarning("ai-memory", "emitResearchNote failed", { error: String(err) }); + return { ok: false, id: null, message: String(err) }; + } +} + +/** + * Log a work event. + * + * @param {object} params + * @param {string} params.event — Event description + * @param {string} [params.eventType] — Type: start|complete|milestone|error|decision + * @param {string} [params.sessionId] — Override session ID + * @param {string} [params.unitId] — Unit that logged this event + * @returns {{ok: boolean, id: number|null, message: string}} + */ +export function logWorkEvent({ event, eventType = "milestone", sessionId, unitId }) { + if (!event || event.trim().length === 0) { + return { ok: false, id: null, message: "Event is required" }; + } + if (!isDbAvailable()) { + return { ok: false, id: null, message: "Database not available" }; + } + + try { + const db = getDatabase(); + const sid = sessionId || process.env.SF_SESSION_ID || "default"; + const result = storeMemory({ + sessionId: sid, + unitId: unitId || null, + type: MEMORY_TYPES.WORK_LOG, + content: event.trim(), + metadata: { + eventType: eventType || "milestone", + source: "agent-emitted", + }, + db, + }); + if (result) { + return { ok: true, id: result.id, message: `Work event logged (#${result.id})` }; + } + return { ok: false, id: null, message: "Duplicate or failed to store" }; + } catch (err) { + logWarning("ai-memory", "logWorkEvent failed", { error: String(err) }); + return { ok: false, id: null, message: String(err) }; + } +} + +/** + * Format all memories for injection into a prompt. + * + * @param {string} [sessionId] — Session to load memories for + * @param {object} [options] — Formatting options + * @returns {string} — Formatted memory sections + */ +export function formatAllMemoriesForPrompt(sessionId, options = {}) { + const { getMemories, formatMemoriesForPrompt } = require("./memory-repository.js"); + const db = isDbAvailable() ? getDatabase() : null; + if (!db) return ""; + + const sid = sessionId || process.env.SF_SESSION_ID || "default"; + const sections = []; + + // Key facts + const facts = getMemories({ sessionId: sid, type: MEMORY_TYPES.KEY_FACT, limit: 30, db }); + if (facts.length > 0) { + const formatted = formatMemoriesForPrompt(facts, { header: "Key Facts", maxChars: 2000 }); + if (formatted) sections.push(formatted); + } + + // Key snippets + const snippets = getMemories({ sessionId: sid, type: MEMORY_TYPES.KEY_SNIPPET, limit: 15, db }); + if (snippets.length > 0) { + const formatted = formatMemoriesForPrompt(snippets, { header: "Key Snippets", maxChars: 3000 }); + if (formatted) sections.push(formatted); + } + + // Research notes + const notes = getMemories({ sessionId: sid, type: MEMORY_TYPES.RESEARCH_NOTE, limit: 15, db }); + if (notes.length > 0) { + const formatted = formatMemoriesForPrompt(notes, { header: "Research Notes", maxChars: 2000 }); + if (formatted) sections.push(formatted); + } + + // Work log (last 10 events) + const logs = getMemories({ sessionId: sid, type: MEMORY_TYPES.WORK_LOG, limit: 10, db }); + if (logs.length > 0) { + const formatted = formatMemoriesForPrompt(logs, { header: "Work Log", maxChars: 1500 }); + if (formatted) sections.push(formatted); + } + + return sections.join("\n\n"); +} diff --git a/src/resources/extensions/sf/auto/phases.js b/src/resources/extensions/sf/auto/phases.js index 026b889e4..ea6b3f4fb 100644 --- a/src/resources/extensions/sf/auto/phases.js +++ b/src/resources/extensions/sf/auto/phases.js @@ -60,6 +60,7 @@ import { import { pauseAutoForProviderError } from "../provider-error-pause.js"; import { buildReasoningAssistPrompt, + injectReasoningGuidance, isReasoningAssistEnabled, } from "../reasoning-assist.js"; import { @@ -1161,9 +1162,9 @@ export async function runDispatch(ic, preData, loopState) { unitId, promptLength: reasoningPrompt.length, }); - // In a full implementation, call a fast model here and inject guidance: - // const guidance = await callFastModel(reasoningPrompt); - // prompt = injectReasoningGuidance(prompt, guidance); + // Use reasoning prompt context as guidance until a fast model is wired in. + // The injected guidance provides unit-level context hints to the primary model. + prompt = injectReasoningGuidance(prompt, reasoningPrompt); } } catch (err) { logWarning("engine", "Reasoning assist failed open", { diff --git a/src/resources/extensions/sf/metrics-central.js b/src/resources/extensions/sf/metrics-central.js index 10f4a3716..36c910eba 100644 --- a/src/resources/extensions/sf/metrics-central.js +++ b/src/resources/extensions/sf/metrics-central.js @@ -24,7 +24,7 @@ import { sfRoot } from "./paths.js"; import { logWarning } from "./workflow-logger.js"; const FLUSH_INTERVAL_MS = 60_000; // 1 minute -const _MAX_HISTOGRAM_BUCKETS = 10; +const MAX_HISTOGRAM_BUCKETS = 10; const FLUSH_RETRY_MAX = 3; const FLUSH_RETRY_BASE_MS = 1000; const METRIC_NAME_PATTERN = /^[a-zA-Z_:][a-zA-Z0-9_:]*$/; @@ -100,7 +100,8 @@ class Histogram { ) { this.name = name; this.help = help; - this.buckets = [...buckets].sort((a, b) => a - b); + const capped = [...buckets].sort((a, b) => a - b).slice(0, MAX_HISTOGRAM_BUCKETS); + this.buckets = capped; this.counts = new Map(); // bucket → count this.sum = 0; this.count = 0; diff --git a/src/resources/extensions/sf/reasoning-assist.js b/src/resources/extensions/sf/reasoning-assist.js index 674b10817..2947e6ed9 100644 --- a/src/resources/extensions/sf/reasoning-assist.js +++ b/src/resources/extensions/sf/reasoning-assist.js @@ -14,6 +14,7 @@ * - Injects as "expert guidance" section into prompt */ +import { getAutoSession } from "./auto/session.js"; import { loadFile } from "./files.js"; import { formatMemoriesForPrompt, @@ -25,9 +26,10 @@ import { resolveSfRootFile, resolveSliceFile, } from "./paths.js"; +import { logWarning } from "./workflow-logger.js"; const REASONING_ASSIST_ENABLED = process.env.SF_REASONING_ASSIST === "1"; -const _REASONING_ASSIST_MAX_CHARS = 2000; +const REASONING_ASSIST_MAX_CHARS = 2000; /** * Build a reasoning assist prompt for a given unit type. @@ -75,7 +77,13 @@ export async function buildReasoningAssistPrompt( } } - return parts.join("\n"); + const result = parts.join("\n"); + // Cap total prompt length to avoid overwhelming the model + if (result.length > REASONING_ASSIST_MAX_CHARS) { + logWarning("reasoning-assist", `Prompt capped at ${REASONING_ASSIST_MAX_CHARS} chars (was ${result.length})`); + return result.slice(0, REASONING_ASSIST_MAX_CHARS); + } + return result; } async function loadRelevantContext(unitType, unitId, basePath, ctx) { @@ -217,6 +225,12 @@ discrepancy. */ export function isReasoningAssistEnabled(unitType) { if (!REASONING_ASSIST_ENABLED) return false; + // Respect auto session mode — don't assist when paused + const autoSession = getAutoSession(); + if (autoSession && !autoSession.isRunning()) { + logWarning("reasoning-assist", "Skipping: auto session not running"); + return false; + } // Only enable for complex unit types const enabledTypes = [ "research-milestone", diff --git a/src/resources/extensions/sf/remote-steering.js b/src/resources/extensions/sf/remote-steering.js index 45485cd43..1ee41e8e1 100644 --- a/src/resources/extensions/sf/remote-steering.js +++ b/src/resources/extensions/sf/remote-steering.js @@ -18,6 +18,7 @@ import { RUN_CONTROL_MODES, WORK_MODES, } from "./operating-model.js"; +import { logWarning } from "./workflow-logger.js"; /** * Parse a remote answer for steering directives. @@ -139,6 +140,7 @@ function recordThrottle(source) { */ export function applyRemoteSteeringDirectives(directives, source = "default") { if (isThrottled(source)) { + logWarning("remote-steering", `Steering throttled for source: ${source}`); return directives.map((d) => ({ ...d, applied: false, diff --git a/src/resources/extensions/sf/sf-db.js b/src/resources/extensions/sf/sf-db.js index c0f64743e..f85c5d809 100644 --- a/src/resources/extensions/sf/sf-db.js +++ b/src/resources/extensions/sf/sf-db.js @@ -86,8 +86,13 @@ function createAdapter(rawDb) { /** * Execute a database query with timeout protection. * Falls back to empty result if query exceeds timeout. + * + * Purpose: Prevent hanging reads from blocking autonomous dispatch. + * + * Consumer: memory-repository.js, context-store.js, and any read query + * that needs a safety ceiling. */ -function _withQueryTimeout( +export function withQueryTimeout( operation, fallbackValue, timeoutMs = DB_QUERY_TIMEOUT_MS, @@ -1358,7 +1363,10 @@ function populateSpecTablesFromExisting(db) { `).run(now); } function migrateSchema(db) { - const row = db.prepare("SELECT MAX(version) as v FROM schema_version").get(); + const row = withQueryTimeout( + () => db.prepare("SELECT MAX(version) as v FROM schema_version").get(), + null, + ); const currentVersion = row ? row["v"] : 0; if (currentVersion >= SCHEMA_VERSION) return; // Backup database before migration so a mid-migration crash doesn't diff --git a/src/resources/extensions/sf/trajectory-command.js b/src/resources/extensions/sf/trajectory-command.js index 95dbe5e5d..3e2055094 100644 --- a/src/resources/extensions/sf/trajectory-command.js +++ b/src/resources/extensions/sf/trajectory-command.js @@ -21,7 +21,7 @@ import { * @param {object} ctx — command context * @param {string} basePath — project root */ -export async function handleTrajectory(args, ctx, _basePath) { +export async function handleTrajectory(args, ctx, basePath) { if (!isDbAvailable()) { ctx.ui.notify( "Trajectory recording requires a database connection.", @@ -31,7 +31,10 @@ export async function handleTrajectory(args, ctx, _basePath) { } const db = getDatabase(); - const sessionId = ctx.sessionManager?.getSessionId?.() || "default"; + // Resolve session from basePath-scoped DB or fall back to context + const sessionId = basePath + ? `${basePath}-session` + : ctx.sessionManager?.getSessionId?.() || "default"; // Parse flags const flags = args.split(/\s+/).filter(Boolean); diff --git a/src/resources/extensions/sf/trajectory-recorder.js b/src/resources/extensions/sf/trajectory-recorder.js index d86bc645a..fefb8e647 100644 --- a/src/resources/extensions/sf/trajectory-recorder.js +++ b/src/resources/extensions/sf/trajectory-recorder.js @@ -14,6 +14,7 @@ * - Exportable for analysis and debugging */ +import { debugLog } from "./debug-logger.js"; import { isDbAvailable } from "./sf-db.js"; import { logWarning } from "./workflow-logger.js";