From 3e52f4d66b17cc7e9f5ebd045b1868a7a95c0e72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=82CHES?= Date: Mon, 16 Mar 2026 22:29:35 -0600 Subject: [PATCH] feat: incremental memory system for auto-mode (#795) * chore: add .audits/ to .gitignore Co-Authored-By: Claude Opus 4.6 (1M context) * feat: add auto-learned project memory system Extracts durable knowledge from completed unit activity logs via background LLM calls (Haiku-preferred) and injects ranked memories into the system prompt. Includes DB schema v3 migration, memory store CRUD with confidence/hit-count ranking, secret redaction, decay, and cap enforcement. Co-Authored-By: Claude Opus 4.6 (1M context) * fix: add timestamp to UserMessage in memory extractor Co-Authored-By: Claude Opus 4.6 (1M context) * fix: update schema version assertion in md-importer test Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- .gitignore | 1 + src/resources/extensions/gsd/activity-log.ts | 8 +- src/resources/extensions/gsd/auto.ts | 46 +- src/resources/extensions/gsd/gsd-db.ts | 64 ++- src/resources/extensions/gsd/index.ts | 15 +- .../extensions/gsd/memory-extractor.ts | 352 ++++++++++++++ src/resources/extensions/gsd/memory-store.ts | 441 ++++++++++++++++++ .../extensions/gsd/tests/gsd-db.test.ts | 4 +- .../extensions/gsd/tests/md-importer.test.ts | 5 +- .../gsd/tests/memory-extractor.test.ts | 180 +++++++ .../extensions/gsd/tests/memory-store.test.ts | 345 ++++++++++++++ 11 files changed, 1447 insertions(+), 14 deletions(-) create mode 100644 src/resources/extensions/gsd/memory-extractor.ts create mode 100644 src/resources/extensions/gsd/memory-store.ts create mode 100644 src/resources/extensions/gsd/tests/memory-extractor.test.ts create mode 100644 src/resources/extensions/gsd/tests/memory-store.test.ts diff --git a/.gitignore b/.gitignore index 5a0355593..5e04ce633 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,7 @@ AGENTS.md .bg-shell/ TODOS.md .planning/ +.audits/ # ── GSD baseline (auto-generated) ── .gsd/activity/ diff --git a/src/resources/extensions/gsd/activity-log.ts b/src/resources/extensions/gsd/activity-log.ts index aa69192c6..53f792937 100644 --- a/src/resources/extensions/gsd/activity-log.ts +++ b/src/resources/extensions/gsd/activity-log.ts @@ -103,10 +103,10 @@ export function saveActivityLog( basePath: string, unitType: string, unitId: string, -): void { +): string | null { try { const entries = ctx.sessionManager.getEntries(); - if (!entries || entries.length === 0) return; + if (!entries || entries.length === 0) return null; const activityDir = join(gsdRoot(basePath), "activity"); mkdirSync(activityDir, { recursive: true }); @@ -116,7 +116,7 @@ export function saveActivityLog( const unitKey = `${unitType}\0${safeUnitId}`; // Use lightweight fingerprint instead of serializing all entries (#611) const key = snapshotKey(unitType, safeUnitId, entries); - if (state.lastSnapshotKeyByUnit.get(unitKey) === key) return; + if (state.lastSnapshotKeyByUnit.get(unitKey) === key) return null; const filePath = nextActivityFilePath(activityDir, state, unitType, safeUnitId); // Stream entries to disk line-by-line instead of building one massive string (#611). @@ -131,9 +131,11 @@ export function saveActivityLog( } state.nextSeq += 1; state.lastSnapshotKeyByUnit.set(unitKey, key); + return filePath; } catch (e) { // Don't let logging failures break auto-mode void e; + return null; } } diff --git a/src/resources/extensions/gsd/auto.ts b/src/resources/extensions/gsd/auto.ts index 8e95668b2..785f05f3a 100644 --- a/src/resources/extensions/gsd/auto.ts +++ b/src/resources/extensions/gsd/auto.ts @@ -1504,7 +1504,16 @@ export async function handleAgentEnd( if (currentUnit) { const modelId = ctx.model?.id ?? "unknown"; snapshotUnitMetrics(ctx, currentUnit.type, currentUnit.id, currentUnit.startedAt, modelId, { promptCharCount: lastPromptCharCount, baselineCharCount: lastBaselineCharCount, ...(currentUnitRouting ?? {}) }); - saveActivityLog(ctx, basePath, currentUnit.type, currentUnit.id); + const hookActivityFile = saveActivityLog(ctx, basePath, currentUnit.type, currentUnit.id); + if (hookActivityFile) { + try { + const { buildMemoryLLMCall, extractMemoriesFromUnit } = await import('./memory-extractor.js'); + const llmCallFn = buildMemoryLLMCall(ctx); + if (llmCallFn) { + extractMemoriesFromUnit(hookActivityFile, currentUnit.type, currentUnit.id, llmCallFn).catch(() => {}); + } + } catch { /* non-fatal */ } + } } currentUnit = { type: hookUnit.unitType, id: hookUnit.unitId, startedAt: hookStartedAt }; writeUnitRuntimeRecord(basePath, hookUnit.unitType, hookUnit.unitId, hookStartedAt, { @@ -1646,7 +1655,16 @@ export async function handleAgentEnd( if (currentUnit) { const modelId = ctx.model?.id ?? "unknown"; snapshotUnitMetrics(ctx, currentUnit.type, currentUnit.id, currentUnit.startedAt, modelId); - saveActivityLog(ctx, basePath, currentUnit.type, currentUnit.id); + const triageActivityFile = saveActivityLog(ctx, basePath, currentUnit.type, currentUnit.id); + if (triageActivityFile) { + try { + const { buildMemoryLLMCall, extractMemoriesFromUnit } = await import('./memory-extractor.js'); + const llmCallFn = buildMemoryLLMCall(ctx); + if (llmCallFn) { + extractMemoriesFromUnit(triageActivityFile, currentUnit.type, currentUnit.id, llmCallFn).catch(() => {}); + } + } catch { /* non-fatal */ } + } } // Dispatch triage as a new unit (early-dispatch-and-return) @@ -1724,7 +1742,16 @@ export async function handleAgentEnd( if (currentUnit) { const modelId = ctx.model?.id ?? "unknown"; snapshotUnitMetrics(ctx, currentUnit.type, currentUnit.id, currentUnit.startedAt, modelId); - saveActivityLog(ctx, basePath, currentUnit.type, currentUnit.id); + const qtActivityFile = saveActivityLog(ctx, basePath, currentUnit.type, currentUnit.id); + if (qtActivityFile) { + try { + const { buildMemoryLLMCall, extractMemoriesFromUnit } = await import('./memory-extractor.js'); + const llmCallFn = buildMemoryLLMCall(ctx); + if (llmCallFn) { + extractMemoriesFromUnit(qtActivityFile, currentUnit.type, currentUnit.id, llmCallFn).catch(() => {}); + } + } catch { /* non-fatal */ } + } } // Dispatch quick-task as a new unit @@ -2686,7 +2713,18 @@ async function dispatchNextUnit( if (currentUnit) { const modelId = ctx.model?.id ?? "unknown"; snapshotUnitMetrics(ctx, currentUnit.type, currentUnit.id, currentUnit.startedAt, modelId, { promptCharCount: lastPromptCharCount, baselineCharCount: lastBaselineCharCount, ...(currentUnitRouting ?? {}) }); - saveActivityLog(ctx, basePath, currentUnit.type, currentUnit.id); + const activityFile = saveActivityLog(ctx, basePath, currentUnit.type, currentUnit.id); + + // Fire-and-forget memory extraction from completed unit + if (activityFile) { + try { + const { buildMemoryLLMCall, extractMemoriesFromUnit } = await import('./memory-extractor.js'); + const llmCallFn = buildMemoryLLMCall(ctx); + if (llmCallFn) { + extractMemoriesFromUnit(activityFile, currentUnit.type, currentUnit.id, llmCallFn).catch(() => {}); + } + } catch { /* non-fatal */ } + } // Record routing outcome for adaptive learning if (currentUnitRouting) { diff --git a/src/resources/extensions/gsd/gsd-db.ts b/src/resources/extensions/gsd/gsd-db.ts index 22a36504f..1a1adb708 100644 --- a/src/resources/extensions/gsd/gsd-db.ts +++ b/src/resources/extensions/gsd/gsd-db.ts @@ -161,7 +161,7 @@ function openRawDb(path: string): unknown { // ─── Schema ──────────────────────────────────────────────────────────────── -const SCHEMA_VERSION = 2; +const SCHEMA_VERSION = 3; function initSchema(db: DbAdapter, fileBacked: boolean): void { // WAL mode for file-backed databases (must be outside transaction) @@ -221,9 +221,36 @@ function initSchema(db: DbAdapter, fileBacked: boolean): void { ) `); + db.exec(` + CREATE TABLE IF NOT EXISTS memories ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + id TEXT NOT NULL UNIQUE, + category TEXT NOT NULL, + content TEXT NOT NULL, + confidence REAL NOT NULL DEFAULT 0.8, + source_unit_type TEXT, + source_unit_id TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + superseded_by TEXT DEFAULT NULL, + hit_count INTEGER NOT NULL DEFAULT 0 + ) + `); + + db.exec(` + CREATE TABLE IF NOT EXISTS memory_processed_units ( + unit_key TEXT PRIMARY KEY, + activity_file TEXT, + processed_at TEXT NOT NULL + ) + `); + + db.exec('CREATE INDEX IF NOT EXISTS idx_memories_active ON memories(superseded_by)'); + // Views — DROP + CREATE since CREATE VIEW IF NOT EXISTS doesn't update definitions db.exec(`CREATE VIEW IF NOT EXISTS active_decisions AS SELECT * FROM decisions WHERE superseded_by IS NULL`); db.exec(`CREATE VIEW IF NOT EXISTS active_requirements AS SELECT * FROM requirements WHERE superseded_by IS NULL`); + db.exec(`CREATE VIEW IF NOT EXISTS active_memories AS SELECT * FROM memories WHERE superseded_by IS NULL`); // Insert schema version if not already present const existing = db.prepare('SELECT count(*) as cnt FROM schema_version').get(); @@ -274,6 +301,41 @@ function migrateSchema(db: DbAdapter): void { ); } + // v2 → v3: add memories + memory_processed_units tables + if (currentVersion < 3) { + db.exec(` + CREATE TABLE IF NOT EXISTS memories ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + id TEXT NOT NULL UNIQUE, + category TEXT NOT NULL, + content TEXT NOT NULL, + confidence REAL NOT NULL DEFAULT 0.8, + source_unit_type TEXT, + source_unit_id TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + superseded_by TEXT DEFAULT NULL, + hit_count INTEGER NOT NULL DEFAULT 0 + ) + `); + + db.exec(` + CREATE TABLE IF NOT EXISTS memory_processed_units ( + unit_key TEXT PRIMARY KEY, + activity_file TEXT, + processed_at TEXT NOT NULL + ) + `); + + db.exec('CREATE INDEX IF NOT EXISTS idx_memories_active ON memories(superseded_by)'); + db.exec('DROP VIEW IF EXISTS active_memories'); + db.exec('CREATE VIEW active_memories AS SELECT * FROM memories WHERE superseded_by IS NULL'); + + db.prepare('INSERT INTO schema_version (version, applied_at) VALUES (:version, :applied_at)').run( + { ':version': 3, ':applied_at': new Date().toISOString() }, + ); + } + db.exec('COMMIT'); } catch (err) { db.exec('ROLLBACK'); diff --git a/src/resources/extensions/gsd/index.ts b/src/resources/extensions/gsd/index.ts index 615dd6e38..f387f5f5f 100644 --- a/src/resources/extensions/gsd/index.ts +++ b/src/resources/extensions/gsd/index.ts @@ -566,6 +566,19 @@ export default function (pi: ExtensionAPI) { } } + // Inject auto-learned project memories + let memoryBlock = ""; + try { + const { getActiveMemoriesRanked, formatMemoriesForPrompt } = await import("./memory-store.js"); + const memories = getActiveMemoriesRanked(30); + if (memories.length > 0) { + const formatted = formatMemoriesForPrompt(memories, 2000); + if (formatted) { + memoryBlock = `\n\n${formatted}`; + } + } + } catch { /* non-fatal */ } + // Detect skills installed during this auto-mode session let newSkillsBlock = ""; if (hasSkillSnapshot()) { @@ -625,7 +638,7 @@ export default function (pi: ExtensionAPI) { ].join("\n"); } - const fullSystem = `${event.systemPrompt}\n\n[SYSTEM CONTEXT — GSD]\n\n${systemContent}${preferenceBlock}${agentInstructionsBlock}${knowledgeBlock}${newSkillsBlock}${worktreeBlock}`; + const fullSystem = `${event.systemPrompt}\n\n[SYSTEM CONTEXT — GSD]\n\n${systemContent}${preferenceBlock}${agentInstructionsBlock}${knowledgeBlock}${memoryBlock}${newSkillsBlock}${worktreeBlock}`; stopContextTimer({ systemPromptSize: fullSystem.length, injectionSize: injection?.length ?? 0, diff --git a/src/resources/extensions/gsd/memory-extractor.ts b/src/resources/extensions/gsd/memory-extractor.ts new file mode 100644 index 000000000..c63a385a5 --- /dev/null +++ b/src/resources/extensions/gsd/memory-extractor.ts @@ -0,0 +1,352 @@ +// GSD Memory Extractor — Background LLM extraction from activity logs +// +// After each unit completes, extracts durable knowledge from the session +// transcript and stores it as memory entries. One extraction at a time +// (mutex guard). Fire-and-forget — never blocks auto-mode. + +import { readFileSync, statSync } from 'node:fs'; +import type { ExtensionContext } from '@gsd/pi-coding-agent'; +import type { Api, AssistantMessage, Model } from '@gsd/pi-ai'; +import { + getActiveMemories, + isUnitProcessed, + markUnitProcessed, + applyMemoryActions, + decayStaleMemories, +} from './memory-store.js'; +import type { MemoryAction } from './memory-store.js'; + +// ─── Types ────────────────────────────────────────────────────────────────── + +export type LLMCallFn = (system: string, user: string) => Promise; + +// ─── Concurrency Guard ────────────────────────────────────────────────────── + +let _extracting = false; +let _lastExtractionTime = 0; + +const MIN_EXTRACTION_INTERVAL_MS = 30_000; + +// ─── Skip Conditions ──────────────────────────────────────────────────────── + +const SKIP_TYPES = new Set([ + 'complete-slice', + 'rewrite-docs', + 'triage-captures', +]); + +const MIN_ACTIVITY_SIZE = 1024; // 1KB + +// ─── Secret Redaction ─────────────────────────────────────────────────────── + +const SECRET_PATTERNS = [ + /(?:sk|pk|api[_-]?key|token|secret|password|credential|auth)[_-]?\w*[\s:=]+['"]?[\w\-./+=]{20,}['"]?/gi, + /AKIA[0-9A-Z]{16}/g, + /gh[pousr]_[A-Za-z0-9_]{36,}/g, + /[rsp]k_(?:live|test)_[A-Za-z0-9]{20,}/g, + /eyJ[A-Za-z0-9_-]{20,}\.eyJ[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]+/g, + /-----BEGIN (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----[\s\S]*?-----END (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----/g, + /(?:Bearer\s+)[A-Za-z0-9\-._~+/]+=*/gi, + /npm_[A-Za-z0-9]{36,}/g, + /sk-ant-[A-Za-z0-9\-_]{20,}/g, + /sk-[A-Za-z0-9]{40,}/g, +]; + +function redactSecrets(text: string): string { + let result = text; + for (const pattern of SECRET_PATTERNS) { + // Reset lastIndex for global regexes + pattern.lastIndex = 0; + result = result.replace(pattern, '[REDACTED]'); + } + return result; +} + +// ─── Model Selection ──────────────────────────────────────────────────────── + +/** + * Build an LLM call function using the cheapest available model (preferring Haiku). + * Returns null if no models available. + */ +export function buildMemoryLLMCall(ctx: ExtensionContext): LLMCallFn | null { + try { + const available = ctx.modelRegistry.getAvailable(); + if (!available || available.length === 0) return null; + + // Prefer Haiku by ID substring match + let model = available.find(m => + m.id.toLowerCase().includes('haiku'), + ); + + // Fallback: cheapest by input cost + if (!model) { + model = [...available].sort((a, b) => a.cost.input - b.cost.input)[0]; + } + + if (!model) return null; + + const selectedModel = model as Model; + + return async (system: string, user: string): Promise => { + const { completeSimple } = await import('@gsd/pi-ai'); + const result: AssistantMessage = await completeSimple(selectedModel, { + systemPrompt: system, + messages: [{ role: 'user', content: [{ type: 'text', text: user }], timestamp: Date.now() }], + }, { + maxTokens: 2048, + temperature: 0, + }); + + // Extract text from response + const textParts = result.content + .filter((c): c is { type: 'text'; text: string } => c.type === 'text') + .map(c => c.text); + return textParts.join(''); + }; + } catch { + return null; + } +} + +// ─── Extraction Prompts ───────────────────────────────────────────────────── + +const EXTRACTION_SYSTEM = `You are a memory extraction agent for a software project. Analyze the session +transcript and identify durable knowledge worth remembering for future sessions. + +Categories: architecture, convention, gotcha, preference, environment, pattern + +Actions (return JSON array): +- CREATE: {"action": "CREATE", "category": "", "content": "", "confidence": <0.6-0.95>} +- UPDATE: {"action": "UPDATE", "id": "", "content": ""} +- REINFORCE: {"action": "REINFORCE", "id": ""} +- SUPERSEDE: {"action": "SUPERSEDE", "id": "", "superseded_by": ""} + +Rules: +- Don't create memories for one-off bug fixes or temporary state +- Don't duplicate existing memories — use REINFORCE or UPDATE +- Keep content to 1-3 sentences +- Confidence: 0.6 tentative, 0.8 solid, 0.95 well-confirmed +- Prefer fewer high-quality memories over many low-quality ones +- Return empty array [] if nothing worth remembering +- NEVER include secrets, API keys, or passwords + +Return ONLY a valid JSON array.`; + +function buildExtractionUserPrompt( + unitType: string, + unitId: string, + existingMemories: { id: string; category: string; content: string }[], + transcript: string, +): string { + let memoriesSection: string; + if (existingMemories.length === 0) { + memoriesSection = '(none yet)'; + } else { + memoriesSection = existingMemories + .map((m, i) => `${i + 1}. [${m.id}] (${m.category}) ${m.content}`) + .join('\n'); + } + + return `## Current Active Memories\n${memoriesSection}\n\n## Session Transcript (${unitType}: ${unitId})\n${transcript}`; +} + +// ─── Activity JSONL Parsing ───────────────────────────────────────────────── + +/** + * Extract assistant message text from activity JSONL. + * Returns concatenated text content from assistant role entries. + */ +function extractTranscriptFromActivity(raw: string, maxChars = 30_000): string { + const lines = raw.split('\n'); + const parts: string[] = []; + let totalChars = 0; + + for (const line of lines) { + if (!line.trim()) continue; + try { + const entry = JSON.parse(line); + if (entry.role !== 'assistant') continue; + + // Handle content array or direct text + if (Array.isArray(entry.content)) { + for (const block of entry.content) { + if (block.type === 'text' && block.text) { + const text = block.text; + if (totalChars + text.length > maxChars) { + parts.push(text.substring(0, maxChars - totalChars)); + return parts.join('\n\n'); + } + parts.push(text); + totalChars += text.length; + } + } + } else if (typeof entry.content === 'string') { + const text = entry.content; + if (totalChars + text.length > maxChars) { + parts.push(text.substring(0, maxChars - totalChars)); + return parts.join('\n\n'); + } + parts.push(text); + totalChars += text.length; + } + } catch { + // Skip malformed lines + } + } + + return parts.join('\n\n'); +} + +// ─── Response Parsing ─────────────────────────────────────────────────────── + +/** + * Parse the LLM response into memory actions. + * Strips markdown fences, validates required fields. + * Returns [] on any parse failure. + */ +export function parseMemoryResponse(raw: string): MemoryAction[] { + try { + // Strip markdown code fences + let cleaned = raw.trim(); + if (cleaned.startsWith('```')) { + cleaned = cleaned.replace(/^```(?:json)?\s*\n?/, '').replace(/\n?```\s*$/, ''); + } + + const parsed = JSON.parse(cleaned); + if (!Array.isArray(parsed)) return []; + + const actions: MemoryAction[] = []; + for (const item of parsed) { + if (!item || typeof item !== 'object' || !item.action) continue; + + switch (item.action) { + case 'CREATE': + if (typeof item.category === 'string' && typeof item.content === 'string') { + actions.push({ + action: 'CREATE', + category: item.category, + content: item.content, + confidence: typeof item.confidence === 'number' ? item.confidence : undefined, + }); + } + break; + case 'UPDATE': + if (typeof item.id === 'string' && typeof item.content === 'string') { + actions.push({ + action: 'UPDATE', + id: item.id, + content: item.content, + confidence: typeof item.confidence === 'number' ? item.confidence : undefined, + }); + } + break; + case 'REINFORCE': + if (typeof item.id === 'string') { + actions.push({ action: 'REINFORCE', id: item.id }); + } + break; + case 'SUPERSEDE': + if (typeof item.id === 'string' && typeof item.superseded_by === 'string') { + actions.push({ + action: 'SUPERSEDE', + id: item.id, + superseded_by: item.superseded_by, + }); + } + break; + } + } + + return actions; + } catch { + return []; + } +} + +// ─── Main Extraction Function ─────────────────────────────────────────────── + +/** + * Extract memories from a completed unit's activity log. + * Fire-and-forget — never throws, mutex-guarded, respects rate limiting. + */ +export async function extractMemoriesFromUnit( + activityFile: string, + unitType: string, + unitId: string, + llmCallFn: LLMCallFn, +): Promise { + // Mutex guard + if (_extracting) return; + + // Rate limit + const now = Date.now(); + if (now - _lastExtractionTime < MIN_EXTRACTION_INTERVAL_MS) return; + + // Skip certain unit types + if (SKIP_TYPES.has(unitType)) return; + + const unitKey = `${unitType}/${unitId}`; + + // Already processed + if (isUnitProcessed(unitKey)) return; + + // Check file size + try { + const stat = statSync(activityFile); + if (stat.size < MIN_ACTIVITY_SIZE) return; + } catch { + return; + } + + _extracting = true; + _lastExtractionTime = now; + + try { + // Read and parse activity file + const raw = readFileSync(activityFile, 'utf-8'); + const transcript = extractTranscriptFromActivity(raw); + if (!transcript.trim()) return; + + // Redact secrets + const safeTranscript = redactSecrets(transcript); + + // Get current memories for context + const activeMemories = getActiveMemories().map(m => ({ + id: m.id, + category: m.category, + content: m.content, + })); + + // Build prompts + const userPrompt = buildExtractionUserPrompt(unitType, unitId, activeMemories, safeTranscript); + + // Call LLM + const response = await llmCallFn(EXTRACTION_SYSTEM, userPrompt); + + // Parse response + const actions = parseMemoryResponse(response); + + // Apply actions + if (actions.length > 0) { + applyMemoryActions(actions, unitType, unitId); + } + + // Decay stale memories periodically + decayStaleMemories(20); + + // Mark unit as processed + markUnitProcessed(unitKey, activityFile); + } catch { + // Non-fatal — memory extraction failure should never affect auto-mode + } finally { + _extracting = false; + } +} + +// ─── Testing Helpers ──────────────────────────────────────────────────────── + +/** Reset extraction state (testing only). */ +export function _resetExtractionState(): void { + _extracting = false; + _lastExtractionTime = 0; +} diff --git a/src/resources/extensions/gsd/memory-store.ts b/src/resources/extensions/gsd/memory-store.ts new file mode 100644 index 000000000..b7d0094d4 --- /dev/null +++ b/src/resources/extensions/gsd/memory-store.ts @@ -0,0 +1,441 @@ +// GSD Memory Store — CRUD, ranked queries, maintenance, and prompt formatting +// +// Storage layer for auto-learned project memories. Follows context-store.ts patterns. +// All functions degrade gracefully: return empty results when DB unavailable, never throw. + +import { isDbAvailable, _getAdapter, transaction } from './gsd-db.js'; + +// ─── Types ────────────────────────────────────────────────────────────────── + +export interface Memory { + seq: number; + id: string; + category: string; + content: string; + confidence: number; + source_unit_type: string | null; + source_unit_id: string | null; + created_at: string; + updated_at: string; + superseded_by: string | null; + hit_count: number; +} + +export type MemoryActionCreate = { + action: 'CREATE'; + category: string; + content: string; + confidence?: number; +}; + +export type MemoryActionUpdate = { + action: 'UPDATE'; + id: string; + content: string; + confidence?: number; +}; + +export type MemoryActionReinforce = { + action: 'REINFORCE'; + id: string; +}; + +export type MemoryActionSupersede = { + action: 'SUPERSEDE'; + id: string; + superseded_by: string; +}; + +export type MemoryAction = + | MemoryActionCreate + | MemoryActionUpdate + | MemoryActionReinforce + | MemoryActionSupersede; + +// ─── Category Display Order ───────────────────────────────────────────────── + +const CATEGORY_PRIORITY: Record = { + gotcha: 0, + convention: 1, + architecture: 2, + pattern: 3, + environment: 4, + preference: 5, +}; + +// ─── Row Mapping ──────────────────────────────────────────────────────────── + +function rowToMemory(row: Record): Memory { + return { + seq: row['seq'] as number, + id: row['id'] as string, + category: row['category'] as string, + content: row['content'] as string, + confidence: row['confidence'] as number, + source_unit_type: (row['source_unit_type'] as string) ?? null, + source_unit_id: (row['source_unit_id'] as string) ?? null, + created_at: row['created_at'] as string, + updated_at: row['updated_at'] as string, + superseded_by: (row['superseded_by'] as string) ?? null, + hit_count: row['hit_count'] as number, + }; +} + +// ─── Query Functions ──────────────────────────────────────────────────────── + +/** + * Get all memories where superseded_by IS NULL. + * Returns [] if DB is not available. Never throws. + */ +export function getActiveMemories(): Memory[] { + if (!isDbAvailable()) return []; + const adapter = _getAdapter(); + if (!adapter) return []; + + try { + const rows = adapter.prepare('SELECT * FROM memories WHERE superseded_by IS NULL').all(); + return rows.map(rowToMemory); + } catch { + return []; + } +} + +/** + * Get active memories ordered by ranking score: confidence * (1 + hit_count * 0.1). + * Higher-scored memories are more relevant and frequently confirmed. + */ +export function getActiveMemoriesRanked(limit = 30): Memory[] { + if (!isDbAvailable()) return []; + const adapter = _getAdapter(); + if (!adapter) return []; + + try { + const rows = adapter.prepare( + `SELECT * FROM memories + WHERE superseded_by IS NULL + ORDER BY (confidence * (1.0 + hit_count * 0.1)) DESC + LIMIT :limit`, + ).all({ ':limit': limit }); + return rows.map(rowToMemory); + } catch { + return []; + } +} + +/** + * Generate the next memory ID: MEM + zero-padded 3-digit from MAX(seq). + * Returns MEM001 if no memories exist. + */ +export function nextMemoryId(): string { + if (!isDbAvailable()) return 'MEM001'; + const adapter = _getAdapter(); + if (!adapter) return 'MEM001'; + + try { + const row = adapter + .prepare('SELECT MAX(seq) as max_seq FROM memories') + .get(); + const maxSeq = row ? (row['max_seq'] as number | null) : null; + if (maxSeq == null || isNaN(maxSeq)) return 'MEM001'; + const next = maxSeq + 1; + return `MEM${String(next).padStart(3, '0')}`; + } catch { + return 'MEM001'; + } +} + +// ─── Mutation Functions ───────────────────────────────────────────────────── + +/** + * Insert a new memory with auto-assigned ID. + * Returns the assigned ID, or null on failure. + */ +export function createMemory(fields: { + category: string; + content: string; + confidence?: number; + source_unit_type?: string; + source_unit_id?: string; +}): string | null { + if (!isDbAvailable()) return null; + const adapter = _getAdapter(); + if (!adapter) return null; + + try { + const id = nextMemoryId(); + const now = new Date().toISOString(); + adapter.prepare( + `INSERT INTO memories (id, category, content, confidence, source_unit_type, source_unit_id, created_at, updated_at) + VALUES (:id, :category, :content, :confidence, :source_unit_type, :source_unit_id, :created_at, :updated_at)`, + ).run({ + ':id': id, + ':category': fields.category, + ':content': fields.content, + ':confidence': fields.confidence ?? 0.8, + ':source_unit_type': fields.source_unit_type ?? null, + ':source_unit_id': fields.source_unit_id ?? null, + ':created_at': now, + ':updated_at': now, + }); + return id; + } catch { + return null; + } +} + +/** + * Update a memory's content and optionally its confidence. + */ +export function updateMemoryContent(id: string, content: string, confidence?: number): boolean { + if (!isDbAvailable()) return false; + const adapter = _getAdapter(); + if (!adapter) return false; + + try { + const now = new Date().toISOString(); + if (confidence != null) { + adapter.prepare( + 'UPDATE memories SET content = :content, confidence = :confidence, updated_at = :updated_at WHERE id = :id', + ).run({ ':content': content, ':confidence': confidence, ':updated_at': now, ':id': id }); + } else { + adapter.prepare( + 'UPDATE memories SET content = :content, updated_at = :updated_at WHERE id = :id', + ).run({ ':content': content, ':updated_at': now, ':id': id }); + } + return true; + } catch { + return false; + } +} + +/** + * Reinforce a memory: increment hit_count, update timestamp. + */ +export function reinforceMemory(id: string): boolean { + if (!isDbAvailable()) return false; + const adapter = _getAdapter(); + if (!adapter) return false; + + try { + adapter.prepare( + 'UPDATE memories SET hit_count = hit_count + 1, updated_at = :updated_at WHERE id = :id', + ).run({ ':updated_at': new Date().toISOString(), ':id': id }); + return true; + } catch { + return false; + } +} + +/** + * Mark a memory as superseded by another. + */ +export function supersedeMemory(oldId: string, newId: string): boolean { + if (!isDbAvailable()) return false; + const adapter = _getAdapter(); + if (!adapter) return false; + + try { + adapter.prepare( + 'UPDATE memories SET superseded_by = :new_id, updated_at = :updated_at WHERE id = :old_id', + ).run({ ':new_id': newId, ':updated_at': new Date().toISOString(), ':old_id': oldId }); + return true; + } catch { + return false; + } +} + +// ─── Processed Unit Tracking ──────────────────────────────────────────────── + +/** + * Check if a unit has already been processed for memory extraction. + */ +export function isUnitProcessed(unitKey: string): boolean { + if (!isDbAvailable()) return false; + const adapter = _getAdapter(); + if (!adapter) return false; + + try { + const row = adapter.prepare( + 'SELECT 1 FROM memory_processed_units WHERE unit_key = :key', + ).get({ ':key': unitKey }); + return row != null; + } catch { + return false; + } +} + +/** + * Record that a unit has been processed for memory extraction. + */ +export function markUnitProcessed(unitKey: string, activityFile: string): boolean { + if (!isDbAvailable()) return false; + const adapter = _getAdapter(); + if (!adapter) return false; + + try { + adapter.prepare( + `INSERT OR IGNORE INTO memory_processed_units (unit_key, activity_file, processed_at) + VALUES (:key, :file, :at)`, + ).run({ ':key': unitKey, ':file': activityFile, ':at': new Date().toISOString() }); + return true; + } catch { + return false; + } +} + +// ─── Maintenance ──────────────────────────────────────────────────────────── + +/** + * Reduce confidence for memories not updated within the last N processed units. + * "Stale" = updated_at is older than the Nth most recent processed_at. + */ +export function decayStaleMemories(thresholdUnits = 20): void { + if (!isDbAvailable()) return; + const adapter = _getAdapter(); + if (!adapter) return; + + try { + // Find the timestamp of the Nth most recent processed unit + const row = adapter.prepare( + `SELECT processed_at FROM memory_processed_units + ORDER BY processed_at DESC + LIMIT 1 OFFSET :offset`, + ).get({ ':offset': thresholdUnits - 1 }); + + if (!row) return; // not enough processed units yet + + const cutoff = row['processed_at'] as string; + adapter.prepare( + `UPDATE memories + SET confidence = MAX(0.1, confidence - 0.1), updated_at = :now + WHERE superseded_by IS NULL AND updated_at < :cutoff AND confidence > 0.1`, + ).run({ ':now': new Date().toISOString(), ':cutoff': cutoff }); + } catch { + // non-fatal + } +} + +/** + * Supersede lowest-ranked memories when count exceeds cap. + */ +export function enforceMemoryCap(max = 50): void { + if (!isDbAvailable()) return; + const adapter = _getAdapter(); + if (!adapter) return; + + try { + const countRow = adapter.prepare( + 'SELECT count(*) as cnt FROM memories WHERE superseded_by IS NULL', + ).get(); + const count = (countRow?.['cnt'] as number) ?? 0; + if (count <= max) return; + + const excess = count - max; + // Find the IDs of the lowest-ranked active memories + const rows = adapter.prepare( + `SELECT id FROM memories + WHERE superseded_by IS NULL + ORDER BY (confidence * (1.0 + hit_count * 0.1)) ASC + LIMIT :limit`, + ).all({ ':limit': excess }); + + const now = new Date().toISOString(); + for (const row of rows) { + adapter.prepare( + 'UPDATE memories SET superseded_by = :reason, updated_at = :now WHERE id = :id', + ).run({ ':reason': 'CAP_EXCEEDED', ':now': now, ':id': row['id'] as string }); + } + } catch { + // non-fatal + } +} + +// ─── Action Application ───────────────────────────────────────────────────── + +/** + * Process an array of memory actions in a transaction. + * Calls enforceMemoryCap at the end. + */ +export function applyMemoryActions( + actions: MemoryAction[], + unitType?: string, + unitId?: string, +): void { + if (!isDbAvailable() || actions.length === 0) return; + + try { + transaction(() => { + for (const action of actions) { + switch (action.action) { + case 'CREATE': + createMemory({ + category: action.category, + content: action.content, + confidence: action.confidence, + source_unit_type: unitType, + source_unit_id: unitId, + }); + break; + case 'UPDATE': + updateMemoryContent(action.id, action.content, action.confidence); + break; + case 'REINFORCE': + reinforceMemory(action.id); + break; + case 'SUPERSEDE': + supersedeMemory(action.id, action.superseded_by); + break; + } + } + enforceMemoryCap(); + }); + } catch { + // non-fatal — transaction will have rolled back + } +} + +// ─── Prompt Formatting ────────────────────────────────────────────────────── + +/** + * Format memories as categorized markdown for system prompt injection. + * Truncates to token budget (~4 chars per token). + */ +export function formatMemoriesForPrompt(memories: Memory[], tokenBudget = 2000): string { + if (memories.length === 0) return ''; + + const charBudget = tokenBudget * 4; + const header = '## Project Memory (auto-learned)\n'; + let output = header; + let remaining = charBudget - header.length; + + // Group by category + const grouped = new Map(); + for (const m of memories) { + const list = grouped.get(m.category) ?? []; + list.push(m); + grouped.set(m.category, list); + } + + // Sort categories by priority + const sortedCategories = [...grouped.keys()].sort( + (a, b) => (CATEGORY_PRIORITY[a] ?? 99) - (CATEGORY_PRIORITY[b] ?? 99), + ); + + for (const category of sortedCategories) { + const items = grouped.get(category)!; + const catHeader = `\n### ${category.charAt(0).toUpperCase() + category.slice(1)}\n`; + + if (remaining < catHeader.length + 10) break; + output += catHeader; + remaining -= catHeader.length; + + for (const item of items) { + const bullet = `- ${item.content}\n`; + if (remaining < bullet.length) break; + output += bullet; + remaining -= bullet.length; + } + } + + return output.trimEnd(); +} diff --git a/src/resources/extensions/gsd/tests/gsd-db.test.ts b/src/resources/extensions/gsd/tests/gsd-db.test.ts index 872f125c7..7d4053c58 100644 --- a/src/resources/extensions/gsd/tests/gsd-db.test.ts +++ b/src/resources/extensions/gsd/tests/gsd-db.test.ts @@ -65,8 +65,8 @@ console.log('\n=== gsd-db: fresh DB schema init (memory) ==='); // Check schema_version table const adapter = _getAdapter()!; - const version = adapter.prepare('SELECT version FROM schema_version').get(); - assertEq(version?.['version'], 2, 'schema version should be 2'); + const version = adapter.prepare('SELECT MAX(version) as version FROM schema_version').get(); + assertEq(version?.['version'], 3, 'schema version should be 3'); // Check tables exist by querying them const dRows = adapter.prepare('SELECT count(*) as cnt FROM decisions').get(); diff --git a/src/resources/extensions/gsd/tests/md-importer.test.ts b/src/resources/extensions/gsd/tests/md-importer.test.ts index a91844e59..7cdb49d1a 100644 --- a/src/resources/extensions/gsd/tests/md-importer.test.ts +++ b/src/resources/extensions/gsd/tests/md-importer.test.ts @@ -350,12 +350,11 @@ console.log('=== md-importer: missing file handling ==='); console.log('=== md-importer: schema v1→v2 migration ==='); { - // This test verifies that opening a v1 DB auto-migrates to v2 - // (The actual migration is tested via the gsd-db.test.ts schema version assertion = 2) + // This test verifies that opening a fresh DB auto-migrates to current schema version openDatabase(':memory:'); const adapter = _getAdapter(); const version = adapter?.prepare('SELECT MAX(version) as v FROM schema_version').get(); - assertEq(version?.v, 2, 'new DB should be at schema version 2'); + assertEq(version?.v, 3, 'new DB should be at schema version 3'); // Artifacts table should exist const tableCheck = adapter?.prepare("SELECT count(*) as c FROM sqlite_master WHERE type='table' AND name='artifacts'").get(); diff --git a/src/resources/extensions/gsd/tests/memory-extractor.test.ts b/src/resources/extensions/gsd/tests/memory-extractor.test.ts new file mode 100644 index 000000000..a4e4f7031 --- /dev/null +++ b/src/resources/extensions/gsd/tests/memory-extractor.test.ts @@ -0,0 +1,180 @@ +import { createTestContext } from './test-helpers.ts'; +import { parseMemoryResponse, _resetExtractionState } from '../memory-extractor.ts'; +import { + openDatabase, + closeDatabase, +} from '../gsd-db.ts'; +import { + getActiveMemories, + applyMemoryActions, + getActiveMemoriesRanked, +} from '../memory-store.ts'; +import type { MemoryAction } from '../memory-store.ts'; + +const { assertEq, assertTrue, report } = createTestContext(); + +// ═══════════════════════════════════════════════════════════════════════════ +// memory-extractor: parse valid JSON response +// ═══════════════════════════════════════════════════════════════════════════ + +console.log('\n=== memory-extractor: parse valid JSON ==='); +{ + const response = JSON.stringify([ + { action: 'CREATE', category: 'gotcha', content: 'esbuild drops binaries', confidence: 0.85 }, + { action: 'REINFORCE', id: 'MEM001' }, + { action: 'UPDATE', id: 'MEM002', content: 'revised content' }, + { action: 'SUPERSEDE', id: 'MEM003', superseded_by: 'MEM004' }, + ]); + + const actions = parseMemoryResponse(response); + assertEq(actions.length, 4, 'should parse 4 actions'); + assertEq(actions[0].action, 'CREATE', 'first action should be CREATE'); + assertEq((actions[0] as any).category, 'gotcha', 'CREATE category'); + assertEq((actions[0] as any).confidence, 0.85, 'CREATE confidence'); + assertEq(actions[1].action, 'REINFORCE', 'second action should be REINFORCE'); + assertEq(actions[2].action, 'UPDATE', 'third action should be UPDATE'); + assertEq(actions[3].action, 'SUPERSEDE', 'fourth action should be SUPERSEDE'); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// memory-extractor: parse fenced JSON response +// ═══════════════════════════════════════════════════════════════════════════ + +console.log('\n=== memory-extractor: parse fenced JSON ==='); +{ + const response = '```json\n[\n {"action": "CREATE", "category": "convention", "content": "test memory"}\n]\n```'; + + const actions = parseMemoryResponse(response); + assertEq(actions.length, 1, 'should parse 1 action from fenced JSON'); + assertEq(actions[0].action, 'CREATE', 'action should be CREATE'); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// memory-extractor: parse empty array response +// ═══════════════════════════════════════════════════════════════════════════ + +console.log('\n=== memory-extractor: parse empty array ==='); +{ + const actions = parseMemoryResponse('[]'); + assertEq(actions.length, 0, 'empty array should parse to empty actions'); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// memory-extractor: parse malformed response +// ═══════════════════════════════════════════════════════════════════════════ + +console.log('\n=== memory-extractor: malformed responses ==='); +{ + assertEq(parseMemoryResponse('not json at all'), [], 'garbage text should return []'); + assertEq(parseMemoryResponse('{"action": "CREATE"}'), [], 'non-array should return []'); + assertEq(parseMemoryResponse(''), [], 'empty string should return []'); + assertEq(parseMemoryResponse('```\nbroken\n```'), [], 'fenced non-JSON should return []'); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// memory-extractor: validation of required fields +// ═══════════════════════════════════════════════════════════════════════════ + +console.log('\n=== memory-extractor: field validation ==='); +{ + const response = JSON.stringify([ + // Valid CREATE + { action: 'CREATE', category: 'gotcha', content: 'valid' }, + // Invalid CREATE — missing content + { action: 'CREATE', category: 'gotcha' }, + // Invalid CREATE — missing category + { action: 'CREATE', content: 'no category' }, + // Valid REINFORCE + { action: 'REINFORCE', id: 'MEM001' }, + // Invalid REINFORCE — missing id + { action: 'REINFORCE' }, + // Valid UPDATE + { action: 'UPDATE', id: 'MEM002', content: 'new content' }, + // Invalid UPDATE — missing content + { action: 'UPDATE', id: 'MEM002' }, + // Valid SUPERSEDE + { action: 'SUPERSEDE', id: 'MEM001', superseded_by: 'MEM002' }, + // Invalid SUPERSEDE — missing superseded_by + { action: 'SUPERSEDE', id: 'MEM001' }, + // Unknown action + { action: 'DELETE', id: 'MEM001' }, + // Null entry + null, + ]); + + const actions = parseMemoryResponse(response); + assertEq(actions.length, 4, 'should only accept 4 valid actions'); + assertEq(actions[0].action, 'CREATE', 'first valid is CREATE'); + assertEq(actions[1].action, 'REINFORCE', 'second valid is REINFORCE'); + assertEq(actions[2].action, 'UPDATE', 'third valid is UPDATE'); + assertEq(actions[3].action, 'SUPERSEDE', 'fourth valid is SUPERSEDE'); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// Integration: applyMemoryActions with mixed actions +// ═══════════════════════════════════════════════════════════════════════════ + +console.log('\n=== integration: mixed action lifecycle ==='); +{ + openDatabase(':memory:'); + + // Phase 1: Create initial memories + applyMemoryActions([ + { action: 'CREATE', category: 'gotcha', content: 'npm run build needs tsc first', confidence: 0.7 }, + { action: 'CREATE', category: 'convention', content: 'all DB queries use named params', confidence: 0.8 }, + { action: 'CREATE', category: 'architecture', content: 'extensions loaded from two paths', confidence: 0.85 }, + ], 'plan-slice', 'M001/S01'); + + let active = getActiveMemoriesRanked(30); + assertEq(active.length, 3, 'phase 1: 3 active memories'); + + // Phase 2: Reinforce one, update another, create new + applyMemoryActions([ + { action: 'REINFORCE', id: 'MEM002' }, + { action: 'UPDATE', id: 'MEM001', content: 'npm run build requires tsc --noEmit first' }, + { action: 'CREATE', category: 'pattern', content: 'use INSERT OR IGNORE for idempotency', confidence: 0.75 }, + ], 'execute-task', 'M001/S01/T01'); + + active = getActiveMemoriesRanked(30); + assertEq(active.length, 4, 'phase 2: 4 active memories'); + assertEq( + active.find(m => m.id === 'MEM001')?.content, + 'npm run build requires tsc --noEmit first', + 'MEM001 content should be updated', + ); + assertEq(active.find(m => m.id === 'MEM002')?.hit_count, 1, 'MEM002 should be reinforced'); + + // Phase 3: Supersede MEM001 with MEM005 + applyMemoryActions([ + { action: 'CREATE', category: 'gotcha', content: 'build script handles tsc automatically now', confidence: 0.9 }, + { action: 'SUPERSEDE', id: 'MEM001', superseded_by: 'MEM005' }, + ], 'execute-task', 'M001/S01/T02'); + + active = getActiveMemoriesRanked(30); + assertEq(active.length, 4, 'phase 3: 4 active (1 superseded, 1 created)'); + assertTrue(!active.find(m => m.id === 'MEM001'), 'MEM001 should be superseded'); + assertTrue(!!active.find(m => m.id === 'MEM005'), 'MEM005 should be active'); + + // Verify ranking: MEM003 (0.85) > MEM005 (0.9) but MEM002 has 1 hit + // MEM002: 0.8 * (1 + 1*0.1) = 0.88 + // MEM003: 0.85 * 1.0 = 0.85 + // MEM005: 0.9 * 1.0 = 0.9 + // MEM004: 0.75 * 1.0 = 0.75 + assertEq(active[0].id, 'MEM005', 'MEM005 should rank first (0.9)'); + assertEq(active[1].id, 'MEM002', 'MEM002 should rank second (0.88)'); + + closeDatabase(); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// memory-extractor: _resetExtractionState +// ═══════════════════════════════════════════════════════════════════════════ + +console.log('\n=== memory-extractor: reset extraction state ==='); +{ + // Just verify it doesn't throw + _resetExtractionState(); + assertTrue(true, '_resetExtractionState should not throw'); +} + +report(); diff --git a/src/resources/extensions/gsd/tests/memory-store.test.ts b/src/resources/extensions/gsd/tests/memory-store.test.ts new file mode 100644 index 000000000..5ba71f732 --- /dev/null +++ b/src/resources/extensions/gsd/tests/memory-store.test.ts @@ -0,0 +1,345 @@ +import { createTestContext } from './test-helpers.ts'; +import { + openDatabase, + closeDatabase, + isDbAvailable, + _getAdapter, +} from '../gsd-db.ts'; +import { + getActiveMemories, + getActiveMemoriesRanked, + nextMemoryId, + createMemory, + updateMemoryContent, + reinforceMemory, + supersedeMemory, + isUnitProcessed, + markUnitProcessed, + decayStaleMemories, + enforceMemoryCap, + applyMemoryActions, + formatMemoriesForPrompt, +} from '../memory-store.ts'; +import type { MemoryAction } from '../memory-store.ts'; + +const { assertEq, assertTrue, assertMatch, report } = createTestContext(); + +// ═══════════════════════════════════════════════════════════════════════════ +// memory-store: fallback when DB not open +// ═══════════════════════════════════════════════════════════════════════════ + +console.log('\n=== memory-store: fallback returns empty when DB not open ==='); +{ + closeDatabase(); + assertTrue(!isDbAvailable(), 'DB should not be available'); + + assertEq(getActiveMemories(), [], 'getActiveMemories returns [] when DB closed'); + assertEq(getActiveMemoriesRanked(), [], 'getActiveMemoriesRanked returns [] when DB closed'); + assertEq(nextMemoryId(), 'MEM001', 'nextMemoryId returns MEM001 when DB closed'); + assertEq(createMemory({ category: 'test', content: 'test' }), null, 'createMemory returns null when DB closed'); + assertTrue(!reinforceMemory('MEM001'), 'reinforceMemory returns false when DB closed'); + assertTrue(!isUnitProcessed('test/key'), 'isUnitProcessed returns false when DB closed'); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// memory-store: CRUD operations +// ═══════════════════════════════════════════════════════════════════════════ + +console.log('\n=== memory-store: create and query memories ==='); +{ + openDatabase(':memory:'); + + // Create memories + const id1 = createMemory({ category: 'gotcha', content: 'esbuild drops .node binaries' }); + assertTrue(id1 !== null, 'createMemory should return an ID'); + assertEq(id1, 'MEM001', 'first memory ID should be MEM001'); + + const id2 = createMemory({ category: 'convention', content: 'use :memory: for tests', confidence: 0.9 }); + assertEq(id2, 'MEM002', 'second memory ID should be MEM002'); + + const id3 = createMemory({ category: 'architecture', content: 'extensions discovered from src/resources/' }); + assertEq(id3, 'MEM003', 'third memory ID should be MEM003'); + + // Query all active + const active = getActiveMemories(); + assertEq(active.length, 3, 'should have 3 active memories'); + assertEq(active[0].category, 'gotcha', 'first memory category'); + assertEq(active[0].content, 'esbuild drops .node binaries', 'first memory content'); + assertEq(active[1].confidence, 0.9, 'second memory confidence'); + + closeDatabase(); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// memory-store: update and reinforce +// ═══════════════════════════════════════════════════════════════════════════ + +console.log('\n=== memory-store: update and reinforce ==='); +{ + openDatabase(':memory:'); + + createMemory({ category: 'gotcha', content: 'original content' }); + + // Update content + const updated = updateMemoryContent('MEM001', 'revised content', 0.95); + assertTrue(updated, 'updateMemoryContent should return true'); + + const active = getActiveMemories(); + assertEq(active[0].content, 'revised content', 'content should be updated'); + assertEq(active[0].confidence, 0.95, 'confidence should be updated'); + + // Reinforce + const reinforced = reinforceMemory('MEM001'); + assertTrue(reinforced, 'reinforceMemory should return true'); + + const after = getActiveMemories(); + assertEq(after[0].hit_count, 1, 'hit_count should be 1 after reinforce'); + + // Reinforce again + reinforceMemory('MEM001'); + const after2 = getActiveMemories(); + assertEq(after2[0].hit_count, 2, 'hit_count should be 2 after second reinforce'); + + closeDatabase(); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// memory-store: supersede +// ═══════════════════════════════════════════════════════════════════════════ + +console.log('\n=== memory-store: supersede ==='); +{ + openDatabase(':memory:'); + + createMemory({ category: 'convention', content: 'old convention' }); + createMemory({ category: 'convention', content: 'new convention' }); + + supersedeMemory('MEM001', 'MEM002'); + + const active = getActiveMemories(); + assertEq(active.length, 1, 'should have 1 active memory after supersede'); + assertEq(active[0].id, 'MEM002', 'active memory should be MEM002'); + + closeDatabase(); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// memory-store: ranked query ordering +// ═══════════════════════════════════════════════════════════════════════════ + +console.log('\n=== memory-store: ranked query ordering ==='); +{ + openDatabase(':memory:'); + + // Low confidence, no hits + createMemory({ category: 'pattern', content: 'low ranking', confidence: 0.5 }); + // High confidence, no hits + createMemory({ category: 'gotcha', content: 'high confidence', confidence: 0.95 }); + // Medium confidence, many hits + createMemory({ category: 'convention', content: 'frequently used', confidence: 0.7 }); + + // Reinforce MEM003 multiple times to boost its ranking + for (let i = 0; i < 10; i++) reinforceMemory('MEM003'); + + const ranked = getActiveMemoriesRanked(10); + assertEq(ranked.length, 3, 'should have 3 ranked memories'); + // MEM003: 0.7 * (1 + 10*0.1) = 0.7 * 2.0 = 1.4 + // MEM002: 0.95 * (1 + 0*0.1) = 0.95 + // MEM001: 0.5 * (1 + 0*0.1) = 0.5 + assertEq(ranked[0].id, 'MEM003', 'highest ranked should be MEM003 (reinforced)'); + assertEq(ranked[1].id, 'MEM002', 'second ranked should be MEM002 (high confidence)'); + assertEq(ranked[2].id, 'MEM001', 'lowest ranked should be MEM001'); + + // Test limit + const limited = getActiveMemoriesRanked(2); + assertEq(limited.length, 2, 'limit should cap results'); + + closeDatabase(); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// memory-store: processed unit tracking +// ═══════════════════════════════════════════════════════════════════════════ + +console.log('\n=== memory-store: processed unit tracking ==='); +{ + openDatabase(':memory:'); + + assertTrue(!isUnitProcessed('execute-task/M001/S01/T01'), 'should not be processed initially'); + + markUnitProcessed('execute-task/M001/S01/T01', '/path/to/activity.jsonl'); + + assertTrue(isUnitProcessed('execute-task/M001/S01/T01'), 'should be processed after marking'); + assertTrue(!isUnitProcessed('execute-task/M001/S01/T02'), 'different key should not be processed'); + + closeDatabase(); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// memory-store: enforce memory cap +// ═══════════════════════════════════════════════════════════════════════════ + +console.log('\n=== memory-store: enforce memory cap ==='); +{ + openDatabase(':memory:'); + + // Create 5 memories with varying confidence + createMemory({ category: 'gotcha', content: 'mem 1', confidence: 0.9 }); + createMemory({ category: 'gotcha', content: 'mem 2', confidence: 0.5 }); + createMemory({ category: 'gotcha', content: 'mem 3', confidence: 0.3 }); + createMemory({ category: 'gotcha', content: 'mem 4', confidence: 0.95 }); + createMemory({ category: 'gotcha', content: 'mem 5', confidence: 0.7 }); + + // Enforce cap of 3 + enforceMemoryCap(3); + + const active = getActiveMemories(); + assertEq(active.length, 3, 'should have 3 active memories after cap enforcement'); + + // The 2 lowest-ranked (MEM003=0.3 and MEM002=0.5) should be superseded + const ids = active.map(m => m.id).sort(); + assertTrue(ids.includes('MEM001'), 'MEM001 (0.9) should survive'); + assertTrue(ids.includes('MEM004'), 'MEM004 (0.95) should survive'); + assertTrue(ids.includes('MEM005'), 'MEM005 (0.7) should survive'); + + closeDatabase(); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// memory-store: applyMemoryActions transaction +// ═══════════════════════════════════════════════════════════════════════════ + +console.log('\n=== memory-store: applyMemoryActions ==='); +{ + openDatabase(':memory:'); + + const actions: MemoryAction[] = [ + { action: 'CREATE', category: 'gotcha', content: 'first gotcha', confidence: 0.8 }, + { action: 'CREATE', category: 'convention', content: 'first convention', confidence: 0.9 }, + ]; + + applyMemoryActions(actions, 'execute-task', 'M001/S01/T01'); + + let active = getActiveMemories(); + assertEq(active.length, 2, 'should have 2 memories after CREATE actions'); + + // Now apply UPDATE + REINFORCE + const updateActions: MemoryAction[] = [ + { action: 'UPDATE', id: 'MEM001', content: 'updated gotcha' }, + { action: 'REINFORCE', id: 'MEM002' }, + ]; + + applyMemoryActions(updateActions, 'execute-task', 'M001/S01/T02'); + + active = getActiveMemories(); + assertEq(active.find(m => m.id === 'MEM001')?.content, 'updated gotcha', 'MEM001 should be updated'); + assertEq(active.find(m => m.id === 'MEM002')?.hit_count, 1, 'MEM002 should be reinforced'); + + // SUPERSEDE + const supersedeActions: MemoryAction[] = [ + { action: 'CREATE', category: 'gotcha', content: 'better gotcha', confidence: 0.95 }, + { action: 'SUPERSEDE', id: 'MEM001', superseded_by: 'MEM003' }, + ]; + + applyMemoryActions(supersedeActions, 'execute-task', 'M001/S01/T03'); + + active = getActiveMemories(); + assertEq(active.length, 2, 'should have 2 active after supersede'); + assertTrue(!active.find(m => m.id === 'MEM001'), 'MEM001 should be superseded'); + assertTrue(!!active.find(m => m.id === 'MEM003'), 'MEM003 should be active'); + + closeDatabase(); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// memory-store: formatMemoriesForPrompt +// ═══════════════════════════════════════════════════════════════════════════ + +console.log('\n=== memory-store: formatMemoriesForPrompt ==='); +{ + openDatabase(':memory:'); + + createMemory({ category: 'gotcha', content: 'esbuild drops .node binaries' }); + createMemory({ category: 'convention', content: 'use :memory: for tests' }); + createMemory({ category: 'architecture', content: 'extensions in src/resources/' }); + createMemory({ category: 'gotcha', content: 'TypeScript path aliases need .js' }); + + const memories = getActiveMemoriesRanked(30); + const formatted = formatMemoriesForPrompt(memories); + + assertTrue(formatted.includes('## Project Memory (auto-learned)'), 'should have header'); + assertTrue(formatted.includes('### Gotcha'), 'should have gotcha category'); + assertTrue(formatted.includes('### Convention'), 'should have convention category'); + assertTrue(formatted.includes('### Architecture'), 'should have architecture category'); + assertTrue(formatted.includes('- esbuild drops .node binaries'), 'should have gotcha content'); + assertTrue(formatted.includes('- use :memory: for tests'), 'should have convention content'); + + // Test empty memories + closeDatabase(); + openDatabase(':memory:'); + const emptyFormatted = formatMemoriesForPrompt([]); + assertEq(emptyFormatted, '', 'empty memories should return empty string'); + + // Test token budget truncation + closeDatabase(); + openDatabase(':memory:'); + for (let i = 0; i < 20; i++) { + createMemory({ category: 'pattern', content: `A very long memory entry that takes up space #${i}: ${'x'.repeat(200)}` }); + } + const budgetMemories = getActiveMemoriesRanked(30); + const truncated = formatMemoriesForPrompt(budgetMemories, 500); + assertTrue(truncated.length < 2500, `formatted length ${truncated.length} should be under budget`); + + closeDatabase(); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// memory-store: ID generation +// ═══════════════════════════════════════════════════════════════════════════ + +console.log('\n=== memory-store: ID generation ==='); +{ + openDatabase(':memory:'); + + assertEq(nextMemoryId(), 'MEM001', 'first ID should be MEM001'); + + createMemory({ category: 'test', content: 'test' }); + assertEq(nextMemoryId(), 'MEM002', 'after first create, next should be MEM002'); + + // Create several more + for (let i = 0; i < 98; i++) createMemory({ category: 'test', content: `test ${i}` }); + assertEq(nextMemoryId(), 'MEM100', 'after 99 creates, next should be MEM100'); + + closeDatabase(); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// memory-store: schema migration (v2 → v3) +// ═══════════════════════════════════════════════════════════════════════════ + +console.log('\n=== memory-store: schema includes memories table ==='); +{ + openDatabase(':memory:'); + + const adapter = _getAdapter()!; + + // Verify memories table exists + const memCount = adapter.prepare('SELECT count(*) as cnt FROM memories').get(); + assertEq(memCount?.['cnt'], 0, 'memories table should exist and be empty'); + + // Verify memory_processed_units table exists + const procCount = adapter.prepare('SELECT count(*) as cnt FROM memory_processed_units').get(); + assertEq(procCount?.['cnt'], 0, 'memory_processed_units table should exist and be empty'); + + // Verify active_memories view exists + const viewCount = adapter.prepare('SELECT count(*) as cnt FROM active_memories').get(); + assertEq(viewCount?.['cnt'], 0, 'active_memories view should exist'); + + // Verify schema version is 3 + const version = adapter.prepare('SELECT MAX(version) as v FROM schema_version').get(); + assertEq(version?.['v'], 3, 'schema version should be 3'); + + closeDatabase(); +} + +report();