// SF Memory Extractor — Background LLM extraction from activity logs
//
// After each unit completes, extracts durable knowledge from the session
// transcript and stores it as memory entries. One extraction at a time
// (mutex guard). Fire-and-forget — never blocks autonomous mode.

import { readFileSync, statSync } from "node:fs";
import { delay } from "./atomic-write.js";
import { debugLog } from "./debug-logger.js";
import {
	applyConsolidationActions,
	decayStaleMemories,
	getActiveMemories,
	isUnitProcessed,
	markUnitProcessed,
	recordMemoryExtractionAttempt,
} from "./memory-store.js";

// ─── Concurrency Guard ──────────────────────────────────────────────────────
let _extracting = false;
let _lastExtractionTime = 0;
const MIN_EXTRACTION_INTERVAL_MS = 30_000;
// ─── Skip Conditions ────────────────────────────────────────────────────────
const SKIP_TYPES = new Set([
	"complete-slice",
	"rewrite-docs",
	"triage-captures",
]);
const MIN_ACTIVITY_SIZE = 1024; // 1KB
// ─── Secret Redaction ───────────────────────────────────────────────────────
const SECRET_PATTERNS = [
	/(?:sk|pk|api[_-]?key|token|secret|password|credential|auth)[_-]?\w*[\s:=]+['"]?[\w\-./+=]{20,}['"]?/gi,
	/AKIA[0-9A-Z]{16}/g,
	/gh[pousr]_[A-Za-z0-9_]{36,}/g,
	/[rsp]k_(?:live|test)_[A-Za-z0-9]{20,}/g,
	/eyJ[A-Za-z0-9_-]{20,}\.eyJ[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]+/g,
	/-----BEGIN (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----[\s\S]*?-----END (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----/g,
	/(?:Bearer\s+)[A-Za-z0-9\-._~+/]+=*/gi,
	/npm_[A-Za-z0-9]{36,}/g,
	/sk-ant-[A-Za-z0-9\-_]{20,}/g,
	/sk-[A-Za-z0-9]{40,}/g,
];
function redactSecrets(text) {
	let result = text;
	for (const pattern of SECRET_PATTERNS) {
		// Reset lastIndex for global regexes
		pattern.lastIndex = 0;
		result = result.replace(pattern, "[REDACTED]");
	}
	return result;
}
// ─── Model Selection ────────────────────────────────────────────────────────
/**
 * Build an LLM call function using the cheapest available model (preferring Haiku).
 * Returns null if no models available.
 */
export function buildMemoryLLMCall(ctx) {
	try {
		const available = ctx.modelRegistry.getAvailable();
		if (!available || available.length === 0) return null;
		// Prefer Haiku by ID substring match
		let model = available.find((m) => m.id.toLowerCase().includes("haiku"));
		// Fallback: cheapest by input cost
		if (!model) {
			model = [...available].sort((a, b) => a.cost.input - b.cost.input)[0];
		}
		if (!model) return null;
		const selectedModel = model;
		return async (system, user) => {
			const { completeSimple } = await import("@singularity-forge/ai");
			// Resolve API key inside the async body on each invocation so that
			// rotated or revoked credentials are picked up without rebuilding the
			// LLM call function. See: https://github.com/singularity-forge/sf-run/issues/2959
			const resolvedApiKey = await ctx.modelRegistry
				.getApiKey(selectedModel)
				.catch(() => undefined);
			const result = await completeSimple(
				selectedModel,
				{
					systemPrompt: system,
					messages: [
						{
							role: "user",
							content: [{ type: "text", text: user }],
							timestamp: Date.now(),
						},
					],
				},
				{
					maxTokens: 2048,
					temperature: 0,
					...(resolvedApiKey ? { apiKey: resolvedApiKey } : {}),
				},
			);
			// Extract text from response
			const textParts = result.content
				.filter((c) => c.type === "text")
				.map((c) => c.text);
			return textParts.join("");
		};
	} catch {
		return null;
	}
}
// ─── Extraction Prompts ─────────────────────────────────────────────────────
const EXTRACTION_SYSTEM = `You are a memory extraction agent for a software project. Analyze the session
transcript and identify durable knowledge worth remembering for future sessions.

Categories: architecture, convention, gotcha, preference, environment, pattern

Actions (return JSON array):
- add:   {"action": "add", "category": "<cat>", "content": "<text>", "confidence": <0.6-0.95>}
- prune: {"action": "prune", "id": "<MEM###>"}

Action rules:
- Use "add" to record new durable knowledge that will be useful in future sessions.
- Use "prune" to remove a memory that is now incorrect, outdated, or stale.
- Do NOT emit "update", "UPDATE", "supersede", or any other action kinds — they are rejected.
- To change an existing memory entry: prune it and add a fresh replacement. Every
  change must be visible as an explicit add or prune.

Content rules:
- Don't create memories for one-off bug fixes or temporary state
- Don't duplicate existing memories — if an entry already covers it, skip
- Keep content to 1-3 sentences
- Confidence: 0.6 tentative, 0.8 solid, 0.95 well-confirmed
- Prefer fewer high-quality memories over many low-quality ones
- Return empty array [] if nothing worth remembering
- NEVER include secrets, API keys, or passwords

Return ONLY a valid JSON array.`;
function buildExtractionUserPrompt(
	unitType,
	unitId,
	existingMemories,
	transcript,
) {
	let memoriesSection;
	if (existingMemories.length === 0) {
		memoriesSection = "(none yet)";
	} else {
		memoriesSection = existingMemories
			.map((m, i) => `${i + 1}. [${m.id}] (${m.category}) ${m.content}`)
			.join("\n");
	}
	return `## Current Active Memories\n${memoriesSection}\n\n## Session Transcript (${unitType}: ${unitId})\n${transcript}`;
}
// ─── Activity JSONL Parsing ─────────────────────────────────────────────────
/**
 * Extract assistant message text from activity JSONL.
 * Returns concatenated text content from assistant role entries.
 *
 * Truncation strategy (changed from front-preserving): the front of
 * an execute-task transcript carries the goal/context (already in
 * the prompt), while the tail carries verification, final summary,
 * and the decisions actually committed — the highest-signal slices
 * for memory extraction. Front-preserving truncation discarded
 * exactly the content the LLM is best at extracting from.
 *
 * The fix: collect every assistant text block first, then if the
 * total exceeds maxChars, keep ~25% from the front (goal echo,
 * early decisions) and ~75% from the back (where the durable signal
 * lives). The two halves are joined with a `[…truncated middle…]`
 * marker so the LLM knows content was elided.
 */
function extractTranscriptFromActivity(raw, maxChars = 30_000) {
	const lines = raw.split("\n");
	const parts = [];
	function appendText(text) {
		parts.push(text);
	}
	for (const line of lines) {
		if (!line.trim()) continue;
		try {
			const entry = JSON.parse(line);
			// Modern activity logs use custom_message with customType "sf-auto"
			// for assistant reasoning; legacy logs use role === "assistant".
			const isAssistant =
				entry.role === "assistant" ||
				(entry.type === "custom_message" &&
					entry.customType?.startsWith("sf-"));
			if (!isAssistant) continue;
			// Handle content array or direct text
			if (Array.isArray(entry.content)) {
				for (const block of entry.content) {
					if (block.type === "text" && block.text) {
						appendText(block.text);
					}
				}
			} else if (typeof entry.content === "string" && entry.content) {
				appendText(entry.content);
			}
			// Also read plain text/content field on custom_message entries
			if (entry.text && typeof entry.text === "string") {
				appendText(entry.text);
			}
		} catch {
			// Skip malformed lines
		}
	}
	const joined = parts.join("\n\n");
	if (joined.length <= maxChars) return joined;
	return truncateMiddle(joined, maxChars);
}

/**
 * Keep N from front + M from back of `text`, drop the middle.
 *
 * Split ratio is back-weighted (default 25/75) because in an
 * execute-task transcript the durable signal — what was committed,
 * what verified, what the operator decided — concentrates at the
 * tail. The front share keeps enough goal/early-reasoning context
 * to anchor the extraction; the back share keeps the outcomes.
 *
 * Exported (named export below) for the dedicated unit test that
 * pins the ratio + the truncation marker.
 */
export function truncateMiddle(text, maxChars, frontShare = 0.25) {
	if (text.length <= maxChars) return text;
	const marker = "\n\n[…truncated middle…]\n\n";
	const budget = Math.max(0, maxChars - marker.length);
	const frontLen = Math.floor(budget * frontShare);
	const backLen = budget - frontLen;
	return text.slice(0, frontLen) + marker + text.slice(text.length - backLen);
}
// ─── Response Parsing ───────────────────────────────────────────────────────
class MemoryExtractionFailure extends Error {
	constructor(message, failureClass, reason, cause) {
		super(message);
		this.name = "MemoryExtractionFailure";
		this.failureClass = failureClass;
		this.reason = reason;
		this.cause = cause;
	}
}

/**
 * Classify memory extraction failures for retry and operator diagnostics.
 *
 * Purpose: keep transient provider failures, deterministic parse failures,
 * filesystem faults, and apply/storage failures distinct in DB-backed memory
 * closeout attempts.
 *
 * Consumer: extractMemoriesFromUnit when deciding whether a failed extraction
 * should retry immediately or surface as an actionable diagnostic.
 */
export function classifyMemoryExtractionFailure(error) {
	const inherited = error?.failureClass;
	if (typeof inherited === "string" && inherited) return inherited;
	const message = String(error?.message ?? error ?? "").toLowerCase();
	const code = String(error?.code ?? "").toLowerCase();
	if (
		code === "enoent" ||
		code === "eacces" ||
		code === "eperm" ||
		message.includes("no such file") ||
		message.includes("permission denied")
	) {
		return "filesystem";
	}
	if (
		message.includes("json") ||
		message.includes("parse") ||
		message.includes("invalid response")
	) {
		return "parse";
	}
	if (
		code === "etimedout" ||
		code === "econnreset" ||
		code === "econnrefused" ||
		message.includes("timeout") ||
		message.includes("timed out") ||
		message.includes("rate limit") ||
		message.includes("429") ||
		/\b5\d\d\b/.test(message) ||
		message.includes("overloaded") ||
		message.includes("service unavailable") ||
		message.includes("network")
	) {
		return "transient";
	}
	if (
		message.includes("unauthorized") ||
		message.includes("forbidden") ||
		message.includes("api key") ||
		message.includes("401") ||
		message.includes("403")
	) {
		return "auth";
	}
	if (
		message.includes("sqlite") ||
		message.includes("constraint") ||
		message.includes("database")
	) {
		return "apply";
	}
	return "unknown";
}

/**
 * Parse the LLM response into memory actions.
 * Strips markdown fences, validates required fields.
 * Returns [] on any parse failure.
 */
export function parseMemoryResponse(raw) {
	return parseMemoryResponseDetailed(raw).actions;
}

/**
 * Parse the LLM response into actions plus an explicit failure diagnostic.
 *
 * Purpose: distinguish a legitimate empty memory response from malformed JSON
 * or unsupported action shapes so closeout diagnostics do not report parser
 * failures as successful no-op extractions.
 *
 * Consumer: extractMemoriesFromUnit failure classification.
 */
export function parseMemoryResponseDetailed(raw) {
	try {
		// Strip markdown code fences
		if (typeof raw !== "string" || raw.trim() === "") {
			return {
				actions: [],
				failureClass: "parse",
				reason: "empty-response",
				error: "LLM returned an empty memory extraction response",
			};
		}
		let cleaned = raw.trim();
		if (cleaned.startsWith("```")) {
			cleaned = cleaned
				.replace(/^```(?:json)?\s*\n?/, "")
				.replace(/\n?```\s*$/, "");
		}
		const parsed = JSON.parse(cleaned);
		if (!Array.isArray(parsed)) {
			return {
				actions: [],
				failureClass: "parse",
				reason: "non-array-response",
				error: "Memory extraction response must be a JSON array",
			};
		}
		const actions = [];
		let invalidItems = 0;
		for (const item of parsed) {
			if (!item || typeof item !== "object" || !item.action) {
				invalidItems++;
				continue;
			}
			switch (item.action) {
				case "add":
					if (
						typeof item.category === "string" &&
						typeof item.content === "string"
					) {
						actions.push({
							action: "add",
							category: item.category,
							content: item.content,
							confidence:
								typeof item.confidence === "number"
									? item.confidence
									: undefined,
						});
					} else {
						invalidItems++;
					}
					break;
				case "prune":
					if (typeof item.id === "string") {
						actions.push({ action: "prune", id: item.id });
					} else {
						invalidItems++;
					}
					break;
				// Silently drop any legacy uppercase action variants the model may emit
				// despite the prompt instructions — they will be rejected downstream by
				// applyConsolidationActions anyway, but filtering here keeps the error
				// surface clean and avoids a throw on every legacy response.
				default:
					invalidItems++;
					break;
			}
		}
		if (parsed.length > 0 && actions.length === 0 && invalidItems > 0) {
			return {
				actions: [],
				failureClass: "parse",
				reason: "invalid-actions",
				error: "Memory extraction response contained no valid actions",
			};
		}
		return { actions };
	} catch (error) {
		return {
			actions: [],
			failureClass: "parse",
			reason: "invalid-json",
			error: error?.message ?? String(error),
		};
	}
}
// ─── Main Extraction Function ───────────────────────────────────────────────
/**
 * Extract memories from a completed unit's activity log.
 * Fire-and-forget — never throws, mutex-guarded, respects rate limiting.
 */
export async function extractMemoriesFromUnit(
	activityFile,
	unitType,
	unitId,
	llmCallFn,
) {
	const unitKey = `${unitType}/${unitId}`;
	const recordAttempt = (status, reason, error, failureClass) =>
		recordMemoryExtractionAttempt({
			unitKey,
			unitType,
			unitId,
			activityFile,
			status,
			reason,
			failureClass:
				failureClass ??
				(status === "error" || error
					? classifyMemoryExtractionFailure(error)
					: null),
			error: error?.message ?? error,
		});
	// Mutex guard
	if (_extracting) {
		recordAttempt("skipped", "mutex-busy");
		debugLog("memory-extract", {
			phase: "skip",
			reason: "mutex-busy",
			unitKey,
		});
		return;
	}
	// Rate limit
	const now = Date.now();
	if (now - _lastExtractionTime < MIN_EXTRACTION_INTERVAL_MS) {
		recordAttempt("skipped", "rate-limited");
		debugLog("memory-extract", {
			phase: "skip",
			reason: "rate-limited",
			unitKey: `${unitType}/${unitId}`,
			lastExtraction: _lastExtractionTime,
		});
		return;
	}
	// Skip certain unit types
	if (SKIP_TYPES.has(unitType)) {
		recordAttempt("skipped", "skip-type");
		debugLog("memory-extract", {
			phase: "skip",
			reason: "skip-type",
			unitType,
		});
		return;
	}
	// Already processed
	if (isUnitProcessed(unitKey)) {
		recordAttempt("skipped", "already-processed");
		debugLog("memory-extract", {
			phase: "skip",
			reason: "already-processed",
			unitKey,
		});
		return;
	}
	// Check file size
	try {
		const stat = statSync(activityFile);
		if (stat.size < MIN_ACTIVITY_SIZE) {
			recordAttempt("skipped", "file-too-small");
			debugLog("memory-extract", {
				phase: "skip",
				reason: "file-too-small",
				unitKey,
				size: stat.size,
				min: MIN_ACTIVITY_SIZE,
			});
			return;
		}
	} catch (error) {
		recordAttempt("skipped", "stat-failed", error, "filesystem");
		debugLog("memory-extract", {
			phase: "skip",
			reason: "stat-failed",
			unitKey,
			file: activityFile,
		});
		return;
	}
	_extracting = true;
	_lastExtractionTime = now;
	recordAttempt("started");
	debugLog("memory-extract", { phase: "start", unitKey, file: activityFile });
	let userPrompt;
	try {
		// Read and parse activity file
		const raw = readFileSync(activityFile, "utf-8");
		const transcript = extractTranscriptFromActivity(raw);
		if (!transcript.trim()) {
			recordAttempt("skipped", "empty-transcript");
			debugLog("memory-extract", {
				phase: "skip",
				reason: "empty-transcript",
				unitKey,
			});
			return;
		}
		// Redact secrets
		const safeTranscript = redactSecrets(transcript);
		// Get current memories for context
		const activeMemories = getActiveMemories().map((m) => ({
			id: m.id,
			category: m.category,
			content: m.content,
		}));
		// Build prompts
		userPrompt = buildExtractionUserPrompt(
			unitType,
			unitId,
			activeMemories,
			safeTranscript,
		);
		// Call LLM
		debugLog("memory-extract", {
			phase: "llm-call",
			unitKey,
			transcriptChars: safeTranscript.length,
		});
		const response = await llmCallFn(EXTRACTION_SYSTEM, userPrompt);
		// Parse response
		const parsed = parseMemoryResponseDetailed(response);
		if (parsed.failureClass) {
			throw new MemoryExtractionFailure(
				parsed.error ?? parsed.reason,
				parsed.failureClass,
				parsed.reason,
			);
		}
		const actions = parsed.actions;
		debugLog("memory-extract", {
			phase: "parsed",
			unitKey,
			actions: actions.length,
		});
		// Apply actions (consolidation-path only — add/prune discipline enforced)
		if (actions.length > 0) {
			applyConsolidationActions(actions, unitType, unitId);
			debugLog("memory-extract", {
				phase: "applied",
				unitKey,
				actions: actions.length,
			});
		}
		// Decay stale memories periodically
		decayStaleMemories(20);
		// Mark unit as processed
		markUnitProcessed(unitKey, activityFile);
		recordAttempt(
			"processed",
			actions.length > 0 ? "actions-applied" : "no-actions",
		);
		debugLog("memory-extract", { phase: "done", unitKey });
	} catch (err) {
		const failureClass = classifyMemoryExtractionFailure(err);
		const reason = err?.reason ?? `${failureClass}-failure`;
		recordAttempt("error", reason, err, failureClass);
		debugLog("memory-extract", {
			phase: "error",
			unitKey,
			failureClass,
			error: err?.message || String(err),
		});
		// Retry once only for transient provider or network failures. Parser and
		// apply/storage failures need a different input or code fix, so immediate
		// retry just burns tokens while hiding the real class from diagnostics.
		if (userPrompt && failureClass === "transient") {
			try {
				await delay(2000);
				const response2 = await llmCallFn(EXTRACTION_SYSTEM, userPrompt);
				const parsed2 = parseMemoryResponseDetailed(response2);
				if (parsed2.failureClass) {
					throw new MemoryExtractionFailure(
						parsed2.error ?? parsed2.reason,
						parsed2.failureClass,
						parsed2.reason,
					);
				}
				const actions2 = parsed2.actions;
				if (actions2.length > 0)
					applyConsolidationActions(actions2, unitType, unitId);
				markUnitProcessed(unitKey, activityFile);
				recordAttempt("processed", "retry-success");
				debugLog("memory-extract", { phase: "retry-success", unitKey });
			} catch (err2) {
				const retryFailureClass = classifyMemoryExtractionFailure(err2);
				recordAttempt(
					"error",
					err2?.reason ?? "retry-failed",
					err2,
					retryFailureClass,
				);
				debugLog("memory-extract", {
					phase: "retry-failed",
					unitKey,
					failureClass: retryFailureClass,
					error: err2?.message || String(err2),
				});
				// Non-fatal — memory extraction failure should never affect autonomous mode
			}
		}
	} finally {
		_extracting = false;
	}
}
// ─── Testing Helpers ────────────────────────────────────────────────────────
/** Reset extraction state (testing only). */
export function _resetExtractionState() {
	_extracting = false;
	_lastExtractionTime = 0;
}
