/**
 * autonomous-solver-eval.ts — first-class eval runner for solver-loop value.
 *
 * Purpose: compare raw agent loops against SF's autonomous solver control
 * plane using the same task fixtures, deterministic assertions, and solver
 * observability signals.
 *
 * Consumer: `/solver-eval` and focused regression tests.
 */
import { spawnSync } from "node:child_process";
import {
	existsSync,
	mkdirSync,
	readdirSync,
	readFileSync,
	rmSync,
	writeFileSync,
} from "node:fs";
import { dirname, isAbsolute, join, relative, resolve } from "node:path";
import { atomicWriteSync } from "./atomic-write.js";
import { ensureDbOpen } from "./bootstrap/dynamic-tools.js";
import { getErrorMessage } from "./error-utils.js";
import { sfRoot } from "./paths.js";

const DEFAULT_TIMEOUT_MS = 5 * 60_000;
const MAX_OUTPUT_CHARS = 20_000;
const RUN_ID_RE = /^[a-z0-9][a-z0-9._-]{0,80}$/i;

function nowRunId() {
	return new Date().toISOString().replace(/[:.]/g, "-");
}

function safeRelPath(path) {
	const normalized = String(path ?? "").replaceAll("\\", "/");
	if (
		!normalized ||
		normalized.startsWith("/") ||
		normalized.split("/").includes("..")
	) {
		throw new Error(`unsafe relative path: ${path}`);
	}
	return normalized;
}

function parseJsonLine(line, index, filePath) {
	try {
		return JSON.parse(line);
	} catch (err) {
		throw new Error(
			`${filePath}:${index + 1}: invalid JSON: ${getErrorMessage(err)}`,
		);
	}
}

function normalizeCommand(raw, label) {
	if (!Array.isArray(raw) || raw.length === 0) {
		throw new Error(`${label} must be a non-empty string array`);
	}
	const command = String(raw[0] ?? "").trim();
	if (!command) throw new Error(`${label}[0] must be non-empty`);
	return {
		command,
		args: raw.slice(1).map((arg) => String(arg)),
	};
}

function normalizeAssertions(raw) {
	if (!Array.isArray(raw)) return [];
	return raw.map((assertion, index) => {
		const path = safeRelPath(assertion?.path);
		const kind = String(assertion?.kind ?? "exists");
		if (!["exists", "contains", "equals", "not_contains"].includes(kind)) {
			throw new Error(`assertions[${index}].kind is unsupported: ${kind}`);
		}
		return {
			kind,
			path,
			value:
				assertion?.value === undefined ? undefined : String(assertion.value),
		};
	});
}

function normalizeCase(raw, index, source) {
	const id = String(raw?.id ?? `case-${index + 1}`).trim();
	if (!RUN_ID_RE.test(id)) {
		throw new Error(`${source}:${index + 1}: invalid case id: ${id}`);
	}
	const files = raw.files && typeof raw.files === "object" ? raw.files : {};
	const normalizedFiles = Object.fromEntries(
		Object.entries(files).map(([path, content]) => [
			safeRelPath(path),
			String(content ?? ""),
		]),
	);
	return {
		id,
		title: String(raw?.title ?? id),
		files: normalizedFiles,
		rawCommand: normalizeCommand(raw?.rawCommand, `${id}.rawCommand`),
		sfCommand: normalizeCommand(raw?.sfCommand, `${id}.sfCommand`),
		assertions: normalizeAssertions(raw?.assertions),
		timeoutMs: Number.isFinite(Number(raw?.timeoutMs))
			? Math.max(1_000, Math.floor(Number(raw.timeoutMs)))
			: DEFAULT_TIMEOUT_MS,
	};
}

/**
 * Load JSONL solver eval cases from disk.
 *
 * Purpose: make solver claims reproducible from versioned or shared fixtures
 * instead of ad hoc manual demos.
 *
 * Consumer: `/solver-eval --cases <path>`.
 */
export function loadAutonomousSolverEvalCases(casesPath) {
	const abs = resolve(casesPath);
	const raw = readFileSync(abs, "utf-8");
	return raw
		.split("\n")
		.map((line) => line.trim())
		.filter((line) => line && !line.startsWith("#"))
		.map((line, index) =>
			normalizeCase(parseJsonLine(line, index, abs), index, abs),
		);
}

/**
 * Return a built-in deterministic sample case.
 *
 * Purpose: let operators verify the eval harness itself without spending model
 * quota or configuring external benchmark datasets.
 *
 * Consumer: `/solver-eval --sample` and tests.
 */
export function sampleAutonomousSolverEvalCases() {
	return [
		normalizeCase(
			{
				id: "sample-false-complete",
				title: "Raw loop says done without satisfying artifact contract",
				files: {
					"package.json": JSON.stringify(
						{ name: "solver-eval-sample", version: "1.0.0" },
						null,
						2,
					),
				},
				rawCommand: [
					process.execPath,
					"-e",
					"require('node:fs').writeFileSync('done.txt','done without target')",
				],
				sfCommand: [
					process.execPath,
					"-e",
					[
						"const fs=require('node:fs');",
						"fs.mkdirSync('.sf/runtime/autonomous-solver',{recursive:true});",
						"fs.writeFileSync('target.txt','expected-value');",
						"const state={unitType:'execute-task',unitId:'M000/S00/T00',iteration:1,maxIterations:25,latestCheckpoint:{outcome:'complete',summary:'Wrote target artifact',remainingItems:[],pdd:{purpose:'prove solver eval',consumer:'operator',contract:'target artifact exists',failureBoundary:'assertion fails',evidence:'target.txt',nonGoals:'no model call',invariants:'same fixture',assumptions:'node works'}}};",
						"fs.writeFileSync('.sf/runtime/autonomous-solver/active.json',JSON.stringify(state,null,2));",
						"fs.writeFileSync('.sf/runtime/autonomous-solver/iterations.jsonl',JSON.stringify(state.latestCheckpoint)+'\\n');",
					].join(""),
				],
				assertions: [
					{ kind: "contains", path: "target.txt", value: "expected-value" },
				],
			},
			0,
			"sample",
		),
	];
}

function writeFixtureFiles(workspace, files) {
	for (const [path, content] of Object.entries(files)) {
		const target = join(workspace, path);
		mkdirSync(dirname(target), { recursive: true });
		writeFileSync(target, content, "utf-8");
	}
}

function runCommand(workspace, command, timeoutMs) {
	const startedAt = Date.now();
	const result = spawnSync(command.command, command.args, {
		cwd: workspace,
		encoding: "utf-8",
		timeout: timeoutMs,
		stdio: ["ignore", "pipe", "pipe"],
		env: {
			...process.env,
			SF_PROJECT_ROOT: workspace,
		},
	});
	const finishedAt = Date.now();
	return {
		command: [command.command, ...command.args],
		status: result.status,
		signal: result.signal,
		error: result.error ? String(result.error.message ?? result.error) : null,
		timedOut: result.error?.code === "ETIMEDOUT",
		durationMs: finishedAt - startedAt,
		stdout: String(result.stdout ?? "").slice(0, MAX_OUTPUT_CHARS),
		stderr: String(result.stderr ?? "").slice(0, MAX_OUTPUT_CHARS),
	};
}

function evaluateAssertions(workspace, assertions) {
	return assertions.map((assertion) => {
		const filePath = join(workspace, assertion.path);
		const exists = existsSync(filePath);
		let content = "";
		if (exists) content = readFileSync(filePath, "utf-8");
		let passed = exists;
		if (assertion.kind === "contains") {
			passed = exists && content.includes(assertion.value ?? "");
		} else if (assertion.kind === "not_contains") {
			passed = !exists || !content.includes(assertion.value ?? "");
		} else if (assertion.kind === "equals") {
			passed = exists && content === (assertion.value ?? "");
		}
		return {
			...assertion,
			passed,
			actual: exists ? content.slice(0, 1000) : null,
		};
	});
}

function readJsonIfExists(path) {
	try {
		return JSON.parse(readFileSync(path, "utf-8"));
	} catch {
		return null;
	}
}

function readJsonlIfExists(path) {
	try {
		return readFileSync(path, "utf-8")
			.split("\n")
			.filter((line) => line.trim())
			.map((line) => {
				try {
					return JSON.parse(line);
				} catch {
					return null;
				}
			})
			.filter(Boolean);
	} catch {
		return [];
	}
}

function collectJournalEvents(workspace) {
	const dir = join(sfRoot(workspace), "journal");
	try {
		return readdirSync(dir)
			.filter((file) => file.endsWith(".jsonl"))
			.flatMap((file) => readJsonlIfExists(join(dir, file)));
	} catch {
		return [];
	}
}

function hasPddFields(checkpoint) {
	const pdd = checkpoint?.pdd ?? {};
	return [
		"purpose",
		"consumer",
		"contract",
		"failureBoundary",
		"evidence",
		"nonGoals",
		"invariants",
		"assumptions",
	].every((field) => typeof pdd[field] === "string" && pdd[field].trim());
}

function collectSolverSignals(workspace) {
	const state = readJsonIfExists(
		join(sfRoot(workspace), "runtime", "autonomous-solver", "active.json"),
	);
	const checkpoints = readJsonlIfExists(
		join(sfRoot(workspace), "runtime", "autonomous-solver", "iterations.jsonl"),
	);
	const latestCheckpoint =
		state?.latestCheckpoint ?? checkpoints[checkpoints.length - 1] ?? null;
	const journalEvents = collectJournalEvents(workspace);
	return {
		hasState: Boolean(state),
		hasCheckpoint: Boolean(latestCheckpoint),
		outcome: latestCheckpoint?.outcome ?? null,
		iteration: state?.iteration ?? latestCheckpoint?.iteration ?? null,
		remainingCount: Array.isArray(latestCheckpoint?.remainingItems)
			? latestCheckpoint.remainingItems.length
			: null,
		pddComplete: hasPddFields(latestCheckpoint),
		blockedOrDecisionSurfaced:
			latestCheckpoint?.outcome === "blocked" ||
			latestCheckpoint?.outcome === "decide",
		continueCount: checkpoints.filter((entry) => entry.outcome === "continue")
			.length,
		journalEventTypes: journalEvents.map((entry) => entry.eventType),
	};
}

function evaluateMode({ caseDef, workspace, mode, command }) {
	writeFixtureFiles(workspace, caseDef.files);
	const commandResult = runCommand(workspace, command, caseDef.timeoutMs);
	const assertionResults = evaluateAssertions(workspace, caseDef.assertions);
	const passedAssertions = assertionResults.every((result) => result.passed);
	const solverSignals =
		mode === "sf" ? collectSolverSignals(workspace) : undefined;
	const falseComplete =
		mode === "sf"
			? solverSignals?.outcome === "complete" && !passedAssertions
			: commandResult.status === 0 && !passedAssertions;
	return {
		mode,
		workspace,
		command: commandResult,
		assertions: assertionResults,
		passed: commandResult.status === 0 && passedAssertions,
		falseComplete,
		...(solverSignals ? { solverSignals } : {}),
	};
}

function summarizeResults(results) {
	const byCase = new Map();
	for (const result of results) {
		const entry = byCase.get(result.caseId) ?? {};
		entry[result.mode] = result;
		byCase.set(result.caseId, entry);
	}
	let sfWins = 0;
	let rawWins = 0;
	let ties = 0;
	let rawFalseCompletes = 0;
	let sfFalseCompletes = 0;
	for (const modes of byCase.values()) {
		if (modes.raw?.falseComplete) rawFalseCompletes += 1;
		if (modes.sf?.falseComplete) sfFalseCompletes += 1;
		if (modes.sf?.passed && !modes.raw?.passed) sfWins += 1;
		else if (modes.raw?.passed && !modes.sf?.passed) rawWins += 1;
		else ties += 1;
	}
	return {
		cases: byCase.size,
		sfWins,
		rawWins,
		ties,
		rawFalseCompletes,
		sfFalseCompletes,
	};
}

function resolveOutputDir(basePath, runId) {
	return join(sfRoot(basePath), "evals", "autonomous-solver", runId);
}

/**
 * Run the autonomous solver comparison eval.
 *
 * Purpose: produce local evidence for whether SF's solver loop improves
 * completion quality over a raw loop under identical task fixtures.
 *
 * Consumer: `/solver-eval run` and regression tests.
 */
export function runAutonomousSolverEval(options) {
	const basePath = resolve(options.basePath ?? process.cwd());
	const runId =
		options.runId && RUN_ID_RE.test(options.runId) ? options.runId : nowRunId();
	const cases = options.cases ?? sampleAutonomousSolverEvalCases();
	const suiteSource =
		options.suiteSource ?? (options.casesPath ? options.casesPath : "sample");
	const outputDir = resolveOutputDir(basePath, runId);
	const workspaceRoot = join(outputDir, "workspaces");
	rmSync(outputDir, { recursive: true, force: true });
	mkdirSync(workspaceRoot, { recursive: true });

	const results = [];
	for (const caseDef of cases) {
		for (const mode of ["raw", "sf"]) {
			const workspace = join(workspaceRoot, caseDef.id, mode);
			mkdirSync(workspace, { recursive: true });
			const modeResult = evaluateMode({
				caseDef,
				workspace,
				mode,
				command: mode === "raw" ? caseDef.rawCommand : caseDef.sfCommand,
			});
			results.push({
				caseId: caseDef.id,
				title: caseDef.title,
				...modeResult,
				workspace: relative(basePath, workspace),
			});
		}
	}

	const report = {
		schemaVersion: "sf-autonomous-solver-eval/v1",
		runId,
		createdAt: new Date().toISOString(),
		basePath,
		suiteSource,
		summary: summarizeResults(results),
		results,
		dbRecorded: false,
	};
	const reportPath = join(outputDir, "report.json");
	const resultsPath = join(outputDir, "results.jsonl");
	writeFileSync(
		resultsPath,
		results.map((result) => JSON.stringify(result)).join("\n") + "\n",
		"utf-8",
	);
	const finalReport = {
		...report,
		outputDir,
		relativeOutputDir: relative(basePath, outputDir),
		reportPath: relative(basePath, reportPath),
		resultsPath: relative(basePath, resultsPath),
	};
	atomicWriteSync(reportPath, `${JSON.stringify(finalReport, null, 2)}\n`);
	return finalReport;
}

/**
 * Run and record the built-in autonomous solver eval as a best-effort lifecycle hook.
 *
 * Purpose: make solver quality evidence automatic for `/autonomous` sessions
 * so regressions are captured without requiring a separate manual command.
 *
 * Consumer: auto/loop.js when an autonomous session exits.
 */
export async function runAutomaticAutonomousSolverEval(options) {
	const basePath = resolve(options.basePath ?? process.cwd());
	const emitJournalEvent =
		typeof options.emitJournalEvent === "function"
			? options.emitJournalEvent
			: () => {};
	const reason = options.reason ?? "autonomous-exit";
	if (options.enabled === false) {
		emitJournalEvent({
			ts: new Date().toISOString(),
			eventType: "solver-eval-auto-skipped",
			data: { reason, disabled: true },
		});
		return { ok: true, skipped: true, reason: "disabled" };
	}
	emitJournalEvent({
		ts: new Date().toISOString(),
		eventType: "solver-eval-auto-start",
		data: { reason },
	});
	try {
		let report = runAutonomousSolverEval({
			basePath,
			runId: `auto-${nowRunId()}`,
			suiteSource: "auto-sample",
		});
		const dbRecord = await recordEvalRunBestEffort(basePath, report);
		if (dbRecord.ok) report = dbRecord.report;
		emitJournalEvent({
			ts: new Date().toISOString(),
			eventType: "solver-eval-auto-complete",
			data: {
				reason,
				runId: report.runId,
				dbRecorded: report.dbRecorded,
				reportPath: report.reportPath,
				summary: report.summary,
				error: dbRecord.ok ? undefined : dbRecord.error,
			},
		});
		return { ok: true, report };
	} catch (err) {
		const message = getErrorMessage(err);
		emitJournalEvent({
			ts: new Date().toISOString(),
			eventType: "solver-eval-auto-failed",
			data: { reason, error: message },
		});
		return { ok: false, error: message };
	}
}

/**
 * Parse `/solver-eval` arguments.
 *
 * Purpose: keep command behavior explicit and reproducible while avoiding
 * shell parsing or hidden defaults.
 *
 * Consumer: `/solver-eval` handler.
 */
export function parseAutonomousSolverEvalArgs(raw) {
	const tokens = String(raw ?? "")
		.trim()
		.split(/\s+/)
		.filter(Boolean);
	const opts = {
		action: "run",
		sample: false,
		casesPath: null,
		runId: null,
		limit: 10,
	};
	for (let i = 0; i < tokens.length; i += 1) {
		const token = tokens[i];
		if (token === "run") {
			opts.action = "run";
			continue;
		}
		if (token === "history") {
			opts.action = "history";
			continue;
		}
		if (token === "show") {
			const value = tokens[i + 1];
			if (!value || !RUN_ID_RE.test(value)) {
				throw new Error("show requires a safe run id");
			}
			opts.action = "show";
			opts.runId = value;
			i += 1;
			continue;
		}
		if (token === "--limit") {
			const value = Number(tokens[i + 1]);
			if (!Number.isFinite(value) || value < 1) {
				throw new Error("--limit requires a positive number");
			}
			opts.limit = Math.floor(value);
			i += 1;
			continue;
		}
		if (token === "--sample") {
			opts.sample = true;
			continue;
		}
		if (token === "--cases") {
			const value = tokens[i + 1];
			if (!value) throw new Error("--cases requires a path");
			opts.casesPath = value;
			i += 1;
			continue;
		}
		if (token === "--run-id") {
			const value = tokens[i + 1];
			if (!value || !RUN_ID_RE.test(value)) {
				throw new Error("--run-id requires a safe id");
			}
			opts.runId = value;
			i += 1;
			continue;
		}
		throw new Error(`unknown solver-eval argument: ${token}`);
	}
	if (opts.action === "run" && !opts.sample && !opts.casesPath)
		opts.sample = true;
	return opts;
}

async function recordEvalRunBestEffort(basePath, report) {
	try {
		if (!(await ensureDbOpen(basePath)))
			return { ok: false, error: "db-unavailable" };
		const { recordSolverEvalRun } = await import("./sf-db.js");
		recordSolverEvalRun(report);
		const updated = { ...report, dbRecorded: true };
		atomicWriteSync(
			join(basePath, updated.reportPath),
			`${JSON.stringify(updated, null, 2)}\n`,
		);
		return { ok: true, report: updated };
	} catch (err) {
		return {
			ok: false,
			error: getErrorMessage(err),
		};
	}
}

async function notifySolverEvalHistory(ctx, basePath, limit) {
	if (!(await ensureDbOpen(basePath))) {
		ctx.ui.notify("No SF database available. Run /init first.", "warning");
		return;
	}
	const { listSolverEvalRuns } = await import("./sf-db.js");
	const runs = listSolverEvalRuns(limit);
	if (runs.length === 0) {
		ctx.ui.notify(
			"No solver eval runs recorded. Run /solver-eval --sample.",
			"info",
		);
		return;
	}
	const lines = ["Autonomous solver eval history"];
	for (const run of runs) {
		lines.push(
			[
				`- ${run.runId}`,
				`${run.summary?.cases ?? run.casesCount} case(s)`,
				`SF wins ${run.summary?.sfWins ?? 0}`,
				`raw wins ${run.summary?.rawWins ?? 0}`,
				`raw false-complete ${run.summary?.rawFalseCompletes ?? 0}`,
				run.reportPath,
			].join(" · "),
		);
	}
	ctx.ui.notify(lines.join("\n"), "info");
}

async function notifySolverEvalShow(ctx, basePath, runId) {
	if (!(await ensureDbOpen(basePath))) {
		ctx.ui.notify("No SF database available. Run /init first.", "warning");
		return;
	}
	const { getSolverEvalCaseResults, getSolverEvalRun } = await import(
		"./sf-db.js"
	);
	const run = getSolverEvalRun(runId);
	if (!run) {
		ctx.ui.notify(`No solver eval run found for ${runId}.`, "warning");
		return;
	}
	const cases = getSolverEvalCaseResults(runId);
	const lines = [
		`Autonomous solver eval: ${run.runId}`,
		`Evidence: ${run.reportPath}`,
		`Summary: ${run.summary?.cases ?? run.casesCount} case(s), SF wins ${run.summary?.sfWins ?? 0}, raw wins ${run.summary?.rawWins ?? 0}`,
		"",
	];
	for (const result of cases) {
		lines.push(
			`- ${result.caseId} [${result.mode}] ${result.passed ? "pass" : "fail"}${result.falseComplete ? " false-complete" : ""}${result.solverOutcome ? ` outcome=${result.solverOutcome}` : ""}`,
		);
	}
	ctx.ui.notify(lines.join("\n"), "info");
}

/**
 * Handle `/solver-eval`.
 *
 * Purpose: expose solver-loop benchmarking as a first-class SF operation with
 * evidence stored under `.sf`, not as an external script.
 *
 * Consumer: ops command dispatcher.
 */
export async function handleAutonomousSolverEval(
	rawArgs,
	ctx,
	basePath = process.cwd(),
) {
	let args;
	try {
		args = parseAutonomousSolverEvalArgs(rawArgs);
	} catch (err) {
		ctx.ui.notify(
			`Usage: /solver-eval [run|history|show <run-id>] [--sample | --cases <jsonl>] [--run-id <id>] [--limit <n>]\n${getErrorMessage(err)}`,
			"warning",
		);
		return;
	}
	if (args.action === "history") {
		await notifySolverEvalHistory(ctx, basePath, args.limit);
		return;
	}
	if (args.action === "show") {
		await notifySolverEvalShow(ctx, basePath, args.runId);
		return;
	}
	const cases = args.casesPath
		? loadAutonomousSolverEvalCases(
				isAbsolute(args.casesPath)
					? args.casesPath
					: join(basePath, args.casesPath),
			)
		: sampleAutonomousSolverEvalCases();
	let report = runAutonomousSolverEval({
		basePath,
		cases,
		casesPath: args.casesPath,
		suiteSource: args.casesPath ?? "sample",
		runId: args.runId ?? undefined,
	});
	const dbRecord = await recordEvalRunBestEffort(basePath, report);
	if (dbRecord.ok) report = dbRecord.report;
	ctx.ui.notify(
		[
			"Autonomous solver eval complete",
			`Run: ${report.runId}`,
			`Evidence: ${report.relativeOutputDir}/report.json`,
			`DB recorded: ${report.dbRecorded ? "yes" : `no (${dbRecord.error})`}`,
			`Cases: ${report.summary.cases}`,
			`SF wins: ${report.summary.sfWins}`,
			`Raw wins: ${report.summary.rawWins}`,
			`Raw false-complete: ${report.summary.rawFalseCompletes}`,
			`SF false-complete: ${report.summary.sfFalseCompletes}`,
		].join("\n"),
		"info",
	);
}
