singularity-forge/src/cli-stats.ts

import { existsSync } from "node:fs";
import { createRequire } from "node:module";
import { join } from "node:path";

export interface ModelStatsRow {
	model_id: string;
	provider: string;
	unit_type: string;
	attempts: number;
	successes: number;
	avg_ms: number | null;
	avg_cost: number | null;
	avg_retries: number | null;
}

export interface QueryModelStatsOptions {
	sinceSeconds: number;
	unitType?: string;
}

interface SqliteStatement {
	all(...params: unknown[]): Record<string, unknown>[];
}

interface SqliteDb {
	prepare(sql: string): SqliteStatement;
	close?: () => void;
}

interface ParsedStatsArgs {
	command: "models";
	sinceSeconds: number;
	unitType?: string;
}

const require = createRequire(import.meta.url);

export function parseDurationSeconds(value: string): number {
	const match = value.trim().match(/^(\d+(?:\.\d+)?)([smhdw])$/i);
	if (!match) {
		throw new Error(`Invalid duration: ${value}`);
	}
	const amount = Number(match[1]);
	const unit = match[2].toLowerCase();
	const multiplier =
		unit === "s"
			? 1
			: unit === "m"
				? 60
				: unit === "h"
					? 60 * 60
					: unit === "d"
						? 24 * 60 * 60
						: 7 * 24 * 60 * 60;
	return Math.max(1, Math.floor(amount * multiplier));
}

function toNumber(value: unknown, fallback = 0): number {
	const n = Number(value);
	return Number.isFinite(n) ? n : fallback;
}

function toNullableNumber(value: unknown): number | null {
	if (value === null || value === undefined) return null;
	const n = Number(value);
	return Number.isFinite(n) ? n : null;
}

export function queryModelStats(
	db: SqliteDb,
	options: QueryModelStatsOptions,
): ModelStatsRow[] {
	const where = ["recorded_at > unixepoch() - ?"];
	const params: unknown[] = [options.sinceSeconds];
	if (options.unitType) {
		where.push("unit_type = ?");
		params.push(options.unitType);
	}

	const sql = `
SELECT model_id, provider, unit_type, COUNT(*) AS attempts,
       SUM(succeeded) AS successes,
       AVG(duration_ms) AS avg_ms,
       AVG(cost_usd) AS avg_cost,
       AVG(retries) AS avg_retries
FROM llm_task_outcomes
WHERE ${where.join(" AND ")}
GROUP BY model_id, unit_type
ORDER BY attempts DESC
`;

	return db
		.prepare(sql)
		.all(...params)
		.map((row) => ({
			model_id: String(row.model_id ?? ""),
			provider: String(row.provider ?? ""),
			unit_type: String(row.unit_type ?? ""),
			attempts: toNumber(row.attempts),
			successes: toNumber(row.successes),
			avg_ms: toNullableNumber(row.avg_ms),
			avg_cost: toNullableNumber(row.avg_cost),
			avg_retries: toNullableNumber(row.avg_retries),
		}));
}

function formatSuccessRate(row: ModelStatsRow): string {
	if (row.attempts <= 0) return "0.0%";
	return `${((row.successes / row.attempts) * 100).toFixed(1)}%`;
}

function formatMs(value: number | null): string {
	return value === null ? "n/a" : value.toFixed(0);
}

function formatCost(value: number | null): string {
	return value === null ? "n/a" : `$${value.toFixed(4)}`;
}

function formatRetries(value: number | null): string {
	return value === null ? "n/a" : value.toFixed(2);
}

function pad(value: string, width: number, align: "left" | "right"): string {
	return align === "right" ? value.padStart(width) : value.padEnd(width);
}

export function formatModelStatsTable(rows: ModelStatsRow[]): string {
	const headers = [
		"model_id",
		"provider",
		"unit_type",
		"attempts",
		"success_rate",
		"avg_ms",
		"avg_cost",
		"avg_retries",
	];
	const body = rows.map((row) => [
		row.model_id,
		row.provider,
		row.unit_type,
		String(row.attempts),
		formatSuccessRate(row),
		formatMs(row.avg_ms),
		formatCost(row.avg_cost),
		formatRetries(row.avg_retries),
	]);
	const widths = headers.map((header, index) =>
		Math.max(header.length, ...body.map((row) => row[index].length)),
	);
	const numeric = new Set([3, 4, 5, 6, 7]);

	const separator = `+${widths.map((width) => "-".repeat(width + 2)).join("+")}+`;
	const renderRow = (row: string[]) =>
		`| ${row.map((cell, index) => pad(cell, widths[index], numeric.has(index) ? "right" : "left")).join(" | ")} |`;

	return (
		[
			separator,
			renderRow(headers),
			separator,
			...body.map(renderRow),
			separator,
		].join("\n") + "\n"
	);
}

function parseStatsArgs(argv: string[]): ParsedStatsArgs | null {
	const args = argv.slice(1);
	if (args[0] !== "models") return null;
	const parsed: ParsedStatsArgs = {
		command: "models",
		sinceSeconds: parseDurationSeconds("7d"),
	};

	for (let i = 1; i < args.length; i++) {
		const arg = args[i];
		if (arg === "--unit-type" && i + 1 < args.length) {
			parsed.unitType = args[++i];
		} else if (arg.startsWith("--unit-type=")) {
			parsed.unitType = arg.slice("--unit-type=".length);
		} else if (arg === "--since" && i + 1 < args.length) {
			parsed.sinceSeconds = parseDurationSeconds(args[++i]);
		} else if (arg.startsWith("--since=")) {
			parsed.sinceSeconds = parseDurationSeconds(arg.slice("--since=".length));
		}
	}

	return parsed;
}

function usage(): string {
	return (
		[
			"Usage: sf stats models [--unit-type <type>] [--since <duration>]",
			"",
			"Examples:",
			"  sf stats models --since 7d",
			"  sf stats models --unit-type execute-task --since 24h",
		].join("\n") + "\n"
	);
}

function openSqliteDb(dbPath: string): SqliteDb {
	try {
		const sqlite = require("node:sqlite") as {
			DatabaseSync?: new (
				path: string,
				options?: Record<string, unknown>,
			) => SqliteDb;
		};
		if (sqlite.DatabaseSync) {
			return new sqlite.DatabaseSync(dbPath, { readOnly: true });
		}
	} catch {
		// Try better-sqlite3 below.
	}

	try {
		const mod = require("better-sqlite3") as
			| {
					default?: new (
						path: string,
						options?: Record<string, unknown>,
					) => SqliteDb;
			  }
			| (new (
					path: string,
					options?: Record<string, unknown>,
			  ) => SqliteDb);
		const Database = typeof mod === "function" ? mod : mod.default;
		if (Database)
			return new Database(dbPath, { readonly: true, fileMustExist: true });
	} catch {
		// Report a single actionable error below.
	}

	throw new Error(
		"No SQLite provider available (tried node:sqlite, better-sqlite3)",
	);
}

export async function runStatsCli(
	argv: string[],
	deps: {
		basePath: string;
		stdout?: Pick<typeof process.stdout, "write">;
		stderr?: Pick<typeof process.stderr, "write">;
	},
): Promise<number> {
	const stdout = deps.stdout ?? process.stdout;
	const stderr = deps.stderr ?? process.stderr;
	let parsed: ParsedStatsArgs | null;
	try {
		parsed = parseStatsArgs(argv);
	} catch (err) {
		stderr.write(
			`sf stats: ${err instanceof Error ? err.message : String(err)}\n`,
		);
		return 1;
	}
	if (!parsed) {
		stderr.write(usage());
		return 1;
	}

	const dbPath = join(deps.basePath, ".sf", "sf.db");
	if (!existsSync(dbPath)) {
		stderr.write(`sf stats: database not found at ${dbPath}\n`);
		return 1;
	}

	let db: SqliteDb | null = null;
	try {
		db = openSqliteDb(dbPath);
		const rows = queryModelStats(db, parsed);
		if (rows.length === 0) {
			stdout.write("No model outcomes recorded for the selected window.\n");
		} else {
			stdout.write(formatModelStatsTable(rows));
		}
		return 0;
	} catch (err) {
		const message = err instanceof Error ? err.message : String(err);
		if (/no such table/i.test(message)) {
			stdout.write("No model outcomes recorded yet.\n");
			return 0;
		}
		stderr.write(`sf stats: ${message}\n`);
		return 1;
	} finally {
		db?.close?.();
	}
}