import { existsSync } from "node:fs"; import { join } from "node:path"; import { DatabaseSync } from "node:sqlite"; export interface ModelStatsRow { model_id: string; provider: string; unit_type: string; attempts: number; successes: number; avg_ms: number | null; avg_cost: number | null; avg_retries: number | null; } export interface QueryModelStatsOptions { sinceSeconds: number; unitType?: string; } interface SqliteStatement { all(...params: unknown[]): Record[]; } interface SqliteDb { prepare(sql: string): SqliteStatement; close?: () => void; } interface ParsedStatsArgs { command: "models"; sinceSeconds: number; unitType?: string; } export function parseDurationSeconds(value: string): number { const match = value.trim().match(/^(\d+(?:\.\d+)?)([smhdw])$/i); if (!match) { throw new Error(`Invalid duration: ${value}`); } const amount = Number(match[1]); const unit = match[2].toLowerCase(); const multiplier = unit === "s" ? 1 : unit === "m" ? 60 : unit === "h" ? 60 * 60 : unit === "d" ? 24 * 60 * 60 : 7 * 24 * 60 * 60; return Math.max(1, Math.floor(amount * multiplier)); } function toNumber(value: unknown, fallback = 0): number { const n = Number(value); return Number.isFinite(n) ? n : fallback; } function toNullableNumber(value: unknown): number | null { if (value === null || value === undefined) return null; const n = Number(value); return Number.isFinite(n) ? n : null; } export function queryModelStats( db: SqliteDb, options: QueryModelStatsOptions, ): ModelStatsRow[] { const where = ["recorded_at > unixepoch() - ?"]; const params: unknown[] = [options.sinceSeconds]; if (options.unitType) { where.push("unit_type = ?"); params.push(options.unitType); } const sql = ` SELECT model_id, provider, unit_type, COUNT(*) AS attempts, SUM(succeeded) AS successes, AVG(duration_ms) AS avg_ms, AVG(cost_usd) AS avg_cost, AVG(retries) AS avg_retries FROM llm_task_outcomes WHERE ${where.join(" AND ")} GROUP BY model_id, unit_type ORDER BY attempts DESC `; return db .prepare(sql) .all(...params) .map((row) => ({ model_id: String(row.model_id ?? ""), provider: String(row.provider ?? ""), unit_type: String(row.unit_type ?? ""), attempts: toNumber(row.attempts), successes: toNumber(row.successes), avg_ms: toNullableNumber(row.avg_ms), avg_cost: toNullableNumber(row.avg_cost), avg_retries: toNullableNumber(row.avg_retries), })); } function formatSuccessRate(row: ModelStatsRow): string { if (row.attempts <= 0) return "0.0%"; return `${((row.successes / row.attempts) * 100).toFixed(1)}%`; } function formatMs(value: number | null): string { return value === null ? "n/a" : value.toFixed(0); } function formatCost(value: number | null): string { return value === null ? "n/a" : `$${value.toFixed(4)}`; } function formatRetries(value: number | null): string { return value === null ? "n/a" : value.toFixed(2); } function pad(value: string, width: number, align: "left" | "right"): string { return align === "right" ? value.padStart(width) : value.padEnd(width); } export function formatModelStatsTable(rows: ModelStatsRow[]): string { const headers = [ "model_id", "provider", "unit_type", "attempts", "success_rate", "avg_ms", "avg_cost", "avg_retries", ]; const body = rows.map((row) => [ row.model_id, row.provider, row.unit_type, String(row.attempts), formatSuccessRate(row), formatMs(row.avg_ms), formatCost(row.avg_cost), formatRetries(row.avg_retries), ]); const widths = headers.map((header, index) => Math.max(header.length, ...body.map((row) => row[index].length)), ); const numeric = new Set([3, 4, 5, 6, 7]); const separator = `+${widths.map((width) => "-".repeat(width + 2)).join("+")}+`; const renderRow = (row: string[]) => `| ${row.map((cell, index) => pad(cell, widths[index], numeric.has(index) ? "right" : "left")).join(" | ")} |`; return ( [ separator, renderRow(headers), separator, ...body.map(renderRow), separator, ].join("\n") + "\n" ); } function parseStatsArgs(argv: string[]): ParsedStatsArgs | null { const args = argv.slice(1); if (args[0] !== "models") return null; const parsed: ParsedStatsArgs = { command: "models", sinceSeconds: parseDurationSeconds("7d"), }; for (let i = 1; i < args.length; i++) { const arg = args[i]; if (arg === "--unit-type" && i + 1 < args.length) { parsed.unitType = args[++i]; } else if (arg.startsWith("--unit-type=")) { parsed.unitType = arg.slice("--unit-type=".length); } else if (arg === "--since" && i + 1 < args.length) { parsed.sinceSeconds = parseDurationSeconds(args[++i]); } else if (arg.startsWith("--since=")) { parsed.sinceSeconds = parseDurationSeconds(arg.slice("--since=".length)); } } return parsed; } function usage(): string { return ( [ "Usage: sf stats models [--unit-type ] [--since ]", "", "Examples:", " sf stats models --since 7d", " sf stats models --unit-type execute-task --since 24h", ].join("\n") + "\n" ); } function openSqliteDb(dbPath: string): SqliteDb { return new DatabaseSync(dbPath, { readOnly: true }) as SqliteDb; } export async function runStatsCli( argv: string[], deps: { basePath: string; stdout?: Pick; stderr?: Pick; }, ): Promise { const stdout = deps.stdout ?? process.stdout; const stderr = deps.stderr ?? process.stderr; let parsed: ParsedStatsArgs | null; try { parsed = parseStatsArgs(argv); } catch (err) { stderr.write( `sf stats: ${err instanceof Error ? err.message : String(err)}\n`, ); return 1; } if (!parsed) { stderr.write(usage()); return 1; } const dbPath = join(deps.basePath, ".sf", "sf.db"); if (!existsSync(dbPath)) { stderr.write(`sf stats: database not found at ${dbPath}\n`); return 1; } let db: SqliteDb | null = null; try { db = openSqliteDb(dbPath); const rows = queryModelStats(db, parsed); if (rows.length === 0) { stdout.write("No model outcomes recorded for the selected window.\n"); } else { stdout.write(formatModelStatsTable(rows)); } return 0; } catch (err) { const message = err instanceof Error ? err.message : String(err); if (/no such table/i.test(message)) { stdout.write("No model outcomes recorded yet.\n"); return 0; } stderr.write(`sf stats: ${message}\n`); return 1; } finally { db?.close?.(); } }