singularity-forge/scripts/validate-model-cost-table.mjs

#!/usr/bin/env node
/**
 * validate-model-cost-table.mjs
 *
 * Purpose: verify that every model in packages/ai/src/models.generated.ts that is
 * also in src/resources/extensions/sf/model-cost-table.js has a matching entry and
 * report models present in models.generated.ts but missing from the extension table.
 *
 * The two tables intentionally use different schemas:
 *   models.generated.ts : cost.input / cost.output in $/million tokens (auto-generated)
 *   model-cost-table.js : inputPer1k / outputPer1k in $/1K tokens (hand-maintained)
 *
 * This script catches coverage gaps and price divergence > 5% so engineers know when
 * the extension table needs updating after a models.generated.ts regeneration.
 *
 * Usage:
 *   node scripts/validate-model-cost-table.mjs
 *   node scripts/validate-model-cost-table.mjs --threshold 10  # % diff threshold
 */

import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { resolve, dirname } from "node:path";

const __dirname = dirname(fileURLToPath(import.meta.url));
const root = resolve(__dirname, "..");

// ---------------------------------------------------------------------------
// Parse the extension cost table (JS module — use dynamic import)
// ---------------------------------------------------------------------------

const costTablePath = resolve(
	root,
	"src/resources/extensions/sf/model-cost-table.js",
);

// The cost table uses ES module exports — eval it via a data: URL import
const rawCostTable = readFileSync(costTablePath, "utf8");

// Strip JSDoc comments and extract the array literal safely using regex
// This avoids spinning up a full module system just for a static data file.
const arrayMatch = rawCostTable.match(
	/export const BUNDLED_COST_TABLE\s*=\s*(\[[\s\S]*?\]);/,
);
if (!arrayMatch) {
	console.error(
		"❌ Could not parse BUNDLED_COST_TABLE from model-cost-table.js",
	);
	process.exit(1);
}

let extensionTable;
try {
	// biome-ignore lint/security/noEval: parsing a static, local-file data literal
	extensionTable = eval(`(${arrayMatch[1]})`);
} catch (err) {
	console.error("❌ Failed to evaluate BUNDLED_COST_TABLE:", err.message);
	process.exit(1);
}

/** @type {Map<string, { inputPer1k: number; outputPer1k: number }>} */
const extensionById = new Map(
	extensionTable.map((entry) => [
		entry.id,
		{ inputPer1k: entry.inputPer1k, outputPer1k: entry.outputPer1k },
	]),
);

// ---------------------------------------------------------------------------
// Parse models.generated.ts (text scan — avoid TS compilation)
// ---------------------------------------------------------------------------

const generatedPath = resolve(root, "packages/ai/src/models.generated.ts");
const generatedSrc = readFileSync(generatedPath, "utf8");

// Extract all model blocks: "model-id": { ... cost: { input: N, output: N ... } ... }
// Use a simple regex to find id + cost pairs; not a full parser but sufficient for CI.
const modelBlocks = [];
// Match patterns like: "model-id": { ... cost: { input: X, output: Y ... }
const blockRe =
	/"([a-zA-Z0-9._:/@-]+)":\s*\{[^{}]*?cost:\s*\{[^{}]*?input:\s*([\d.]+)[^{}]*?output:\s*([\d.]+)[^{}]*?\}/gs;
let match;
while ((match = blockRe.exec(generatedSrc)) !== null) {
	const [, id, inputStr, outputStr] = match;
	const input = parseFloat(inputStr);
	const output = parseFloat(outputStr);
	if (!Number.isNaN(input) && !Number.isNaN(output)) {
		modelBlocks.push({ id, inputPerM: input, outputPerM: output });
	}
}

// De-duplicate (same model id can appear under multiple providers in generated file)
/** @type {Map<string, { inputPerM: number; outputPerM: number }>} */
const generatedById = new Map();
for (const { id, inputPerM, outputPerM } of modelBlocks) {
	if (!generatedById.has(id)) {
		generatedById.set(id, { inputPerM, outputPerM });
	}
}

// ---------------------------------------------------------------------------
// Compare coverage and prices
// ---------------------------------------------------------------------------

const thresholdArg = process.argv.indexOf("--threshold");
const threshold =
	thresholdArg !== -1 ? parseFloat(process.argv[thresholdArg + 1]) : 5;

const missing = [];
const diverged = [];
let checked = 0;

for (const [id, ext] of extensionById) {
	const gen = generatedById.get(id);
	if (!gen) {
		// Model in extension table but not in generated — that's OK (extension can have extras)
		continue;
	}

	checked++;
	// Convert generated per-million to per-1K for comparison
	const genInputPer1k = gen.inputPerM / 1000;
	const genOutputPer1k = gen.outputPerM / 1000;

	const inputDiff =
		genInputPer1k > 0
			? Math.abs(ext.inputPer1k - genInputPer1k) / genInputPer1k
			: 0;
	const outputDiff =
		genOutputPer1k > 0
			? Math.abs(ext.outputPer1k - genOutputPer1k) / genOutputPer1k
			: 0;

	if (inputDiff > threshold / 100 || outputDiff > threshold / 100) {
		diverged.push({
			id,
			ext,
			gen: { inputPer1k: genInputPer1k, outputPer1k: genOutputPer1k },
			inputDiff: (inputDiff * 100).toFixed(1),
			outputDiff: (outputDiff * 100).toFixed(1),
		});
	}
}

// Also report models in generated that are missing from extension table
// (only for non-zero-cost models since zero-cost models like local/self-hosted are OK to omit)
for (const [id, gen] of generatedById) {
	if (!extensionById.has(id) && gen.inputPerM > 0) {
		missing.push({ id, gen });
	}
}

// ---------------------------------------------------------------------------
// Report
// ---------------------------------------------------------------------------

let hasIssues = false;

if (missing.length > 0) {
	console.log(
		`\n⚠️  ${missing.length} model(s) in models.generated.ts missing from model-cost-table.js:\n`,
	);
	for (const { id, gen } of missing) {
		const inputPer1k = (gen.inputPerM / 1000).toFixed(6);
		const outputPer1k = (gen.outputPerM / 1000).toFixed(6);
		console.log(
			`  ${id}  →  inputPer1k: ${inputPer1k}, outputPer1k: ${outputPer1k}`,
		);
	}
	hasIssues = true;
}

if (diverged.length > 0) {
	console.log(
		`\n⚠️  ${diverged.length} model(s) with price divergence > ${threshold}% between tables:\n`,
	);
	for (const { id, ext, gen, inputDiff, outputDiff } of diverged) {
		console.log(`  ${id}`);
		console.log(
			`    extension:  inputPer1k=${ext.inputPer1k}, outputPer1k=${ext.outputPer1k}`,
		);
		console.log(
			`    generated:  inputPer1k=${gen.inputPer1k.toFixed(6)}, outputPer1k=${gen.outputPer1k.toFixed(6)}`,
		);
		console.log(`    diff: input=${inputDiff}%, output=${outputDiff}%`);
	}
	hasIssues = true;
}

if (!hasIssues) {
	console.log(
		`✅ model-cost-table.js is consistent with models.generated.ts (${checked} models checked, ${missing.length} missing, divergence threshold ${threshold}%)`,
	);
}

// Exit 0 even with issues — this is informational, not a hard gate.
// Set STRICT=1 to fail CI on divergence.
if (hasIssues && process.env.STRICT === "1") {
	process.exit(1);
}