/**
 * benchmark-coverage.test.mjs
 *
 * Tests that computeBenchmarkCoverage partitions the dispatchable model set
 * against the static benchmark file, applies the user policy filters
 * (allowed_providers, provider_model_allow, provider_model_block, plus the
 * built-in per-provider policy), and that writeBenchmarkCoverage /
 * detectCoverageChange round-trip and notice change correctly.
 */
import assert from "node:assert/strict";
import {
	existsSync,
	mkdirSync,
	mkdtempSync,
	readFileSync,
	rmSync,
	writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, beforeEach, describe, test } from "vitest";

// Touch preferences.js so the prefs loader / lazy circular dep is wired.
import "../preferences.js";
import {
	computeBenchmarkCoverage,
	detectCoverageChange,
	normalizeForBenchmarkLookup,
	writeBenchmarkCoverage,
} from "../benchmark-coverage.js";

// ─── Test isolation ──────────────────────────────────────────────────────────

const tmpDirs = [];
let originalSfHome;

beforeEach(() => {
	originalSfHome = process.env.SF_HOME;
});

afterEach(() => {
	while (tmpDirs.length > 0) {
		rmSync(tmpDirs.pop(), { recursive: true, force: true });
	}
	if (originalSfHome === undefined) delete process.env.SF_HOME;
	else process.env.SF_HOME = originalSfHome;
});

function tempSfHome() {
	const dir = mkdtempSync(join(tmpdir(), "sf-benchmark-coverage-test-"));
	tmpDirs.push(dir);
	process.env.SF_HOME = dir;
	return dir;
}

function writeCatalog(sfHome, providerId, modelEntries) {
	const dir = join(sfHome, "model-catalog");
	mkdirSync(dir, { recursive: true });
	writeFileSync(
		join(dir, `${providerId}.json`),
		JSON.stringify({
			fetchedAt: new Date().toISOString(),
			modelIds: modelEntries,
		}),
		"utf-8",
	);
}

// ─── normalizeForBenchmarkLookup ─────────────────────────────────────────────

describe("normalizeForBenchmarkLookup", () => {
	test("lowercases plain ids", () => {
		assert.equal(normalizeForBenchmarkLookup("MiniMax-M2.7"), "minimax-m2.7");
	});
	test("strips provider/ prefix (openrouter)", () => {
		assert.equal(
			normalizeForBenchmarkLookup("deepseek/deepseek-v4-flash"),
			"deepseek-v4-flash",
		);
	});
	test("strips :free suffix", () => {
		assert.equal(
			normalizeForBenchmarkLookup("qwen/qwen3-coder:free"),
			"qwen3-coder",
		);
	});
	test("preserves :Nb size suffix", () => {
		assert.equal(
			normalizeForBenchmarkLookup("deepseek-v3.1:671b"),
			"deepseek-v3.1:671b",
		);
	});
});

// ─── computeBenchmarkCoverage ────────────────────────────────────────────────

describe("computeBenchmarkCoverage", () => {
	test("partitions covered vs uncovered using the real benchmark file", () => {
		const home = tempSfHome();
		// glm-4.5 is in the static benchmark file; bogus-not-in-bench-2026 is not.
		writeCatalog(home, "zai", [
			{ id: "glm-4.5" },
			{ id: "bogus-not-in-bench-2026" },
		]);

		const result = computeBenchmarkCoverage({
			allowed_providers: ["zai"],
		});

		const coveredIds = result.covered.map((m) => m.id);
		const uncoveredIds = result.uncovered.map((m) => m.id);
		assert.ok(coveredIds.includes("glm-4.5"), "glm-4.5 should be covered");
		assert.ok(
			uncoveredIds.includes("bogus-not-in-bench-2026"),
			"bogus model should be uncovered",
		);
		assert.equal(result.summary.total, 2);
		assert.equal(result.summary.coveredCount, 1);
		assert.equal(result.summary.uncoveredCount, 1);
		assert.equal(result.summary.coverageRatio, 0.5);
	});

	test("allowed_providers excludes models from non-listed providers", () => {
		const home = tempSfHome();
		writeCatalog(home, "zai", [{ id: "glm-4.5" }]);
		writeCatalog(home, "anthropic", [{ id: "claude-sonnet-4-6" }]);

		const result = computeBenchmarkCoverage({
			allowed_providers: ["zai"],
		});

		const allIds = [...result.covered, ...result.uncovered].map(
			(m) => `${m.provider}/${m.id}`,
		);
		assert.ok(allIds.includes("zai/glm-4.5"));
		assert.ok(
			!allIds.some((id) => id.startsWith("anthropic/")),
			"anthropic must not appear when not in allowed_providers",
		);
	});

	test("provider_model_block: ollama-cloud gemini-* is excluded", () => {
		const home = tempSfHome();
		writeCatalog(home, "ollama-cloud", [
			{ id: "deepseek-v4-flash" },
			{ id: "gemini-3-flash-preview" },
		]);

		const result = computeBenchmarkCoverage({
			allowed_providers: ["ollama-cloud"],
			provider_model_block: { "ollama-cloud": ["gemini-*"] },
		});

		const ids = [...result.covered, ...result.uncovered].map((m) => m.id);
		assert.ok(ids.includes("deepseek-v4-flash"));
		assert.ok(
			!ids.includes("gemini-3-flash-preview"),
			"gemini-3-flash-preview must be blocked",
		);
	});

	test("openrouter built-in policy keeps only :free / zero-cost", () => {
		const home = tempSfHome();
		writeCatalog(home, "openrouter", [
			{ id: "anthropic/claude-opus-4-7" }, // paid → blocked by built-in
			{ id: "qwen/qwen3-coder:free" }, // :free → allowed
			{
				id: "openrouter/zero-cost-model",
				cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
			}, // zero-cost → allowed
		]);

		const result = computeBenchmarkCoverage({
			allowed_providers: ["openrouter"],
		});

		const ids = [...result.covered, ...result.uncovered].map((m) => m.id);
		assert.ok(
			!ids.includes("anthropic/claude-opus-4-7"),
			"paid claude blocked",
		);
		assert.ok(ids.includes("qwen/qwen3-coder:free"), ":free allowed");
		assert.ok(ids.includes("openrouter/zero-cost-model"), "zero-cost allowed");
	});

	test("returns empty when catalog dir is missing", () => {
		tempSfHome(); // pinned but no catalog dir written
		const result = computeBenchmarkCoverage({});
		assert.deepEqual(result.covered, []);
		assert.deepEqual(result.uncovered, []);
		assert.equal(result.summary.total, 0);
		assert.equal(result.summary.coverageRatio, 0);
	});
});

// ─── writeBenchmarkCoverage + detectCoverageChange ───────────────────────────

describe("writeBenchmarkCoverage / detectCoverageChange", () => {
	test("write + read round-trips", () => {
		const home = tempSfHome();
		const coverage = {
			covered: [{ provider: "zai", id: "glm-4.5" }],
			uncovered: [{ provider: "zai", id: "bogus" }],
			summary: {
				total: 2,
				coveredCount: 1,
				uncoveredCount: 1,
				coverageRatio: 0.5,
			},
		};
		writeBenchmarkCoverage(coverage);
		const path = join(home, "benchmark-coverage.json");
		assert.ok(existsSync(path), "coverage file should be written");
		const parsed = JSON.parse(readFileSync(path, "utf-8"));
		assert.equal(parsed.schemaVersion, 1);
		assert.equal(parsed.summary.total, 2);
		assert.deepEqual(parsed.uncovered, [{ provider: "zai", id: "bogus" }]);
	});

	test("detectCoverageChange returns true on first write (no prior file)", () => {
		tempSfHome();
		assert.equal(
			detectCoverageChange({ uncovered: [{ provider: "zai", id: "x" }] }),
			true,
		);
	});

	test("detectCoverageChange returns false when uncovered set is unchanged", () => {
		tempSfHome();
		const coverage = {
			covered: [],
			uncovered: [
				{ provider: "zai", id: "x" },
				{ provider: "kimi-coding", id: "y" },
			],
			summary: {
				total: 2,
				coveredCount: 0,
				uncoveredCount: 2,
				coverageRatio: 0,
			},
		};
		writeBenchmarkCoverage(coverage);
		assert.equal(detectCoverageChange(coverage), false);
	});

	test("detectCoverageChange returns true when a new uncovered id appears", () => {
		tempSfHome();
		writeBenchmarkCoverage({
			covered: [],
			uncovered: [{ provider: "zai", id: "x" }],
			summary: {
				total: 1,
				coveredCount: 0,
				uncoveredCount: 1,
				coverageRatio: 0,
			},
		});
		assert.equal(
			detectCoverageChange({
				covered: [],
				uncovered: [
					{ provider: "zai", id: "x" },
					{ provider: "zai", id: "y" },
				],
			}),
			true,
		);
	});
});
