/**
 * Tests for the auto-benchmark model selector.
 *
 * All tests pass an in-memory benchmark fixture so they don't depend on
 * real `learning/data/model-benchmarks.json` drift. The real JSON's
 * numbers change as vendors publish new cards; the selector's logic
 * must remain stable.
 */

import assert from "node:assert/strict";
import { test, describe } from 'vitest';
import type { CandidateModel } from "../benchmark-selector.ts";
import {
	_resetBenchmarkCache,
	selectByBenchmarks,
} from "../benchmark-selector.ts";

// Synthetic fixture. Real schema but handpicked numbers designed to make
// the assertions below exercise each selector feature.
const FIXTURE = {
	_meta: { version: "test" },
	"alpha-pro": {
		// reasoning champion
		swe_bench: 60,
		live_code_bench: 70,
		human_eval: 80,
		hle: 55,
		aime_2026: 90,
		gpqa: 80,
		mmlu_pro: 85,
		instruction_following: 80,
	},
	"beta-coder": {
		// coding champion
		swe_bench: 85,
		swe_bench_verified: null,
		live_code_bench: 90,
		human_eval: 95,
		hle: 30,
		aime_2026: 60,
		gpqa: 65,
		mmlu_pro: 70,
		instruction_following: 90,
	},
	"gamma-verified": {
		// publishes swe_bench_verified only
		swe_bench: null,
		swe_bench_verified: 80,
		live_code_bench: 80,
		human_eval: 88,
		hle: 25,
		aime_2026: 50,
		gpqa: 60,
		mmlu_pro: 65,
		instruction_following: 85,
	},
	"delta-fast": {
		// lightweight specialist
		human_eval: 92,
		instruction_following: 95,
		long_context_ruler: 90,
	},
	"ministral-3b-latest": {
		// Tiny model with excellent fast+correct scores should still be below
		// the completion/validation floor.
		human_eval: 99,
		instruction_following: 99,
		long_context_ruler: 99,
	},
	"mistral-small-latest": {
		human_eval: 78,
		instruction_following: 82,
		long_context_ruler: 80,
	},
	"ministral-3b-2512": {
		human_eval: 99,
		instruction_following: 99,
		long_context_ruler: 99,
	},
	"empty-shell": {
		// all nulls
		swe_bench: null,
		live_code_bench: null,
		human_eval: null,
		hle: null,
	},
};

describe("benchmark-selector", () => {
	test.beforeEach(() => _resetBenchmarkCache());

	test("returns null for empty candidate pool", () => {
		assert.equal(
			selectByBenchmarks("plan-milestone", [], { benchmarks: FIXTURE }),
			null,
		);
	});

	test("reasoning unit prefers the reasoning champion", () => {
		const candidates: CandidateModel[] = [
			{ provider: "p1", id: "alpha-pro" },
			{ provider: "p2", id: "beta-coder" },
			{ provider: "p3", id: "delta-fast" },
		];
		const r = selectByBenchmarks("validate-milestone", candidates, {
			benchmarks: FIXTURE,
		});
		assert.ok(r);
		assert.equal(
			r.primary,
			"p1/alpha-pro",
			`scores: ${JSON.stringify(r.scores)}`,
		);
	});

	test("coding unit prefers the coding champion", () => {
		const candidates: CandidateModel[] = [
			{ provider: "p1", id: "alpha-pro" },
			{ provider: "p2", id: "beta-coder" },
			{ provider: "p3", id: "delta-fast" },
		];
		const r = selectByBenchmarks("execute-task", candidates, {
			benchmarks: FIXTURE,
		});
		assert.ok(r);
		assert.equal(r.primary, "p2/beta-coder");
	});

	test("lightweight unit prefers the fast specialist", () => {
		const candidates: CandidateModel[] = [
			{ provider: "p1", id: "alpha-pro" },
			{ provider: "p2", id: "beta-coder" },
			{ provider: "p3", id: "delta-fast" },
		];
		const r = selectByBenchmarks("complete-slice", candidates, {
			benchmarks: FIXTURE,
		});
		assert.ok(r);
		assert.equal(r.primary, "p3/delta-fast");
	});

	test("completion units enforce a standard-tier floor before benchmark ranking", () => {
		const r = selectByBenchmarks(
			"complete-slice",
			[
				{ provider: "mistral", id: "ministral-3b-latest" },
				{ provider: "mistral", id: "mistral-small-latest" },
			],
			{ benchmarks: FIXTURE },
		);
		assert.ok(r);
		assert.equal(
			r.primary,
			"mistral/mistral-small-latest",
			`scores: ${JSON.stringify(r.scores)}`,
		);
		assert.ok(
			!("mistral/ministral-3b-latest" in r.scores),
			"tiny light-tier model should be excluded before scoring",
		);
	});

	test("completion tier floor filters provider-specific tiny model ids", () => {
		const r = selectByBenchmarks(
			"complete-slice",
			[
				{ provider: "openrouter", id: "mistralai/ministral-3b-2512" },
				{ provider: "mistral", id: "mistral-small-latest" },
			],
			{ benchmarks: FIXTURE },
		);
		assert.ok(r);
		assert.equal(r.primary, "mistral/mistral-small-latest");
		assert.ok(
			!("openrouter/mistralai/ministral-3b-2512" in r.scores),
			"provider-specific 3B models should be excluded before scoring",
		);
	});

	test("simple execution may still select a light model when benchmarks justify it", () => {
		const r = selectByBenchmarks(
			"execute-task-simple",
			[
				{ provider: "mistral", id: "ministral-3b-latest" },
				{ provider: "mistral", id: "mistral-small-latest" },
			],
			{ benchmarks: FIXTURE },
		);
		assert.ok(r);
		assert.equal(r.primary, "mistral/ministral-3b-latest");
	});

	test("swe_bench and swe_bench_verified are treated as equivalent", () => {
		// gamma-verified only publishes swe_bench_verified; a selector that
		// treats them as different keys would score gamma at 0 for units
		// that request "swe_bench". Expect a non-zero score.
		const r = selectByBenchmarks(
			"execute-task",
			[{ provider: "p", id: "gamma-verified" }],
			{ benchmarks: FIXTURE },
		);
		assert.ok(r);
		assert.ok(
			r.scores["p/gamma-verified"] > 50,
			`expected nonzero score, got ${r.scores["p/gamma-verified"]}`,
		);
	});

	test("models with all-null benchmarks score 0 but still appear", () => {
		const r = selectByBenchmarks(
			"execute-task",
			[
				{ provider: "p1", id: "beta-coder" },
				{ provider: "p2", id: "empty-shell" },
			],
			{ benchmarks: FIXTURE },
		);
		assert.ok(r);
		assert.equal(r.primary, "p1/beta-coder");
		assert.equal(r.scores["p2/empty-shell"], 0);
		// empty-shell still shows in fallbacks — it's a candidate, just ranked low
		assert.ok(r.fallbacks.includes("p2/empty-shell"));
	});

	test("normalises by populated weight — sparse-but-strong beats dense-but-weak", () => {
		// "delta-fast" has only 3 dimensions populated, but each ≥ 90. It
		// should outscore a hypothetical dense-mediocre peer on the
		// complete-slice (fast+correct) profile.
		const fixture = {
			...FIXTURE,
			"dense-mediocre": {
				human_eval: 50,
				instruction_following: 50,
				long_context_ruler: 50,
				swe_bench: 50,
				live_code_bench: 50,
				hle: 50,
			},
		};
		const r = selectByBenchmarks(
			"complete-slice",
			[
				{ provider: "p1", id: "delta-fast" },
				{ provider: "p2", id: "dense-mediocre" },
			],
			{ benchmarks: fixture },
		);
		assert.ok(r);
		assert.equal(r.primary, "p1/delta-fast");
	});

	test("provider diversity: fallbacks don't collapse into one provider", () => {
		// Three models all from provider p1, one from p2. Fallback list
		// should surface p2 even if p1's second+third models score higher.
		const candidates: CandidateModel[] = [
			{ provider: "p1", id: "alpha-pro" },
			{ provider: "p1", id: "beta-coder" },
			{ provider: "p1", id: "gamma-verified" },
			{ provider: "p2", id: "delta-fast" },
		];
		const r = selectByBenchmarks("plan-milestone", candidates, {
			benchmarks: FIXTURE,
			maxEntries: 2,
		});
		assert.ok(r);
		const providersInResult = new Set([
			r.primary.split("/")[0],
			...r.fallbacks.map((f) => f.split("/")[0]),
		]);
		// Expect at least 2 distinct providers once maxEntries >= 2
		assert.ok(
			providersInResult.size >= 2,
			`providers: ${[...providersInResult].join(",")}`,
		);
	});

	test("deterministic tie-breaking by coverage then lexical id", () => {
		const fixture = {
			_meta: {},
			"same-a": { swe_bench: 80, live_code_bench: 80, hle: 80, gpqa: 80 },
			"same-b": { swe_bench: 80, live_code_bench: 80, hle: 80, gpqa: 80 },
		};
		const r = selectByBenchmarks(
			"plan-milestone",
			[
				{ provider: "p2", id: "same-b" },
				{ provider: "p1", id: "same-a" },
			],
			{ benchmarks: fixture },
		);
		assert.ok(r);
		// Identical scores + coverage → fall back to lexical "p1/same-a" < "p2/same-b"
		assert.equal(r.primary, "p1/same-a");
	});

	test("near benchmark ties prefer lower blended token cost", () => {
		const fixture = {
			_meta: {},
			"expensive-coder": {
				swe_bench: 80,
				live_code_bench: 80,
				human_eval: 80,
				instruction_following: 80,
			},
			"cheap-coder": {
				swe_bench: 79,
				live_code_bench: 79,
				human_eval: 79,
				instruction_following: 79,
			},
		};
		const r = selectByBenchmarks(
			"execute-task",
			[
				{
					provider: "p1",
					id: "expensive-coder",
					cost: { input: 30, output: 90 },
				},
				{
					provider: "p2",
					id: "cheap-coder",
					cost: { input: 0.5, output: 2 },
				},
			],
			{ benchmarks: fixture },
		);
		assert.ok(r);
		assert.equal(r.primary, "p2/cheap-coder");
		assert.ok(
			(r.costEstimates["p2/cheap-coder"] ?? Infinity) <
				(r.costEstimates["p1/expensive-coder"] ?? 0),
		);
	});

	test("capability metadata breaks exact benchmark and cost ties", () => {
		const fixture = {
			_meta: {},
			plain: { hle: 70, gpqa: 70, mmlu_pro: 70, instruction_following: 70 },
			capable: {
				hle: 70,
				gpqa: 70,
				mmlu_pro: 70,
				instruction_following: 70,
			},
		};
		const r = selectByBenchmarks(
			"discuss-milestone",
			[
				{
					provider: "a",
					id: "plain",
					cost: { input: 1, output: 3 },
					contextWindow: 8192,
					maxTokens: 8192,
					reasoning: false,
				},
				{
					provider: "z",
					id: "capable",
					cost: { input: 1, output: 3 },
					contextWindow: 1048576,
					maxTokens: 131072,
					reasoning: true,
					capabilities: { supportsXhigh: true, thinkingNoBudget: true },
				},
			],
			{ benchmarks: fixture },
		);
		assert.ok(r);
		assert.equal(r.primary, "z/capable");
		assert.ok(r.capabilitySignals["z/capable"] > r.capabilitySignals["a/plain"]);
	});

	test("returns full score map for UI / debug", () => {
		const r = selectByBenchmarks(
			"execute-task",
			[
				{ provider: "p", id: "beta-coder" },
				{ provider: "p", id: "alpha-pro" },
			],
			{ benchmarks: FIXTURE },
		);
		assert.ok(r);
		assert.ok(typeof r.scores["p/beta-coder"] === "number");
		assert.ok(typeof r.scores["p/alpha-pro"] === "number");
		assert.ok(r.scores["p/beta-coder"] > r.scores["p/alpha-pro"]);
	});

	test("unknown unit type uses default coding profile without throwing", () => {
		const r = selectByBenchmarks(
			"some-brand-new-unit",
			[
				{ provider: "p1", id: "beta-coder" },
				{ provider: "p2", id: "alpha-pro" },
			],
			{ benchmarks: FIXTURE },
		);
		assert.ok(r);
		// Default profile is coding-leaning, so beta-coder should win
		assert.equal(r.primary, "p1/beta-coder");
		assert.match(r.profile, /default/);
	});

	test("Kimi Code wire id uses Kimi K2.6 benchmark identity", () => {
		const fixture = {
			_meta: {},
			"kimi-k2.6": {
				swe_bench: 100,
				live_code_bench: 100,
				human_eval: 100,
				aime_2026: 100,
				gpqa: 100,
			},
			"kimi-for-coding": {
				swe_bench: 1,
				live_code_bench: 1,
				human_eval: 1,
				aime_2026: 1,
				gpqa: 1,
			},
		};
		const r = selectByBenchmarks(
			"execute-task",
			[{ provider: "kimi-coding", id: "kimi-for-coding" }],
			{ benchmarks: fixture },
		);
		assert.ok(r);
		assert.equal(r.primary, "kimi-coding/kimi-for-coding");
		assert.ok(
			r.scores["kimi-coding/kimi-for-coding"] > 90,
			"expected Kimi Code route to score from kimi-k2.6, not wire-id row",
		);
	});

	test("Kimi K2.5 wire ids use Kimi K2.5 benchmark identity", () => {
		const fixture = {
			_meta: {},
			"kimi-k2.5": {
				swe_bench: 99,
				live_code_bench: 99,
				human_eval: 99,
				aime_2026: 99,
				gpqa: 99,
			},
			"kimi-for-coding": {
				swe_bench: 1,
				live_code_bench: 1,
				human_eval: 1,
				aime_2026: 1,
				gpqa: 1,
			},
		};
		const r = selectByBenchmarks(
			"execute-task",
			[
				{ provider: "opencode-go", id: "kimi-k2.5" },
				{ provider: "openrouter", id: "moonshotai/kimi-k2.5" },
			],
			{ benchmarks: fixture },
		);
		assert.ok(r);
		assert.ok(
			r.scores["opencode-go/kimi-k2.5"] > 90,
			"expected provider Kimi K2.5 route to score from kimi-k2.5",
		);
		assert.ok(
			r.scores["openrouter/moonshotai/kimi-k2.5"] > 90,
			"expected provider-specific Kimi K2.5 route to score from kimi-k2.5",
		);
	});

	test("findBenchmarkKey matches date-suffixed model variants", () => {
		const fixture = {
			_meta: {},
			"devstral-medium-latest": { swe_bench: 60 },
		};
		// Real-world: pi-ai catalog has "devstral-medium-2507" but benchmarks
		// only key the "latest" family. Selector should match.
		const r = selectByBenchmarks(
			"execute-task",
			[{ provider: "mistral", id: "devstral-medium-2507" }],
			{ benchmarks: fixture },
		);
		assert.ok(r);
		assert.ok(
			r.scores["mistral/devstral-medium-2507"] > 0,
			"expected non-zero via family match",
		);
	});
});
