/**
 * Unit tests for model-learner.js
 *
 * Purpose: verify per-task-type model performance tracking, failure analysis,
 * and A/B testing candidate identification work correctly.
 */

import { mkdirSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, beforeEach, describe, expect, test } from "vitest";
import {
	FailureAnalyzer,
	ModelLearner,
	ModelPerformanceTracker,
} from "../model-learner.js";

describe("ModelPerformanceTracker", () => {
	let tracker: ModelPerformanceTracker;

	beforeEach(() => {
		tracker = new ModelPerformanceTracker();
	});

	test("tracks success and failure counts", () => {
		tracker.recordOutcome("execute-task", "gpt-4o", true, false, 100, 0.05);
		tracker.recordOutcome("execute-task", "gpt-4o", true, false, 120, 0.06);
		tracker.recordOutcome("execute-task", "gpt-4o", false, false, 100, 0.05);

		const stats = tracker.getStats("execute-task", "gpt-4o");
		expect(stats.successes).toBe(2);
		expect(stats.failures).toBe(1);
		expect(stats.total).toBe(3);
	});

	test("computes success rate correctly", () => {
		tracker.recordOutcome("plan-slice", "claude-opus", true, false, 50, 0.02);
		tracker.recordOutcome("plan-slice", "claude-opus", true, false, 60, 0.03);
		tracker.recordOutcome("plan-slice", "claude-opus", true, false, 55, 0.025);

		const stats = tracker.getStats("plan-slice", "claude-opus");
		expect(stats.successRate).toBe(1.0);
	});

	test("detects demotion when failure rate exceeds threshold", () => {
		// Record 6 failures out of 10 attempts (60% failure rate)
		for (let i = 0; i < 4; i++) {
			tracker.recordOutcome(
				"execute-task",
				"bad-model",
				true,
				false,
				100,
				0.05,
			);
		}
		for (let i = 0; i < 6; i++) {
			tracker.recordOutcome(
				"execute-task",
				"bad-model",
				false,
				false,
				100,
				0.05,
			);
		}

		const shouldDemote = tracker.shouldDemote("execute-task", "bad-model", 0.5);
		expect(shouldDemote).toBe(true);
	});

	test("does not demote when failure rate below threshold", () => {
		// Record 2 failures out of 10 (20% failure rate)
		for (let i = 0; i < 8; i++) {
			tracker.recordOutcome(
				"execute-task",
				"good-model",
				true,
				false,
				100,
				0.05,
			);
		}
		for (let i = 0; i < 2; i++) {
			tracker.recordOutcome(
				"execute-task",
				"good-model",
				false,
				false,
				100,
				0.05,
			);
		}

		const shouldDemote = tracker.shouldDemote(
			"execute-task",
			"good-model",
			0.5,
		);
		expect(shouldDemote).toBe(false);
	});

	test("returns ranked models sorted by success rate", () => {
		// Model A: 90% success
		for (let i = 0; i < 9; i++) {
			tracker.recordOutcome("execute-task", "model-a", true, false, 100, 0.05);
		}
		tracker.recordOutcome("execute-task", "model-a", false, false, 100, 0.05);

		// Model B: 100% success
		for (let i = 0; i < 5; i++) {
			tracker.recordOutcome("execute-task", "model-b", true, false, 100, 0.05);
		}

		// Model C: 50% success
		tracker.recordOutcome("execute-task", "model-c", true, false, 100, 0.05);
		tracker.recordOutcome("execute-task", "model-c", false, false, 100, 0.05);

		const ranked = tracker.getRankedModels("execute-task", 0);
		expect(ranked.length).toBeGreaterThan(0);
		// Model B should rank higher than A, A higher than C
		const bIdx = ranked.findIndex((r) => r.modelId === "model-b");
		const aIdx = ranked.findIndex((r) => r.modelId === "model-a");
		const cIdx = ranked.findIndex((r) => r.modelId === "model-c");
		expect(bIdx).toBeLessThan(aIdx);
		expect(aIdx).toBeLessThan(cIdx);
	});

	test("accumulates tokens and cost correctly", () => {
		tracker.recordOutcome("execute-task", "gpt-4o", true, false, 1000, 0.5);
		tracker.recordOutcome("execute-task", "gpt-4o", true, false, 2000, 1.0);

		const stats = tracker.getStats("execute-task", "gpt-4o");
		expect(stats.totalTokens).toBe(3000);
		expect(stats.totalCost).toBe(1.5);
	});
});

describe("FailureAnalyzer", () => {
	let analyzer: FailureAnalyzer;

	beforeEach(() => {
		analyzer = new FailureAnalyzer();
	});

	test("categorizes failures by reason", () => {
		analyzer.logFailure(
			"execute-task",
			"gpt-4o",
			"quality_check_failed",
			false,
			{},
		);
		analyzer.logFailure("execute-task", "gpt-4o", "timeout", true, {});
		analyzer.logFailure(
			"execute-task",
			"claude-opus",
			"quality_check_failed",
			false,
			{},
		);

		const summary = analyzer.getFailureSummary("execute-task", "gpt-4o");
		expect(summary.reasons).toBeDefined();
		expect(summary.reasons.quality_check_failed).toBe(1);
		expect(summary.reasons.timeout).toBe(1);
	});

	test("detects timeout patterns", () => {
		analyzer.logFailure("execute-task", "slow-model", "timeout", true, {});
		analyzer.logFailure("execute-task", "slow-model", "timeout", true, {});
		analyzer.logFailure("execute-task", "slow-model", "timeout", true, {});

		const summary = analyzer.getFailureSummary("execute-task", "slow-model");
		expect(summary.patterns).toBeDefined();
		expect(summary.patterns.includes("timeout_prone")).toBe(true);
	});

	test("detects quality check failures", () => {
		for (let i = 0; i < 5; i++) {
			analyzer.logFailure(
				"execute-task",
				"bad-quality-model",
				"quality_check_failed",
				false,
				{},
			);
		}

		const summary = analyzer.getFailureSummary(
			"execute-task",
			"bad-quality-model",
		);
		expect(summary.patterns).toBeDefined();
		expect(summary.patterns.includes("quality_issues")).toBe(true);
	});

	test("tracks failure counts per model", () => {
		analyzer.logFailure(
			"plan-slice",
			"model-x",
			"quality_check_failed",
			false,
			{},
		);
		analyzer.logFailure(
			"plan-slice",
			"model-x",
			"quality_check_failed",
			false,
			{},
		);
		analyzer.logFailure("execute-task", "model-x", "timeout", true, {});

		const planSummary = analyzer.getFailureSummary("plan-slice", "model-x");
		const execSummary = analyzer.getFailureSummary("execute-task", "model-x");

		expect(planSummary.failureCount).toBe(2);
		expect(execSummary.failureCount).toBe(1);
	});
});

describe("ModelLearner (integration)", () => {
	let tmpDir: string;
	let learner: ModelLearner;

	beforeEach(() => {
		tmpDir = join(tmpdir(), `test-model-learner-${Date.now()}`);
		mkdirSync(tmpDir, { recursive: true });
		learner = new ModelLearner(tmpDir);
	});

	afterEach(() => {
		if (tmpDir) {
			rmSync(tmpDir, { recursive: true, force: true });
		}
	});

	test("records outcomes to storage", () => {
		learner.recordOutcome("execute-task", "gpt-4o", {
			success: true,
			timeout: false,
			tokensUsed: 5000,
			costUsd: 0.15,
		});

		const rankedModels = learner.getRankedModels("execute-task");
		expect(rankedModels.length).toBeGreaterThan(0);
		expect(rankedModels[0].modelId).toBe("gpt-4o");
	});

	test("logs failures with context", () => {
		learner.logFailure("plan-slice", "claude-opus", {
			reason: "quality_check_failed",
			timeout: false,
			tokensUsed: 3000,
			context: { unitId: "M001/S01" },
		});

		const summary = learner.getFailureSummary("plan-slice", "claude-opus");
		expect(summary.failureCount).toBeGreaterThan(0);
	});

	test("identifies demotion candidates", () => {
		// Create high-failure-rate model
		for (let i = 0; i < 3; i++) {
			learner.recordOutcome("execute-task", "unreliable", {
				success: false,
				timeout: false,
				tokensUsed: 2000,
				costUsd: 0.1,
			});
		}
		for (let i = 0; i < 1; i++) {
			learner.recordOutcome("execute-task", "unreliable", {
				success: true,
				timeout: false,
				tokensUsed: 2000,
				costUsd: 0.1,
			});
		}

		const shouldDemote = learner.shouldDemote(
			"execute-task",
			"unreliable",
			0.5,
		);
		expect(shouldDemote).toBe(true);
	});

	test("identifies A/B test candidates", () => {
		// Incumbent model with moderate success
		for (let i = 0; i < 8; i++) {
			learner.recordOutcome("execute-task", "incumbent", {
				success: true,
				timeout: false,
				tokensUsed: 3000,
				costUsd: 0.2,
			});
		}
		for (let i = 0; i < 2; i++) {
			learner.recordOutcome("execute-task", "incumbent", {
				success: false,
				timeout: false,
				tokensUsed: 3000,
				costUsd: 0.2,
			});
		}

		// Challenger with limited data
		learner.recordOutcome("execute-task", "challenger", {
			success: true,
			timeout: false,
			tokensUsed: 2500,
			costUsd: 0.1,
		});

		const abCandidates = learner.getABTestCandidates("execute-task");
		expect(abCandidates).toBeDefined();
		expect(abCandidates.incumbent).toBe("incumbent");
	});

	test("persists data to filesystem in canonical schema", () => {
		learner.recordOutcome("execute-task", "gpt-4o", {
			success: true,
			timeout: false,
			tokensUsed: 5000,
			costUsd: 0.15,
		});

		const perfFile = join(tmpDir, ".sf", "model-performance.json");
		const content = readFileSync(perfFile, "utf-8");
		const data = JSON.parse(content);

		// Without a registry resolver, bare model IDs go to _unmapped.by_route.
		// The canonical schema places unmappable routes in _unmapped.
		const unmapped = data["execute-task"]?.["_unmapped"];
		expect(unmapped).toBeDefined();
		expect(unmapped?.by_route?.["gpt-4o"]).toBeDefined();
		expect(unmapped.by_route["gpt-4o"].successes).toBe(1);
	});

	test("gracefully handles missing storage directory", () => {
		// Use path that doesn't exist
		const badLearner = new ModelLearner("/nonexistent/path");

		// Should not throw
		expect(() => {
			badLearner.recordOutcome("execute-task", "model-x", {
				success: true,
				timeout: false,
				tokensUsed: 1000,
				costUsd: 0.05,
			});
		}).not.toThrow();
	});

	test("computes per-task-type rankings independently", () => {
		// Create different success rates per task type
		for (let i = 0; i < 9; i++) {
			learner.recordOutcome("execute-task", "model-a", {
				success: true,
				timeout: false,
				tokensUsed: 1000,
				costUsd: 0.05,
			});
		}
		learner.recordOutcome("execute-task", "model-a", {
			success: false,
			timeout: false,
			tokensUsed: 1000,
			costUsd: 0.05,
		});

		// Model A is poor at plan-slice
		for (let i = 0; i < 3; i++) {
			learner.recordOutcome("plan-slice", "model-a", {
				success: false,
				timeout: false,
				tokensUsed: 1000,
				costUsd: 0.05,
			});
		}
		for (let i = 0; i < 1; i++) {
			learner.recordOutcome("plan-slice", "model-a", {
				success: true,
				timeout: false,
				tokensUsed: 1000,
				costUsd: 0.05,
			});
		}
		for (let i = 0; i < 4; i++) {
			learner.recordOutcome("plan-slice", "model-b", {
				success: true,
				timeout: false,
				tokensUsed: 1000,
				costUsd: 0.05,
			});
		}

		const execRanked = learner.getRankedModels("execute-task");
		const planRanked = learner.getRankedModels("plan-slice");

		// Model A should rank high for execute-task, low for plan-slice
		const execAIdx = execRanked.findIndex((r) => r.modelId === "model-a");
		const planAIdx = planRanked.findIndex((r) => r.modelId === "model-a");

		expect(execAIdx).toBeLessThan(planAIdx);
	});
});
