singularity-forge/src/resources/extensions/sf/visualizer-data.js

// Data loader for workflow visualizer overlay — aggregates state + metrics.
import { existsSync, readFileSync, statSync } from "node:fs";
import { join } from "node:path";
import { countPendingCaptures, loadAllCaptures } from "./captures.js";
import { runEnvironmentChecks } from "./doctor-environment.js";
import { runProviderChecks } from "./doctor-providers.js";
import { loadFile, parseSummary } from "./files.js";
import {
	aggregateByModel,
	aggregateByPhase,
	aggregateBySlice,
	aggregateByTier,
	formatTierSavings,
	getLedger,
	getProjectTotals,
	loadLedgerFromDisk,
} from "./metrics.js";
import { findMilestoneIds } from "./milestone-ids.js";
import { parsePlan, parseRoadmap } from "./parsers.js";
import {
	resolveMilestoneFile,
	resolveSfRootFile,
	resolveSliceFile,
	sfRoot,
} from "./paths.js";
import { loadEffectiveSFPreferences } from "./preferences.js";
import { getMilestoneSlices, getSliceTasks, isDbAvailable } from "./sf-db.js";
import { generateSkillHealthReport } from "./skill-health.js";
import { deriveState } from "./state.js";

const DOCTOR_HISTORY_SCHEMA_VERSION = 1;

// ─── Critical Path ────────────────────────────────────────────────────────────
export function computeCriticalPath(milestones) {
	const empty = {
		milestonePath: [],
		slicePath: [],
		milestoneSlack: new Map(),
		sliceSlack: new Map(),
	};
	if (milestones.length === 0) return empty;
	// Milestone-level critical path (weight = number of incomplete slices)
	const msMap = new Map(milestones.map((m) => [m.id, m]));
	const msIds = milestones.map((m) => m.id);
	const msAdj = new Map();
	const msWeight = new Map();
	for (const ms of milestones) {
		msAdj.set(ms.id, []);
		const incomplete = ms.slices.filter((s) => !s.done).length;
		msWeight.set(ms.id, ms.status === "complete" ? 0 : Math.max(1, incomplete));
	}
	for (const ms of milestones) {
		for (const dep of ms.dependsOn) {
			if (msMap.has(dep)) {
				const adj = msAdj.get(dep);
				if (adj) adj.push(ms.id);
			}
		}
	}
	// Topological sort (Kahn's algorithm)
	const inDegree = new Map();
	for (const id of msIds) inDegree.set(id, 0);
	for (const ms of milestones) {
		for (const dep of ms.dependsOn) {
			if (msMap.has(dep)) inDegree.set(ms.id, (inDegree.get(ms.id) ?? 0) + 1);
		}
	}
	const queue = [];
	for (const [id, deg] of inDegree) {
		if (deg === 0) queue.push(id);
	}
	const topoOrder = [];
	while (queue.length > 0) {
		const node = queue.shift();
		topoOrder.push(node);
		for (const next of msAdj.get(node) ?? []) {
			const d = (inDegree.get(next) ?? 1) - 1;
			inDegree.set(next, d);
			if (d === 0) queue.push(next);
		}
	}
	// Longest path from each root
	const dist = new Map();
	const prev = new Map();
	for (const id of msIds) {
		dist.set(id, 0);
		prev.set(id, null);
	}
	for (const node of topoOrder) {
		const w = msWeight.get(node) ?? 1;
		const nodeDist = dist.get(node) + w;
		for (const next of msAdj.get(node) ?? []) {
			if (nodeDist > dist.get(next)) {
				dist.set(next, nodeDist);
				prev.set(next, node);
			}
		}
	}
	// Find the end of the critical path (node with max dist + own weight)
	let maxDist = 0;
	let endNode = msIds[0];
	for (const id of msIds) {
		const totalDist = dist.get(id) + (msWeight.get(id) ?? 1);
		if (totalDist > maxDist) {
			maxDist = totalDist;
			endNode = id;
		}
	}
	// Trace back
	const milestonePath = [];
	let cur = endNode;
	while (cur !== null) {
		milestonePath.unshift(cur);
		cur = prev.get(cur) ?? null;
	}
	// Compute milestone slack
	const milestoneSlack = new Map();
	const criticalSet = new Set(milestonePath);
	for (const id of msIds) {
		if (criticalSet.has(id)) {
			milestoneSlack.set(id, 0);
		} else {
			const nodeTotal = dist.get(id) + (msWeight.get(id) ?? 1);
			milestoneSlack.set(id, Math.max(0, maxDist - nodeTotal));
		}
	}
	// Slice-level critical path within active milestone
	const activeMs = milestones.find((m) => m.status === "active");
	const slicePath = [];
	const sliceSlack = new Map();
	if (activeMs && activeMs.slices.length > 0) {
		const slMap = new Map(activeMs.slices.map((s) => [s.id, s]));
		const slAdj = new Map();
		for (const s of activeMs.slices) slAdj.set(s.id, []);
		for (const s of activeMs.slices) {
			for (const dep of s.depends) {
				if (slMap.has(dep)) {
					const adj = slAdj.get(dep);
					if (adj) adj.push(s.id);
				}
			}
		}
		// Topo sort slices
		const slIn = new Map();
		for (const s of activeMs.slices) slIn.set(s.id, 0);
		for (const s of activeMs.slices) {
			for (const dep of s.depends) {
				if (slMap.has(dep)) slIn.set(s.id, (slIn.get(s.id) ?? 0) + 1);
			}
		}
		const slQueue = [];
		for (const [id, d] of slIn) {
			if (d === 0) slQueue.push(id);
		}
		const slTopo = [];
		while (slQueue.length > 0) {
			const n = slQueue.shift();
			slTopo.push(n);
			for (const next of slAdj.get(n) ?? []) {
				const d = (slIn.get(next) ?? 1) - 1;
				slIn.set(next, d);
				if (d === 0) slQueue.push(next);
			}
		}
		const slDist = new Map();
		const slPrev = new Map();
		for (const s of activeMs.slices) {
			const _w = s.done ? 0 : 1;
			slDist.set(s.id, 0);
			slPrev.set(s.id, null);
		}
		for (const n of slTopo) {
			const w = slMap.get(n)?.done ? 0 : 1;
			const nd = slDist.get(n) + w;
			for (const next of slAdj.get(n) ?? []) {
				if (nd > slDist.get(next)) {
					slDist.set(next, nd);
					slPrev.set(next, n);
				}
			}
		}
		let slMax = 0;
		let slEnd = activeMs.slices[0].id;
		for (const s of activeMs.slices) {
			const totalDist = slDist.get(s.id) + (s.done ? 0 : 1);
			if (totalDist > slMax) {
				slMax = totalDist;
				slEnd = s.id;
			}
		}
		let slCur = slEnd;
		while (slCur !== null) {
			slicePath.unshift(slCur);
			slCur = slPrev.get(slCur) ?? null;
		}
		const slCritSet = new Set(slicePath);
		for (const s of activeMs.slices) {
			if (slCritSet.has(s.id)) {
				sliceSlack.set(s.id, 0);
			} else {
				const nodeTotal = slDist.get(s.id) + (s.done ? 0 : 1);
				sliceSlack.set(s.id, Math.max(0, slMax - nodeTotal));
			}
		}
	}
	return { milestonePath, slicePath, milestoneSlack, sliceSlack };
}
// ─── Agent Activity ──────────────────────────────────────────────────────────
function loadAgentActivity(units, milestones) {
	if (units.length === 0) return null;
	// Find currently running unit (finishedAt === 0)
	const running = units.find((u) => u.finishedAt === 0);
	const now = Date.now();
	const completedUnits = units.filter((u) => u.finishedAt > 0).length;
	const totalSlices = milestones.reduce((sum, m) => sum + m.slices.length, 0);
	// Completion rate from finished units
	const finished = units.filter((u) => u.finishedAt > 0);
	let completionRate = 0;
	if (finished.length >= 2) {
		const earliest = Math.min(...finished.map((u) => u.startedAt));
		const latest = Math.max(...finished.map((u) => u.finishedAt));
		const totalHours = (latest - earliest) / 3_600_000;
		completionRate = totalHours > 0 ? finished.length / totalHours : 0;
	}
	const sessionCost = units.reduce((sum, u) => sum + u.cost, 0);
	const sessionTokens = units.reduce((sum, u) => sum + u.tokens.total, 0);
	return {
		currentUnit: running
			? { type: running.type, id: running.id, startedAt: running.startedAt }
			: null,
		elapsed: running ? now - running.startedAt : 0,
		completedUnits,
		totalSlices,
		completionRate,
		active: !!running,
		sessionCost,
		sessionTokens,
	};
}
// ─── Changelog & Verifications ────────────────────────────────────────────────
const changelogCache = new Map();
async function loadChangelogAndVerifications(basePath, milestones) {
	const entries = [];
	const verifications = [];
	for (const ms of milestones) {
		for (const sl of ms.slices) {
			if (!sl.done) continue;
			const summaryFile = resolveSliceFile(basePath, ms.id, sl.id, "SUMMARY");
			if (!summaryFile) continue;
			const cacheKey = `${ms.id}/${sl.id}`;
			const cached = changelogCache.get(cacheKey);
			let mtime = 0;
			try {
				mtime = statSync(summaryFile).mtimeMs;
			} catch {
				continue;
			}
			if (cached && cached.mtime === mtime) {
				entries.push(cached.entry);
				verifications.push(cached.verification);
				continue;
			}
			const content = await loadFile(summaryFile);
			if (!content) continue;
			const summary = parseSummary(content);
			const entry = {
				milestoneId: ms.id,
				sliceId: sl.id,
				title: sl.title,
				oneLiner: summary.oneLiner,
				filesModified: summary.filesModified.map((f) => ({
					path: f.path,
					description: f.description,
				})),
				completedAt: String(summary.frontmatter.completed_at ?? ""),
			};
			const verification = {
				milestoneId: ms.id,
				sliceId: sl.id,
				verificationResult: summary.frontmatter.verification_result || "",
				blockerDiscovered: summary.frontmatter.blocker_discovered,
				keyDecisions: summary.frontmatter.key_decisions || [],
				patternsEstablished: summary.frontmatter.patterns_established || [],
				provides: summary.frontmatter.provides || [],
				requires: (summary.frontmatter.requires || []).map((r) => ({
					slice: r.slice,
					provides: r.provides,
				})),
			};
			changelogCache.set(cacheKey, { mtime, entry, verification });
			entries.push(entry);
			verifications.push(verification);
		}
	}
	entries.sort((a, b) =>
		String(b.completedAt || "").localeCompare(String(a.completedAt || "")),
	);
	return { changelog: { entries }, verifications };
}
// ─── Knowledge Loader ─────────────────────────────────────────────────────────
function loadKnowledge(basePath) {
	const knowledgePath = resolveSfRootFile(basePath, "KNOWLEDGE");
	if (!existsSync(knowledgePath)) {
		return { rules: [], patterns: [], lessons: [], exists: false };
	}
	let content;
	try {
		content = readFileSync(knowledgePath, "utf-8");
	} catch {
		return { rules: [], patterns: [], lessons: [], exists: false };
	}
	const rules = [];
	const patterns = [];
	const lessons = [];
	const lines = content.split("\n");
	let currentSection = "";
	for (const line of lines) {
		if (line.startsWith("## Rules")) {
			currentSection = "rules";
			continue;
		}
		if (line.startsWith("## Patterns")) {
			currentSection = "patterns";
			continue;
		}
		if (line.startsWith("## Lessons")) {
			currentSection = "lessons";
			continue;
		}
		if (line.startsWith("## ")) {
			currentSection = "";
			continue;
		}
		if (
			!line.startsWith("| ") ||
			line.startsWith("| ---") ||
			line.startsWith("| ID")
		)
			continue;
		const cols = line
			.split("|")
			.map((c) => c.trim())
			.filter((c) => c.length > 0);
		if (cols.length < 2) continue;
		if (currentSection === "rules" && cols.length >= 3) {
			rules.push({ id: cols[0], scope: cols[1], content: cols[2] });
		} else if (currentSection === "patterns" && cols.length >= 2) {
			patterns.push({ id: cols[0], content: cols[1] });
		} else if (currentSection === "lessons" && cols.length >= 2) {
			lessons.push({ id: cols[0], content: cols[1] });
		}
	}
	return { rules, patterns, lessons, exists: true };
}
// ─── Health Loader ────────────────────────────────────────────────────────────
async function loadHealth(units, totals, basePath) {
	const prefs = loadEffectiveSFPreferences();
	const budgetCeiling = prefs?.preferences?.budget_ceiling;
	const tokenProfile = prefs?.preferences?.token_profile ?? "standard";
	let truncationRate = 0;
	let continueHereRate = 0;
	if (totals && totals.units > 0) {
		truncationRate = (totals.totalTruncationSections / totals.units) * 100;
		continueHereRate = (totals.continueHereFiredCount / totals.units) * 100;
	}
	const tierBreakdown = aggregateByTier(units);
	const tierSavingsLine = formatTierSavings(units);
	// Provider checks — fast (auth.json + env vars only, no network)
	let providers = [];
	try {
		providers = runProviderChecks().map((r) => ({
			name: r.name,
			label: r.label,
			category: r.category,
			ok: r.status === "ok" || r.status === "unconfigured",
			required: r.required,
			message: r.message,
		}));
	} catch {
		/* non-fatal */
	}
	// Skill health summary
	let skillSummary = {
		total: 0,
		warningCount: 0,
		criticalCount: 0,
		topIssue: null,
	};
	try {
		const report = generateSkillHealthReport(basePath);
		const warnings = report.suggestions.filter((s) => s.severity === "warning");
		const criticals = report.suggestions.filter(
			(s) => s.severity === "critical",
		);
		skillSummary = {
			total: report.skills.length,
			warningCount: warnings.length,
			criticalCount: criticals.length,
			topIssue: report.suggestions[0]?.message ?? null,
		};
	} catch {
		/* non-fatal */
	}
	// Environment issues (from doctor-environment.ts, #1221)
	let environmentIssues = [];
	try {
		environmentIssues = runEnvironmentChecks(basePath).filter(
			(r) => r.status !== "ok",
		);
	} catch {
		/* non-fatal */
	}
	// Doctor run history — persisted across sessions (sync read to keep loadHealth sync).
	// Parse each line independently so a single corrupt row doesn't discard the
	// surrounding 19 valid entries.
	let doctorHistory = [];
	try {
		const historyPath = join(sfRoot(basePath), "doctor-history.jsonl");
		if (existsSync(historyPath)) {
			const lines = readFileSync(historyPath, "utf-8")
				.split("\n")
				.filter((l) => l.trim());
			doctorHistory = lines
				.slice(-20)
				.reverse()
				.flatMap((l) => {
					try {
						const entry = normalizeDoctorHistoryEntry(JSON.parse(l));
						return entry ? [entry] : [];
					} catch {
						return [];
					}
				});
		}
	} catch {
		/* non-fatal */
	}
	// Current progress score — only meaningful when autonomous mode has health data
	let progressScore = null;
	try {
		const { getHealthHistory } = await import("./doctor-proactive.js");
		const history = getHealthHistory();
		if (history.length > 0) {
			const { computeProgressScore } = await import("./progress-score.js");
			const score = computeProgressScore();
			progressScore = {
				level: score.level,
				summary: score.summary,
				signals: score.signals,
			};
		}
	} catch {
		/* non-fatal */
	}
	return {
		budgetCeiling,
		tokenProfile,
		truncationRate,
		continueHereRate,
		tierBreakdown,
		tierSavingsLine,
		toolCalls: totals?.toolCalls ?? 0,
		assistantMessages: totals?.assistantMessages ?? 0,
		userMessages: totals?.userMessages ?? 0,
		providers,
		skillSummary,
		environmentIssues,
		doctorHistory,
		progressScore,
	};
}

function normalizeDoctorHistoryEntry(entry) {
	if (!entry || typeof entry !== "object" || Array.isArray(entry)) return null;
	const schemaVersion = entry.schemaVersion ?? DOCTOR_HISTORY_SCHEMA_VERSION;
	if (schemaVersion !== DOCTOR_HISTORY_SCHEMA_VERSION) return null;
	return {
		...entry,
		schemaVersion,
	};
}
const RECENT_ENTRY_LIMIT = 3;
const FEATURE_PREVIEW_LIMIT = 5;
const UPDATED_WINDOW_MS = 7 * 24 * 60 * 60 * 1000;
function buildVisualizerStats(milestones, entries) {
	const missing = [];
	for (const ms of milestones) {
		for (const sl of ms.slices) {
			if (!sl.done)
				missing.push({ milestoneId: ms.id, sliceId: sl.id, title: sl.title });
		}
	}
	const missingCount = missing.length;
	const missingSlices = missing.slice(0, FEATURE_PREVIEW_LIMIT);
	const now = Date.now();
	const updatedEntries = entries.filter((entry) => {
		if (!entry.completedAt) return false;
		const parsed = Date.parse(entry.completedAt);
		return !Number.isNaN(parsed) && now - parsed <= UPDATED_WINDOW_MS;
	});
	const updatedCount = updatedEntries.length;
	const updatedSlices = updatedEntries
		.slice(0, FEATURE_PREVIEW_LIMIT)
		.map((entry) => ({
			milestoneId: entry.milestoneId,
			sliceId: entry.sliceId,
			title: entry.title,
			completedAt: entry.completedAt,
		}));
	const recentEntries = entries.slice(0, RECENT_ENTRY_LIMIT);
	return {
		missingCount,
		missingSlices,
		updatedCount,
		updatedSlices,
		recentEntries,
	};
}
function loadDiscussionState(basePath, milestones) {
	const states = [];
	for (const ms of milestones) {
		const contextPath = resolveMilestoneFile(basePath, ms.id, "CONTEXT");
		const draftPath = resolveMilestoneFile(basePath, ms.id, "CONTEXT-DRAFT");
		const state = contextPath
			? "discussed"
			: draftPath
				? "draft"
				: "undiscussed";
		let lastUpdated = null;
		const target = contextPath ?? draftPath;
		if (target) {
			try {
				lastUpdated = new Date(statSync(target).mtimeMs).toISOString();
			} catch {
				lastUpdated = null;
			}
		}
		states.push({
			milestoneId: ms.id,
			title: ms.title,
			state,
			hasContext: !!contextPath,
			hasDraft: !!draftPath,
			lastUpdated,
		});
	}
	return states;
}
// ─── File Fingerprint Cache ───────────────────────────────────────────────────
/**
 * Mtime-based cache for parsed file contents. Avoids re-reading and re-parsing
 * roadmap/plan files whose mtime hasn't changed since the last load.
 */
const fileContentCache = new Map();
function readFileCached(filePath) {
	try {
		const mtime = statSync(filePath).mtimeMs;
		const cached = fileContentCache.get(filePath);
		if (cached && cached.mtime === mtime) {
			return cached.content;
		}
		const content = readFileSync(filePath, "utf-8");
		fileContentCache.set(filePath, { mtime, content });
		return content;
	} catch {
		return null;
	}
}
// ─── Loader ───────────────────────────────────────────────────────────────────
export async function loadVisualizerData(basePath) {
	const state = await deriveState(basePath);
	const milestoneIds = findMilestoneIds(basePath);
	const milestones = [];
	for (const mid of milestoneIds) {
		const entry = state.registry.find((r) => r.id === mid);
		const status = entry?.status ?? "pending";
		const dependsOn = entry?.dependsOn ?? [];
		const slices = [];
		const roadmapFile = resolveMilestoneFile(basePath, mid, "ROADMAP");
		const roadmapContent = roadmapFile ? readFileCached(roadmapFile) : null;
		if (roadmapContent || isDbAvailable()) {
			let normSlices = null;
			if (isDbAvailable()) {
				const dbSlices = getMilestoneSlices(mid);
				if (dbSlices.length > 0) {
					normSlices = dbSlices.map((s) => ({
						id: s.id,
						done: s.status === "complete",
						title: s.title,
						risk: s.risk || "medium",
						depends: s.depends,
						demo: s.demo,
					}));
				}
			}
			if (!normSlices && roadmapContent) {
				// File-based fallback: parse roadmap for slice entries
				const parsed = parseRoadmap(roadmapContent);
				normSlices = parsed.slices.map((s) => ({
					id: s.id,
					done: s.done,
					title: s.title,
					risk: s.risk || "medium",
					depends: s.depends,
					demo: "",
				}));
			}
			if (!normSlices) normSlices = [];
			for (const s of normSlices) {
				const isActiveSlice =
					state.activeMilestone?.id === mid && state.activeSlice?.id === s.id;
				const tasks = [];
				if (isActiveSlice) {
					// Normalize tasks from DB, fall back to file parsing when DB has no data
					let usedDbTasks = false;
					if (isDbAvailable()) {
						const dbTasks = getSliceTasks(mid, s.id);
						if (dbTasks.length > 0) {
							usedDbTasks = true;
							for (const t of dbTasks) {
								tasks.push({
									id: t.id,
									title: t.title,
									done: t.status === "complete" || t.status === "done",
									active: state.activeTask?.id === t.id,
									estimate: t.estimate || undefined,
								});
							}
						}
					}
					if (!usedDbTasks) {
						// File-based fallback: parse slice plan for task entries
						const slicePlanFile = resolveSliceFile(basePath, mid, s.id, "PLAN");
						if (slicePlanFile) {
							const planContent = readFileCached(slicePlanFile);
							if (planContent) {
								const parsed = parsePlan(planContent);
								for (const t of parsed.tasks) {
									tasks.push({
										id: t.id,
										title: t.title,
										done: t.done,
										active: state.activeTask?.id === t.id,
										estimate: t.estimate || undefined,
									});
								}
							}
						}
					}
				}
				slices.push({
					id: s.id,
					title: s.title,
					done: s.done,
					active: isActiveSlice,
					risk: s.risk,
					depends: s.depends,
					tasks,
				});
			}
		}
		milestones.push({
			id: mid,
			title: entry?.title ?? mid,
			status,
			dependsOn,
			slices,
		});
	}
	// Metrics
	let totals = null;
	let byPhase = [];
	let bySlice = [];
	let byModel = [];
	let byTier = [];
	let tierSavingsLine = "";
	let units = [];
	const ledger = getLedger() ?? loadLedgerFromDisk(basePath);
	if (ledger && ledger.units.length > 0) {
		units = [...ledger.units].sort((a, b) => a.startedAt - b.startedAt);
		totals = getProjectTotals(units);
		byPhase = aggregateByPhase(units);
		bySlice = aggregateBySlice(units);
		byModel = aggregateByModel(units);
		byTier = aggregateByTier(units);
		tierSavingsLine = formatTierSavings(units);
	}
	// Compute new fields
	const criticalPath = computeCriticalPath(milestones);
	let remainingSliceCount = 0;
	for (const ms of milestones) {
		for (const sl of ms.slices) {
			if (!sl.done) remainingSliceCount++;
		}
	}
	const agentActivity = loadAgentActivity(units, milestones);
	const { changelog, verifications: sliceVerifications } =
		await loadChangelogAndVerifications(basePath, milestones);
	const knowledge = loadKnowledge(basePath);
	const allCaptures = loadAllCaptures(basePath);
	const pendingCount = countPendingCaptures(basePath);
	const captures = {
		entries: allCaptures,
		pendingCount,
		totalCount: allCaptures.length,
	};
	const health = await loadHealth(units, totals, basePath);
	const stats = buildVisualizerStats(milestones, changelog.entries);
	const discussion = loadDiscussionState(basePath, milestones);
	return {
		milestones,
		phase: state.phase,
		totals,
		byPhase,
		bySlice,
		byModel,
		byTier,
		tierSavingsLine,
		units,
		criticalPath,
		remainingSliceCount,
		agentActivity,
		changelog,
		sliceVerifications,
		knowledge,
		captures,
		health,
		discussion,
		stats,
	};
}