singularity-forge/src/headless-context.ts

/**
 * Headless Context Loading — stdin reading, file context, and project bootstrapping
 *
 * Handles loading context from files or stdin for headless new-milestone,
 * and bootstraps the .sf/ directory structure when needed.
 */

import {
	type Dirent,
	existsSync,
	mkdirSync,
	readdirSync,
	readFileSync,
	statSync,
} from "node:fs";
import { join, relative, resolve } from "node:path";
import { ensureAgenticDocsScaffold } from "./resources/extensions/sf/agentic-docs-scaffold.js";
import { ensureSiftIndexWarmup } from "./resources/extensions/sf/code-intelligence.js";
import {
	checkDocsScaffold,
	formatDocCheckReport,
} from "./resources/extensions/sf/doc-checker.js";
import {
	ensureGitignore,
	ensurePreferences,
	untrackRuntimeFiles,
} from "./resources/extensions/sf/gitignore.js";
import {
	nativeInit,
	nativeIsRepo,
} from "./resources/extensions/sf/native-git-bridge.js";
import { sfRoot } from "./resources/extensions/sf/paths.js";
import { isInheritedRepo } from "./resources/extensions/sf/repo-identity.js";

// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------

interface ContextOptions {
	context?: string; // file path or '-' for stdin
	contextText?: string; // inline text
}

const AUTO_BOOTSTRAP_MAX_BYTES = 180_000;
const AUTO_BOOTSTRAP_MAX_FILE_BYTES = 40_000;
const AUTO_BOOTSTRAP_SF_SPEC_FILES = [
	".sf/PROJECT.md",
	".sf/REQUIREMENTS.md",
	".sf/DECISIONS.md",
	".sf/KNOWLEDGE.md",
	".sf/RUNTIME.md",
	".sf/STATE.md",
	".sf/PRINCIPLES.md",
	".sf/TASTE.md",
	".sf/PREFERENCES.md",
	".sf/ANTI-GOALS.md",
	".sf/CODEBASE.md",
];
const AUTO_BOOTSTRAP_ORIENTATION_FILES = [
	"TODO.md",
	"README.md",
	"AGENTS.md",
	"CLAUDE.md",
	"CONTRIBUTING.md",
];
const AUTO_BOOTSTRAP_PRIORITY_FILES = [
	...AUTO_BOOTSTRAP_SF_SPEC_FILES,
	...AUTO_BOOTSTRAP_ORIENTATION_FILES,
];
const AUTO_BOOTSTRAP_SOURCE_EXTENSIONS = new Set([
	".go",
	".ts",
	".tsx",
	".js",
	".jsx",
	".mjs",
	".cjs",
	".py",
	".rs",
	".java",
	".kt",
	".kts",
	".rb",
	".php",
	".cs",
	".c",
	".cc",
	".cpp",
	".h",
	".hpp",
	".swift",
	".scala",
	".sh",
	".bash",
	".zsh",
	".fish",
	".sql",
	".yaml",
	".yml",
	".toml",
	".json",
	".jsonc",
	".xml",
	".html",
	".css",
	".scss",
	".sass",
	".vue",
	".svelte",
	".lua",
	".ex",
	".exs",
	".erl",
	".hrl",
	".clj",
	".cljs",
	".nix",
	".proto",
]);
const AUTO_BOOTSTRAP_EXCLUDED_DIRS = new Set([
	".git",
	".sf",
	"node_modules",
	"vendor",
	"dist",
	"build",
	"target",
	".next",
	".cache",
]);

// ---------------------------------------------------------------------------
// Stdin Reader
// ---------------------------------------------------------------------------

export async function readStdin(): Promise<string> {
	const chunks: Buffer[] = [];
	for await (const chunk of process.stdin) {
		chunks.push(chunk as Buffer);
	}
	return Buffer.concat(chunks).toString("utf-8");
}

// ---------------------------------------------------------------------------
// Context Loading
// ---------------------------------------------------------------------------

export async function loadContext(options: ContextOptions): Promise<string> {
	if (options.contextText) return options.contextText;
	if (options.context === "-") {
		return readStdin();
	}
	if (options.context) {
		return readFileSync(resolve(options.context), "utf-8");
	}
	throw new Error(
		"No context provided. Use --context <file> or --context-text <text>",
	);
}

export function hasMilestones(basePath: string): boolean {
	const milestonesDir = join(basePath, ".sf", "milestones");
	if (!existsSync(milestonesDir)) return false;
	try {
		return readdirSync(milestonesDir, { withFileTypes: true }).some((entry) =>
			entry.isDirectory(),
		);
	} catch {
		return false;
	}
}

export async function hasProjectMilestones(basePath: string): Promise<boolean> {
	if (hasMilestones(basePath)) return true;
	try {
		const dynamicToolsPath =
			"./resources/extensions/sf/bootstrap/dynamic-tools.js";
		const { ensureDbOpen } = await import(dynamicToolsPath);
		if (!(await ensureDbOpen(basePath))) return false;
		const sfDbPath = "./resources/extensions/sf/sf-db.js";
		const { getAllMilestones, isDbAvailable } = await import(sfDbPath);
		return isDbAvailable() && getAllMilestones().length > 0;
	} catch {
		return false;
	}
}

export function buildAutoBootstrapContext(basePath: string): string {
	const selectedFiles = collectAutoBootstrapFiles(basePath);
	const sourceFiles = collectSourceFiles(basePath);
	const chunks: string[] = [
		"# Autonomous Repo Bootstrap",
		"",
		"SF headless autonomous found no milestones. Use the repository files below as the seed context.",
		"Research SF working specs first, then every relevant markdown document and every source file path before creating the initial milestone plan.",
		"Use tool-based repository inspection for source contents; do not assume the seed excerpt is complete.",
		"Treat .sf/PROJECT.md, .sf/REQUIREMENTS.md, .sf/DECISIONS.md, .sf/KNOWLEDGE.md, and .sf/RUNTIME.md as review/export or recovery surfaces when present; `.sf/sf.db` remains the canonical structured runtime state.",
		"Treat any root-level SPEC.md, BASE_SPEC.md, PRODUCT_SPEC.md, docs/specs files, or other docs as repo evidence for humans. Project facts SF needs later into SF's .sf working model and DB-backed state; do not create a parallel base-spec system.",
		"For product-facing or workflow-facing work, research the product category and representative competitors before locking requirements or slices. Capture table stakes, differentiators, common failure modes, and what not to copy.",
		"Extract the project purpose, vision, architecture, constraints, current TODOs, risks, eval/gate ideas, and implementation backlog.",
		"Apply the ACE spec-first TDD shape when planning: purpose and consumer first, behavior contract before implementation, tests as specs, evidence after gates.",
		"Every proposed milestone, slice, and task must state its purpose before implementation detail.",
		"For each proposed slice, capture Observed/Inferred/Proposed facts, a falsifier, acceptance criteria, and the verification command or eval that proves it.",
		"Use explorer-style subagents or equivalent high-context research passes before planning when the runtime supports them.",
		"Recommended explorer passes: docs/purpose/vision; source architecture and dependency map; tests/gates/tooling; risks/backlog/eval candidates.",
		"Merge explorer findings into one repo map with cited file paths before creating milestones.",
		"Follow harness-engineering principles: keep AGENTS.md short as a table of contents, use docs/ for human exports and git-history reports, capture operational knowledge in .sf/DB-backed state, prefer mechanically enforced architecture/taste rules, and add cleanup/gardening work when repo knowledge is stale.",
		"Optimize for agent legibility: every milestone should improve the next agent's ability to understand, validate, and safely modify the repo.",
		"Create actionable milestones and slices from the repo's docs and source tree rather than asking the user to restate them.",
		"",
	];

	let used = chunks.join("\n").length;
	for (const filePath of selectedFiles) {
		let content: string;
		try {
			content = readFileSync(filePath, "utf-8");
		} catch {
			continue;
		}
		if (content.length > AUTO_BOOTSTRAP_MAX_FILE_BYTES) {
			content =
				content.slice(0, AUTO_BOOTSTRAP_MAX_FILE_BYTES) +
				"\n\n[truncated by SF headless autonomous bootstrap]\n";
		}

		const relPath = relative(basePath, filePath);
		const block = `\n\n## ${relPath}\n\n${content.trim()}\n`;
		if (used + block.length > AUTO_BOOTSTRAP_MAX_BYTES) break;
		chunks.push(block);
		used += block.length;
	}

	if (sourceFiles.length > 0) {
		const inventoryLines = [
			"\n\n## Source File Inventory\n",
			"Inspect these source/config/test files during repo research before finalizing the plan.\n",
			...sourceFiles.map((filePath) => `- ${relative(basePath, filePath)}`),
			"",
		];
		const block = inventoryLines.join("\n");
		if (used + block.length <= AUTO_BOOTSTRAP_MAX_BYTES) {
			chunks.push(block);
			used += block.length;
		} else {
			const remaining = AUTO_BOOTSTRAP_MAX_BYTES - used;
			if (remaining > 1000) {
				chunks.push(block.slice(0, remaining));
			}
		}
	}

	if (selectedFiles.length === 0) {
		chunks.push(
			"No markdown docs were found. Inspect the repository directly and create an initial milestone from source layout, package metadata, tests, and git status.",
		);
	}

	return chunks.join("\n").trim() + "\n";
}

function collectAutoBootstrapFiles(basePath: string): string[] {
	const seen = new Set<string>();
	const files: string[] = [];

	for (const name of AUTO_BOOTSTRAP_PRIORITY_FILES) {
		const path = join(basePath, name);
		if (existsMarkdownFile(path)) {
			seen.add(path);
			files.push(path);
		}
	}

	for (const path of walkMarkdownFiles(basePath)) {
		if (seen.has(path)) continue;
		seen.add(path);
		files.push(path);
	}

	return files;
}

function existsMarkdownFile(path: string): boolean {
	try {
		const stat = statSync(path);
		return stat.isFile() && path.toLowerCase().endsWith(".md");
	} catch {
		return false;
	}
}

function collectSourceFiles(basePath: string): string[] {
	return walkFiles(basePath, (path) => {
		const lower = path.toLowerCase();
		if (lower.endsWith(".md")) return false;
		const dot = lower.lastIndexOf(".");
		return dot !== -1 && AUTO_BOOTSTRAP_SOURCE_EXTENSIONS.has(lower.slice(dot));
	});
}

function walkMarkdownFiles(root: string): string[] {
	return walkFiles(root, (path) => path.toLowerCase().endsWith(".md"));
}

function walkFiles(
	root: string,
	includeFile: (path: string) => boolean,
): string[] {
	const found: string[] = [];
	const visit = (dir: string) => {
		let entries: Dirent[];
		try {
			entries = readdirSync(dir, { withFileTypes: true }) as Dirent[];
		} catch {
			return;
		}
		for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
			const path = join(dir, entry.name);
			if (entry.isDirectory()) {
				if (!AUTO_BOOTSTRAP_EXCLUDED_DIRS.has(entry.name)) {
					visit(path);
				}
				continue;
			}
			if (entry.isFile() && includeFile(path)) {
				found.push(path);
			}
		}
	};
	visit(root);
	return found;
}

/**
 * Bootstrap .sf/ directory structure for headless new-milestone.
 * Mirrors the bootstrap logic from guided-flow.ts showSmartEntry().
 */
export function bootstrapProject(basePath: string): void {
	if (!nativeIsRepo(basePath) || isInheritedRepo(basePath)) {
		nativeInit(basePath, "main");
	}

	const root = sfRoot(basePath);
	mkdirSync(join(root, "milestones"), { recursive: true });
	mkdirSync(join(root, "runtime"), { recursive: true });

	ensureGitignore(basePath);
	ensurePreferences(basePath);
	ensureAgenticDocsScaffold(basePath);
	ensureSiftIndexWarmup(basePath, {});
	untrackRuntimeFiles(basePath);

	// Run scaffold check after init — surfaces which files need real content
	const report = checkDocsScaffold(basePath);
	if (report.summary.stub > 0 || report.summary.missing > 0) {
		process.stderr.write(`\n${formatDocCheckReport(report)}\n`);
	}
}