9d739dfa5d
- All gsdDir/gsdRoot/gsdHome → sfDir/sfRootDir/sfHome - GSDWorkspace* → SFWorkspace* interfaces - bootstrapGsdProject → bootstrapProject - runGSDDoctor → runSFDoctor - GsdClient → SfClient, gsd-client.ts → sf-client.ts - .gsd/ → .sf/ in all tests, docs, docker, native, vscode - Auto-migration: headless detects .gsd/ → renames to .sf/ - Deleted gsd-phase-state.ts backward-compat re-export - Renamed bin/gsd-from-source → bin/sf-from-source - Updated mintlify docs, github workflows, docker configs
315 lines
13 KiB
TypeScript
315 lines
13 KiB
TypeScript
/**
|
|
* Live Regression Test Harness — Post-Build Pipeline Validation
|
|
*
|
|
* These tests run AFTER `npm publish` against the installed `sf` binary.
|
|
* They exercise the dispatch loop state machine end-to-end by:
|
|
*
|
|
* 1. Creating real `.sf/` directory structures with milestone artifacts
|
|
* 2. Calling `sf headless query` to verify state derivation
|
|
* 3. Verifying phase transitions match expected outcomes
|
|
* 4. Testing crash recovery (lock file lifecycle)
|
|
* 5. Testing worktree identity hash consistency
|
|
*
|
|
* These tests DO NOT require LLM API keys — they test the state machine
|
|
* and infrastructure, not the LLM execution.
|
|
*
|
|
* Run from CI pipeline after `npm install -g sf-run@<version>`:
|
|
* node --experimental-strip-types tests/live-regression/run.ts
|
|
*
|
|
* Or locally:
|
|
* SF_SMOKE_BINARY=dist/loader.js node --experimental-strip-types tests/live-regression/run.ts
|
|
*/
|
|
|
|
import { execFileSync, execSync } from "child_process";
|
|
import { mkdtempSync, mkdirSync, writeFileSync, existsSync, readFileSync, rmSync, unlinkSync } from "fs";
|
|
import { join, dirname } from "path";
|
|
import { tmpdir } from "os";
|
|
|
|
// ─── Config ───────────────────────────────────────────────────────────────
|
|
|
|
const binary = process.env.SF_SMOKE_BINARY || process.env.SF_SMOKE_BINARY || "sf";
|
|
let passed = 0;
|
|
let failed = 0;
|
|
|
|
function run(label: string, fn: () => void): void {
|
|
try {
|
|
fn();
|
|
console.log(` ✅ ${label}`);
|
|
passed++;
|
|
} catch (err: any) {
|
|
console.error(` ❌ ${label}`);
|
|
console.error(` ${err.message || err}`);
|
|
failed++;
|
|
}
|
|
}
|
|
|
|
function assert(condition: boolean, message: string): void {
|
|
if (!condition) throw new Error(message);
|
|
}
|
|
|
|
function sfCli(args: string[], cwd: string, env?: Record<string, string>): { stdout: string; stderr: string; code: number } {
|
|
try {
|
|
const stdout = execFileSync(binary === "sf" ? "sf" : "node",
|
|
binary === "sf" ? args : [binary, ...args], {
|
|
cwd,
|
|
encoding: "utf-8",
|
|
timeout: 30_000,
|
|
stdio: ["pipe", "pipe", "pipe"],
|
|
env: { ...process.env, ...env, SF_NON_INTERACTIVE: "1", GSD_NON_INTERACTIVE: "1" },
|
|
});
|
|
return { stdout, stderr: "", code: 0 };
|
|
} catch (err: any) {
|
|
return { stdout: err.stdout || "", stderr: err.stderr || "", code: err.status ?? 1 };
|
|
}
|
|
}
|
|
|
|
function createTempProject(name: string): string {
|
|
const dir = mkdtempSync(join(tmpdir(), `sf-live-${name}-`));
|
|
try { execSync("git init && git config user.email test@test.com && git config user.name Test && git commit --allow-empty -m init", { cwd: dir, stdio: "pipe" }); } catch {}
|
|
return dir;
|
|
}
|
|
|
|
function buildMinimalRoadmap(slices: Array<{ id: string; title: string; done: boolean }>): string {
|
|
const lines = ["# M001: Test Milestone", "", "## Slices", ""];
|
|
for (const s of slices) {
|
|
const cb = s.done ? "x" : " ";
|
|
lines.push(`- [${cb}] **${s.id}: ${s.title}** \`risk:low\` \`depends:[]\``);
|
|
lines.push(` > Demo for ${s.id}`);
|
|
lines.push("");
|
|
}
|
|
return lines.join("\n");
|
|
}
|
|
|
|
function buildMinimalPlan(tasks: Array<{ id: string; title: string; done: boolean }>): string {
|
|
const lines = ["# S01: Test Slice", "", "**Goal:** test", "", "## Tasks", ""];
|
|
for (const t of tasks) {
|
|
const cb = t.done ? "x" : " ";
|
|
lines.push(`- [${cb}] **${t.id}: ${t.title}** \`est:5m\``);
|
|
}
|
|
return lines.join("\n");
|
|
}
|
|
|
|
function buildTaskSummary(id: string): string {
|
|
return `---\nid: ${id}\nparent: S01\nmilestone: M001\nduration: 5m\nverification_result: passed\ncompleted_at: ${new Date().toISOString()}\n---\n\n# ${id}: Done\n\nCompleted.`;
|
|
}
|
|
|
|
// ─── Test: headless query returns valid JSON ──────────────────────────────
|
|
|
|
run("headless query returns valid JSON on initialized project", () => {
|
|
const dir = createTempProject("query");
|
|
try {
|
|
const sfDir = join(dir, ".sf");
|
|
mkdirSync(join(sfDir, "milestones"), { recursive: true });
|
|
|
|
const result = sfCli(["headless", "query"], dir);
|
|
assert(result.code === 0, `expected exit 0, got ${result.code}: ${result.stderr}`);
|
|
|
|
const json = JSON.parse(result.stdout);
|
|
assert(typeof ((json.state?.phase ?? json.phase)) === "string", "response should have phase field");
|
|
assert(Array.isArray(json.milestones) || json.milestones === undefined, "milestones should be array or undefined");
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
// ─── Test: state derivation — empty project ──────────────────────────────
|
|
|
|
run("headless query: empty project reports pre-planning", () => {
|
|
const dir = createTempProject("empty");
|
|
try {
|
|
mkdirSync(join(dir, ".sf", "milestones"), { recursive: true });
|
|
|
|
const result = sfCli(["headless", "query"], dir);
|
|
assert(result.code === 0, `expected exit 0, got ${result.code}`);
|
|
|
|
const json = JSON.parse(result.stdout);
|
|
assert((json.state?.phase ?? json.phase) === "pre-planning" || (json.state?.phase ?? json.phase) === "idle",
|
|
`expected pre-planning or idle, got: ${(json.state?.phase ?? json.phase)}`);
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
// ─── Test: state derivation — milestone with roadmap ─────────────────────
|
|
|
|
run("headless query: milestone with roadmap reports planning phase", () => {
|
|
const dir = createTempProject("planning");
|
|
try {
|
|
const mDir = join(dir, ".sf", "milestones", "M001");
|
|
mkdirSync(join(mDir, "slices", "S01"), { recursive: true });
|
|
writeFileSync(join(mDir, "M001-CONTEXT.md"), "# M001\n\nContext.");
|
|
writeFileSync(join(mDir, "M001-ROADMAP.md"), buildMinimalRoadmap([
|
|
{ id: "S01", title: "First Slice", done: false },
|
|
]));
|
|
|
|
const result = sfCli(["headless", "query"], dir);
|
|
assert(result.code === 0, `expected exit 0, got ${result.code}`);
|
|
|
|
const json = JSON.parse(result.stdout);
|
|
assert((json.state?.phase ?? json.phase) === "planning", `expected planning, got: ${(json.state?.phase ?? json.phase)}`);
|
|
assert((json.state?.activeMilestone ?? json.activeMilestone) === "M001" || (json.state?.activeMilestone ?? json.activeMilestone)?.id === "M001",
|
|
`expected active milestone M001`);
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
// ─── Test: state derivation — all tasks done ─────────────────────────────
|
|
|
|
run("headless query: all tasks done reports summarizing phase", () => {
|
|
const dir = createTempProject("summarizing");
|
|
try {
|
|
const mDir = join(dir, ".sf", "milestones", "M001");
|
|
const sDir = join(mDir, "slices", "S01");
|
|
mkdirSync(join(sDir, "tasks"), { recursive: true });
|
|
writeFileSync(join(mDir, "M001-CONTEXT.md"), "# M001\n\nContext.");
|
|
writeFileSync(join(mDir, "M001-ROADMAP.md"), buildMinimalRoadmap([
|
|
{ id: "S01", title: "First Slice", done: false },
|
|
]));
|
|
writeFileSync(join(sDir, "S01-PLAN.md"), buildMinimalPlan([
|
|
{ id: "T01", title: "Task One", done: true },
|
|
]));
|
|
writeFileSync(join(sDir, "tasks", "T01-SUMMARY.md"), buildTaskSummary("T01"));
|
|
|
|
const result = sfCli(["headless", "query"], dir);
|
|
assert(result.code === 0, `expected exit 0, got ${result.code}`);
|
|
|
|
const json = JSON.parse(result.stdout);
|
|
assert((json.state?.phase ?? json.phase) === "summarizing", `expected summarizing, got: ${(json.state?.phase ?? json.phase)}`);
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
// ─── Test: state derivation — complete milestone ─────────────────────────
|
|
|
|
run("headless query: milestone with summary reports complete", () => {
|
|
const dir = createTempProject("complete");
|
|
try {
|
|
const mDir = join(dir, ".sf", "milestones", "M001");
|
|
mkdirSync(mDir, { recursive: true });
|
|
writeFileSync(join(mDir, "M001-ROADMAP.md"), buildMinimalRoadmap([
|
|
{ id: "S01", title: "Done", done: true },
|
|
]));
|
|
writeFileSync(join(mDir, "M001-SUMMARY.md"), "# M001 Summary\n\nComplete.");
|
|
|
|
const result = sfCli(["headless", "query"], dir);
|
|
assert(result.code === 0, `expected exit 0, got ${result.code}`);
|
|
|
|
const json = JSON.parse(result.stdout);
|
|
assert((json.state?.phase ?? json.phase) === "complete" || (json.state?.phase ?? json.phase) === "idle" || (json.state?.phase ?? json.phase) === "pre-planning",
|
|
`expected complete/idle/pre-planning, got: ${(json.state?.phase ?? json.phase)}`);
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
// ─── Test: lock file lifecycle ───────────────────────────────────────────
|
|
|
|
run("stale auto.lock with dead PID does not block --version", () => {
|
|
const dir = createTempProject("stale-lock");
|
|
try {
|
|
const sfDir = join(dir, ".sf");
|
|
mkdirSync(sfDir, { recursive: true });
|
|
// Write a lock with a PID that doesn't exist
|
|
writeFileSync(join(sfDir, "auto.lock"), JSON.stringify({
|
|
pid: 99999999,
|
|
startedAt: new Date().toISOString(),
|
|
unitType: "starting",
|
|
unitId: "bootstrap",
|
|
unitStartedAt: new Date().toISOString(),
|
|
completedUnits: 0,
|
|
}));
|
|
|
|
const result = sfCli(["--version"], dir);
|
|
assert(result.code === 0, `--version should succeed even with stale lock, got code ${result.code}`);
|
|
assert(/\d+\.\d+\.\d+/.test(result.stdout.trim()), `should output version, got: ${result.stdout}`);
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
// ─── Test: crash recovery message ────────────────────────────────────────
|
|
|
|
run("crash recovery shows actionable guidance", () => {
|
|
const dir = createTempProject("crash-recovery");
|
|
try {
|
|
const sfDir = join(dir, ".sf");
|
|
mkdirSync(join(sfDir, "milestones"), { recursive: true });
|
|
writeFileSync(join(sfDir, "auto.lock"), JSON.stringify({
|
|
pid: 99999999,
|
|
startedAt: new Date().toISOString(),
|
|
unitType: "execute-task",
|
|
unitId: "M001/S01/T02",
|
|
unitStartedAt: new Date().toISOString(),
|
|
completedUnits: 5,
|
|
}));
|
|
|
|
// headless query should still work — lock is for auto-mode, not query
|
|
const result = sfCli(["headless", "query"], dir);
|
|
assert(result.code === 0, `query should succeed with stale lock`);
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
// ─── Test: TTY check fires before heavy initialization ───────────────────
|
|
|
|
run("non-TTY invocation exits quickly with clean error", () => {
|
|
const dir = createTempProject("tty-check");
|
|
try {
|
|
const start = Date.now();
|
|
const result = sfCli([], dir); // No args, no TTY
|
|
const elapsed = Date.now() - start;
|
|
|
|
assert(result.code === 1, `expected exit 1 for non-TTY, got ${result.code}`);
|
|
assert(elapsed < 5000, `should exit within 5s, took ${elapsed}ms`);
|
|
assert(
|
|
result.stderr.includes("TTY") || result.stderr.includes("terminal") || result.stderr.includes("Interactive"),
|
|
`should mention TTY requirement in stderr`
|
|
);
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
// ─── Test: version skew detection ────────────────────────────────────────
|
|
|
|
run("version skew is detected before TTY check", () => {
|
|
const dir = createTempProject("version-skew");
|
|
try {
|
|
// Create a fake managed-resources.json with a future version
|
|
const agentDir = join(dir, ".sf-test-agent");
|
|
mkdirSync(agentDir, { recursive: true });
|
|
writeFileSync(join(agentDir, "managed-resources.json"), JSON.stringify({
|
|
sfVersion: "999.0.0",
|
|
}));
|
|
|
|
// Set HOME to the temp dir so SF reads the fake agent dir
|
|
const fakeHome = dir;
|
|
mkdirSync(join(fakeHome, ".sf", "agent"), { recursive: true });
|
|
writeFileSync(join(fakeHome, ".sf", "agent", "managed-resources.json"), JSON.stringify({
|
|
sfVersion: "999.0.0",
|
|
}));
|
|
|
|
const result = sfCli([], dir, { HOME: fakeHome });
|
|
// Should either exit with version mismatch or TTY error — both are fine
|
|
assert(result.code === 1, `expected exit 1, got ${result.code}`);
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
// ─── Test: native addon graceful fallback ────────────────────────────────
|
|
|
|
run("sf --help works (native addon loads or falls back gracefully)", () => {
|
|
const result = sfCli(["--help"], process.cwd());
|
|
assert(result.code === 0, `--help should exit 0, got ${result.code}`);
|
|
assert(result.stdout.toLowerCase().includes("sf") || result.stdout.toLowerCase().includes("usage"),
|
|
`help output should contain sf or usage`);
|
|
});
|
|
|
|
// ─── Summary ─────────────────────────────────────────────────────────────
|
|
|
|
console.log(`\nLive regression: ${passed} passed, ${failed} failed`);
|
|
if (failed > 0) process.exit(1);
|