398 lines
13 KiB
TypeScript
398 lines
13 KiB
TypeScript
/**
|
|
* Live Regression Test Harness — Post-Build Pipeline Validation
|
|
*
|
|
* These tests run AFTER `npm publish` against the installed `sf` binary.
|
|
* They exercise the dispatch loop state machine end-to-end by:
|
|
*
|
|
* 1. Creating real `.sf/` directory structures with milestone artifacts
|
|
* 2. Calling `sf headless query` to verify state derivation
|
|
* 3. Verifying phase transitions match expected outcomes
|
|
* 4. Testing crash recovery (lock file lifecycle)
|
|
* 5. Testing worktree identity hash consistency
|
|
*
|
|
* These tests DO NOT require LLM API keys — they test the state machine
|
|
* and infrastructure, not the LLM execution.
|
|
*
|
|
* Run from CI pipeline after `npm install -g sf-run@<version>`:
|
|
* node --experimental-strip-types tests/live-regression/run.ts
|
|
*
|
|
* Or locally:
|
|
* SF_SMOKE_BINARY=dist/loader.js node --experimental-strip-types tests/live-regression/run.ts
|
|
*/
|
|
|
|
import { execFileSync, execSync } from "node:child_process";
|
|
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
|
|
// ─── Config ───────────────────────────────────────────────────────────────
|
|
|
|
const binary =
|
|
process.env.SF_SMOKE_BINARY || process.env.SF_SMOKE_BINARY || "sf";
|
|
let passed = 0;
|
|
let failed = 0;
|
|
|
|
function run(label: string, fn: () => void): void {
|
|
try {
|
|
fn();
|
|
console.log(` ✅ ${label}`);
|
|
passed++;
|
|
} catch (err: any) {
|
|
console.error(` ❌ ${label}`);
|
|
console.error(` ${err.message || err}`);
|
|
failed++;
|
|
}
|
|
}
|
|
|
|
function assert(condition: boolean, message: string): void {
|
|
if (!condition) throw new Error(message);
|
|
}
|
|
|
|
function sfCli(
|
|
args: string[],
|
|
cwd: string,
|
|
env?: Record<string, string>,
|
|
): { stdout: string; stderr: string; code: number } {
|
|
try {
|
|
const stdout = execFileSync(
|
|
binary === "sf" ? "sf" : "node",
|
|
binary === "sf" ? args : [binary, ...args],
|
|
{
|
|
cwd,
|
|
encoding: "utf-8",
|
|
timeout: 30_000,
|
|
stdio: ["pipe", "pipe", "pipe"],
|
|
env: {
|
|
...process.env,
|
|
...env,
|
|
SF_NON_INTERACTIVE: "1",
|
|
},
|
|
},
|
|
);
|
|
return { stdout, stderr: "", code: 0 };
|
|
} catch (err: any) {
|
|
return {
|
|
stdout: err.stdout || "",
|
|
stderr: err.stderr || "",
|
|
code: err.status ?? 1,
|
|
};
|
|
}
|
|
}
|
|
|
|
function createTempProject(name: string): string {
|
|
const dir = mkdtempSync(join(tmpdir(), `sf-live-${name}-`));
|
|
try {
|
|
execSync(
|
|
"git init && git config user.email test@test.com && git config user.name Test && git commit --allow-empty -m init",
|
|
{ cwd: dir, stdio: "pipe" },
|
|
);
|
|
} catch {}
|
|
return dir;
|
|
}
|
|
|
|
function buildMinimalRoadmap(
|
|
slices: Array<{ id: string; title: string; done: boolean }>,
|
|
): string {
|
|
const lines = ["# M001: Test Milestone", "", "## Slices", ""];
|
|
for (const s of slices) {
|
|
const cb = s.done ? "x" : " ";
|
|
lines.push(`- [${cb}] **${s.id}: ${s.title}** \`risk:low\` \`depends:[]\``);
|
|
lines.push(` > Demo for ${s.id}`);
|
|
lines.push("");
|
|
}
|
|
return lines.join("\n");
|
|
}
|
|
|
|
function buildMinimalPlan(
|
|
tasks: Array<{ id: string; title: string; done: boolean }>,
|
|
): string {
|
|
const lines = ["# S01: Test Slice", "", "**Goal:** test", "", "## Tasks", ""];
|
|
for (const t of tasks) {
|
|
const cb = t.done ? "x" : " ";
|
|
lines.push(`- [${cb}] **${t.id}: ${t.title}** \`est:5m\``);
|
|
}
|
|
return lines.join("\n");
|
|
}
|
|
|
|
function buildTaskSummary(id: string): string {
|
|
return `---\nid: ${id}\nparent: S01\nmilestone: M001\nduration: 5m\nverification_result: passed\ncompleted_at: ${new Date().toISOString()}\n---\n\n# ${id}: Done\n\nCompleted.`;
|
|
}
|
|
|
|
// ─── Test: headless query returns valid JSON ──────────────────────────────
|
|
|
|
run("headless query returns valid JSON on initialized project", () => {
|
|
const dir = createTempProject("query");
|
|
try {
|
|
const sfDir = join(dir, ".sf");
|
|
mkdirSync(join(sfDir, "milestones"), { recursive: true });
|
|
|
|
const result = sfCli(["headless", "query"], dir);
|
|
assert(
|
|
result.code === 0,
|
|
`expected exit 0, got ${result.code}: ${result.stderr}`,
|
|
);
|
|
|
|
const json = JSON.parse(result.stdout);
|
|
assert(
|
|
typeof (json.state?.phase ?? json.phase) === "string",
|
|
"response should have phase field",
|
|
);
|
|
assert(
|
|
Array.isArray(json.milestones) || json.milestones === undefined,
|
|
"milestones should be array or undefined",
|
|
);
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
// ─── Test: state derivation — empty project ──────────────────────────────
|
|
|
|
run("headless query: empty project reports pre-planning", () => {
|
|
const dir = createTempProject("empty");
|
|
try {
|
|
mkdirSync(join(dir, ".sf", "milestones"), { recursive: true });
|
|
|
|
const result = sfCli(["headless", "query"], dir);
|
|
assert(result.code === 0, `expected exit 0, got ${result.code}`);
|
|
|
|
const json = JSON.parse(result.stdout);
|
|
assert(
|
|
(json.state?.phase ?? json.phase) === "pre-planning" ||
|
|
(json.state?.phase ?? json.phase) === "idle",
|
|
`expected pre-planning or idle, got: ${json.state?.phase ?? json.phase}`,
|
|
);
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
// ─── Test: state derivation — milestone with roadmap ─────────────────────
|
|
|
|
run("headless query: milestone with roadmap reports planning phase", () => {
|
|
const dir = createTempProject("planning");
|
|
try {
|
|
const mDir = join(dir, ".sf", "milestones", "M001");
|
|
mkdirSync(join(mDir, "slices", "S01"), { recursive: true });
|
|
writeFileSync(join(mDir, "M001-CONTEXT.md"), "# M001\n\nContext.");
|
|
writeFileSync(
|
|
join(mDir, "M001-ROADMAP.md"),
|
|
buildMinimalRoadmap([{ id: "S01", title: "First Slice", done: false }]),
|
|
);
|
|
|
|
const result = sfCli(["headless", "query"], dir);
|
|
assert(result.code === 0, `expected exit 0, got ${result.code}`);
|
|
|
|
const json = JSON.parse(result.stdout);
|
|
assert(
|
|
(json.state?.phase ?? json.phase) === "planning",
|
|
`expected planning, got: ${json.state?.phase ?? json.phase}`,
|
|
);
|
|
assert(
|
|
(json.state?.activeMilestone ?? json.activeMilestone) === "M001" ||
|
|
(json.state?.activeMilestone ?? json.activeMilestone)?.id === "M001",
|
|
`expected active milestone M001`,
|
|
);
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
// ─── Test: state derivation — all tasks done ─────────────────────────────
|
|
|
|
run("headless query: all tasks done reports summarizing phase", () => {
|
|
const dir = createTempProject("summarizing");
|
|
try {
|
|
const mDir = join(dir, ".sf", "milestones", "M001");
|
|
const sDir = join(mDir, "slices", "S01");
|
|
mkdirSync(join(sDir, "tasks"), { recursive: true });
|
|
writeFileSync(join(mDir, "M001-CONTEXT.md"), "# M001\n\nContext.");
|
|
writeFileSync(
|
|
join(mDir, "M001-ROADMAP.md"),
|
|
buildMinimalRoadmap([{ id: "S01", title: "First Slice", done: false }]),
|
|
);
|
|
writeFileSync(
|
|
join(sDir, "S01-PLAN.md"),
|
|
buildMinimalPlan([{ id: "T01", title: "Task One", done: true }]),
|
|
);
|
|
writeFileSync(
|
|
join(sDir, "tasks", "T01-SUMMARY.md"),
|
|
buildTaskSummary("T01"),
|
|
);
|
|
|
|
const result = sfCli(["headless", "query"], dir);
|
|
assert(result.code === 0, `expected exit 0, got ${result.code}`);
|
|
|
|
const json = JSON.parse(result.stdout);
|
|
assert(
|
|
(json.state?.phase ?? json.phase) === "summarizing",
|
|
`expected summarizing, got: ${json.state?.phase ?? json.phase}`,
|
|
);
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
// ─── Test: state derivation — complete milestone ─────────────────────────
|
|
|
|
run("headless query: milestone with summary reports complete", () => {
|
|
const dir = createTempProject("complete");
|
|
try {
|
|
const mDir = join(dir, ".sf", "milestones", "M001");
|
|
mkdirSync(mDir, { recursive: true });
|
|
writeFileSync(
|
|
join(mDir, "M001-ROADMAP.md"),
|
|
buildMinimalRoadmap([{ id: "S01", title: "Done", done: true }]),
|
|
);
|
|
writeFileSync(join(mDir, "M001-SUMMARY.md"), "# M001 Summary\n\nComplete.");
|
|
|
|
const result = sfCli(["headless", "query"], dir);
|
|
assert(result.code === 0, `expected exit 0, got ${result.code}`);
|
|
|
|
const json = JSON.parse(result.stdout);
|
|
assert(
|
|
(json.state?.phase ?? json.phase) === "complete" ||
|
|
(json.state?.phase ?? json.phase) === "idle" ||
|
|
(json.state?.phase ?? json.phase) === "pre-planning",
|
|
`expected complete/idle/pre-planning, got: ${json.state?.phase ?? json.phase}`,
|
|
);
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
// ─── Test: lock file lifecycle ───────────────────────────────────────────
|
|
|
|
run("stale auto.lock with dead PID does not block --version", () => {
|
|
const dir = createTempProject("stale-lock");
|
|
try {
|
|
const sfDir = join(dir, ".sf");
|
|
mkdirSync(sfDir, { recursive: true });
|
|
// Write a lock with a PID that doesn't exist
|
|
writeFileSync(
|
|
join(sfDir, "auto.lock"),
|
|
JSON.stringify({
|
|
pid: 99999999,
|
|
startedAt: new Date().toISOString(),
|
|
unitType: "starting",
|
|
unitId: "bootstrap",
|
|
unitStartedAt: new Date().toISOString(),
|
|
completedUnits: 0,
|
|
}),
|
|
);
|
|
|
|
const result = sfCli(["--version"], dir);
|
|
assert(
|
|
result.code === 0,
|
|
`--version should succeed even with stale lock, got code ${result.code}`,
|
|
);
|
|
assert(
|
|
/\d+\.\d+\.\d+/.test(result.stdout.trim()),
|
|
`should output version, got: ${result.stdout}`,
|
|
);
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
// ─── Test: crash recovery message ────────────────────────────────────────
|
|
|
|
run("crash recovery shows actionable guidance", () => {
|
|
const dir = createTempProject("crash-recovery");
|
|
try {
|
|
const sfDir = join(dir, ".sf");
|
|
mkdirSync(join(sfDir, "milestones"), { recursive: true });
|
|
writeFileSync(
|
|
join(sfDir, "auto.lock"),
|
|
JSON.stringify({
|
|
pid: 99999999,
|
|
startedAt: new Date().toISOString(),
|
|
unitType: "execute-task",
|
|
unitId: "M001/S01/T02",
|
|
unitStartedAt: new Date().toISOString(),
|
|
completedUnits: 5,
|
|
}),
|
|
);
|
|
|
|
// headless query should still work — lock is for auto-mode, not query
|
|
const result = sfCli(["headless", "query"], dir);
|
|
assert(result.code === 0, `query should succeed with stale lock`);
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
// ─── Test: TTY check fires before heavy initialization ───────────────────
|
|
|
|
run("non-TTY invocation exits quickly with clean error", () => {
|
|
const dir = createTempProject("tty-check");
|
|
try {
|
|
const start = Date.now();
|
|
const result = sfCli([], dir); // No args, no TTY
|
|
const elapsed = Date.now() - start;
|
|
|
|
assert(
|
|
result.code === 1,
|
|
`expected exit 1 for non-TTY, got ${result.code}`,
|
|
);
|
|
assert(elapsed < 5000, `should exit within 5s, took ${elapsed}ms`);
|
|
assert(
|
|
result.stderr.includes("TTY") ||
|
|
result.stderr.includes("terminal") ||
|
|
result.stderr.includes("Interactive"),
|
|
`should mention TTY requirement in stderr`,
|
|
);
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
// ─── Test: version skew detection ────────────────────────────────────────
|
|
|
|
run("version skew is detected before TTY check", () => {
|
|
const dir = createTempProject("version-skew");
|
|
try {
|
|
// Create a fake managed-resources.json with a future version
|
|
const agentDir = join(dir, ".sf-test-agent");
|
|
mkdirSync(agentDir, { recursive: true });
|
|
writeFileSync(
|
|
join(agentDir, "managed-resources.json"),
|
|
JSON.stringify({
|
|
sfVersion: "999.0.0",
|
|
}),
|
|
);
|
|
|
|
// Set HOME to the temp dir so SF reads the fake agent dir
|
|
const fakeHome = dir;
|
|
mkdirSync(join(fakeHome, ".sf", "agent"), { recursive: true });
|
|
writeFileSync(
|
|
join(fakeHome, ".sf", "agent", "managed-resources.json"),
|
|
JSON.stringify({
|
|
sfVersion: "999.0.0",
|
|
}),
|
|
);
|
|
|
|
const result = sfCli([], dir, { HOME: fakeHome });
|
|
// Should either exit with version mismatch or TTY error — both are fine
|
|
assert(result.code === 1, `expected exit 1, got ${result.code}`);
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
// ─── Test: native addon graceful fallback ────────────────────────────────
|
|
|
|
run("sf --help works (native addon loads or falls back gracefully)", () => {
|
|
const result = sfCli(["--help"], process.cwd());
|
|
assert(result.code === 0, `--help should exit 0, got ${result.code}`);
|
|
assert(
|
|
result.stdout.toLowerCase().includes("sf") ||
|
|
result.stdout.toLowerCase().includes("usage"),
|
|
`help output should contain sf or usage`,
|
|
);
|
|
});
|
|
|
|
// ─── Summary ─────────────────────────────────────────────────────────────
|
|
|
|
console.log(`\nLive regression: ${passed} passed, ${failed} failed`);
|
|
if (failed > 0) process.exit(1);
|