test: Add verification gate integration tests for failure catching, cle…
- src/resources/extensions/sf/tests/verification-gate.test.ts SF-Task: S03/T02
This commit is contained in:
Mikael Hugo
parent
a45f873124
commit
37c5db3dd3
1 changed files with 219 additions and 1 deletions
|
|
@ -17,12 +17,13 @@
|
|||
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { afterEach, beforeEach, describe, test } from "node:test";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { validatePreferences } from "../preferences.ts";
|
||||
import { writeVerificationJSON } from "../verification-evidence.ts";
|
||||
import {
|
||||
captureRuntimeErrors,
|
||||
discoverCommands,
|
||||
|
|
@ -1194,3 +1195,220 @@ test("dependency-audit: subdirectory package.json does not trigger audit", () =>
|
|||
);
|
||||
assert.deepStrictEqual(result, []);
|
||||
});
|
||||
|
||||
// ─── Integration Tests — Gate + Evidence JSON (T02) ─────────────────────────
|
||||
|
||||
describe("verification-gate: integration — gate + evidence JSON", () => {
|
||||
let tmp: string;
|
||||
beforeEach(() => {
|
||||
tmp = makeTempDir("vg-integration");
|
||||
});
|
||||
afterEach(() => {
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("gate catches injected failure → evidence JSON has correct structure", () => {
|
||||
const result = runVerificationGate({
|
||||
cwd: tmp,
|
||||
preferenceCommands: ["echo ok", "sh -c 'echo err >&2; exit 1'"],
|
||||
});
|
||||
assert.equal(result.passed, false, "gate should fail");
|
||||
assert.equal(result.checks.length, 2);
|
||||
assert.equal(result.checks[0].exitCode, 0);
|
||||
assert.equal(result.checks[1].exitCode, 1);
|
||||
|
||||
writeVerificationJSON(result, tmp, "T02", "M004/S03/T02");
|
||||
|
||||
const json = JSON.parse(
|
||||
readFileSync(join(tmp, "T02-VERIFY.json"), "utf-8"),
|
||||
);
|
||||
assert.equal(json.schemaVersion, 1);
|
||||
assert.equal(json.taskId, "T02");
|
||||
assert.equal(json.unitId, "M004/S03/T02");
|
||||
assert.equal(json.passed, false);
|
||||
assert.equal(json.discoverySource, "preference");
|
||||
assert.equal(Array.isArray(json.checks), true);
|
||||
assert.equal(json.checks.length, 2);
|
||||
assert.equal(json.checks[0].command, "echo ok");
|
||||
assert.equal(json.checks[0].exitCode, 0);
|
||||
assert.equal(json.checks[0].verdict, "pass");
|
||||
assert.equal(typeof json.checks[0].durationMs, "number");
|
||||
assert.equal(json.checks[1].command, "sh -c 'echo err >&2; exit 1'");
|
||||
assert.equal(json.checks[1].exitCode, 1);
|
||||
assert.equal(json.checks[1].verdict, "fail");
|
||||
// stdout/stderr must NOT appear in JSON
|
||||
assert.ok(!("stdout" in json.checks[0]));
|
||||
assert.ok(!("stderr" in json.checks[0]));
|
||||
});
|
||||
|
||||
test("gate passes on clean commands → evidence JSON has passed:true", () => {
|
||||
const result = runVerificationGate({
|
||||
cwd: tmp,
|
||||
preferenceCommands: ["echo hello", "echo world"],
|
||||
});
|
||||
assert.equal(result.passed, true, "gate should pass");
|
||||
assert.equal(result.checks.length, 2);
|
||||
|
||||
writeVerificationJSON(result, tmp, "T02-pass");
|
||||
|
||||
const json = JSON.parse(
|
||||
readFileSync(join(tmp, "T02-pass-VERIFY.json"), "utf-8"),
|
||||
);
|
||||
assert.equal(json.passed, true);
|
||||
assert.equal(json.checks.length, 2);
|
||||
assert.equal(json.checks[0].verdict, "pass");
|
||||
assert.equal(json.checks[1].verdict, "pass");
|
||||
assert.equal(json.discoverySource, "preference");
|
||||
});
|
||||
|
||||
test("discovery order D003 respected in integration — preference > task-plan > package.json", () => {
|
||||
writeFileSync(
|
||||
join(tmp, "package.json"),
|
||||
JSON.stringify({ scripts: { test: "vitest" } }),
|
||||
);
|
||||
|
||||
// Only package.json available
|
||||
const pkgOnly = runVerificationGate({ cwd: tmp });
|
||||
assert.equal(pkgOnly.discoverySource, "package-json");
|
||||
assert.deepStrictEqual(pkgOnly.checks.map((c) => c.command), [
|
||||
"npm run test",
|
||||
]);
|
||||
|
||||
// Task plan verify beats package.json
|
||||
const taskPlan = runVerificationGate({
|
||||
cwd: tmp,
|
||||
taskPlanVerify: "echo from-task-plan",
|
||||
});
|
||||
assert.equal(taskPlan.discoverySource, "task-plan");
|
||||
assert.deepStrictEqual(taskPlan.checks.map((c) => c.command), [
|
||||
"echo from-task-plan",
|
||||
]);
|
||||
|
||||
// Preference beats task plan and package.json
|
||||
const pref = runVerificationGate({
|
||||
cwd: tmp,
|
||||
preferenceCommands: ["echo from-preference"],
|
||||
taskPlanVerify: "echo from-task-plan",
|
||||
});
|
||||
assert.equal(pref.discoverySource, "preference");
|
||||
assert.deepStrictEqual(pref.checks.map((c) => c.command), [
|
||||
"echo from-preference",
|
||||
]);
|
||||
});
|
||||
|
||||
test("commandTimeoutMs causes timeout on slow command", () => {
|
||||
const result = runVerificationGate({
|
||||
cwd: tmp,
|
||||
preferenceCommands: ["sleep 5"],
|
||||
commandTimeoutMs: 100, // 100 ms — way too short for sleep 5
|
||||
});
|
||||
assert.equal(result.passed, false, "timed-out command should fail");
|
||||
assert.equal(result.checks.length, 1);
|
||||
// Exit code for timeout is typically null (killed by signal) which gate maps to 1
|
||||
assert.notEqual(result.checks[0].exitCode, 0, "exit code should be non-zero");
|
||||
assert.ok(
|
||||
result.checks[0].durationMs >= 0 && result.checks[0].durationMs < 500,
|
||||
"duration should be short (<500ms) because of timeout",
|
||||
);
|
||||
});
|
||||
|
||||
test("shell injection in taskPlanVerify is sanitized and rejected", () => {
|
||||
writeFileSync(
|
||||
join(tmp, "package.json"),
|
||||
JSON.stringify({ scripts: { test: "vitest" } }),
|
||||
);
|
||||
|
||||
// Semicolon injection should be rejected, falling through to package.json
|
||||
const result = runVerificationGate({
|
||||
cwd: tmp,
|
||||
taskPlanVerify: "echo ok; rm -rf /",
|
||||
});
|
||||
assert.equal(result.discoverySource, "package-json");
|
||||
assert.deepStrictEqual(result.checks.map((c) => c.command), [
|
||||
"npm run test",
|
||||
]);
|
||||
});
|
||||
|
||||
test("backtick injection in taskPlanVerify is sanitized and rejected", () => {
|
||||
writeFileSync(
|
||||
join(tmp, "package.json"),
|
||||
JSON.stringify({ scripts: { test: "vitest" } }),
|
||||
);
|
||||
|
||||
const result = runVerificationGate({
|
||||
cwd: tmp,
|
||||
taskPlanVerify: "echo `rm -rf /`",
|
||||
});
|
||||
assert.equal(result.discoverySource, "package-json");
|
||||
});
|
||||
|
||||
test("$() injection in taskPlanVerify is sanitized and rejected", () => {
|
||||
writeFileSync(
|
||||
join(tmp, "package.json"),
|
||||
JSON.stringify({ scripts: { test: "vitest" } }),
|
||||
);
|
||||
|
||||
const result = runVerificationGate({
|
||||
cwd: tmp,
|
||||
taskPlanVerify: "echo $(rm -rf /)",
|
||||
});
|
||||
assert.equal(result.discoverySource, "package-json");
|
||||
});
|
||||
|
||||
test("pipe injection in taskPlanVerify is sanitized and rejected", () => {
|
||||
writeFileSync(
|
||||
join(tmp, "package.json"),
|
||||
JSON.stringify({ scripts: { test: "vitest" } }),
|
||||
);
|
||||
|
||||
const result = runVerificationGate({
|
||||
cwd: tmp,
|
||||
taskPlanVerify: "echo ok | rm -rf /",
|
||||
});
|
||||
assert.equal(result.discoverySource, "package-json");
|
||||
});
|
||||
|
||||
test("integration: gate with runtimeErrors and auditWarnings → evidence JSON includes them", () => {
|
||||
const result = runVerificationGate({
|
||||
cwd: tmp,
|
||||
preferenceCommands: ["echo ok"],
|
||||
});
|
||||
assert.equal(result.passed, true);
|
||||
|
||||
// Inject runtimeErrors and auditWarnings into the result
|
||||
result.runtimeErrors = [
|
||||
{
|
||||
source: "bg-shell",
|
||||
severity: "crash",
|
||||
message: "Server crashed",
|
||||
blocking: true,
|
||||
},
|
||||
];
|
||||
result.auditWarnings = [
|
||||
{
|
||||
name: "lodash",
|
||||
severity: "high",
|
||||
title: "Prototype Pollution",
|
||||
url: "https://github.com/advisories/GHSA-1234",
|
||||
fixAvailable: true,
|
||||
},
|
||||
];
|
||||
|
||||
writeVerificationJSON(result, tmp, "T02-rich");
|
||||
|
||||
const json = JSON.parse(
|
||||
readFileSync(join(tmp, "T02-rich-VERIFY.json"), "utf-8"),
|
||||
);
|
||||
assert.equal(json.passed, true);
|
||||
assert.equal(Array.isArray(json.runtimeErrors), true);
|
||||
assert.equal(json.runtimeErrors.length, 1);
|
||||
assert.equal(json.runtimeErrors[0].source, "bg-shell");
|
||||
assert.equal(json.runtimeErrors[0].severity, "crash");
|
||||
assert.equal(json.runtimeErrors[0].blocking, true);
|
||||
assert.equal(Array.isArray(json.auditWarnings), true);
|
||||
assert.equal(json.auditWarnings.length, 1);
|
||||
assert.equal(json.auditWarnings[0].name, "lodash");
|
||||
assert.equal(json.auditWarnings[0].severity, "high");
|
||||
assert.equal(json.auditWarnings[0].fixAvailable, true);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue