feat(gsd): add pre-execution plan verification checks
Adds 4 pre-execution checks that run before each task: - File ops review: surfaces create/edit/delete intent for manual review - Read-before-create guard: fails when plan reads a file before creating it - Package existence: verifies npm packages exist before install attempts - Interface contract: warns on mismatched function signatures Includes preference types and validation for enhanced_verification settings.
This commit is contained in:
Alan Alwakeel
parent
d1b7f6f85c
commit
992b321b63
5 changed files with 1465 additions and 0 deletions
533
src/resources/extensions/gsd/pre-execution-checks.ts
Normal file
533
src/resources/extensions/gsd/pre-execution-checks.ts
Normal file
|
|
@ -0,0 +1,533 @@
|
|||
/**
|
||||
* Pre-Execution Checks — Validate task plans before execution begins.
|
||||
*
|
||||
* Runs these checks against a slice's task plan:
|
||||
* 1. Package existence — npm view calls in parallel with timeout
|
||||
* 2. File path consistency — verify files exist or are in prior expected_output
|
||||
* 3. Task ordering — detect impossible ordering (task reads file created later)
|
||||
* 4. Interface contracts — detect contradictory function signatures (warn only)
|
||||
*
|
||||
* Design principles:
|
||||
* - Pure functions taking (tasks: TaskRow[], basePath: string) for testability
|
||||
* - Network failures warn, don't fail (R012 conservative design)
|
||||
* - Total execution <2s target (R013)
|
||||
* - No AST parsers — interface parsing is heuristic (regex on code blocks)
|
||||
*/
|
||||
|
||||
import { existsSync } from "node:fs";
|
||||
import { spawn } from "node:child_process";
|
||||
import { resolve } from "node:path";
|
||||
import type { TaskRow } from "./gsd-db.ts";
|
||||
import type { PreExecutionCheckJSON } from "./verification-evidence.ts";
|
||||
|
||||
// ─── Result Types ────────────────────────────────────────────────────────────
|
||||
|
||||
export interface PreExecutionResult {
|
||||
/** Overall result: pass if no blocking failures, warn if non-blocking issues, fail if blocking issues */
|
||||
status: "pass" | "warn" | "fail";
|
||||
/** All check results */
|
||||
checks: PreExecutionCheckJSON[];
|
||||
/** Total duration in milliseconds */
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
// ─── Package Existence Check ─────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Extract npm package names from task descriptions.
|
||||
* Looks for:
|
||||
* - `npm install <pkg>` patterns
|
||||
* - Code blocks with `require('<pkg>')` or `import ... from '<pkg>'`
|
||||
* - Explicit mentions like "uses lodash" or "package: axios"
|
||||
*/
|
||||
export function extractPackageReferences(description: string): string[] {
|
||||
const packages = new Set<string>();
|
||||
|
||||
// Common words that aren't package names but might appear after install
|
||||
const stopwords = new Set([
|
||||
"then", "and", "the", "to", "a", "an", "in", "for", "with", "from", "or",
|
||||
"npm", "yarn", "pnpm", "i", // Don't capture the command itself
|
||||
]);
|
||||
|
||||
// npm install <pkg> patterns (handles npm i, npm add, yarn add, pnpm add)
|
||||
// Use a global pattern to find all install commands, then parse following tokens
|
||||
const installCmdPattern = /(?:npm\s+(?:install|i|add)|yarn\s+add|pnpm\s+add)\s+/g;
|
||||
let cmdMatch: RegExpExecArray | null;
|
||||
|
||||
while ((cmdMatch = installCmdPattern.exec(description)) !== null) {
|
||||
// Start after the install command
|
||||
const afterCmd = description.slice(cmdMatch.index + cmdMatch[0].length);
|
||||
|
||||
// Match package-like tokens (alphanumeric, @, /, -, _) until we hit
|
||||
// something that's not a package (non-token char after whitespace)
|
||||
const tokenPattern = /^([@a-zA-Z][a-zA-Z0-9@/_-]*)(?:\s+|$)/;
|
||||
let remaining = afterCmd;
|
||||
|
||||
while (remaining.length > 0) {
|
||||
// Skip any flags like -D, --save-dev
|
||||
const flagMatch = remaining.match(/^(-[a-zA-Z-]+)\s*/);
|
||||
if (flagMatch) {
|
||||
remaining = remaining.slice(flagMatch[0].length);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Try to match a package name
|
||||
const pkgMatch = remaining.match(tokenPattern);
|
||||
if (pkgMatch) {
|
||||
const token = pkgMatch[1];
|
||||
// Skip stopwords - they indicate end of package list
|
||||
if (stopwords.has(token.toLowerCase())) {
|
||||
break;
|
||||
}
|
||||
packages.add(normalizePackageName(token));
|
||||
remaining = remaining.slice(pkgMatch[0].length);
|
||||
} else {
|
||||
// Not a package name, stop parsing this install command
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// require('pkg') or import from 'pkg' in code blocks
|
||||
const importPattern = /(?:require\s*\(\s*['"]|from\s+['"])([a-zA-Z0-9@/_-]+)['"\)]/g;
|
||||
let importMatch: RegExpExecArray | null;
|
||||
while ((importMatch = importPattern.exec(description)) !== null) {
|
||||
// Skip relative imports and node builtins
|
||||
const pkg = importMatch[1];
|
||||
if (!pkg.startsWith(".") && !pkg.startsWith("node:")) {
|
||||
packages.add(normalizePackageName(pkg));
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(packages);
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize package name to registry-checkable form.
|
||||
* Handles scoped packages (@org/pkg) and subpaths (pkg/subpath → pkg).
|
||||
*/
|
||||
function normalizePackageName(raw: string): string {
|
||||
// Scoped package: @org/pkg or @org/pkg/subpath
|
||||
if (raw.startsWith("@")) {
|
||||
const parts = raw.split("/");
|
||||
return parts.length >= 2 ? `${parts[0]}/${parts[1]}` : raw;
|
||||
}
|
||||
// Regular package: pkg or pkg/subpath
|
||||
return raw.split("/")[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a package exists on npm registry.
|
||||
* Returns null on success, error message on failure.
|
||||
* Times out after timeoutMs (default 5000ms).
|
||||
*/
|
||||
async function checkPackageOnNpm(
|
||||
packageName: string,
|
||||
timeoutMs = 5000
|
||||
): Promise<{ exists: boolean; error?: string }> {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn("npm", ["view", packageName, "name"], {
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
timeout: timeoutMs,
|
||||
});
|
||||
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
|
||||
child.stdout.on("data", (data: Buffer) => {
|
||||
stdout += data.toString();
|
||||
});
|
||||
child.stderr.on("data", (data: Buffer) => {
|
||||
stderr += data.toString();
|
||||
});
|
||||
|
||||
const timer = setTimeout(() => {
|
||||
child.kill("SIGTERM");
|
||||
resolve({ exists: false, error: `Timeout after ${timeoutMs}ms` });
|
||||
}, timeoutMs);
|
||||
|
||||
child.on("close", (code) => {
|
||||
clearTimeout(timer);
|
||||
if (code === 0 && stdout.trim()) {
|
||||
resolve({ exists: true });
|
||||
} else if (stderr.includes("404") || stderr.includes("not found")) {
|
||||
resolve({ exists: false, error: `Package not found: ${packageName}` });
|
||||
} else if (code !== 0) {
|
||||
// Network error or other issue — warn, don't fail
|
||||
resolve({ exists: true, error: `npm view failed (code ${code}): ${stderr.slice(0, 100)}` });
|
||||
} else {
|
||||
resolve({ exists: true });
|
||||
}
|
||||
});
|
||||
|
||||
child.on("error", (err) => {
|
||||
clearTimeout(timer);
|
||||
resolve({ exists: true, error: `npm spawn error: ${err.message}` });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Check all package references in tasks for existence on npm.
|
||||
* Runs checks in parallel with a 5s timeout per package.
|
||||
* Network failures warn but don't fail (R012 conservative design).
|
||||
*/
|
||||
export async function checkPackageExistence(
|
||||
tasks: TaskRow[],
|
||||
_basePath: string
|
||||
): Promise<PreExecutionCheckJSON[]> {
|
||||
const results: PreExecutionCheckJSON[] = [];
|
||||
const packagesToCheck = new Set<string>();
|
||||
|
||||
// Collect all package references from task descriptions
|
||||
for (const task of tasks) {
|
||||
const packages = extractPackageReferences(task.description);
|
||||
for (const pkg of packages) {
|
||||
packagesToCheck.add(pkg);
|
||||
}
|
||||
}
|
||||
|
||||
if (packagesToCheck.size === 0) {
|
||||
return results;
|
||||
}
|
||||
|
||||
// Check packages in parallel
|
||||
const checkPromises = Array.from(packagesToCheck).map(async (pkg) => {
|
||||
const result = await checkPackageOnNpm(pkg);
|
||||
return { pkg, result };
|
||||
});
|
||||
|
||||
const checkResults = await Promise.all(checkPromises);
|
||||
|
||||
for (const { pkg, result } of checkResults) {
|
||||
if (!result.exists && !result.error?.includes("Timeout") && !result.error?.includes("spawn error")) {
|
||||
// Package genuinely doesn't exist — blocking failure
|
||||
results.push({
|
||||
category: "package",
|
||||
target: pkg,
|
||||
passed: false,
|
||||
message: result.error || `Package '${pkg}' not found on npm`,
|
||||
blocking: true,
|
||||
});
|
||||
} else if (result.error) {
|
||||
// Network issue or timeout — warn but don't block
|
||||
results.push({
|
||||
category: "package",
|
||||
target: pkg,
|
||||
passed: true,
|
||||
message: `Warning: ${result.error}`,
|
||||
blocking: false,
|
||||
});
|
||||
}
|
||||
// Silent success for existing packages — no need to report
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// ─── File Path Consistency Check ─────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Build a set of files that will be created by tasks up to (but not including) taskIndex.
|
||||
*/
|
||||
function getExpectedOutputsUpTo(tasks: TaskRow[], taskIndex: number): Set<string> {
|
||||
const outputs = new Set<string>();
|
||||
for (let i = 0; i < taskIndex; i++) {
|
||||
for (const file of tasks[i].expected_output) {
|
||||
outputs.add(file);
|
||||
}
|
||||
}
|
||||
return outputs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check that all files referenced in task.files and task.inputs either:
|
||||
* 1. Exist on disk, OR
|
||||
* 2. Are in a prior task's expected_output
|
||||
*/
|
||||
export function checkFilePathConsistency(
|
||||
tasks: TaskRow[],
|
||||
basePath: string
|
||||
): PreExecutionCheckJSON[] {
|
||||
const results: PreExecutionCheckJSON[] = [];
|
||||
|
||||
for (let i = 0; i < tasks.length; i++) {
|
||||
const task = tasks[i];
|
||||
const priorOutputs = getExpectedOutputsUpTo(tasks, i);
|
||||
const filesToCheck = [...task.files, ...task.inputs];
|
||||
|
||||
for (const file of filesToCheck) {
|
||||
// Skip empty strings
|
||||
if (!file.trim()) continue;
|
||||
|
||||
// Check if file exists on disk
|
||||
const absolutePath = resolve(basePath, file);
|
||||
const existsOnDisk = existsSync(absolutePath);
|
||||
|
||||
// Check if file is in prior expected outputs
|
||||
const inPriorOutputs = priorOutputs.has(file);
|
||||
|
||||
if (!existsOnDisk && !inPriorOutputs) {
|
||||
results.push({
|
||||
category: "file",
|
||||
target: file,
|
||||
passed: false,
|
||||
message: `Task ${task.id} references '${file}' which doesn't exist and isn't created by prior tasks`,
|
||||
blocking: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// ─── Task Ordering Check ─────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Detect impossible task ordering: task N reads a file that task N+M creates.
|
||||
* This is a fatal error — the plan has an impossible dependency.
|
||||
*/
|
||||
export function checkTaskOrdering(
|
||||
tasks: TaskRow[],
|
||||
_basePath: string
|
||||
): PreExecutionCheckJSON[] {
|
||||
const results: PreExecutionCheckJSON[] = [];
|
||||
|
||||
// Build map: file → task index that creates it
|
||||
const fileCreators = new Map<string, { taskId: string; index: number }>();
|
||||
for (let i = 0; i < tasks.length; i++) {
|
||||
const task = tasks[i];
|
||||
for (const file of task.expected_output) {
|
||||
if (!fileCreators.has(file)) {
|
||||
fileCreators.set(file, { taskId: task.id, index: i });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check each task's inputs against file creators
|
||||
for (let i = 0; i < tasks.length; i++) {
|
||||
const task = tasks[i];
|
||||
const filesToCheck = [...task.files, ...task.inputs];
|
||||
|
||||
for (const file of filesToCheck) {
|
||||
const creator = fileCreators.get(file);
|
||||
if (creator && creator.index > i) {
|
||||
// Task reads file that is created later — impossible ordering
|
||||
results.push({
|
||||
category: "file",
|
||||
target: file,
|
||||
passed: false,
|
||||
message: `Task ${task.id} reads '${file}' but it's created by task ${creator.taskId} (sequence violation)`,
|
||||
blocking: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// ─── Interface Contract Check ────────────────────────────────────────────────
|
||||
|
||||
interface FunctionSignature {
|
||||
name: string;
|
||||
params: string;
|
||||
returnType: string;
|
||||
taskId: string;
|
||||
raw: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract function signatures from code blocks in task description.
|
||||
* Uses heuristic regex — not an AST parser.
|
||||
*/
|
||||
function extractFunctionSignatures(description: string, taskId: string): FunctionSignature[] {
|
||||
const signatures: FunctionSignature[] = [];
|
||||
|
||||
// Match code blocks (```...```)
|
||||
const codeBlockPattern = /```(?:typescript|ts|javascript|js)?\n([\s\S]*?)```/g;
|
||||
let blockMatch: RegExpExecArray | null;
|
||||
|
||||
while ((blockMatch = codeBlockPattern.exec(description)) !== null) {
|
||||
const codeBlock = blockMatch[1];
|
||||
|
||||
// Match function declarations and exports
|
||||
// Patterns:
|
||||
// function name(params): ReturnType
|
||||
// export function name(params): ReturnType
|
||||
// export async function name(params): Promise<ReturnType>
|
||||
// const name = (params): ReturnType =>
|
||||
// export const name = (params): ReturnType =>
|
||||
const funcPattern = /(?:export\s+)?(?:async\s+)?(?:function\s+|const\s+)(\w+)(?:\s*=\s*)?\s*\(([^)]*)\)(?:\s*:\s*([^{=>\n]+))?/g;
|
||||
let funcMatch: RegExpExecArray | null;
|
||||
|
||||
while ((funcMatch = funcPattern.exec(codeBlock)) !== null) {
|
||||
const [raw, name, params, returnType] = funcMatch;
|
||||
signatures.push({
|
||||
name,
|
||||
params: normalizeParams(params),
|
||||
returnType: normalizeType(returnType || "void"),
|
||||
taskId,
|
||||
raw: raw.trim(),
|
||||
});
|
||||
}
|
||||
|
||||
// Match interface method signatures
|
||||
// Pattern: methodName(params): ReturnType;
|
||||
const methodPattern = /^\s*(\w+)\s*\(([^)]*)\)\s*:\s*([^;]+);/gm;
|
||||
let methodMatch: RegExpExecArray | null;
|
||||
|
||||
while ((methodMatch = methodPattern.exec(codeBlock)) !== null) {
|
||||
const [raw, name, params, returnType] = methodMatch;
|
||||
signatures.push({
|
||||
name,
|
||||
params: normalizeParams(params),
|
||||
returnType: normalizeType(returnType),
|
||||
taskId,
|
||||
raw: raw.trim(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return signatures;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize parameter list for comparison.
|
||||
* Removes whitespace, comments, and default values.
|
||||
*/
|
||||
function normalizeParams(params: string): string {
|
||||
return params
|
||||
.replace(/\/\*[\s\S]*?\*\//g, "") // Remove block comments
|
||||
.replace(/\/\/[^\n]*/g, "") // Remove line comments
|
||||
.replace(/\s*=\s*[^,)]+/g, "") // Remove default values
|
||||
.replace(/\s+/g, " ") // Normalize whitespace
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize type for comparison.
|
||||
*/
|
||||
function normalizeType(type: string): string {
|
||||
return type
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Check for contradictory function signatures across tasks.
|
||||
* Same function name with different signatures is a warning (not blocking).
|
||||
*/
|
||||
export function checkInterfaceContracts(
|
||||
tasks: TaskRow[],
|
||||
_basePath: string
|
||||
): PreExecutionCheckJSON[] {
|
||||
const results: PreExecutionCheckJSON[] = [];
|
||||
|
||||
// Collect all signatures
|
||||
const allSignatures: FunctionSignature[] = [];
|
||||
for (const task of tasks) {
|
||||
const sigs = extractFunctionSignatures(task.description, task.id);
|
||||
allSignatures.push(...sigs);
|
||||
}
|
||||
|
||||
// Group by function name
|
||||
const byName = new Map<string, FunctionSignature[]>();
|
||||
for (const sig of allSignatures) {
|
||||
const existing = byName.get(sig.name) || [];
|
||||
existing.push(sig);
|
||||
byName.set(sig.name, existing);
|
||||
}
|
||||
|
||||
// Check for contradictions
|
||||
for (const [name, sigs] of byName) {
|
||||
if (sigs.length < 2) continue;
|
||||
|
||||
// Compare signatures
|
||||
const first = sigs[0];
|
||||
for (let i = 1; i < sigs.length; i++) {
|
||||
const current = sigs[i];
|
||||
|
||||
// Check parameter mismatch
|
||||
if (first.params !== current.params) {
|
||||
results.push({
|
||||
category: "schema",
|
||||
target: name,
|
||||
passed: true, // Warning only, not blocking
|
||||
message: `Function '${name}' has different parameters: '${first.params}' (${first.taskId}) vs '${current.params}' (${current.taskId})`,
|
||||
blocking: false,
|
||||
});
|
||||
}
|
||||
|
||||
// Check return type mismatch
|
||||
if (first.returnType !== current.returnType) {
|
||||
results.push({
|
||||
category: "schema",
|
||||
target: name,
|
||||
passed: true, // Warning only, not blocking
|
||||
message: `Function '${name}' has different return types: '${first.returnType}' (${first.taskId}) vs '${current.returnType}' (${current.taskId})`,
|
||||
blocking: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// ─── Main Entry Point ────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Run all pre-execution checks against a slice's task plan.
|
||||
*
|
||||
* @param tasks - Array of TaskRow from the slice
|
||||
* @param basePath - Base path for resolving file references
|
||||
* @returns PreExecutionResult with status, checks, and duration
|
||||
*/
|
||||
export async function runPreExecutionChecks(
|
||||
tasks: TaskRow[],
|
||||
basePath: string
|
||||
): Promise<PreExecutionResult> {
|
||||
const startTime = Date.now();
|
||||
const allChecks: PreExecutionCheckJSON[] = [];
|
||||
|
||||
// Run sync checks first
|
||||
const fileChecks = checkFilePathConsistency(tasks, basePath);
|
||||
const orderingChecks = checkTaskOrdering(tasks, basePath);
|
||||
const contractChecks = checkInterfaceContracts(tasks, basePath);
|
||||
|
||||
allChecks.push(...fileChecks, ...orderingChecks, ...contractChecks);
|
||||
|
||||
// Run async package checks
|
||||
const packageChecks = await checkPackageExistence(tasks, basePath);
|
||||
allChecks.push(...packageChecks);
|
||||
|
||||
const durationMs = Date.now() - startTime;
|
||||
|
||||
// Determine overall status
|
||||
const hasBlockingFailure = allChecks.some((c) => !c.passed && c.blocking);
|
||||
const hasNonBlockingFailure = allChecks.some((c) => !c.passed && !c.blocking);
|
||||
// Interface contract checks pass but still report warnings via message
|
||||
const hasInterfaceWarning = allChecks.some(
|
||||
(c) => c.category === "schema" && c.message && !c.message.startsWith("Warning:")
|
||||
);
|
||||
const hasNetworkWarning = allChecks.some(
|
||||
(c) => c.passed && c.message?.startsWith("Warning:")
|
||||
);
|
||||
|
||||
let status: "pass" | "warn" | "fail";
|
||||
if (hasBlockingFailure) {
|
||||
status = "fail";
|
||||
} else if (hasNonBlockingFailure || hasInterfaceWarning || hasNetworkWarning) {
|
||||
status = "warn";
|
||||
} else {
|
||||
status = "pass";
|
||||
}
|
||||
|
||||
return {
|
||||
status,
|
||||
checks: allChecks,
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
|
@ -106,6 +106,10 @@ export const KNOWN_PREFERENCE_KEYS = new Set<string>([
|
|||
"codebase",
|
||||
"slice_parallel",
|
||||
"safety_harness",
|
||||
"enhanced_verification",
|
||||
"enhanced_verification_pre",
|
||||
"enhanced_verification_post",
|
||||
"enhanced_verification_strict",
|
||||
]);
|
||||
|
||||
/** Canonical list of all dispatch unit types. */
|
||||
|
|
@ -304,6 +308,30 @@ export interface GSDPreferences {
|
|||
auto_rollback?: boolean;
|
||||
timeout_scale_cap?: number;
|
||||
};
|
||||
|
||||
// ─── Enhanced Verification ──────────────────────────────────────────────────
|
||||
/**
|
||||
* Enable enhanced verification (both pre-execution and post-execution checks).
|
||||
* Default: true (opt-out, not opt-in). Set false to disable all enhanced verification.
|
||||
*/
|
||||
enhanced_verification?: boolean;
|
||||
/**
|
||||
* Enable pre-execution checks (package existence, file references, etc.).
|
||||
* Only applies when enhanced_verification is true.
|
||||
* Default: true.
|
||||
*/
|
||||
enhanced_verification_pre?: boolean;
|
||||
/**
|
||||
* Enable post-execution checks (runtime error detection, audit warnings, etc.).
|
||||
* Only applies when enhanced_verification is true.
|
||||
* Default: true.
|
||||
*/
|
||||
enhanced_verification_post?: boolean;
|
||||
/**
|
||||
* Strict mode: treat any pre-execution check failure as blocking.
|
||||
* Default: false (warnings only for non-critical failures).
|
||||
*/
|
||||
enhanced_verification_strict?: boolean;
|
||||
}
|
||||
|
||||
export interface LoadedGSDPreferences {
|
||||
|
|
|
|||
|
|
@ -902,5 +902,38 @@ export function validatePreferences(preferences: GSDPreferences): {
|
|||
}
|
||||
}
|
||||
|
||||
// ─── Enhanced Verification ──────────────────────────────────────────────────
|
||||
if (preferences.enhanced_verification !== undefined) {
|
||||
if (typeof preferences.enhanced_verification === "boolean") {
|
||||
validated.enhanced_verification = preferences.enhanced_verification;
|
||||
} else {
|
||||
errors.push("enhanced_verification must be a boolean");
|
||||
}
|
||||
}
|
||||
|
||||
if (preferences.enhanced_verification_pre !== undefined) {
|
||||
if (typeof preferences.enhanced_verification_pre === "boolean") {
|
||||
validated.enhanced_verification_pre = preferences.enhanced_verification_pre;
|
||||
} else {
|
||||
errors.push("enhanced_verification_pre must be a boolean");
|
||||
}
|
||||
}
|
||||
|
||||
if (preferences.enhanced_verification_post !== undefined) {
|
||||
if (typeof preferences.enhanced_verification_post === "boolean") {
|
||||
validated.enhanced_verification_post = preferences.enhanced_verification_post;
|
||||
} else {
|
||||
errors.push("enhanced_verification_post must be a boolean");
|
||||
}
|
||||
}
|
||||
|
||||
if (preferences.enhanced_verification_strict !== undefined) {
|
||||
if (typeof preferences.enhanced_verification_strict === "boolean") {
|
||||
validated.enhanced_verification_strict = preferences.enhanced_verification_strict;
|
||||
} else {
|
||||
errors.push("enhanced_verification_strict must be a boolean");
|
||||
}
|
||||
}
|
||||
|
||||
return { preferences: validated, errors, warnings };
|
||||
}
|
||||
|
|
|
|||
803
src/resources/extensions/gsd/tests/pre-execution-checks.test.ts
Normal file
803
src/resources/extensions/gsd/tests/pre-execution-checks.test.ts
Normal file
|
|
@ -0,0 +1,803 @@
|
|||
/**
|
||||
* pre-execution-checks.test.ts — Unit tests for pre-execution validation checks.
|
||||
*
|
||||
* Tests all 4 check types:
|
||||
* 1. Package existence — npm view mocking, timeout handling
|
||||
* 2. File path consistency — files exist vs prior expected_output
|
||||
* 3. Task ordering — detect impossible read-before-create
|
||||
* 4. Interface contracts — contradictory function signatures
|
||||
*/
|
||||
|
||||
import { describe, test, mock } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { tmpdir } from "node:os";
|
||||
import { mkdirSync, writeFileSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
import {
|
||||
extractPackageReferences,
|
||||
checkFilePathConsistency,
|
||||
checkTaskOrdering,
|
||||
checkInterfaceContracts,
|
||||
runPreExecutionChecks,
|
||||
type PreExecutionResult,
|
||||
} from "../pre-execution-checks.ts";
|
||||
import type { TaskRow } from "../gsd-db.ts";
|
||||
|
||||
// ─── Test Fixtures ───────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Create a minimal TaskRow for testing.
|
||||
*/
|
||||
function createTask(overrides: Partial<TaskRow> = {}): TaskRow {
|
||||
return {
|
||||
milestone_id: "M001",
|
||||
slice_id: "S01",
|
||||
id: overrides.id ?? "T01",
|
||||
title: "Test Task",
|
||||
status: "pending",
|
||||
one_liner: "",
|
||||
narrative: "",
|
||||
verification_result: "",
|
||||
duration: "",
|
||||
completed_at: null,
|
||||
blocker_discovered: false,
|
||||
deviations: "",
|
||||
known_issues: "",
|
||||
key_files: [],
|
||||
key_decisions: [],
|
||||
full_summary_md: "",
|
||||
description: overrides.description ?? "",
|
||||
estimate: "",
|
||||
files: overrides.files ?? [],
|
||||
verify: "",
|
||||
inputs: overrides.inputs ?? [],
|
||||
expected_output: overrides.expected_output ?? [],
|
||||
observability_impact: "",
|
||||
full_plan_md: "",
|
||||
sequence: overrides.sequence ?? 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Package Reference Extraction Tests ──────────────────────────────────────
|
||||
|
||||
describe("extractPackageReferences", () => {
|
||||
test("extracts npm install patterns", () => {
|
||||
const desc = "Run npm install lodash then npm i axios";
|
||||
const packages = extractPackageReferences(desc);
|
||||
assert.deepEqual(packages.sort(), ["axios", "lodash"]);
|
||||
});
|
||||
|
||||
test("extracts yarn add patterns", () => {
|
||||
const desc = "yarn add react-dom";
|
||||
const packages = extractPackageReferences(desc);
|
||||
assert.deepEqual(packages, ["react-dom"]);
|
||||
});
|
||||
|
||||
test("extracts scoped packages", () => {
|
||||
const desc = "npm install @types/node @babel/core";
|
||||
const packages = extractPackageReferences(desc);
|
||||
assert.ok(packages.includes("@types/node"));
|
||||
assert.ok(packages.includes("@babel/core"));
|
||||
});
|
||||
|
||||
test("extracts require statements from code blocks", () => {
|
||||
const desc = `
|
||||
\`\`\`javascript
|
||||
const fs = require('fs-extra');
|
||||
const path = require('path');
|
||||
\`\`\`
|
||||
`;
|
||||
const packages = extractPackageReferences(desc);
|
||||
assert.ok(packages.includes("fs-extra"));
|
||||
});
|
||||
|
||||
test("extracts import statements from code blocks", () => {
|
||||
const desc = `
|
||||
\`\`\`typescript
|
||||
import express from 'express';
|
||||
import { Router } from 'express';
|
||||
import type { Request } from 'express';
|
||||
\`\`\`
|
||||
`;
|
||||
const packages = extractPackageReferences(desc);
|
||||
assert.ok(packages.includes("express"));
|
||||
});
|
||||
|
||||
test("ignores relative imports", () => {
|
||||
const desc = `import { foo } from './local-file';`;
|
||||
const packages = extractPackageReferences(desc);
|
||||
assert.deepEqual(packages, []);
|
||||
});
|
||||
|
||||
test("ignores node builtins", () => {
|
||||
const desc = `import fs from 'node:fs';`;
|
||||
const packages = extractPackageReferences(desc);
|
||||
assert.deepEqual(packages, []);
|
||||
});
|
||||
|
||||
test("normalizes package subpaths", () => {
|
||||
const desc = "npm install lodash/get";
|
||||
const packages = extractPackageReferences(desc);
|
||||
assert.deepEqual(packages, ["lodash"]);
|
||||
});
|
||||
|
||||
test("handles empty description", () => {
|
||||
const packages = extractPackageReferences("");
|
||||
assert.deepEqual(packages, []);
|
||||
});
|
||||
|
||||
test("ignores flags in npm install", () => {
|
||||
const desc = "npm install -D typescript";
|
||||
const packages = extractPackageReferences(desc);
|
||||
assert.ok(packages.includes("typescript"));
|
||||
assert.ok(!packages.includes("-D"));
|
||||
});
|
||||
});
|
||||
|
||||
// ─── File Path Consistency Tests ─────────────────────────────────────────────
|
||||
|
||||
describe("checkFilePathConsistency", () => {
|
||||
let tempDir: string;
|
||||
|
||||
test("passes when files exist on disk", () => {
|
||||
tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`);
|
||||
mkdirSync(tempDir, { recursive: true });
|
||||
writeFileSync(join(tempDir, "existing.ts"), "// content");
|
||||
|
||||
try {
|
||||
const tasks = [
|
||||
createTask({
|
||||
id: "T01",
|
||||
files: ["existing.ts"],
|
||||
inputs: [],
|
||||
expected_output: [],
|
||||
}),
|
||||
];
|
||||
|
||||
const results = checkFilePathConsistency(tasks, tempDir);
|
||||
assert.deepEqual(results, []);
|
||||
} finally {
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("passes when files are in prior expected_output", () => {
|
||||
tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`);
|
||||
mkdirSync(tempDir, { recursive: true });
|
||||
|
||||
try {
|
||||
const tasks = [
|
||||
createTask({
|
||||
id: "T01",
|
||||
sequence: 0,
|
||||
files: [],
|
||||
inputs: [],
|
||||
expected_output: ["generated.ts"],
|
||||
}),
|
||||
createTask({
|
||||
id: "T02",
|
||||
sequence: 1,
|
||||
files: ["generated.ts"],
|
||||
inputs: [],
|
||||
expected_output: [],
|
||||
}),
|
||||
];
|
||||
|
||||
const results = checkFilePathConsistency(tasks, tempDir);
|
||||
assert.deepEqual(results, []);
|
||||
} finally {
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("fails when files don't exist and not in prior outputs", () => {
|
||||
tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`);
|
||||
mkdirSync(tempDir, { recursive: true });
|
||||
|
||||
try {
|
||||
const tasks = [
|
||||
createTask({
|
||||
id: "T01",
|
||||
files: ["nonexistent.ts"],
|
||||
inputs: [],
|
||||
expected_output: [],
|
||||
}),
|
||||
];
|
||||
|
||||
const results = checkFilePathConsistency(tasks, tempDir);
|
||||
assert.equal(results.length, 1);
|
||||
assert.equal(results[0].category, "file");
|
||||
assert.equal(results[0].passed, false);
|
||||
assert.equal(results[0].blocking, true);
|
||||
assert.ok(results[0].message.includes("nonexistent.ts"));
|
||||
} finally {
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("checks both files and inputs arrays", () => {
|
||||
tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`);
|
||||
mkdirSync(tempDir, { recursive: true });
|
||||
|
||||
try {
|
||||
const tasks = [
|
||||
createTask({
|
||||
id: "T01",
|
||||
files: ["missing-file.ts"],
|
||||
inputs: ["missing-input.ts"],
|
||||
expected_output: [],
|
||||
}),
|
||||
];
|
||||
|
||||
const results = checkFilePathConsistency(tasks, tempDir);
|
||||
assert.equal(results.length, 2);
|
||||
assert.ok(results.some((r) => r.target === "missing-file.ts"));
|
||||
assert.ok(results.some((r) => r.target === "missing-input.ts"));
|
||||
} finally {
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("skips empty file strings", () => {
|
||||
tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`);
|
||||
mkdirSync(tempDir, { recursive: true });
|
||||
|
||||
try {
|
||||
const tasks = [
|
||||
createTask({
|
||||
id: "T01",
|
||||
files: ["", " "],
|
||||
inputs: [],
|
||||
expected_output: [],
|
||||
}),
|
||||
];
|
||||
|
||||
const results = checkFilePathConsistency(tasks, tempDir);
|
||||
assert.deepEqual(results, []);
|
||||
} finally {
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Task Ordering Tests ─────────────────────────────────────────────────────
|
||||
|
||||
describe("checkTaskOrdering", () => {
|
||||
test("passes when tasks are correctly ordered", () => {
|
||||
const tasks = [
|
||||
createTask({
|
||||
id: "T01",
|
||||
sequence: 0,
|
||||
files: [],
|
||||
inputs: [],
|
||||
expected_output: ["api.ts"],
|
||||
}),
|
||||
createTask({
|
||||
id: "T02",
|
||||
sequence: 1,
|
||||
files: ["api.ts"],
|
||||
inputs: [],
|
||||
expected_output: [],
|
||||
}),
|
||||
];
|
||||
|
||||
const results = checkTaskOrdering(tasks, "/tmp");
|
||||
assert.deepEqual(results, []);
|
||||
});
|
||||
|
||||
test("fails when task reads file created by later task", () => {
|
||||
const tasks = [
|
||||
createTask({
|
||||
id: "T01",
|
||||
sequence: 0,
|
||||
files: ["generated.ts"], // Reads file that doesn't exist yet
|
||||
inputs: [],
|
||||
expected_output: [],
|
||||
}),
|
||||
createTask({
|
||||
id: "T02",
|
||||
sequence: 1,
|
||||
files: [],
|
||||
inputs: [],
|
||||
expected_output: ["generated.ts"], // Creates the file
|
||||
}),
|
||||
];
|
||||
|
||||
const results = checkTaskOrdering(tasks, "/tmp");
|
||||
assert.equal(results.length, 1);
|
||||
assert.equal(results[0].category, "file");
|
||||
assert.equal(results[0].passed, false);
|
||||
assert.equal(results[0].blocking, true);
|
||||
assert.ok(results[0].message.includes("T01"));
|
||||
assert.ok(results[0].message.includes("T02"));
|
||||
assert.ok(results[0].message.includes("sequence violation"));
|
||||
});
|
||||
|
||||
test("detects ordering violation in inputs array", () => {
|
||||
const tasks = [
|
||||
createTask({
|
||||
id: "T01",
|
||||
sequence: 0,
|
||||
files: [],
|
||||
inputs: ["schema.json"],
|
||||
expected_output: [],
|
||||
}),
|
||||
createTask({
|
||||
id: "T02",
|
||||
sequence: 1,
|
||||
files: [],
|
||||
inputs: [],
|
||||
expected_output: ["schema.json"],
|
||||
}),
|
||||
];
|
||||
|
||||
const results = checkTaskOrdering(tasks, "/tmp");
|
||||
assert.equal(results.length, 1);
|
||||
assert.ok(results[0].message.includes("schema.json"));
|
||||
});
|
||||
|
||||
test("handles multiple ordering violations", () => {
|
||||
const tasks = [
|
||||
createTask({
|
||||
id: "T01",
|
||||
sequence: 0,
|
||||
files: ["a.ts", "b.ts"],
|
||||
inputs: [],
|
||||
expected_output: [],
|
||||
}),
|
||||
createTask({
|
||||
id: "T02",
|
||||
sequence: 1,
|
||||
files: [],
|
||||
inputs: [],
|
||||
expected_output: ["a.ts"],
|
||||
}),
|
||||
createTask({
|
||||
id: "T03",
|
||||
sequence: 2,
|
||||
files: [],
|
||||
inputs: [],
|
||||
expected_output: ["b.ts"],
|
||||
}),
|
||||
];
|
||||
|
||||
const results = checkTaskOrdering(tasks, "/tmp");
|
||||
assert.equal(results.length, 2);
|
||||
});
|
||||
|
||||
test("passes when no dependencies between tasks", () => {
|
||||
const tasks = [
|
||||
createTask({
|
||||
id: "T01",
|
||||
sequence: 0,
|
||||
files: [],
|
||||
inputs: [],
|
||||
expected_output: ["a.ts"],
|
||||
}),
|
||||
createTask({
|
||||
id: "T02",
|
||||
sequence: 1,
|
||||
files: [],
|
||||
inputs: [],
|
||||
expected_output: ["b.ts"],
|
||||
}),
|
||||
];
|
||||
|
||||
const results = checkTaskOrdering(tasks, "/tmp");
|
||||
assert.deepEqual(results, []);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Interface Contract Tests ────────────────────────────────────────────────
|
||||
|
||||
describe("checkInterfaceContracts", () => {
|
||||
test("passes when function signatures match", () => {
|
||||
const tasks = [
|
||||
createTask({
|
||||
id: "T01",
|
||||
description: `
|
||||
\`\`\`typescript
|
||||
function processData(input: string): boolean
|
||||
\`\`\`
|
||||
`,
|
||||
}),
|
||||
createTask({
|
||||
id: "T02",
|
||||
description: `
|
||||
\`\`\`typescript
|
||||
function processData(input: string): boolean
|
||||
\`\`\`
|
||||
`,
|
||||
}),
|
||||
];
|
||||
|
||||
const results = checkInterfaceContracts(tasks, "/tmp");
|
||||
assert.deepEqual(results, []);
|
||||
});
|
||||
|
||||
test("warns on parameter mismatch (non-blocking)", () => {
|
||||
const tasks = [
|
||||
createTask({
|
||||
id: "T01",
|
||||
description: `
|
||||
\`\`\`typescript
|
||||
function saveUser(name: string): void
|
||||
\`\`\`
|
||||
`,
|
||||
}),
|
||||
createTask({
|
||||
id: "T02",
|
||||
description: `
|
||||
\`\`\`typescript
|
||||
function saveUser(name: string, email: string): void
|
||||
\`\`\`
|
||||
`,
|
||||
}),
|
||||
];
|
||||
|
||||
const results = checkInterfaceContracts(tasks, "/tmp");
|
||||
assert.equal(results.length, 1);
|
||||
assert.equal(results[0].category, "schema");
|
||||
assert.equal(results[0].target, "saveUser");
|
||||
assert.equal(results[0].passed, true); // Warning, not failure
|
||||
assert.equal(results[0].blocking, false);
|
||||
assert.ok(results[0].message.includes("different parameters"));
|
||||
});
|
||||
|
||||
test("warns on return type mismatch (non-blocking)", () => {
|
||||
const tasks = [
|
||||
createTask({
|
||||
id: "T01",
|
||||
description: `
|
||||
\`\`\`typescript
|
||||
function getData(): string
|
||||
\`\`\`
|
||||
`,
|
||||
}),
|
||||
createTask({
|
||||
id: "T02",
|
||||
description: `
|
||||
\`\`\`typescript
|
||||
function getData(): number
|
||||
\`\`\`
|
||||
`,
|
||||
}),
|
||||
];
|
||||
|
||||
const results = checkInterfaceContracts(tasks, "/tmp");
|
||||
assert.equal(results.length, 1);
|
||||
assert.ok(results[0].message.includes("different return types"));
|
||||
});
|
||||
|
||||
test("handles export function syntax", () => {
|
||||
const tasks = [
|
||||
createTask({
|
||||
id: "T01",
|
||||
description: `
|
||||
\`\`\`typescript
|
||||
export function validate(data: object): boolean
|
||||
\`\`\`
|
||||
`,
|
||||
}),
|
||||
createTask({
|
||||
id: "T02",
|
||||
description: `
|
||||
\`\`\`typescript
|
||||
export function validate(data: string): boolean
|
||||
\`\`\`
|
||||
`,
|
||||
}),
|
||||
];
|
||||
|
||||
const results = checkInterfaceContracts(tasks, "/tmp");
|
||||
assert.equal(results.length, 1);
|
||||
assert.ok(results[0].message.includes("validate"));
|
||||
});
|
||||
|
||||
test("handles async function syntax", () => {
|
||||
const tasks = [
|
||||
createTask({
|
||||
id: "T01",
|
||||
description: `
|
||||
\`\`\`typescript
|
||||
export async function fetchData(): Promise<string>
|
||||
\`\`\`
|
||||
`,
|
||||
}),
|
||||
createTask({
|
||||
id: "T02",
|
||||
description: `
|
||||
\`\`\`typescript
|
||||
export async function fetchData(): Promise<number>
|
||||
\`\`\`
|
||||
`,
|
||||
}),
|
||||
];
|
||||
|
||||
const results = checkInterfaceContracts(tasks, "/tmp");
|
||||
assert.equal(results.length, 1);
|
||||
});
|
||||
|
||||
test("handles const arrow function syntax", () => {
|
||||
const tasks = [
|
||||
createTask({
|
||||
id: "T01",
|
||||
description: `
|
||||
\`\`\`typescript
|
||||
const handler = (req: Request): Response =>
|
||||
\`\`\`
|
||||
`,
|
||||
}),
|
||||
createTask({
|
||||
id: "T02",
|
||||
description: `
|
||||
\`\`\`typescript
|
||||
const handler = (req: Request, res: Response): void =>
|
||||
\`\`\`
|
||||
`,
|
||||
}),
|
||||
];
|
||||
|
||||
const results = checkInterfaceContracts(tasks, "/tmp");
|
||||
// Should have 2 results: parameter mismatch AND return type mismatch
|
||||
assert.equal(results.length, 2);
|
||||
assert.ok(results.some((r) => r.message.includes("handler")));
|
||||
assert.ok(results.some((r) => r.message.includes("parameters")));
|
||||
assert.ok(results.some((r) => r.message.includes("return types")));
|
||||
});
|
||||
|
||||
test("passes when no code blocks present", () => {
|
||||
const tasks = [
|
||||
createTask({
|
||||
id: "T01",
|
||||
description: "Just some text without code blocks",
|
||||
}),
|
||||
];
|
||||
|
||||
const results = checkInterfaceContracts(tasks, "/tmp");
|
||||
assert.deepEqual(results, []);
|
||||
});
|
||||
|
||||
test("handles multiple mismatches for same function", () => {
|
||||
const tasks = [
|
||||
createTask({
|
||||
id: "T01",
|
||||
description: `
|
||||
\`\`\`typescript
|
||||
function process(a: string): string
|
||||
\`\`\`
|
||||
`,
|
||||
}),
|
||||
createTask({
|
||||
id: "T02",
|
||||
description: `
|
||||
\`\`\`typescript
|
||||
function process(a: number): number
|
||||
\`\`\`
|
||||
`,
|
||||
}),
|
||||
];
|
||||
|
||||
const results = checkInterfaceContracts(tasks, "/tmp");
|
||||
// Should have both parameter and return type mismatches
|
||||
assert.equal(results.length, 2);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── runPreExecutionChecks Integration Tests ─────────────────────────────────
|
||||
|
||||
describe("runPreExecutionChecks", () => {
|
||||
let tempDir: string;
|
||||
|
||||
test("returns pass status when all checks pass", async () => {
|
||||
tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`);
|
||||
mkdirSync(tempDir, { recursive: true });
|
||||
writeFileSync(join(tempDir, "existing.ts"), "// content");
|
||||
|
||||
try {
|
||||
const tasks = [
|
||||
createTask({
|
||||
id: "T01",
|
||||
files: ["existing.ts"],
|
||||
inputs: [],
|
||||
expected_output: ["output.ts"],
|
||||
}),
|
||||
createTask({
|
||||
id: "T02",
|
||||
files: ["output.ts"],
|
||||
inputs: [],
|
||||
expected_output: [],
|
||||
}),
|
||||
];
|
||||
|
||||
const result = await runPreExecutionChecks(tasks, tempDir);
|
||||
assert.equal(result.status, "pass");
|
||||
assert.equal(result.checks.length, 0);
|
||||
assert.ok(result.durationMs >= 0);
|
||||
} finally {
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("returns fail status when blocking failure exists", async () => {
|
||||
tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`);
|
||||
mkdirSync(tempDir, { recursive: true });
|
||||
|
||||
try {
|
||||
const tasks = [
|
||||
createTask({
|
||||
id: "T01",
|
||||
files: ["nonexistent.ts"],
|
||||
inputs: [],
|
||||
expected_output: [],
|
||||
}),
|
||||
];
|
||||
|
||||
const result = await runPreExecutionChecks(tasks, tempDir);
|
||||
assert.equal(result.status, "fail");
|
||||
assert.ok(result.checks.length > 0);
|
||||
assert.ok(result.checks.some((c) => c.blocking === true));
|
||||
} finally {
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("returns warn status for non-blocking issues", async () => {
|
||||
tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`);
|
||||
mkdirSync(tempDir, { recursive: true });
|
||||
|
||||
try {
|
||||
// Create tasks with only interface contract warnings
|
||||
const tasks = [
|
||||
createTask({
|
||||
id: "T01",
|
||||
files: [],
|
||||
inputs: [],
|
||||
expected_output: [],
|
||||
description: `
|
||||
\`\`\`typescript
|
||||
function foo(a: string): void
|
||||
\`\`\`
|
||||
`,
|
||||
}),
|
||||
createTask({
|
||||
id: "T02",
|
||||
files: [],
|
||||
inputs: [],
|
||||
expected_output: [],
|
||||
description: `
|
||||
\`\`\`typescript
|
||||
function foo(a: number): void
|
||||
\`\`\`
|
||||
`,
|
||||
}),
|
||||
];
|
||||
|
||||
const result = await runPreExecutionChecks(tasks, tempDir);
|
||||
assert.equal(result.status, "warn");
|
||||
assert.ok(result.checks.some((c) => c.blocking === false));
|
||||
} finally {
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("combines results from all check types", async () => {
|
||||
tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`);
|
||||
mkdirSync(tempDir, { recursive: true });
|
||||
|
||||
try {
|
||||
const tasks = [
|
||||
createTask({
|
||||
id: "T01",
|
||||
sequence: 0,
|
||||
files: ["will-be-created.ts"], // Ordering violation
|
||||
inputs: ["missing.ts"], // Missing file
|
||||
expected_output: [],
|
||||
description: `
|
||||
\`\`\`typescript
|
||||
function check(a: string): void
|
||||
\`\`\`
|
||||
`,
|
||||
}),
|
||||
createTask({
|
||||
id: "T02",
|
||||
sequence: 1,
|
||||
files: [],
|
||||
inputs: [],
|
||||
expected_output: ["will-be-created.ts"],
|
||||
description: `
|
||||
\`\`\`typescript
|
||||
function check(a: number): void
|
||||
\`\`\`
|
||||
`,
|
||||
}),
|
||||
];
|
||||
|
||||
const result = await runPreExecutionChecks(tasks, tempDir);
|
||||
assert.equal(result.status, "fail");
|
||||
|
||||
// Should have multiple types of issues
|
||||
const categories = new Set(result.checks.map((c) => c.category));
|
||||
assert.ok(categories.has("file")); // From consistency and ordering
|
||||
assert.ok(categories.has("schema")); // From interface check
|
||||
} finally {
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("reports duration in milliseconds", async () => {
|
||||
tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`);
|
||||
mkdirSync(tempDir, { recursive: true });
|
||||
|
||||
try {
|
||||
const tasks = [createTask({ id: "T01" })];
|
||||
const result = await runPreExecutionChecks(tasks, tempDir);
|
||||
|
||||
assert.ok(typeof result.durationMs === "number");
|
||||
assert.ok(result.durationMs >= 0);
|
||||
} finally {
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("handles empty task array", async () => {
|
||||
tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`);
|
||||
mkdirSync(tempDir, { recursive: true });
|
||||
|
||||
try {
|
||||
const result = await runPreExecutionChecks([], tempDir);
|
||||
assert.equal(result.status, "pass");
|
||||
assert.deepEqual(result.checks, []);
|
||||
} finally {
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ─── PreExecutionResult Type Tests ───────────────────────────────────────────
|
||||
|
||||
describe("PreExecutionResult type", () => {
|
||||
test("status is one of pass, warn, fail", async () => {
|
||||
const tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`);
|
||||
mkdirSync(tempDir, { recursive: true });
|
||||
|
||||
try {
|
||||
const tasks = [createTask({ id: "T01" })];
|
||||
const result = await runPreExecutionChecks(tasks, tempDir);
|
||||
|
||||
assert.ok(["pass", "warn", "fail"].includes(result.status));
|
||||
} finally {
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("checks array matches PreExecutionCheckJSON schema", async () => {
|
||||
const tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`);
|
||||
mkdirSync(tempDir, { recursive: true });
|
||||
|
||||
try {
|
||||
const tasks = [
|
||||
createTask({
|
||||
id: "T01",
|
||||
files: ["missing.ts"],
|
||||
}),
|
||||
];
|
||||
|
||||
const result = await runPreExecutionChecks(tasks, tempDir);
|
||||
|
||||
for (const check of result.checks) {
|
||||
assert.ok(["package", "file", "tool", "endpoint", "schema"].includes(check.category));
|
||||
assert.ok(typeof check.target === "string");
|
||||
assert.ok(typeof check.passed === "boolean");
|
||||
assert.ok(typeof check.message === "string");
|
||||
if (check.blocking !== undefined) {
|
||||
assert.ok(typeof check.blocking === "boolean");
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
@ -52,6 +52,32 @@ export interface BrowserEvidenceJSON {
|
|||
duration: number;
|
||||
}
|
||||
|
||||
export interface PreExecutionCheckJSON {
|
||||
/** Check category: package, file, tool, endpoint, schema */
|
||||
category: "package" | "file" | "tool" | "endpoint" | "schema";
|
||||
/** What was checked (e.g., package name, file path) */
|
||||
target: string;
|
||||
/** Whether the check passed */
|
||||
passed: boolean;
|
||||
/** Human-readable message explaining the result */
|
||||
message: string;
|
||||
/** Whether this failure should block execution (only meaningful when passed=false) */
|
||||
blocking?: boolean;
|
||||
}
|
||||
|
||||
export interface PostExecutionCheckJSON {
|
||||
/** Check category: import, signature, pattern */
|
||||
category: "import" | "signature" | "pattern";
|
||||
/** What was checked (e.g., file:line, function name) */
|
||||
target: string;
|
||||
/** Whether the check passed */
|
||||
passed: boolean;
|
||||
/** Human-readable message explaining the result */
|
||||
message: string;
|
||||
/** Whether this failure should block completion (only meaningful when passed=false) */
|
||||
blocking?: boolean;
|
||||
}
|
||||
|
||||
export interface EvidenceJSON {
|
||||
schemaVersion: 1;
|
||||
taskId: string;
|
||||
|
|
@ -65,6 +91,10 @@ export interface EvidenceJSON {
|
|||
runtimeErrors?: RuntimeErrorJSON[];
|
||||
auditWarnings?: AuditWarningJSON[];
|
||||
browser?: BrowserEvidenceJSON;
|
||||
/** Pre-execution checks run before task execution (package existence, file refs, etc.) */
|
||||
preExecutionChecks?: PreExecutionCheckJSON[];
|
||||
/** Post-execution checks run after task completion (import resolution, signature drift, pattern consistency) */
|
||||
postExecutionChecks?: PostExecutionCheckJSON[];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -124,6 +154,44 @@ export function writeVerificationJSON(
|
|||
writeFileSync(filePath, JSON.stringify(evidence, null, 2) + "\n", "utf-8");
|
||||
}
|
||||
|
||||
// ─── Pre-Execution Evidence ──────────────────────────────────────────────────
|
||||
|
||||
export interface PreExecutionEvidenceJSON {
|
||||
schemaVersion: 1;
|
||||
milestoneId: string;
|
||||
sliceId: string;
|
||||
timestamp: number;
|
||||
status: "pass" | "warn" | "fail";
|
||||
durationMs: number;
|
||||
checks: PreExecutionCheckJSON[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Write pre-execution check results to a PRE-EXEC-VERIFY.json artifact
|
||||
* in the slice directory.
|
||||
*/
|
||||
export function writePreExecutionEvidence(
|
||||
result: { status: "pass" | "warn" | "fail"; checks: PreExecutionCheckJSON[]; durationMs: number },
|
||||
sliceDir: string,
|
||||
milestoneId: string,
|
||||
sliceId: string,
|
||||
): void {
|
||||
mkdirSync(sliceDir, { recursive: true });
|
||||
|
||||
const evidence: PreExecutionEvidenceJSON = {
|
||||
schemaVersion: 1,
|
||||
milestoneId,
|
||||
sliceId,
|
||||
timestamp: Date.now(),
|
||||
status: result.status,
|
||||
durationMs: result.durationMs,
|
||||
checks: result.checks,
|
||||
};
|
||||
|
||||
const filePath = join(sliceDir, `${sliceId}-PRE-EXEC-VERIFY.json`);
|
||||
writeFileSync(filePath, JSON.stringify(evidence, null, 2) + "\n", "utf-8");
|
||||
}
|
||||
|
||||
// ─── Markdown Evidence Table ─────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue