From 992b321b6327e99238c0112e319d46e825418972 Mon Sep 17 00:00:00 2001 From: Alan Alwakeel Date: Fri, 3 Apr 2026 16:17:25 -0400 Subject: [PATCH] feat(gsd): add pre-execution plan verification checks Adds 4 pre-execution checks that run before each task: - File ops review: surfaces create/edit/delete intent for manual review - Read-before-create guard: fails when plan reads a file before creating it - Package existence: verifies npm packages exist before install attempts - Interface contract: warns on mismatched function signatures Includes preference types and validation for enhanced_verification settings. --- .../extensions/gsd/pre-execution-checks.ts | 533 ++++++++++++ .../extensions/gsd/preferences-types.ts | 28 + .../extensions/gsd/preferences-validation.ts | 33 + .../gsd/tests/pre-execution-checks.test.ts | 803 ++++++++++++++++++ .../extensions/gsd/verification-evidence.ts | 68 ++ 5 files changed, 1465 insertions(+) create mode 100644 src/resources/extensions/gsd/pre-execution-checks.ts create mode 100644 src/resources/extensions/gsd/tests/pre-execution-checks.test.ts diff --git a/src/resources/extensions/gsd/pre-execution-checks.ts b/src/resources/extensions/gsd/pre-execution-checks.ts new file mode 100644 index 000000000..83c269ef1 --- /dev/null +++ b/src/resources/extensions/gsd/pre-execution-checks.ts @@ -0,0 +1,533 @@ +/** + * Pre-Execution Checks — Validate task plans before execution begins. + * + * Runs these checks against a slice's task plan: + * 1. Package existence — npm view calls in parallel with timeout + * 2. File path consistency — verify files exist or are in prior expected_output + * 3. Task ordering — detect impossible ordering (task reads file created later) + * 4. Interface contracts — detect contradictory function signatures (warn only) + * + * Design principles: + * - Pure functions taking (tasks: TaskRow[], basePath: string) for testability + * - Network failures warn, don't fail (R012 conservative design) + * - Total execution <2s target (R013) + * - No AST parsers — interface parsing is heuristic (regex on code blocks) + */ + +import { existsSync } from "node:fs"; +import { spawn } from "node:child_process"; +import { resolve } from "node:path"; +import type { TaskRow } from "./gsd-db.ts"; +import type { PreExecutionCheckJSON } from "./verification-evidence.ts"; + +// ─── Result Types ──────────────────────────────────────────────────────────── + +export interface PreExecutionResult { + /** Overall result: pass if no blocking failures, warn if non-blocking issues, fail if blocking issues */ + status: "pass" | "warn" | "fail"; + /** All check results */ + checks: PreExecutionCheckJSON[]; + /** Total duration in milliseconds */ + durationMs: number; +} + +// ─── Package Existence Check ───────────────────────────────────────────────── + +/** + * Extract npm package names from task descriptions. + * Looks for: + * - `npm install ` patterns + * - Code blocks with `require('')` or `import ... from ''` + * - Explicit mentions like "uses lodash" or "package: axios" + */ +export function extractPackageReferences(description: string): string[] { + const packages = new Set(); + + // Common words that aren't package names but might appear after install + const stopwords = new Set([ + "then", "and", "the", "to", "a", "an", "in", "for", "with", "from", "or", + "npm", "yarn", "pnpm", "i", // Don't capture the command itself + ]); + + // npm install patterns (handles npm i, npm add, yarn add, pnpm add) + // Use a global pattern to find all install commands, then parse following tokens + const installCmdPattern = /(?:npm\s+(?:install|i|add)|yarn\s+add|pnpm\s+add)\s+/g; + let cmdMatch: RegExpExecArray | null; + + while ((cmdMatch = installCmdPattern.exec(description)) !== null) { + // Start after the install command + const afterCmd = description.slice(cmdMatch.index + cmdMatch[0].length); + + // Match package-like tokens (alphanumeric, @, /, -, _) until we hit + // something that's not a package (non-token char after whitespace) + const tokenPattern = /^([@a-zA-Z][a-zA-Z0-9@/_-]*)(?:\s+|$)/; + let remaining = afterCmd; + + while (remaining.length > 0) { + // Skip any flags like -D, --save-dev + const flagMatch = remaining.match(/^(-[a-zA-Z-]+)\s*/); + if (flagMatch) { + remaining = remaining.slice(flagMatch[0].length); + continue; + } + + // Try to match a package name + const pkgMatch = remaining.match(tokenPattern); + if (pkgMatch) { + const token = pkgMatch[1]; + // Skip stopwords - they indicate end of package list + if (stopwords.has(token.toLowerCase())) { + break; + } + packages.add(normalizePackageName(token)); + remaining = remaining.slice(pkgMatch[0].length); + } else { + // Not a package name, stop parsing this install command + break; + } + } + } + + // require('pkg') or import from 'pkg' in code blocks + const importPattern = /(?:require\s*\(\s*['"]|from\s+['"])([a-zA-Z0-9@/_-]+)['"\)]/g; + let importMatch: RegExpExecArray | null; + while ((importMatch = importPattern.exec(description)) !== null) { + // Skip relative imports and node builtins + const pkg = importMatch[1]; + if (!pkg.startsWith(".") && !pkg.startsWith("node:")) { + packages.add(normalizePackageName(pkg)); + } + } + + return Array.from(packages); +} + +/** + * Normalize package name to registry-checkable form. + * Handles scoped packages (@org/pkg) and subpaths (pkg/subpath → pkg). + */ +function normalizePackageName(raw: string): string { + // Scoped package: @org/pkg or @org/pkg/subpath + if (raw.startsWith("@")) { + const parts = raw.split("/"); + return parts.length >= 2 ? `${parts[0]}/${parts[1]}` : raw; + } + // Regular package: pkg or pkg/subpath + return raw.split("/")[0]; +} + +/** + * Check if a package exists on npm registry. + * Returns null on success, error message on failure. + * Times out after timeoutMs (default 5000ms). + */ +async function checkPackageOnNpm( + packageName: string, + timeoutMs = 5000 +): Promise<{ exists: boolean; error?: string }> { + return new Promise((resolve) => { + const child = spawn("npm", ["view", packageName, "name"], { + stdio: ["ignore", "pipe", "pipe"], + timeout: timeoutMs, + }); + + let stdout = ""; + let stderr = ""; + + child.stdout.on("data", (data: Buffer) => { + stdout += data.toString(); + }); + child.stderr.on("data", (data: Buffer) => { + stderr += data.toString(); + }); + + const timer = setTimeout(() => { + child.kill("SIGTERM"); + resolve({ exists: false, error: `Timeout after ${timeoutMs}ms` }); + }, timeoutMs); + + child.on("close", (code) => { + clearTimeout(timer); + if (code === 0 && stdout.trim()) { + resolve({ exists: true }); + } else if (stderr.includes("404") || stderr.includes("not found")) { + resolve({ exists: false, error: `Package not found: ${packageName}` }); + } else if (code !== 0) { + // Network error or other issue — warn, don't fail + resolve({ exists: true, error: `npm view failed (code ${code}): ${stderr.slice(0, 100)}` }); + } else { + resolve({ exists: true }); + } + }); + + child.on("error", (err) => { + clearTimeout(timer); + resolve({ exists: true, error: `npm spawn error: ${err.message}` }); + }); + }); +} + +/** + * Check all package references in tasks for existence on npm. + * Runs checks in parallel with a 5s timeout per package. + * Network failures warn but don't fail (R012 conservative design). + */ +export async function checkPackageExistence( + tasks: TaskRow[], + _basePath: string +): Promise { + const results: PreExecutionCheckJSON[] = []; + const packagesToCheck = new Set(); + + // Collect all package references from task descriptions + for (const task of tasks) { + const packages = extractPackageReferences(task.description); + for (const pkg of packages) { + packagesToCheck.add(pkg); + } + } + + if (packagesToCheck.size === 0) { + return results; + } + + // Check packages in parallel + const checkPromises = Array.from(packagesToCheck).map(async (pkg) => { + const result = await checkPackageOnNpm(pkg); + return { pkg, result }; + }); + + const checkResults = await Promise.all(checkPromises); + + for (const { pkg, result } of checkResults) { + if (!result.exists && !result.error?.includes("Timeout") && !result.error?.includes("spawn error")) { + // Package genuinely doesn't exist — blocking failure + results.push({ + category: "package", + target: pkg, + passed: false, + message: result.error || `Package '${pkg}' not found on npm`, + blocking: true, + }); + } else if (result.error) { + // Network issue or timeout — warn but don't block + results.push({ + category: "package", + target: pkg, + passed: true, + message: `Warning: ${result.error}`, + blocking: false, + }); + } + // Silent success for existing packages — no need to report + } + + return results; +} + +// ─── File Path Consistency Check ───────────────────────────────────────────── + +/** + * Build a set of files that will be created by tasks up to (but not including) taskIndex. + */ +function getExpectedOutputsUpTo(tasks: TaskRow[], taskIndex: number): Set { + const outputs = new Set(); + for (let i = 0; i < taskIndex; i++) { + for (const file of tasks[i].expected_output) { + outputs.add(file); + } + } + return outputs; +} + +/** + * Check that all files referenced in task.files and task.inputs either: + * 1. Exist on disk, OR + * 2. Are in a prior task's expected_output + */ +export function checkFilePathConsistency( + tasks: TaskRow[], + basePath: string +): PreExecutionCheckJSON[] { + const results: PreExecutionCheckJSON[] = []; + + for (let i = 0; i < tasks.length; i++) { + const task = tasks[i]; + const priorOutputs = getExpectedOutputsUpTo(tasks, i); + const filesToCheck = [...task.files, ...task.inputs]; + + for (const file of filesToCheck) { + // Skip empty strings + if (!file.trim()) continue; + + // Check if file exists on disk + const absolutePath = resolve(basePath, file); + const existsOnDisk = existsSync(absolutePath); + + // Check if file is in prior expected outputs + const inPriorOutputs = priorOutputs.has(file); + + if (!existsOnDisk && !inPriorOutputs) { + results.push({ + category: "file", + target: file, + passed: false, + message: `Task ${task.id} references '${file}' which doesn't exist and isn't created by prior tasks`, + blocking: true, + }); + } + } + } + + return results; +} + +// ─── Task Ordering Check ───────────────────────────────────────────────────── + +/** + * Detect impossible task ordering: task N reads a file that task N+M creates. + * This is a fatal error — the plan has an impossible dependency. + */ +export function checkTaskOrdering( + tasks: TaskRow[], + _basePath: string +): PreExecutionCheckJSON[] { + const results: PreExecutionCheckJSON[] = []; + + // Build map: file → task index that creates it + const fileCreators = new Map(); + for (let i = 0; i < tasks.length; i++) { + const task = tasks[i]; + for (const file of task.expected_output) { + if (!fileCreators.has(file)) { + fileCreators.set(file, { taskId: task.id, index: i }); + } + } + } + + // Check each task's inputs against file creators + for (let i = 0; i < tasks.length; i++) { + const task = tasks[i]; + const filesToCheck = [...task.files, ...task.inputs]; + + for (const file of filesToCheck) { + const creator = fileCreators.get(file); + if (creator && creator.index > i) { + // Task reads file that is created later — impossible ordering + results.push({ + category: "file", + target: file, + passed: false, + message: `Task ${task.id} reads '${file}' but it's created by task ${creator.taskId} (sequence violation)`, + blocking: true, + }); + } + } + } + + return results; +} + +// ─── Interface Contract Check ──────────────────────────────────────────────── + +interface FunctionSignature { + name: string; + params: string; + returnType: string; + taskId: string; + raw: string; +} + +/** + * Extract function signatures from code blocks in task description. + * Uses heuristic regex — not an AST parser. + */ +function extractFunctionSignatures(description: string, taskId: string): FunctionSignature[] { + const signatures: FunctionSignature[] = []; + + // Match code blocks (```...```) + const codeBlockPattern = /```(?:typescript|ts|javascript|js)?\n([\s\S]*?)```/g; + let blockMatch: RegExpExecArray | null; + + while ((blockMatch = codeBlockPattern.exec(description)) !== null) { + const codeBlock = blockMatch[1]; + + // Match function declarations and exports + // Patterns: + // function name(params): ReturnType + // export function name(params): ReturnType + // export async function name(params): Promise + // const name = (params): ReturnType => + // export const name = (params): ReturnType => + const funcPattern = /(?:export\s+)?(?:async\s+)?(?:function\s+|const\s+)(\w+)(?:\s*=\s*)?\s*\(([^)]*)\)(?:\s*:\s*([^{=>\n]+))?/g; + let funcMatch: RegExpExecArray | null; + + while ((funcMatch = funcPattern.exec(codeBlock)) !== null) { + const [raw, name, params, returnType] = funcMatch; + signatures.push({ + name, + params: normalizeParams(params), + returnType: normalizeType(returnType || "void"), + taskId, + raw: raw.trim(), + }); + } + + // Match interface method signatures + // Pattern: methodName(params): ReturnType; + const methodPattern = /^\s*(\w+)\s*\(([^)]*)\)\s*:\s*([^;]+);/gm; + let methodMatch: RegExpExecArray | null; + + while ((methodMatch = methodPattern.exec(codeBlock)) !== null) { + const [raw, name, params, returnType] = methodMatch; + signatures.push({ + name, + params: normalizeParams(params), + returnType: normalizeType(returnType), + taskId, + raw: raw.trim(), + }); + } + } + + return signatures; +} + +/** + * Normalize parameter list for comparison. + * Removes whitespace, comments, and default values. + */ +function normalizeParams(params: string): string { + return params + .replace(/\/\*[\s\S]*?\*\//g, "") // Remove block comments + .replace(/\/\/[^\n]*/g, "") // Remove line comments + .replace(/\s*=\s*[^,)]+/g, "") // Remove default values + .replace(/\s+/g, " ") // Normalize whitespace + .trim(); +} + +/** + * Normalize type for comparison. + */ +function normalizeType(type: string): string { + return type + .replace(/\s+/g, " ") + .trim(); +} + +/** + * Check for contradictory function signatures across tasks. + * Same function name with different signatures is a warning (not blocking). + */ +export function checkInterfaceContracts( + tasks: TaskRow[], + _basePath: string +): PreExecutionCheckJSON[] { + const results: PreExecutionCheckJSON[] = []; + + // Collect all signatures + const allSignatures: FunctionSignature[] = []; + for (const task of tasks) { + const sigs = extractFunctionSignatures(task.description, task.id); + allSignatures.push(...sigs); + } + + // Group by function name + const byName = new Map(); + for (const sig of allSignatures) { + const existing = byName.get(sig.name) || []; + existing.push(sig); + byName.set(sig.name, existing); + } + + // Check for contradictions + for (const [name, sigs] of byName) { + if (sigs.length < 2) continue; + + // Compare signatures + const first = sigs[0]; + for (let i = 1; i < sigs.length; i++) { + const current = sigs[i]; + + // Check parameter mismatch + if (first.params !== current.params) { + results.push({ + category: "schema", + target: name, + passed: true, // Warning only, not blocking + message: `Function '${name}' has different parameters: '${first.params}' (${first.taskId}) vs '${current.params}' (${current.taskId})`, + blocking: false, + }); + } + + // Check return type mismatch + if (first.returnType !== current.returnType) { + results.push({ + category: "schema", + target: name, + passed: true, // Warning only, not blocking + message: `Function '${name}' has different return types: '${first.returnType}' (${first.taskId}) vs '${current.returnType}' (${current.taskId})`, + blocking: false, + }); + } + } + } + + return results; +} + +// ─── Main Entry Point ──────────────────────────────────────────────────────── + +/** + * Run all pre-execution checks against a slice's task plan. + * + * @param tasks - Array of TaskRow from the slice + * @param basePath - Base path for resolving file references + * @returns PreExecutionResult with status, checks, and duration + */ +export async function runPreExecutionChecks( + tasks: TaskRow[], + basePath: string +): Promise { + const startTime = Date.now(); + const allChecks: PreExecutionCheckJSON[] = []; + + // Run sync checks first + const fileChecks = checkFilePathConsistency(tasks, basePath); + const orderingChecks = checkTaskOrdering(tasks, basePath); + const contractChecks = checkInterfaceContracts(tasks, basePath); + + allChecks.push(...fileChecks, ...orderingChecks, ...contractChecks); + + // Run async package checks + const packageChecks = await checkPackageExistence(tasks, basePath); + allChecks.push(...packageChecks); + + const durationMs = Date.now() - startTime; + + // Determine overall status + const hasBlockingFailure = allChecks.some((c) => !c.passed && c.blocking); + const hasNonBlockingFailure = allChecks.some((c) => !c.passed && !c.blocking); + // Interface contract checks pass but still report warnings via message + const hasInterfaceWarning = allChecks.some( + (c) => c.category === "schema" && c.message && !c.message.startsWith("Warning:") + ); + const hasNetworkWarning = allChecks.some( + (c) => c.passed && c.message?.startsWith("Warning:") + ); + + let status: "pass" | "warn" | "fail"; + if (hasBlockingFailure) { + status = "fail"; + } else if (hasNonBlockingFailure || hasInterfaceWarning || hasNetworkWarning) { + status = "warn"; + } else { + status = "pass"; + } + + return { + status, + checks: allChecks, + durationMs, + }; +} diff --git a/src/resources/extensions/gsd/preferences-types.ts b/src/resources/extensions/gsd/preferences-types.ts index a5013c18c..3452e34f3 100644 --- a/src/resources/extensions/gsd/preferences-types.ts +++ b/src/resources/extensions/gsd/preferences-types.ts @@ -106,6 +106,10 @@ export const KNOWN_PREFERENCE_KEYS = new Set([ "codebase", "slice_parallel", "safety_harness", + "enhanced_verification", + "enhanced_verification_pre", + "enhanced_verification_post", + "enhanced_verification_strict", ]); /** Canonical list of all dispatch unit types. */ @@ -304,6 +308,30 @@ export interface GSDPreferences { auto_rollback?: boolean; timeout_scale_cap?: number; }; + + // ─── Enhanced Verification ────────────────────────────────────────────────── + /** + * Enable enhanced verification (both pre-execution and post-execution checks). + * Default: true (opt-out, not opt-in). Set false to disable all enhanced verification. + */ + enhanced_verification?: boolean; + /** + * Enable pre-execution checks (package existence, file references, etc.). + * Only applies when enhanced_verification is true. + * Default: true. + */ + enhanced_verification_pre?: boolean; + /** + * Enable post-execution checks (runtime error detection, audit warnings, etc.). + * Only applies when enhanced_verification is true. + * Default: true. + */ + enhanced_verification_post?: boolean; + /** + * Strict mode: treat any pre-execution check failure as blocking. + * Default: false (warnings only for non-critical failures). + */ + enhanced_verification_strict?: boolean; } export interface LoadedGSDPreferences { diff --git a/src/resources/extensions/gsd/preferences-validation.ts b/src/resources/extensions/gsd/preferences-validation.ts index 21afe285d..33b4fe3f0 100644 --- a/src/resources/extensions/gsd/preferences-validation.ts +++ b/src/resources/extensions/gsd/preferences-validation.ts @@ -902,5 +902,38 @@ export function validatePreferences(preferences: GSDPreferences): { } } + // ─── Enhanced Verification ────────────────────────────────────────────────── + if (preferences.enhanced_verification !== undefined) { + if (typeof preferences.enhanced_verification === "boolean") { + validated.enhanced_verification = preferences.enhanced_verification; + } else { + errors.push("enhanced_verification must be a boolean"); + } + } + + if (preferences.enhanced_verification_pre !== undefined) { + if (typeof preferences.enhanced_verification_pre === "boolean") { + validated.enhanced_verification_pre = preferences.enhanced_verification_pre; + } else { + errors.push("enhanced_verification_pre must be a boolean"); + } + } + + if (preferences.enhanced_verification_post !== undefined) { + if (typeof preferences.enhanced_verification_post === "boolean") { + validated.enhanced_verification_post = preferences.enhanced_verification_post; + } else { + errors.push("enhanced_verification_post must be a boolean"); + } + } + + if (preferences.enhanced_verification_strict !== undefined) { + if (typeof preferences.enhanced_verification_strict === "boolean") { + validated.enhanced_verification_strict = preferences.enhanced_verification_strict; + } else { + errors.push("enhanced_verification_strict must be a boolean"); + } + } + return { preferences: validated, errors, warnings }; } diff --git a/src/resources/extensions/gsd/tests/pre-execution-checks.test.ts b/src/resources/extensions/gsd/tests/pre-execution-checks.test.ts new file mode 100644 index 000000000..6086d78b5 --- /dev/null +++ b/src/resources/extensions/gsd/tests/pre-execution-checks.test.ts @@ -0,0 +1,803 @@ +/** + * pre-execution-checks.test.ts — Unit tests for pre-execution validation checks. + * + * Tests all 4 check types: + * 1. Package existence — npm view mocking, timeout handling + * 2. File path consistency — files exist vs prior expected_output + * 3. Task ordering — detect impossible read-before-create + * 4. Interface contracts — contradictory function signatures + */ + +import { describe, test, mock } from "node:test"; +import assert from "node:assert/strict"; +import { tmpdir } from "node:os"; +import { mkdirSync, writeFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; + +import { + extractPackageReferences, + checkFilePathConsistency, + checkTaskOrdering, + checkInterfaceContracts, + runPreExecutionChecks, + type PreExecutionResult, +} from "../pre-execution-checks.ts"; +import type { TaskRow } from "../gsd-db.ts"; + +// ─── Test Fixtures ─────────────────────────────────────────────────────────── + +/** + * Create a minimal TaskRow for testing. + */ +function createTask(overrides: Partial = {}): TaskRow { + return { + milestone_id: "M001", + slice_id: "S01", + id: overrides.id ?? "T01", + title: "Test Task", + status: "pending", + one_liner: "", + narrative: "", + verification_result: "", + duration: "", + completed_at: null, + blocker_discovered: false, + deviations: "", + known_issues: "", + key_files: [], + key_decisions: [], + full_summary_md: "", + description: overrides.description ?? "", + estimate: "", + files: overrides.files ?? [], + verify: "", + inputs: overrides.inputs ?? [], + expected_output: overrides.expected_output ?? [], + observability_impact: "", + full_plan_md: "", + sequence: overrides.sequence ?? 0, + ...overrides, + }; +} + +// ─── Package Reference Extraction Tests ────────────────────────────────────── + +describe("extractPackageReferences", () => { + test("extracts npm install patterns", () => { + const desc = "Run npm install lodash then npm i axios"; + const packages = extractPackageReferences(desc); + assert.deepEqual(packages.sort(), ["axios", "lodash"]); + }); + + test("extracts yarn add patterns", () => { + const desc = "yarn add react-dom"; + const packages = extractPackageReferences(desc); + assert.deepEqual(packages, ["react-dom"]); + }); + + test("extracts scoped packages", () => { + const desc = "npm install @types/node @babel/core"; + const packages = extractPackageReferences(desc); + assert.ok(packages.includes("@types/node")); + assert.ok(packages.includes("@babel/core")); + }); + + test("extracts require statements from code blocks", () => { + const desc = ` +\`\`\`javascript +const fs = require('fs-extra'); +const path = require('path'); +\`\`\` + `; + const packages = extractPackageReferences(desc); + assert.ok(packages.includes("fs-extra")); + }); + + test("extracts import statements from code blocks", () => { + const desc = ` +\`\`\`typescript +import express from 'express'; +import { Router } from 'express'; +import type { Request } from 'express'; +\`\`\` + `; + const packages = extractPackageReferences(desc); + assert.ok(packages.includes("express")); + }); + + test("ignores relative imports", () => { + const desc = `import { foo } from './local-file';`; + const packages = extractPackageReferences(desc); + assert.deepEqual(packages, []); + }); + + test("ignores node builtins", () => { + const desc = `import fs from 'node:fs';`; + const packages = extractPackageReferences(desc); + assert.deepEqual(packages, []); + }); + + test("normalizes package subpaths", () => { + const desc = "npm install lodash/get"; + const packages = extractPackageReferences(desc); + assert.deepEqual(packages, ["lodash"]); + }); + + test("handles empty description", () => { + const packages = extractPackageReferences(""); + assert.deepEqual(packages, []); + }); + + test("ignores flags in npm install", () => { + const desc = "npm install -D typescript"; + const packages = extractPackageReferences(desc); + assert.ok(packages.includes("typescript")); + assert.ok(!packages.includes("-D")); + }); +}); + +// ─── File Path Consistency Tests ───────────────────────────────────────────── + +describe("checkFilePathConsistency", () => { + let tempDir: string; + + test("passes when files exist on disk", () => { + tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`); + mkdirSync(tempDir, { recursive: true }); + writeFileSync(join(tempDir, "existing.ts"), "// content"); + + try { + const tasks = [ + createTask({ + id: "T01", + files: ["existing.ts"], + inputs: [], + expected_output: [], + }), + ]; + + const results = checkFilePathConsistency(tasks, tempDir); + assert.deepEqual(results, []); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + test("passes when files are in prior expected_output", () => { + tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`); + mkdirSync(tempDir, { recursive: true }); + + try { + const tasks = [ + createTask({ + id: "T01", + sequence: 0, + files: [], + inputs: [], + expected_output: ["generated.ts"], + }), + createTask({ + id: "T02", + sequence: 1, + files: ["generated.ts"], + inputs: [], + expected_output: [], + }), + ]; + + const results = checkFilePathConsistency(tasks, tempDir); + assert.deepEqual(results, []); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + test("fails when files don't exist and not in prior outputs", () => { + tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`); + mkdirSync(tempDir, { recursive: true }); + + try { + const tasks = [ + createTask({ + id: "T01", + files: ["nonexistent.ts"], + inputs: [], + expected_output: [], + }), + ]; + + const results = checkFilePathConsistency(tasks, tempDir); + assert.equal(results.length, 1); + assert.equal(results[0].category, "file"); + assert.equal(results[0].passed, false); + assert.equal(results[0].blocking, true); + assert.ok(results[0].message.includes("nonexistent.ts")); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + test("checks both files and inputs arrays", () => { + tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`); + mkdirSync(tempDir, { recursive: true }); + + try { + const tasks = [ + createTask({ + id: "T01", + files: ["missing-file.ts"], + inputs: ["missing-input.ts"], + expected_output: [], + }), + ]; + + const results = checkFilePathConsistency(tasks, tempDir); + assert.equal(results.length, 2); + assert.ok(results.some((r) => r.target === "missing-file.ts")); + assert.ok(results.some((r) => r.target === "missing-input.ts")); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + test("skips empty file strings", () => { + tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`); + mkdirSync(tempDir, { recursive: true }); + + try { + const tasks = [ + createTask({ + id: "T01", + files: ["", " "], + inputs: [], + expected_output: [], + }), + ]; + + const results = checkFilePathConsistency(tasks, tempDir); + assert.deepEqual(results, []); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); +}); + +// ─── Task Ordering Tests ───────────────────────────────────────────────────── + +describe("checkTaskOrdering", () => { + test("passes when tasks are correctly ordered", () => { + const tasks = [ + createTask({ + id: "T01", + sequence: 0, + files: [], + inputs: [], + expected_output: ["api.ts"], + }), + createTask({ + id: "T02", + sequence: 1, + files: ["api.ts"], + inputs: [], + expected_output: [], + }), + ]; + + const results = checkTaskOrdering(tasks, "/tmp"); + assert.deepEqual(results, []); + }); + + test("fails when task reads file created by later task", () => { + const tasks = [ + createTask({ + id: "T01", + sequence: 0, + files: ["generated.ts"], // Reads file that doesn't exist yet + inputs: [], + expected_output: [], + }), + createTask({ + id: "T02", + sequence: 1, + files: [], + inputs: [], + expected_output: ["generated.ts"], // Creates the file + }), + ]; + + const results = checkTaskOrdering(tasks, "/tmp"); + assert.equal(results.length, 1); + assert.equal(results[0].category, "file"); + assert.equal(results[0].passed, false); + assert.equal(results[0].blocking, true); + assert.ok(results[0].message.includes("T01")); + assert.ok(results[0].message.includes("T02")); + assert.ok(results[0].message.includes("sequence violation")); + }); + + test("detects ordering violation in inputs array", () => { + const tasks = [ + createTask({ + id: "T01", + sequence: 0, + files: [], + inputs: ["schema.json"], + expected_output: [], + }), + createTask({ + id: "T02", + sequence: 1, + files: [], + inputs: [], + expected_output: ["schema.json"], + }), + ]; + + const results = checkTaskOrdering(tasks, "/tmp"); + assert.equal(results.length, 1); + assert.ok(results[0].message.includes("schema.json")); + }); + + test("handles multiple ordering violations", () => { + const tasks = [ + createTask({ + id: "T01", + sequence: 0, + files: ["a.ts", "b.ts"], + inputs: [], + expected_output: [], + }), + createTask({ + id: "T02", + sequence: 1, + files: [], + inputs: [], + expected_output: ["a.ts"], + }), + createTask({ + id: "T03", + sequence: 2, + files: [], + inputs: [], + expected_output: ["b.ts"], + }), + ]; + + const results = checkTaskOrdering(tasks, "/tmp"); + assert.equal(results.length, 2); + }); + + test("passes when no dependencies between tasks", () => { + const tasks = [ + createTask({ + id: "T01", + sequence: 0, + files: [], + inputs: [], + expected_output: ["a.ts"], + }), + createTask({ + id: "T02", + sequence: 1, + files: [], + inputs: [], + expected_output: ["b.ts"], + }), + ]; + + const results = checkTaskOrdering(tasks, "/tmp"); + assert.deepEqual(results, []); + }); +}); + +// ─── Interface Contract Tests ──────────────────────────────────────────────── + +describe("checkInterfaceContracts", () => { + test("passes when function signatures match", () => { + const tasks = [ + createTask({ + id: "T01", + description: ` +\`\`\`typescript +function processData(input: string): boolean +\`\`\` + `, + }), + createTask({ + id: "T02", + description: ` +\`\`\`typescript +function processData(input: string): boolean +\`\`\` + `, + }), + ]; + + const results = checkInterfaceContracts(tasks, "/tmp"); + assert.deepEqual(results, []); + }); + + test("warns on parameter mismatch (non-blocking)", () => { + const tasks = [ + createTask({ + id: "T01", + description: ` +\`\`\`typescript +function saveUser(name: string): void +\`\`\` + `, + }), + createTask({ + id: "T02", + description: ` +\`\`\`typescript +function saveUser(name: string, email: string): void +\`\`\` + `, + }), + ]; + + const results = checkInterfaceContracts(tasks, "/tmp"); + assert.equal(results.length, 1); + assert.equal(results[0].category, "schema"); + assert.equal(results[0].target, "saveUser"); + assert.equal(results[0].passed, true); // Warning, not failure + assert.equal(results[0].blocking, false); + assert.ok(results[0].message.includes("different parameters")); + }); + + test("warns on return type mismatch (non-blocking)", () => { + const tasks = [ + createTask({ + id: "T01", + description: ` +\`\`\`typescript +function getData(): string +\`\`\` + `, + }), + createTask({ + id: "T02", + description: ` +\`\`\`typescript +function getData(): number +\`\`\` + `, + }), + ]; + + const results = checkInterfaceContracts(tasks, "/tmp"); + assert.equal(results.length, 1); + assert.ok(results[0].message.includes("different return types")); + }); + + test("handles export function syntax", () => { + const tasks = [ + createTask({ + id: "T01", + description: ` +\`\`\`typescript +export function validate(data: object): boolean +\`\`\` + `, + }), + createTask({ + id: "T02", + description: ` +\`\`\`typescript +export function validate(data: string): boolean +\`\`\` + `, + }), + ]; + + const results = checkInterfaceContracts(tasks, "/tmp"); + assert.equal(results.length, 1); + assert.ok(results[0].message.includes("validate")); + }); + + test("handles async function syntax", () => { + const tasks = [ + createTask({ + id: "T01", + description: ` +\`\`\`typescript +export async function fetchData(): Promise +\`\`\` + `, + }), + createTask({ + id: "T02", + description: ` +\`\`\`typescript +export async function fetchData(): Promise +\`\`\` + `, + }), + ]; + + const results = checkInterfaceContracts(tasks, "/tmp"); + assert.equal(results.length, 1); + }); + + test("handles const arrow function syntax", () => { + const tasks = [ + createTask({ + id: "T01", + description: ` +\`\`\`typescript +const handler = (req: Request): Response => +\`\`\` + `, + }), + createTask({ + id: "T02", + description: ` +\`\`\`typescript +const handler = (req: Request, res: Response): void => +\`\`\` + `, + }), + ]; + + const results = checkInterfaceContracts(tasks, "/tmp"); + // Should have 2 results: parameter mismatch AND return type mismatch + assert.equal(results.length, 2); + assert.ok(results.some((r) => r.message.includes("handler"))); + assert.ok(results.some((r) => r.message.includes("parameters"))); + assert.ok(results.some((r) => r.message.includes("return types"))); + }); + + test("passes when no code blocks present", () => { + const tasks = [ + createTask({ + id: "T01", + description: "Just some text without code blocks", + }), + ]; + + const results = checkInterfaceContracts(tasks, "/tmp"); + assert.deepEqual(results, []); + }); + + test("handles multiple mismatches for same function", () => { + const tasks = [ + createTask({ + id: "T01", + description: ` +\`\`\`typescript +function process(a: string): string +\`\`\` + `, + }), + createTask({ + id: "T02", + description: ` +\`\`\`typescript +function process(a: number): number +\`\`\` + `, + }), + ]; + + const results = checkInterfaceContracts(tasks, "/tmp"); + // Should have both parameter and return type mismatches + assert.equal(results.length, 2); + }); +}); + +// ─── runPreExecutionChecks Integration Tests ───────────────────────────────── + +describe("runPreExecutionChecks", () => { + let tempDir: string; + + test("returns pass status when all checks pass", async () => { + tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`); + mkdirSync(tempDir, { recursive: true }); + writeFileSync(join(tempDir, "existing.ts"), "// content"); + + try { + const tasks = [ + createTask({ + id: "T01", + files: ["existing.ts"], + inputs: [], + expected_output: ["output.ts"], + }), + createTask({ + id: "T02", + files: ["output.ts"], + inputs: [], + expected_output: [], + }), + ]; + + const result = await runPreExecutionChecks(tasks, tempDir); + assert.equal(result.status, "pass"); + assert.equal(result.checks.length, 0); + assert.ok(result.durationMs >= 0); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + test("returns fail status when blocking failure exists", async () => { + tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`); + mkdirSync(tempDir, { recursive: true }); + + try { + const tasks = [ + createTask({ + id: "T01", + files: ["nonexistent.ts"], + inputs: [], + expected_output: [], + }), + ]; + + const result = await runPreExecutionChecks(tasks, tempDir); + assert.equal(result.status, "fail"); + assert.ok(result.checks.length > 0); + assert.ok(result.checks.some((c) => c.blocking === true)); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + test("returns warn status for non-blocking issues", async () => { + tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`); + mkdirSync(tempDir, { recursive: true }); + + try { + // Create tasks with only interface contract warnings + const tasks = [ + createTask({ + id: "T01", + files: [], + inputs: [], + expected_output: [], + description: ` +\`\`\`typescript +function foo(a: string): void +\`\`\` + `, + }), + createTask({ + id: "T02", + files: [], + inputs: [], + expected_output: [], + description: ` +\`\`\`typescript +function foo(a: number): void +\`\`\` + `, + }), + ]; + + const result = await runPreExecutionChecks(tasks, tempDir); + assert.equal(result.status, "warn"); + assert.ok(result.checks.some((c) => c.blocking === false)); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + test("combines results from all check types", async () => { + tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`); + mkdirSync(tempDir, { recursive: true }); + + try { + const tasks = [ + createTask({ + id: "T01", + sequence: 0, + files: ["will-be-created.ts"], // Ordering violation + inputs: ["missing.ts"], // Missing file + expected_output: [], + description: ` +\`\`\`typescript +function check(a: string): void +\`\`\` + `, + }), + createTask({ + id: "T02", + sequence: 1, + files: [], + inputs: [], + expected_output: ["will-be-created.ts"], + description: ` +\`\`\`typescript +function check(a: number): void +\`\`\` + `, + }), + ]; + + const result = await runPreExecutionChecks(tasks, tempDir); + assert.equal(result.status, "fail"); + + // Should have multiple types of issues + const categories = new Set(result.checks.map((c) => c.category)); + assert.ok(categories.has("file")); // From consistency and ordering + assert.ok(categories.has("schema")); // From interface check + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + test("reports duration in milliseconds", async () => { + tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`); + mkdirSync(tempDir, { recursive: true }); + + try { + const tasks = [createTask({ id: "T01" })]; + const result = await runPreExecutionChecks(tasks, tempDir); + + assert.ok(typeof result.durationMs === "number"); + assert.ok(result.durationMs >= 0); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + test("handles empty task array", async () => { + tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`); + mkdirSync(tempDir, { recursive: true }); + + try { + const result = await runPreExecutionChecks([], tempDir); + assert.equal(result.status, "pass"); + assert.deepEqual(result.checks, []); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); +}); + +// ─── PreExecutionResult Type Tests ─────────────────────────────────────────── + +describe("PreExecutionResult type", () => { + test("status is one of pass, warn, fail", async () => { + const tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`); + mkdirSync(tempDir, { recursive: true }); + + try { + const tasks = [createTask({ id: "T01" })]; + const result = await runPreExecutionChecks(tasks, tempDir); + + assert.ok(["pass", "warn", "fail"].includes(result.status)); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + test("checks array matches PreExecutionCheckJSON schema", async () => { + const tempDir = join(tmpdir(), `pre-exec-test-${Date.now()}`); + mkdirSync(tempDir, { recursive: true }); + + try { + const tasks = [ + createTask({ + id: "T01", + files: ["missing.ts"], + }), + ]; + + const result = await runPreExecutionChecks(tasks, tempDir); + + for (const check of result.checks) { + assert.ok(["package", "file", "tool", "endpoint", "schema"].includes(check.category)); + assert.ok(typeof check.target === "string"); + assert.ok(typeof check.passed === "boolean"); + assert.ok(typeof check.message === "string"); + if (check.blocking !== undefined) { + assert.ok(typeof check.blocking === "boolean"); + } + } + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); +}); diff --git a/src/resources/extensions/gsd/verification-evidence.ts b/src/resources/extensions/gsd/verification-evidence.ts index e6cf431ff..3154ff36c 100644 --- a/src/resources/extensions/gsd/verification-evidence.ts +++ b/src/resources/extensions/gsd/verification-evidence.ts @@ -52,6 +52,32 @@ export interface BrowserEvidenceJSON { duration: number; } +export interface PreExecutionCheckJSON { + /** Check category: package, file, tool, endpoint, schema */ + category: "package" | "file" | "tool" | "endpoint" | "schema"; + /** What was checked (e.g., package name, file path) */ + target: string; + /** Whether the check passed */ + passed: boolean; + /** Human-readable message explaining the result */ + message: string; + /** Whether this failure should block execution (only meaningful when passed=false) */ + blocking?: boolean; +} + +export interface PostExecutionCheckJSON { + /** Check category: import, signature, pattern */ + category: "import" | "signature" | "pattern"; + /** What was checked (e.g., file:line, function name) */ + target: string; + /** Whether the check passed */ + passed: boolean; + /** Human-readable message explaining the result */ + message: string; + /** Whether this failure should block completion (only meaningful when passed=false) */ + blocking?: boolean; +} + export interface EvidenceJSON { schemaVersion: 1; taskId: string; @@ -65,6 +91,10 @@ export interface EvidenceJSON { runtimeErrors?: RuntimeErrorJSON[]; auditWarnings?: AuditWarningJSON[]; browser?: BrowserEvidenceJSON; + /** Pre-execution checks run before task execution (package existence, file refs, etc.) */ + preExecutionChecks?: PreExecutionCheckJSON[]; + /** Post-execution checks run after task completion (import resolution, signature drift, pattern consistency) */ + postExecutionChecks?: PostExecutionCheckJSON[]; } /** @@ -124,6 +154,44 @@ export function writeVerificationJSON( writeFileSync(filePath, JSON.stringify(evidence, null, 2) + "\n", "utf-8"); } +// ─── Pre-Execution Evidence ────────────────────────────────────────────────── + +export interface PreExecutionEvidenceJSON { + schemaVersion: 1; + milestoneId: string; + sliceId: string; + timestamp: number; + status: "pass" | "warn" | "fail"; + durationMs: number; + checks: PreExecutionCheckJSON[]; +} + +/** + * Write pre-execution check results to a PRE-EXEC-VERIFY.json artifact + * in the slice directory. + */ +export function writePreExecutionEvidence( + result: { status: "pass" | "warn" | "fail"; checks: PreExecutionCheckJSON[]; durationMs: number }, + sliceDir: string, + milestoneId: string, + sliceId: string, +): void { + mkdirSync(sliceDir, { recursive: true }); + + const evidence: PreExecutionEvidenceJSON = { + schemaVersion: 1, + milestoneId, + sliceId, + timestamp: Date.now(), + status: result.status, + durationMs: result.durationMs, + checks: result.checks, + }; + + const filePath = join(sliceDir, `${sliceId}-PRE-EXEC-VERIFY.json`); + writeFileSync(filePath, JSON.stringify(evidence, null, 2) + "\n", "utf-8"); +} + // ─── Markdown Evidence Table ───────────────────────────────────────────────── /**