fix: relax milestone validation gate to accept prose evidence (#2779)
* fix: relax milestone validation gate to accept prose evidence The completion gate at auto-dispatch.ts required exact "MET"/"N/A" substrings that renderValidationMarkdown() never emits, causing a deadlock where no validation output could satisfy the gate. The gate now accepts either the structured template format (MET/N/A table) or prose evidence patterns (e.g., "Operational: verified", "Operational checks confirmed") that the validation agent naturally produces. Closes #2739 Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * test: add tests for relaxed validation gate patterns Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
ce23da6718
commit
be7466d99f
2 changed files with 130 additions and 1 deletions
|
|
@ -677,9 +677,14 @@ export const DISPATCH_RULES: DispatchRule[] = [
|
|||
if (validationPath) {
|
||||
const validationContent = await loadFile(validationPath);
|
||||
if (validationContent) {
|
||||
const hasOperationalCheck =
|
||||
// Accept either the structured template format (table with MET/N/A)
|
||||
// or prose evidence patterns the validation agent may emit.
|
||||
const structuredMatch =
|
||||
validationContent.includes("Operational") &&
|
||||
(validationContent.includes("MET") || validationContent.includes("N/A"));
|
||||
const proseMatch =
|
||||
/[Oo]perational[\s:][^\n]*(?:pass|verified|confirmed|met|complete|true|yes|addressed|covered|n\/a|not\s+applicable)/i.test(validationContent);
|
||||
const hasOperationalCheck = structuredMatch || proseMatch;
|
||||
if (!hasOperationalCheck) {
|
||||
return {
|
||||
action: "stop" as const,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,124 @@
|
|||
/**
|
||||
* Unit tests for the milestone completion validation gate pattern matching.
|
||||
*
|
||||
* The gate in auto-dispatch accepts two evidence formats:
|
||||
* 1. Structured template: content contains "Operational" AND ("MET" or "N/A")
|
||||
* 2. Prose evidence: matches /[Oo]perational[\s:][^\n]*(?:pass|verified|...)/i
|
||||
*
|
||||
* These tests exercise the exact same expressions used in auto-dispatch.ts
|
||||
* to ensure both formats are correctly recognized, and that content without
|
||||
* operational evidence is properly rejected.
|
||||
*/
|
||||
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
// ─── Replicate the gate matching logic from auto-dispatch.ts ─────────────────
|
||||
|
||||
/**
|
||||
* Returns true when validation content contains acceptable operational
|
||||
* verification evidence (structured or prose). Mirrors the inline checks
|
||||
* in the "execute → complete-milestone" dispatch rule.
|
||||
*/
|
||||
function hasOperationalEvidence(validationContent: string): boolean {
|
||||
const structuredMatch =
|
||||
validationContent.includes("Operational") &&
|
||||
(validationContent.includes("MET") || validationContent.includes("N/A"));
|
||||
const proseMatch =
|
||||
/[Oo]perational[\s:][^\n]*(?:pass|verified|confirmed|met|complete|true|yes|addressed|covered|n\/a|not\s+applicable)/i.test(
|
||||
validationContent,
|
||||
);
|
||||
return structuredMatch || proseMatch;
|
||||
}
|
||||
|
||||
// ─── Structured format ───────────────────────────────────────────────────────
|
||||
|
||||
test("structured: Operational + MET passes", () => {
|
||||
const content = `| Criteria | Status |
|
||||
| Operational | MET |
|
||||
| Functional | MET |`;
|
||||
assert.ok(hasOperationalEvidence(content));
|
||||
});
|
||||
|
||||
test("structured: Operational + N/A passes", () => {
|
||||
const content = `| Criteria | Status |
|
||||
| Operational | N/A |
|
||||
| Functional | MET |`;
|
||||
assert.ok(hasOperationalEvidence(content));
|
||||
});
|
||||
|
||||
test("structured: Operational present with MET on another row still passes (includes is content-wide)", () => {
|
||||
// The structured check uses .includes() across the entire content,
|
||||
// so "MET" on the Functional row satisfies the condition alongside
|
||||
// "Operational" anywhere in the document.
|
||||
const content = `| Criteria | Status |
|
||||
| Operational | PENDING |
|
||||
| Functional | MET |`;
|
||||
assert.ok(hasOperationalEvidence(content));
|
||||
});
|
||||
|
||||
test("structured: Operational alone without any MET or N/A anywhere fails", () => {
|
||||
const content = `| Criteria | Status |
|
||||
| Operational | PENDING |
|
||||
| Functional | PENDING |`;
|
||||
assert.ok(!hasOperationalEvidence(content));
|
||||
});
|
||||
|
||||
// ─── Prose format ────────────────────────────────────────────────────────────
|
||||
|
||||
test('prose: "Operational: verified" passes', () => {
|
||||
const content = `## Validation Report
|
||||
Operational: verified — all endpoints responsive.
|
||||
Functional: tests pass.`;
|
||||
assert.ok(hasOperationalEvidence(content));
|
||||
});
|
||||
|
||||
test('prose: "Operational checks confirmed" passes', () => {
|
||||
const content = `## Validation Report
|
||||
Operational checks confirmed by smoke test suite.`;
|
||||
assert.ok(hasOperationalEvidence(content));
|
||||
});
|
||||
|
||||
test('prose: "Operational — pass" passes', () => {
|
||||
const content = `Operational — pass (all services healthy)`;
|
||||
assert.ok(hasOperationalEvidence(content));
|
||||
});
|
||||
|
||||
test('prose: "operational: addressed" passes (case-insensitive)', () => {
|
||||
const content = `operational: addressed in CI pipeline run #42.`;
|
||||
assert.ok(hasOperationalEvidence(content));
|
||||
});
|
||||
|
||||
test('prose: "Operational: not applicable" passes', () => {
|
||||
const content = `Operational: not applicable for this library-only change.`;
|
||||
assert.ok(hasOperationalEvidence(content));
|
||||
});
|
||||
|
||||
test('prose: "Operational: n/a" passes', () => {
|
||||
const content = `Operational: n/a — no runtime components.`;
|
||||
assert.ok(hasOperationalEvidence(content));
|
||||
});
|
||||
|
||||
test('prose: "Operational: complete" passes', () => {
|
||||
const content = `Operational: complete — all health checks green.`;
|
||||
assert.ok(hasOperationalEvidence(content));
|
||||
});
|
||||
|
||||
// ─── Rejection cases ─────────────────────────────────────────────────────────
|
||||
|
||||
test("no operational evidence: unrelated content fails", () => {
|
||||
const content = `## Validation Report
|
||||
All functional tests pass.
|
||||
Code coverage at 92%.`;
|
||||
assert.ok(!hasOperationalEvidence(content));
|
||||
});
|
||||
|
||||
test("no operational evidence: word 'operational' buried without qualifying keyword fails", () => {
|
||||
const content = `## Validation Report
|
||||
The operational aspects were not evaluated in this round.`;
|
||||
assert.ok(!hasOperationalEvidence(content));
|
||||
});
|
||||
|
||||
test("no operational evidence: empty content fails", () => {
|
||||
assert.ok(!hasOperationalEvidence(""));
|
||||
});
|
||||
Loading…
Add table
Reference in a new issue