feat(planning-state): DB-first VALIDATION.md migration (proposal MVP)

Implements Phase 1 of docs/dev/proposals/db-first-planning-state.md
(commit f3571475d). VALIDATION.md is now a render target; DB is
canonical.

Three read sites switched to DB:
- tools/complete-milestone.js: getMilestoneValidationAssessment(id)?.status
  replaces readFile + extractVerdict (lines 126-137 → 126-140)
- workspace-index.js: same swap in the indexWorkspace loop (was
  resolveMilestoneFile → loadFile → extractVerdict per milestone)
- state-shared.js:readMilestoneValidationVerdict was already DB-first
  (prefers DB, file fallback only when no DB) — no change needed

Write path regenerates:
- tools/validate-milestone.js:renderValidationMarkdown now prepends
  <!-- generated from .sf/sf.db — do not edit directly; use the
  validate_milestone tool --> so the file is unambiguously a projection
- verdict-parser.js:extractVerdict strips the comment header before
  frontmatter parsing so legacy readers (reflection.js, auto-prompts.js)
  still work on generated files

Doctor check retired (clean delete):
- doctor-engine-checks.js: db_projection_validation_drift detector
  removed entirely. Drift is structurally impossible once the write
  path always regenerates from DB. Comment block explains the removal.

Tests:
- New: db-first-validation.test.mjs — 6 tests covering regeneration,
  three read-site overrides, hand-edit override, doctor non-emission
- Updated: doctor-db-projection-drift.test.mjs now asserts the check is
  NOT emitted (was previously asserting it WAS)

Full suite: 469 passed, 0 failed, 3 skipped. No regressions.

Closes the same class as the self-feedback DB/JSONL divergence pain —
the M001-6377a4-VALIDATION.md doctor warning that's been firing
repeatedly this session is gone by construction. Other planning
artifacts (CONTEXT.md, ROADMAP.md, SUMMARY.md) follow in later phases
per the proposal.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

src/resources/extensions/sf/doctor-engine-checks.js

@@ -14,7 +14,6 @@ import {
 	_getAdapter,
 	getAllMilestones,
 	getMilestoneSlices,
-	getMilestoneValidationAssessment,
 	isDbAvailable,
 	openDatabase,
 } from "./sf-db.js";
@@ -23,7 +22,6 @@ import {
 	summarizeParityHealth,
 	writeParityReport,
 } from "./uok/parity-report.js";
-import { extractVerdict } from "./verdict-parser.js";
 import { readEvents } from "./workflow-events.js";
 import { renderAllProjections } from "./workflow-projections.js";
 import { getErrorMessage } from "./error-utils.js";
@@ -125,29 +123,10 @@ function projectionDriftIssues(basePath, milestoneId) {
 			});
 		}
 	}
-	const validationPath = resolveMilestoneFile(
-		basePath,
-		milestoneId,
-		"VALIDATION",
-	);
-	if (validationPath && existsSync(validationPath)) {
-		const dbAssessment = getMilestoneValidationAssessment(milestoneId);
-		if (dbAssessment?.status) {
-			const fileVerdict = extractVerdict(readFileSync(validationPath, "utf-8"));
-			const dbVerdict = String(dbAssessment.status).trim().toLowerCase();
-			if (fileVerdict && fileVerdict !== dbVerdict) {
-				issues.push({
-					severity: "warning",
-					code: "db_projection_validation_drift",
-					scope: "milestone",
-					unitId: milestoneId,
-					message: `${milestoneId}-VALIDATION.md verdict "${fileVerdict}" differs from DB assessment "${dbVerdict}". DB remains authoritative.`,
-					file: validationPath,
-					fixable: false,
-				});
-			}
-		}
-	}
+	// db_projection_validation_drift check removed (#db-first-validation):
+	// VALIDATION.md is now a generated projection from assessments table.
+	// Drift is structurally impossible — the file is always regenerated from DB
+	// by handleValidateMilestone. No drift check needed.
 	return issues;
 }
 

src/resources/extensions/sf/tests/db-first-validation.test.mjs

@@ -0,0 +1,304 @@
+/**
+ * db-first-validation.test.mjs — VALIDATION.md is a DB projection.
+ *
+ * Purpose: prove that after the DB-first migration:
+ * 1. Writing an assessments row with status=needs-attention and regenerating
+ *    VALIDATION.md produces a file with verdict: needs-attention in frontmatter.
+ * 2. The three read sites (complete-milestone, workspace-index, state-shared)
+ *    return the DB value, not the parsed .md.
+ * 3. If .md is hand-edited to disagree with DB, the next regeneration overrides.
+ * 4. Doctor does NOT register db_projection_validation_drift (check is retired).
+ */
+import assert from "node:assert/strict";
+import {
+	mkdirSync,
+	mkdtempSync,
+	readFileSync,
+	rmSync,
+	writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, test } from "vitest";
+import { checkEngineHealth } from "../doctor-engine-checks.js";
+import {
+	closeDatabase,
+	getMilestoneValidationAssessment,
+	insertAssessment,
+	insertMilestone,
+	insertSlice,
+	insertTask,
+	openDatabase,
+} from "../sf-db.js";
+import { invalidateStateCache } from "../state.js";
+import { readMilestoneValidationVerdict } from "../state-shared.js";
+import { handleCompleteMilestone } from "../tools/complete-milestone.js";
+import { handleValidateMilestone } from "../tools/validate-milestone.js";
+import { indexWorkspace } from "../workspace-index.js";
+
+const tmpDirs = [];
+
+afterEach(() => {
+	closeDatabase();
+	invalidateStateCache();
+	while (tmpDirs.length > 0) {
+		const dir = tmpDirs.pop();
+		if (dir) rmSync(dir, { recursive: true, force: true });
+	}
+});
+
+function makeProject() {
+	const dir = mkdtempSync(join(tmpdir(), "sf-db-first-validation-"));
+	tmpDirs.push(dir);
+	mkdirSync(join(dir, ".sf", "milestones", "M001"), { recursive: true });
+	openDatabase(join(dir, ".sf", "sf.db"));
+	insertMilestone({ id: "M001", title: "DB-first validation", status: "active" });
+	insertSlice({
+		milestoneId: "M001",
+		id: "S01",
+		title: "Done slice",
+		status: "complete",
+		sequence: 1,
+	});
+	insertTask({
+		milestoneId: "M001",
+		sliceId: "S01",
+		id: "T01",
+		title: "Done task",
+		status: "complete",
+	});
+	return dir;
+}
+
+// ── Test 1: regeneration produces correct frontmatter ──────────────────────
+test("handleValidateMilestone_regenerates_validation_md_with_db_verdict_in_frontmatter", async () => {
+	const project = makeProject();
+
+	const result = await handleValidateMilestone(
+		{
+			milestoneId: "M001",
+			verdict: "needs-attention",
+			remediationRound: 1,
+			successCriteriaChecklist: "- [ ] criterion one",
+			sliceDeliveryAudit: "S01: complete",
+			crossSliceIntegration: "no issues",
+			requirementCoverage: "100%",
+			verdictRationale: "minor attention item",
+		},
+		project,
+		{ uokGatesEnabled: false },
+	);
+
+	assert.equal(result.error, undefined, `handleValidateMilestone error: ${result.error}`);
+	assert.equal(result.verdict, "needs-attention");
+
+	// The file must exist and have the generated header
+	const validationPath = join(
+		project,
+		".sf",
+		"milestones",
+		"M001",
+		"M001-VALIDATION.md",
+	);
+	const content = readFileSync(validationPath, "utf-8");
+	assert.ok(
+		content.startsWith("<!-- generated from .sf/sf.db"),
+		"file must start with generated header",
+	);
+	assert.match(content, /verdict:\s*needs-attention/, "frontmatter must carry the verdict");
+
+	// DB row must agree
+	const assessment = getMilestoneValidationAssessment("M001");
+	assert.equal(assessment?.status, "needs-attention");
+});
+
+// ── Test 2a: readMilestoneValidationVerdict returns DB value ───────────────
+test("readMilestoneValidationVerdict_returns_db_status_without_reading_file", async () => {
+	const project = makeProject();
+	const validationPath = join(
+		project,
+		".sf",
+		"milestones",
+		"M001",
+		"M001-VALIDATION.md",
+	);
+
+	// Write STALE file with different verdict than DB
+	writeFileSync(
+		validationPath,
+		"---\nverdict: needs-remediation\nremediation_round: 1\n---\n\n# stale",
+	);
+	// DB says needs-attention
+	insertAssessment({
+		path: validationPath,
+		milestoneId: "M001",
+		scope: "milestone-validation",
+		status: "needs-attention",
+		fullContent: "---\nverdict: needs-attention\n---\n",
+	});
+
+	const { terminal, verdict } = await readMilestoneValidationVerdict(
+		project,
+		"M001",
+		(p) => readFileSync(p, "utf-8"),
+	);
+
+	assert.equal(verdict, "needs-attention", "verdict must come from DB, not stale file");
+	assert.equal(terminal, true);
+});
+
+// ── Test 2b: workspace-index validationVerdict comes from DB ──────────────
+test("buildWorkspaceIndex_populates_validationVerdict_from_db_not_file", async () => {
+	const project = makeProject();
+	const validationPath = join(
+		project,
+		".sf",
+		"milestones",
+		"M001",
+		"M001-VALIDATION.md",
+	);
+
+	// Stale file says needs-remediation, DB says pass
+	writeFileSync(
+		validationPath,
+		"---\nverdict: needs-remediation\nremediation_round: 1\n---\n\n# stale",
+	);
+	insertAssessment({
+		path: validationPath,
+		milestoneId: "M001",
+		scope: "milestone-validation",
+		status: "pass",
+		fullContent: "---\nverdict: pass\n---\n",
+	});
+
+	// Add ROADMAP.md so workspace-index can build the milestone entry
+	writeFileSync(
+		join(project, ".sf", "milestones", "M001", "M001-ROADMAP.md"),
+		"# M001: DB-first validation\n\n## Slice Overview\n| ID | Slice | Risk | Depends | Done | After this |\n|----|-------|------|---------|------|------------|\n| S01 | Done slice | low | - | ✅ | done |\n",
+	);
+
+	const index = await indexWorkspace(project);
+
+	const milestone = index.milestones?.find((m) => m.id === "M001");
+	assert.ok(milestone, "milestone must be in workspace index");
+	assert.equal(
+		milestone.validationVerdict,
+		"pass",
+		"validationVerdict must come from DB (pass), not stale file (needs-remediation)",
+	);
+});
+
+// ── Test 2c: complete-milestone blocks on DB verdict, not file ────────────
+test("handleCompleteMilestone_blocks_when_db_verdict_is_not_pass_regardless_of_file", async () => {
+	const project = makeProject();
+	const validationPath = join(
+		project,
+		".sf",
+		"milestones",
+		"M001",
+		"M001-VALIDATION.md",
+	);
+
+	// File says pass, DB says needs-attention → must block on DB verdict
+	writeFileSync(
+		validationPath,
+		"---\nverdict: pass\nremediation_round: 1\n---\n\n# file says pass but is stale",
+	);
+	insertAssessment({
+		path: validationPath,
+		milestoneId: "M001",
+		scope: "milestone-validation",
+		status: "needs-attention",
+		fullContent: "---\nverdict: needs-attention\n---\n",
+	});
+
+	const result = await handleCompleteMilestone(
+		{
+			milestoneId: "M001",
+			title: "DB-first validation",
+			verificationPassed: true,
+			oneLiner: "Should be blocked",
+			narrative: "DB verdict is needs-attention.",
+			successCriteriaResults: "Checked.",
+			definitionOfDoneResults: "Checked.",
+		},
+		project,
+	);
+
+	assert.ok(
+		result.error?.includes("needs-attention"),
+		`expected block on DB needs-attention verdict, got: ${result.error}`,
+	);
+});
+
+// ── Test 3: stale hand-edit overridden by next regeneration ───────────────
+test("handleValidateMilestone_overwrites_hand_edited_file_with_db_values", async () => {
+	const project = makeProject();
+	const validationPath = join(
+		project,
+		".sf",
+		"milestones",
+		"M001",
+		"M001-VALIDATION.md",
+	);
+
+	// Simulate hand-edit with wrong verdict
+	writeFileSync(
+		validationPath,
+		"---\nverdict: needs-remediation\nremediation_round: 99\n---\n\n# hand edited",
+	);
+
+	// Regenerate via tool (DB-first write then disk render)
+	const result = await handleValidateMilestone(
+		{
+			milestoneId: "M001",
+			verdict: "pass",
+			remediationRound: 1,
+			successCriteriaChecklist: "- [x] all good",
+			sliceDeliveryAudit: "S01: complete",
+			crossSliceIntegration: "none",
+			requirementCoverage: "100%",
+			verdictRationale: "everything passed",
+		},
+		project,
+		{ uokGatesEnabled: false },
+	);
+
+	assert.equal(result.error, undefined);
+	const content = readFileSync(validationPath, "utf-8");
+	assert.match(content, /verdict:\s*pass/, "regeneration must override hand-edit");
+	assert.ok(
+		!content.includes("needs-remediation"),
+		"stale verdict must be gone after regeneration",
+	);
+});
+
+// ── Test 4: db_projection_validation_drift is not registered ──────────────
+test("checkEngineHealth_does_not_report_db_projection_validation_drift", async () => {
+	const project = makeProject();
+	const milestoneDir = join(project, ".sf", "milestones", "M001");
+	const validationPath = join(milestoneDir, "M001-VALIDATION.md");
+
+	// Set up a scenario that would previously have triggered drift:
+	// file says needs-remediation, DB says pass
+	writeFileSync(
+		validationPath,
+		"---\nverdict: needs-remediation\n---\n\n# stale",
+	);
+	insertAssessment({
+		path: validationPath,
+		milestoneId: "M001",
+		scope: "milestone-validation",
+		status: "pass",
+		fullContent: "---\nverdict: pass\n---\n",
+	});
+
+	const issues = [];
+	await checkEngineHealth(project, issues, [], () => false);
+
+	assert.equal(
+		issues.some((i) => i.code === "db_projection_validation_drift"),
+		false,
+		"db_projection_validation_drift must not be emitted — check is retired",
+	);
+});

src/resources/extensions/sf/tests/doctor-db-projection-drift.test.mjs

@@ -104,7 +104,11 @@ function makeProject() {
 	return dir;
 }
 
-test("checkEngineHealth_when_projection_files_disagree_with_db_reports_drift", async () => {
+test("checkEngineHealth_when_roadmap_json_disagrees_with_db_reports_roadmap_drift_but_not_validation_drift", async () => {
+	// db_projection_validation_drift was removed (#db-first-validation):
+	// VALIDATION.md is a generated projection; drift is structurally impossible
+	// after handleValidateMilestone always regenerates from DB.
+	// Only db_projection_roadmap_drift is still checked (ROADMAP.json projection).
 	const project = makeProject();
 	const milestoneDir = join(project, ".sf", "milestones", "M901");
 	const validationPath = join(milestoneDir, "M901-VALIDATION.md");
@@ -120,6 +124,7 @@ test("checkEngineHealth_when_projection_files_disagree_with_db_reports_drift", a
 			2,
 		)}\n`,
 	);
+	// Even with a stale VALIDATION.md file on disk, no drift warning fires.
 	writeFileSync(
 		validationPath,
 		["---", "verdict: needs-remediation", "---", "", "# stale"].join("\n"),
@@ -138,9 +143,11 @@ test("checkEngineHealth_when_projection_files_disagree_with_db_reports_drift", a
 	assert.equal(
 		issues.some((issue) => issue.code === "db_projection_roadmap_drift"),
 		true,
+		"roadmap drift check should still fire",
 	);
 	assert.equal(
 		issues.some((issue) => issue.code === "db_projection_validation_drift"),
-		true,
+		false,
+		"validation drift check is retired — VALIDATION.md is a generated projection",
 	);
 });

src/resources/extensions/sf/tools/complete-milestone.js

@@ -5,7 +5,7 @@
  * renders MILESTONE-SUMMARY.md to disk, stores rendered markdown in DB
  * for recovery, and invalidates caches.
  */
-import { existsSync, mkdirSync, readFileSync } from "node:fs";
+import { mkdirSync } from "node:fs";
 import { join } from "node:path";
 import { clearParseCache, saveFile } from "../files.js";
 import { clearPathCache, resolveMilestonePath } from "../paths.js";
@@ -14,6 +14,7 @@ import { appendScheduleSpecs } from "../schedule/schedule-milestone.js";
 import {
 	getMilestone,
 	getMilestoneSlices,
+	getMilestoneValidationAssessment,
 	getSliceTasks,
 	insertMilestoneEvidence,
 	transaction,
@@ -21,7 +22,6 @@ import {
 } from "../sf-db.js";
 import { invalidateStateCache } from "../state.js";
 import { isClosedStatus } from "../status-guards.js";
-import { extractVerdict } from "../verdict-parser.js";
 import { appendEvent } from "../workflow-events.js";
 import { logError, logWarning } from "../workflow-logger.js";
 import { writeManifest } from "../workflow-manifest.js";
@@ -123,24 +123,20 @@ export async function handleCompleteMilestone(params, basePath) {
 				"verification did not pass — milestone completion blocked. verificationPassed must be explicitly set to true after all verification steps succeed",
 		};
 	}
+	// Resolve the milestone directory (needed for SUMMARY.md path below)
 	const milestoneDir = resolveMilestonePath(basePath, params.milestoneId);
-	const validationPath = milestoneDir
-		? join(milestoneDir, `${params.milestoneId}-VALIDATION.md`)
-		: join(
-				basePath,
-				".sf",
-				"milestones",
+	// ── Validation verdict guard — DB-authoritative (#db-first-validation) ──
+	// Read verdict from assessments table, not from VALIDATION.md on disk.
+	// The file is a generated projection; the DB row is the canonical source.
+	const validationAssessment = getMilestoneValidationAssessment(
 		params.milestoneId,
-				`${params.milestoneId}-VALIDATION.md`,
 	);
-	if (existsSync(validationPath)) {
-		const verdict = extractVerdict(readFileSync(validationPath, "utf-8"));
+	const verdict = validationAssessment?.status ?? undefined;
 	if (verdict && verdict !== "pass") {
 		return {
 			error: `milestone validation verdict is "${verdict}" — only "pass" may be completed automatically`,
 		};
 	}
-	}
 	// ── Guards + DB writes inside a single transaction (prevents TOCTOU) ───
 	const completedAt = new Date().toISOString();
 	let guardError = null;

src/resources/extensions/sf/tools/validate-milestone.js

@@ -31,7 +31,8 @@ import { logWarning } from "../workflow-logger.js";
 import { resolveCanonicalMilestoneRoot } from "../worktree-manager.js";
 
 function renderValidationMarkdown(params) {
-	let md = `---
+	let md = `<!-- generated from .sf/sf.db — do not edit directly; use validate_milestone tool -->
+---
 verdict: ${params.verdict}
 remediation_round: ${params.remediationRound}
 ---

src/resources/extensions/sf/verdict-parser.js

@@ -16,8 +16,13 @@ import { extractUatType } from "./files.js";
  * Returns `undefined` when frontmatter is absent or has no `verdict` field.
  */
 export function extractVerdict(content) {
+	// Strip leading generated-file comment header before parsing frontmatter.
+	// VALIDATION.md is a DB projection; the header is always the first line.
+	const stripped = content.startsWith("<!--")
+		? content.replace(/^<!--[^\n]*-->\n/, "")
+		: content;
 	// Primary: YAML frontmatter verdict (canonical format)
-	const fmMatch = content.match(/^---\n([\s\S]*?)\n---/);
+	const fmMatch = stripped.match(/^---\n([\s\S]*?)\n---/);
 	if (fmMatch) {
 		const verdictMatch = fmMatch[1].match(/verdict:\s*([\w-]+)/i);
 		if (verdictMatch) {
@@ -29,7 +34,7 @@ export function extractVerdict(content) {
 	}
 	// Fallback: detect verdict in markdown body (LLM manual writes, #2960).
 	// Matches patterns like: **Verdict:** PASS, **Verdict:** ✅ PASS, **Verdict** needs-remediation
-	const bodyMatch = content.match(/\*\*Verdict:?\*\*\s*(?:✅\s*)?(\w[\w-]*)/i);
+	const bodyMatch = stripped.match(/\*\*Verdict:?\*\*\s*(?:✅\s*)?(\w[\w-]*)/i);
 	if (bodyMatch) {
 		let v = bodyMatch[1].toLowerCase();
 		if (v === "passed") v = "pass";

src/resources/extensions/sf/workspace-index.js

@@ -7,9 +7,13 @@ import {
 	resolveTaskFile,
 	resolveTasksDir,
 } from "./paths.js";
-import { getMilestoneSlices, getSliceTasks, isDbAvailable } from "./sf-db.js";
+import {
+	getMilestoneSlices,
+	getMilestoneValidationAssessment,
+	getSliceTasks,
+	isDbAvailable,
+} from "./sf-db.js";
 import { deriveState } from "./state.js";
-import { extractVerdict } from "./verdict-parser.js";
 import { detectWorktreeName, getSliceBranchName } from "./worktree.js";
 
 // Extract milestone title from roadmap header without using parsers.
@@ -192,17 +196,12 @@ export async function indexWorkspace(basePath, _opts = {}) {
 			}
 		}
 	}
-	// Populate validationVerdict from VALIDATION files (#2807)
+	// Populate validationVerdict from DB assessments table (#db-first-validation)
+	// DB is authoritative; VALIDATION.md is a generated projection only.
 	for (const milestone of milestones) {
-		const validationPath = resolveMilestoneFile(
-			basePath,
-			milestone.id,
-			"VALIDATION",
-		);
-		if (validationPath) {
-			const validationContent = await loadFile(validationPath);
-			if (validationContent) {
-				const verdict = extractVerdict(validationContent);
+		const assessment = getMilestoneValidationAssessment(milestone.id);
+		if (assessment?.status) {
+			const verdict = String(assessment.status).trim().toLowerCase();
 			if (
 				verdict === "pass" ||
 				verdict === "needs-attention" ||
@@ -212,7 +211,6 @@ export async function indexWorkspace(basePath, _opts = {}) {
 			}
 		}
 	}
-	}
 	const scopes = [{ scope: "project", label: "project", kind: "project" }];
 	for (const milestone of milestones) {
 		scopes.push({