From 87aa04cf05dcb027819718c1d3df42bbbc6bf4f2 Mon Sep 17 00:00:00 2001 From: Mikael Hugo Date: Thu, 7 May 2026 04:20:32 +0200 Subject: [PATCH] Tier 1.3: Add spec/runtime/evidence schema separation (v32) Implements the 3-table normalization model for milestone, slice, and task entities: - 9 new tables: {milestone,slice,task}_{specs,evidence} + runtime tables - milestone_specs: immutable record of intent (vision, goals, risks, proof strategy) - slice_specs: immutable slice-level intent - task_specs: immutable task verification criteria - {entity}_evidence: append-only audit trail with timestamps and phase metadata - Indices on evidence tables for efficient chronological queries Key improvements: - Spec immutability: Write-once specs preserve original intent - Audit trail: Evidence chain enables data archaeology and decision history - Query efficiency: Each table contains only relevant columns - Re-planning clarity: Multiple spec versions can exist for same entity ID - Forensic capability: Timestamp + phase metadata on evidence rows Migration: - Schema version bumped to 32 - Migration runs on first open of existing databases - No data loss; existing milestone/slice/task rows preserved - Creates spec and evidence tables from existing columns (future work) This is Phase 1 of Tier 1.3 implementation (schema definition + basic setup). Phases 2-5 (migration, data layer updates, tool updates, tests) follow in next PRs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...spec-runtime-evidence-schema-separation.md | 270 ++++++++++++++++++ src/resources/extensions/sf/sf-db.js | 141 ++++++++- 2 files changed, 410 insertions(+), 1 deletion(-) create mode 100644 docs/adr/0077-spec-runtime-evidence-schema-separation.md diff --git a/docs/adr/0077-spec-runtime-evidence-schema-separation.md b/docs/adr/0077-spec-runtime-evidence-schema-separation.md new file mode 100644 index 000000000..6992a7131 --- /dev/null +++ b/docs/adr/0077-spec-runtime-evidence-schema-separation.md @@ -0,0 +1,270 @@ +# ADR-0077: Spec/Runtime/Evidence Schema Separation (Tier 1.3) + +**Status:** Proposed (implementation in progress for SF v3.0) +**Date:** 2026-05-07 +**Stakeholders:** SF v3.0 core team, UOK dispatch engine, milestone/slice/task tools + +--- + +## Problem Statement + +**Current state:** Milestone, slice, and task data are stored in wide monolithic tables that mix three distinct concerns: + +1. **Spec data** — immutable record of intent (vision, goals, success criteria, proof strategy) +2. **Runtime state** — current execution state (status, completed_at, blockers, dependencies) +3. **Evidence/narrative** — what happened during execution (verification results, decisions, descriptive summaries) + +**Problems this creates:** + +1. **Spec immutability unclear** — Spec data (vision, goals, risks) can be updated in place, but should represent intent +2. **Re-planning awkwardness** — When a milestone is re-planned, old spec data is overwritten or lost to markdown projections; unclear what was originally intended +3. **Query complexity** — Queries select across many irrelevant columns; indexing and partitioning are hard +4. **Evidence chain missing** — Verification results and narratives are in the same table as specs, making it impossible to audit "why was this decision made?" +5. **Data archaeology disabled** — Cannot reconstruct the decision history when a milestone enters an unexpected state +6. **Table bloat** — As narrative/evidence fields grow, the main runtime table grows unnecessarily + +--- + +## Proposed Solution: 3-Table Schema (Per Entity Type) + +Normalize milestone, slice, and task data from 1 wide table per entity into 3 focused tables: + +### Target Schema: 9 Tables Total + +For each entity type (milestone, slice, task): + +#### 1. **Spec Table** (immutable record of intent) + +Example: `milestone_specs` + +```sql +CREATE TABLE milestone_specs ( + id TEXT PRIMARY KEY, -- matches milestone.id + vision TEXT NOT NULL DEFAULT '', -- immutable spec + success_criteria TEXT DEFAULT '', -- JSON array, immutable spec + key_risks TEXT DEFAULT '', -- JSON array, immutable spec + proof_strategy TEXT DEFAULT '', -- JSON array, immutable spec + verification_contract TEXT DEFAULT '', -- contract spec + verification_integration TEXT DEFAULT '', + verification_operational TEXT DEFAULT '', + verification_uat TEXT DEFAULT '', + definition_of_done TEXT DEFAULT '', -- JSON array + requirement_coverage TEXT DEFAULT '', + boundary_map_markdown TEXT DEFAULT '', + vision_meeting_json TEXT DEFAULT '', -- JSON meeting notes + spec_version INTEGER NOT NULL DEFAULT 1, -- support multi-version specs in future + created_at TEXT NOT NULL, + PRIMARY KEY (id) +); +``` + +**Semantics:** +- Write-once; no UPDATE after initial creation +- Represents what the milestone owner intended when planning began +- When a milestone is re-planned, a new spec version is created (spec_version increments) +- Foreign key to `milestones(id)` ensures referential integrity + +#### 2. **Runtime Table** (current execution state) + +Example: `milestones` (renamed from current — spec removed) + +```sql +CREATE TABLE milestones ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'active', -- active/paused/complete/done/canceled + depends_on TEXT DEFAULT '[]', -- JSON array of milestone IDs + created_at TEXT NOT NULL, + completed_at TEXT DEFAULT NULL, + replan_count INTEGER DEFAULT 0, + PRIMARY KEY (id) +); +``` + +**Semantics:** +- Mutable; represents current state of execution +- Only runtime-relevant columns (status, dependencies, timestamps) +- Foreign key from spec tables (milestone_specs.id → milestones.id) +- Efficient for status queries and state transitions + +#### 3. **Evidence Table** (timestamped audit trail) + +Example: `milestone_evidence` + +```sql +CREATE TABLE milestone_evidence ( + milestone_id TEXT NOT NULL, + evidence_type TEXT NOT NULL, -- enum: verification_contract, verification_integration, verification_operational, verification_uat, narrative, decision, incident + content TEXT NOT NULL, -- markdown, JSON, or structured content + recorded_at TEXT NOT NULL, -- when evidence was recorded + phase_name TEXT DEFAULT '', -- which phase/executor created this + recorded_by TEXT DEFAULT '', -- agent name or "manual" + evidence_id TEXT NOT NULL DEFAULT (lower(hex(randomblob(16)))), + PRIMARY KEY (milestone_id, evidence_id), + FOREIGN KEY (milestone_id) REFERENCES milestones(id) +); +``` + +**Semantics:** +- Append-only; rows are never updated or deleted (unless retention policy triggers archival) +- Timestamped audit trail of decisions, verifications, incidents +- Can be queried chronologically to reconstruct decision history +- Supports data archaeology: "Why did this milestone enter a stuck state?" + +--- + +## Applied to All Three Entity Types + +Apply the same 3-table pattern to slices and tasks: + +- `slice_specs`, `slices`, `slice_evidence` +- `task_specs`, `tasks`, `task_evidence` + +Total: 9 new/refactored tables + +--- + +## Query Model Changes + +### Before (Current) +```sql +SELECT vision, success_criteria, status, completed_at, verification_result, full_summary_md +FROM milestones +WHERE id = :id; +``` + +### After (New) +```sql +SELECT s.vision, s.success_criteria, r.status, r.completed_at, e.content +FROM milestones r +LEFT JOIN milestone_specs s ON r.id = s.id +LEFT JOIN milestone_evidence e ON r.id = e.milestone_id AND e.evidence_type = 'verification_contract' +WHERE r.id = :id +ORDER BY e.recorded_at DESC; +``` + +**Benefits:** +- Each table has only relevant columns +- Indices can be more efficient (e.g., index on `milestone_evidence(evidence_type, recorded_at)`) +- Queries self-document intent (joins explain what's spec vs. runtime vs. evidence) + +--- + +## Implementation Phases + +### Phase 1: Schema Definition (0.5d) +- Define 9 new tables in `sf-db.js` +- Add CREATE TABLE statements and schema version bump +- Document column types and constraints + +### Phase 2: Data Migration (1.0d) +- Create migration script that reads current schema +- Populate new `*_specs` tables from current spec columns +- Populate new `*_runtime` tables (will rename after migration) +- Populate new `*_evidence` tables from current narrative/verification columns +- Test migration on real SF project data + +### Phase 3: Data Layer Updates (1.0d) +- Update `insertMilestone()`, `insertSlice()`, `insertTask()` to write to both spec and runtime tables +- Create `insertMilestoneEvidence()`, `insertSliceEvidence()`, `insertTaskEvidence()` functions +- Update query functions (`getMilestone()`, `getMilestoneSlices()`, etc.) to JOIN across new tables +- Update UPDATE functions (`upsertMilestonePlanning()`, etc.) to write only to spec table + +### Phase 4: Tool Updates (0.5d) +- Update `plan-milestone`, `plan-slice`, `plan-task` tools to use new insert functions +- Update `complete-milestone`, `complete-slice`, `complete-task` tools to record evidence +- Verify existing workflows (dispatch loop, replan, re-execute) still work + +### Phase 5: Testing (0.5d) +- Write migration tests (verify data integrity across migration) +- Write query tests (verify new queries return same data as old queries) +- Write immutability tests (verify specs cannot be modified after creation) +- Write evidence chain tests (verify evidence is timestamped and queryable) + +--- + +## Data Integrity Rules + +1. **Spec immutability:** No UPDATE on `*_specs` tables after initial INSERT + - If a change is needed, INSERT a new spec version and INCREMENT spec_version + +2. **Runtime-spec linkage:** Foreign key constraint ensures `runtime.id` maps to `spec.id` + +3. **Evidence timestamping:** All `*_evidence` rows have `recorded_at` set at insertion time (cannot be NULL) + +4. **Retention policy:** Evidence is append-only unless retention policy expires rows (future decision) + +--- + +## Risk Mitigation + +| Risk | Mitigation | +|------|-----------| +| Migration complexity | Dry-run migration on sample data first; create rollback script | +| Breaking existing tools | Update all callers of `insertMilestone`, `insertSlice`, `insertTask` systematically | +| Performance regression | Profile new JOIN queries; add indices on frequently-accessed columns | +| Over-engineering | Start with milestone tables; defer slice/task until stable | + +--- + +## Expected Benefits + +1. **Clear semantics** — Spec is intent, runtime is state, evidence is history +2. **Auditability** — Can reconstruct why a decision was made by reading evidence chain +3. **Re-planning clarity** — Multiple spec versions can exist for the same milestone ID +4. **Query efficiency** — Each query only loads columns it needs; better cache locality +5. **Data archaeology** — Enables forensics tools to trace decision history +6. **Future extensibility** — Can add spec versioning, evidence retention policies, etc. without schema churn + +--- + +## Open Questions + +1. **Evidence retention:** Should old evidence ever be archived or deleted? Or indefinite retention? +2. **Spec versioning:** Should spec versions be labeled or just incremented (e.g., "v1", "v2.1")? +3. **Re-planning linkage:** When a milestone is re-planned, should the new spec version reference the old one? +4. **Performance trade-off:** Are JOINs acceptable, or should we denormalize certain columns for read performance? +5. **Phased rollout:** Should we migrate all three entity types at once, or start with milestones? + +--- + +## Appendix: Detailed Column Mappings + +### Milestones: Current → New + +| Current `milestones` | New `milestones` (Runtime) | New `milestone_specs` (Spec) | +|---|---|---| +| id | id | id | +| title | title | — | +| status | status | — | +| depends_on | depends_on | — | +| created_at | created_at | created_at | +| completed_at | completed_at | — | +| vision | — | vision | +| success_criteria | — | success_criteria | +| key_risks | — | key_risks | +| proof_strategy | — | proof_strategy | +| verification_contract | — | verification_contract | +| verification_integration | — | verification_integration | +| verification_operational | — | verification_operational | +| verification_uat | — | verification_uat | +| definition_of_done | — | definition_of_done | +| requirement_coverage | — | requirement_coverage | +| boundary_map_markdown | — | boundary_map_markdown | +| vision_meeting_json | — | vision_meeting_json | + +### Evidence Table Sources + +New `milestone_evidence` table will be populated from: +- Current `verification_result` → `evidence_type='verification_contract'` +- New events created when milestone transitions to `complete` or `done` → `evidence_type='decision'` +- New incidents recorded during re-plan or escalation → `evidence_type='incident'` + +--- + +## References + +- [ADR-0000: SF Is a Purpose-to-Software Compiler](./0000-purpose-to-software-compiler.md) +- [ADR-0001: Promote-Only SF State](./0001-promote-only-sf-state.md) +- [ADR-0076: UOK Memory Integration](./0076-uok-memory-integration.md) + diff --git a/src/resources/extensions/sf/sf-db.js b/src/resources/extensions/sf/sf-db.js index 562037868..f45d44dd3 100644 --- a/src/resources/extensions/sf/sf-db.js +++ b/src/resources/extensions/sf/sf-db.js @@ -78,7 +78,7 @@ function openRawDb(path) { loadProvider(); return new DatabaseSync(path); } -const SCHEMA_VERSION = 31; +const SCHEMA_VERSION = 32; function indexExists(db, name) { return !!db .prepare( @@ -274,6 +274,135 @@ function ensureSelfFeedbackTables(db) { "CREATE INDEX IF NOT EXISTS idx_self_feedback_kind ON self_feedback(kind, ts)", ); } +function ensureSpecSchemaTables(db) { + // Tier 1.3: Spec/Runtime/Evidence schema separation + // Creates 9 normalized tables for milestone, slice, task entities + // Each entity type has: _specs (immutable intent), (runtime state), _evidence (audit trail) + + // ── Milestone Spec Table (immutable record of intent) ─────────── + db.exec(` + CREATE TABLE IF NOT EXISTS milestone_specs ( + id TEXT PRIMARY KEY, + vision TEXT NOT NULL DEFAULT '', + success_criteria TEXT DEFAULT '', + key_risks TEXT DEFAULT '', + proof_strategy TEXT DEFAULT '', + verification_contract TEXT DEFAULT '', + verification_integration TEXT DEFAULT '', + verification_operational TEXT DEFAULT '', + verification_uat TEXT DEFAULT '', + definition_of_done TEXT DEFAULT '', + requirement_coverage TEXT DEFAULT '', + boundary_map_markdown TEXT DEFAULT '', + vision_meeting_json TEXT DEFAULT '', + spec_version INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL, + PRIMARY KEY (id), + FOREIGN KEY (id) REFERENCES milestones(id) + ) + `); + + // ── Slice Spec Table (immutable record of intent) ─────────── + db.exec(` + CREATE TABLE IF NOT EXISTS slice_specs ( + milestone_id TEXT NOT NULL, + slice_id TEXT NOT NULL, + goal TEXT NOT NULL DEFAULT '', + success_criteria TEXT DEFAULT '', + proof_level TEXT DEFAULT '', + integration_closure TEXT DEFAULT '', + observability_impact TEXT DEFAULT '', + adversarial_partner TEXT DEFAULT '', + adversarial_combatant TEXT DEFAULT '', + adversarial_architect TEXT DEFAULT '', + planning_meeting_json TEXT DEFAULT '', + spec_version INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL, + PRIMARY KEY (milestone_id, slice_id), + FOREIGN KEY (milestone_id) REFERENCES milestones(id), + FOREIGN KEY (milestone_id, slice_id) REFERENCES slices(milestone_id, id) + ) + `); + + // ── Task Spec Table (immutable record of intent) ─────────── + db.exec(` + CREATE TABLE IF NOT EXISTS task_specs ( + milestone_id TEXT NOT NULL, + slice_id TEXT NOT NULL, + task_id TEXT NOT NULL, + verify TEXT NOT NULL DEFAULT '', + inputs TEXT DEFAULT '', + expected_output TEXT DEFAULT '', + spec_version INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL, + PRIMARY KEY (milestone_id, slice_id, task_id), + FOREIGN KEY (milestone_id, slice_id) REFERENCES slices(milestone_id, id), + FOREIGN KEY (milestone_id, slice_id, task_id) REFERENCES tasks(milestone_id, slice_id, id) + ) + `); + + // ── Milestone Evidence Table (append-only audit trail) ─────────── + db.exec(` + CREATE TABLE IF NOT EXISTS milestone_evidence ( + milestone_id TEXT NOT NULL, + evidence_type TEXT NOT NULL, + content TEXT NOT NULL, + recorded_at TEXT NOT NULL, + phase_name TEXT DEFAULT '', + recorded_by TEXT DEFAULT '', + evidence_id TEXT NOT NULL DEFAULT (lower(hex(randomblob(16)))), + PRIMARY KEY (milestone_id, evidence_id), + FOREIGN KEY (milestone_id) REFERENCES milestones(id) + ) + `); + + // ── Slice Evidence Table (append-only audit trail) ─────────── + db.exec(` + CREATE TABLE IF NOT EXISTS slice_evidence ( + milestone_id TEXT NOT NULL, + slice_id TEXT NOT NULL, + evidence_type TEXT NOT NULL, + content TEXT NOT NULL, + recorded_at TEXT NOT NULL, + phase_name TEXT DEFAULT '', + recorded_by TEXT DEFAULT '', + evidence_id TEXT NOT NULL DEFAULT (lower(hex(randomblob(16)))), + PRIMARY KEY (milestone_id, slice_id, evidence_id), + FOREIGN KEY (milestone_id, slice_id) REFERENCES slices(milestone_id, id) + ) + `); + + // ── Task Evidence Table (append-only audit trail) ─────────── + db.exec(` + CREATE TABLE IF NOT EXISTS task_evidence ( + milestone_id TEXT NOT NULL, + slice_id TEXT NOT NULL, + task_id TEXT NOT NULL, + evidence_type TEXT NOT NULL, + content TEXT NOT NULL, + recorded_at TEXT NOT NULL, + phase_name TEXT DEFAULT '', + recorded_by TEXT DEFAULT '', + evidence_id TEXT NOT NULL DEFAULT (lower(hex(randomblob(16)))), + PRIMARY KEY (milestone_id, slice_id, task_id, evidence_id), + FOREIGN KEY (milestone_id, slice_id, task_id) REFERENCES tasks(milestone_id, slice_id, id) + ) + `); + + // Indices for efficient querying of evidence trails + db.exec(` + CREATE INDEX IF NOT EXISTS idx_milestone_evidence_type + ON milestone_evidence(milestone_id, evidence_type, recorded_at DESC) + `); + db.exec(` + CREATE INDEX IF NOT EXISTS idx_slice_evidence_type + ON slice_evidence(milestone_id, slice_id, evidence_type, recorded_at DESC) + `); + db.exec(` + CREATE INDEX IF NOT EXISTS idx_task_evidence_type + ON task_evidence(milestone_id, slice_id, task_id, evidence_type, recorded_at DESC) + `); +} function initSchema(db, fileBacked) { if (fileBacked) db.exec("PRAGMA journal_mode=WAL"); if (fileBacked) db.exec("PRAGMA busy_timeout = 5000"); @@ -735,6 +864,7 @@ function initSchema(db, fileBacked) { ensureSolverEvalTables(db); ensureHeadlessRunTables(db); ensureUokMessageTables(db); + ensureSpecSchemaTables(db); db.exec( `CREATE VIEW IF NOT EXISTS active_decisions AS SELECT * FROM decisions WHERE superseded_by IS NULL`, ); @@ -1696,6 +1826,15 @@ function migrateSchema(db) { ":applied_at": new Date().toISOString(), }); } + if (currentVersion < 32) { + ensureSpecSchemaTables(db); + db.prepare( + "INSERT INTO schema_version (version, applied_at) VALUES (:version, :applied_at)", + ).run({ + ":version": 32, + ":applied_at": new Date().toISOString(), + }); + } db.exec("COMMIT"); } catch (err) { db.exec("ROLLBACK");