diff --git a/src/resources/extensions/gsd/auto-prompts.ts b/src/resources/extensions/gsd/auto-prompts.ts index 4a2fc476f..8de183301 100644 --- a/src/resources/extensions/gsd/auto-prompts.ts +++ b/src/resources/extensions/gsd/auto-prompts.ts @@ -21,10 +21,7 @@ import type { GSDPreferences } from "./preferences.js"; import { join } from "node:path"; import { existsSync } from "node:fs"; import { computeBudgets, resolveExecutorContextWindow } from "./context-budget.js"; -import { compressToTarget } from "./prompt-compressor.js"; -import { distillSummaries } from "./summary-distiller.js"; import { formatDecisionsCompact, formatRequirementsCompact } from "./structured-data-formatter.js"; -import { chunkByRelevance, formatChunks } from "./semantic-chunker.js"; // ─── Executor Constraints ───────────────────────────────────────────────────── @@ -159,16 +156,10 @@ export async function inlineFileSmart( return `### ${label}\nSource: \`${relPath}\`\n\n${content.trim()}`; } - // Use semantic chunking for large files - const result = chunkByRelevance(content, query, { maxChunks: 5, minScore: 0.05 }); - - // If chunking didn't save much (< 20%), just include full content - if (result.savingsPercent < 20) { - return `### ${label}\nSource: \`${relPath}\`\n\n${content.trim()}`; - } - - const formatted = formatChunks(result, relPath); - return `### ${label} (${result.omittedChunks} sections omitted for relevance)\nSource: \`${relPath}\`\n\n${formatted}`; + // For large files, truncate at section boundary + const { truncateAtSectionBoundary } = await import("./context-budget.js"); + const truncated = truncateAtSectionBoundary(content, threshold).content; + return `### ${label}\nSource: \`${relPath}\`\n\n${truncated}`; } /** @@ -202,20 +193,6 @@ export async function inlineDependencySummaries( const result = sections.join("\n\n"); if (budgetChars !== undefined && result.length > budgetChars) { - // For 3+ summaries, try distillation first (preserves more information) - if (sections.length >= 3) { - const rawSummaries = sections.map(s => { - // Extract content after the header line - const lines = s.split("\n"); - const contentStart = lines.findIndex(l => l.startsWith("Source:")); - return contentStart >= 0 ? lines.slice(contentStart + 1).join("\n").trim() : s; - }); - const distilled = distillSummaries(rawSummaries, budgetChars); - if (distilled.content.length <= budgetChars) { - return distilled.content; - } - } - // Fall back to section-boundary truncation const { truncateAtSectionBoundary } = await import("./context-budget.js"); return truncateAtSectionBoundary(result, budgetChars).content; } @@ -900,15 +877,12 @@ export async function buildExecuteTaskPrompt( const budgets = computeBudgets(contextWindow); const verificationBudget = `~${Math.round(budgets.verificationBudgetChars / 1000)}K chars`; - // Compress carry-forward section when it exceeds 40% of inline context budget. - // Only compress when compression_strategy is "compress" (budget/balanced profiles). + // Truncate carry-forward section when it exceeds 40% of inline context budget. const carryForwardBudget = Math.floor(budgets.inlineContextBudgetChars * 0.4); let finalCarryForward = carryForwardSection; if (carryForwardSection.length > carryForwardBudget) { - const { resolveCompressionStrategy } = await import("./preferences.js"); - if (resolveCompressionStrategy() === "compress") { - finalCarryForward = compressToTarget(carryForwardSection, carryForwardBudget).content; - } + const { truncateAtSectionBoundary } = await import("./context-budget.js"); + finalCarryForward = truncateAtSectionBoundary(carryForwardSection, carryForwardBudget).content; } return loadPrompt("execute-task", { diff --git a/src/resources/extensions/gsd/commands-prefs-wizard.ts b/src/resources/extensions/gsd/commands-prefs-wizard.ts index 4ec4c3dc1..46e4b0a37 100644 --- a/src/resources/extensions/gsd/commands-prefs-wizard.ts +++ b/src/resources/extensions/gsd/commands-prefs-wizard.ts @@ -745,7 +745,7 @@ export function serializePreferencesToFrontmatter(prefs: Record "dynamic_routing", "token_profile", "phases", "parallel", "auto_visualize", "auto_report", "verification_commands", "verification_auto_fix", "verification_max_retries", - "search_provider", "compression_strategy", "context_selection", + "search_provider", "context_selection", ]; const seen = new Set(); diff --git a/src/resources/extensions/gsd/context-budget.ts b/src/resources/extensions/gsd/context-budget.ts index 29bf03836..1788670a0 100644 --- a/src/resources/extensions/gsd/context-budget.ts +++ b/src/resources/extensions/gsd/context-budget.ts @@ -9,7 +9,6 @@ */ import { type TokenProvider, getCharsPerToken } from "./token-counter.js"; -import { compressToTarget } from "./prompt-compressor.js"; // ─── Budget ratio constants ────────────────────────────────────────────────── // Percentages of total context window allocated to each budget category. @@ -202,22 +201,13 @@ export function resolveExecutorContextWindow( } /** - * Smart context reduction: compress first, then truncate if still over budget. - * Returns the content within budget with maximum information preservation. + * Reduce content to fit within budget using section-boundary truncation. */ export function reduceToFit(content: string, budgetChars: number): TruncationResult { if (!content || content.length <= budgetChars) { return { content, droppedSections: 0 }; } - - // Step 1: Try compression - const compressed = compressToTarget(content, budgetChars); - if (compressed.compressedChars <= budgetChars) { - return { content: compressed.content, droppedSections: 0 }; - } - - // Step 2: Truncate the compressed content at section boundaries - return truncateAtSectionBoundary(compressed.content, budgetChars); + return truncateAtSectionBoundary(content, budgetChars); } // ─── Internal helpers ──────────────────────────────────────────────────────── diff --git a/src/resources/extensions/gsd/docs/preferences-reference.md b/src/resources/extensions/gsd/docs/preferences-reference.md index 290eb446c..f3b2ccd0f 100644 --- a/src/resources/extensions/gsd/docs/preferences-reference.md +++ b/src/resources/extensions/gsd/docs/preferences-reference.md @@ -194,8 +194,6 @@ Setting `prefer_skills: []` does **not** disable skill discovery — it just mea - `search_provider`: `"brave"`, `"tavily"`, `"ollama"`, `"native"`, or `"auto"` — selects the search backend for research phases. `"native"` forces Anthropic's built-in web search only; provider values force that backend and disable native search; `"auto"` uses the default heuristic. Default: `"auto"`. -- `compression_strategy`: `"truncate"` or `"compress"` — controls how context that exceeds the budget is reduced. `"truncate"` (default) drops sections from the end. `"compress"` applies heuristic compression before truncating, preserving more content at the cost of some fidelity. Default: `"truncate"`. - - `context_selection`: `"full"` or `"smart"` — controls how files are inlined into context. `"full"` inlines entire files; `"smart"` uses semantic chunking to include only the most relevant sections. Default is derived from `token_profile`. - `parallel`: configures parallel orchestration for running multiple slices concurrently. Keys: diff --git a/src/resources/extensions/gsd/preferences-models.ts b/src/resources/extensions/gsd/preferences-models.ts index 1eeb3c0fe..303c43470 100644 --- a/src/resources/extensions/gsd/preferences-models.ts +++ b/src/resources/extensions/gsd/preferences-models.ts @@ -295,18 +295,6 @@ export function resolveInlineLevel(): InlineLevel { } } -/** - * Resolve the compression strategy from the active token profile. - * budget/balanced -> "compress", quality -> "truncate". - * Explicit preference always wins. - */ -export function resolveCompressionStrategy(): import("./types.js").CompressionStrategy { - const prefs = loadEffectiveGSDPreferences(); - if (prefs?.preferences.compression_strategy) return prefs.preferences.compression_strategy; - const profile = resolveEffectiveProfile(); - return profile === "quality" ? "truncate" : "compress"; -} - /** * Resolve the context selection mode from the active token profile. * budget -> "smart", balanced/quality -> "full". diff --git a/src/resources/extensions/gsd/preferences-types.ts b/src/resources/extensions/gsd/preferences-types.ts index d4b41139b..60f041989 100644 --- a/src/resources/extensions/gsd/preferences-types.ts +++ b/src/resources/extensions/gsd/preferences-types.ts @@ -16,7 +16,6 @@ import type { InlineLevel, PhaseSkipPreferences, ParallelConfig, - CompressionStrategy, ContextSelectionMode, ReactiveExecutionConfig, } from "./types.js"; @@ -84,7 +83,6 @@ export const KNOWN_PREFERENCE_KEYS = new Set([ "verification_auto_fix", "verification_max_retries", "search_provider", - "compression_strategy", "context_selection", "widget_mode", "reactive_execution", @@ -211,8 +209,6 @@ export interface GSDPreferences { verification_max_retries?: number; /** Search provider preference. "brave"/"tavily"/"ollama" force that backend and disable native Anthropic search. "native" forces native only. "auto" = current default behavior. */ search_provider?: "brave" | "tavily" | "ollama" | "native" | "auto"; - /** Compression strategy for context that exceeds budget. "truncate" (default) drops sections, "compress" applies heuristic compression first. */ - compression_strategy?: CompressionStrategy; /** Context selection mode for file inlining. "full" inlines entire files, "smart" uses semantic chunking. Default derived from token profile. */ context_selection?: ContextSelectionMode; /** Default widget display mode for auto-mode dashboard. "full" | "small" | "min" | "off". Default: "full". */ diff --git a/src/resources/extensions/gsd/preferences-validation.ts b/src/resources/extensions/gsd/preferences-validation.ts index 1bdc7a2d6..8f6a2ebcd 100644 --- a/src/resources/extensions/gsd/preferences-validation.ts +++ b/src/resources/extensions/gsd/preferences-validation.ts @@ -686,16 +686,6 @@ export function validatePreferences(preferences: GSDPreferences): { } } - // ─── Compression Strategy ─────────────────────────────────────────── - if (preferences.compression_strategy !== undefined) { - const validStrategies = new Set(["truncate", "compress"]); - if (typeof preferences.compression_strategy === "string" && validStrategies.has(preferences.compression_strategy)) { - validated.compression_strategy = preferences.compression_strategy as GSDPreferences["compression_strategy"]; - } else { - errors.push(`compression_strategy must be one of: truncate, compress`); - } - } - // ─── Context Selection ────────────────────────────────────────────── if (preferences.context_selection !== undefined) { const validModes = new Set(["full", "smart"]); diff --git a/src/resources/extensions/gsd/preferences.ts b/src/resources/extensions/gsd/preferences.ts index d8f005984..bd3e88fb8 100644 --- a/src/resources/extensions/gsd/preferences.ts +++ b/src/resources/extensions/gsd/preferences.ts @@ -77,7 +77,6 @@ export { resolveProfileDefaults, resolveEffectiveProfile, resolveInlineLevel, - resolveCompressionStrategy, resolveContextSelection, resolveSearchProviderFromPreferences, } from "./preferences-models.js"; @@ -269,7 +268,6 @@ function mergePreferences(base: GSDPreferences, override: GSDPreferences): GSDPr verification_auto_fix: override.verification_auto_fix ?? base.verification_auto_fix, verification_max_retries: override.verification_max_retries ?? base.verification_max_retries, search_provider: override.search_provider ?? base.search_provider, - compression_strategy: override.compression_strategy ?? base.compression_strategy, context_selection: override.context_selection ?? base.context_selection, auto_visualize: override.auto_visualize ?? base.auto_visualize, auto_report: override.auto_report ?? base.auto_report, diff --git a/src/resources/extensions/gsd/prompt-compressor.ts b/src/resources/extensions/gsd/prompt-compressor.ts deleted file mode 100644 index 7f72b45ce..000000000 --- a/src/resources/extensions/gsd/prompt-compressor.ts +++ /dev/null @@ -1,508 +0,0 @@ -/** - * Prompt Compressor — deterministic text compression for context reduction. - * - * Applies a series of lossless and near-lossless transformations to reduce - * token count while preserving semantic meaning. No LLM calls, no external - * dependencies. Sub-millisecond for typical prompt sizes. - * - * Compression techniques (applied in order): - * 1. Redundant whitespace normalization - * 2. Markdown formatting reduction (collapse verbose tables, lists) - * 3. Common phrase abbreviation - * 4. Repeated pattern deduplication - * 5. Low-information content removal (empty sections, boilerplate) - */ - -export type CompressionLevel = "light" | "moderate" | "aggressive"; - -export interface CompressionResult { - /** The compressed content */ - content: string; - /** Original character count */ - originalChars: number; - /** Compressed character count */ - compressedChars: number; - /** Savings percentage (0-100) */ - savingsPercent: number; - /** Which compression level was applied */ - level: CompressionLevel; - /** Number of transformations applied */ - transformationsApplied: number; -} - -export interface CompressionOptions { - /** Compression intensity. Default: "moderate" */ - level?: CompressionLevel; - /** Preserve markdown headings (useful for section-boundary truncation). Default: true */ - preserveHeadings?: boolean; - /** Preserve code blocks verbatim. Default: true */ - preserveCodeBlocks?: boolean; - /** Target character count (compression stops when achieved). Default: no target */ - targetChars?: number; -} - -// ─── Phrase Abbreviation Map ──────────────────────────────────────────────── - -/** - * Build a regex that matches a verbose phrase even when split across lines. - * Whitespace between words is matched with \s+ to handle line wrapping. - */ -function phraseRegex(phrase: string): RegExp { - const words = phrase.split(/\s+/); - const pattern = `\\b${words.join("\\s+")}\\b`; - return new RegExp(pattern, "gi"); -} - -const VERBOSE_PHRASES: Array<[RegExp, string]> = [ - [phraseRegex("In order to"), "To"], - [phraseRegex("It is important to note that"), "Note:"], - [phraseRegex("As mentioned previously"), "(see above)"], - [phraseRegex("The following"), "These"], - [phraseRegex("In addition to"), "Also,"], - [phraseRegex("Due to the fact that"), "Because"], - [phraseRegex("At this point in time"), "Now"], - [phraseRegex("For the purpose of"), "For"], - [phraseRegex("In the event that"), "If"], - [phraseRegex("With regard to"), "Re:"], - [phraseRegex("Prior to"), "Before"], - [phraseRegex("Subsequent to"), "After"], - [phraseRegex("In accordance with"), "Per"], - [phraseRegex("A number of"), "Several"], - [phraseRegex("In the case of"), "For"], - [phraseRegex("On the basis of"), "Based on"], -]; - -// ─── Code Block Extraction ────────────────────────────────────────────────── - -interface ExtractedBlocks { - text: string; - blocks: Map; -} - -function extractCodeBlocks(content: string): ExtractedBlocks { - const blocks = new Map(); - let counter = 0; - - const text = content.replace(/```[\s\S]*?```/g, (match) => { - const placeholder = `\x00CODEBLOCK_${counter++}\x00`; - blocks.set(placeholder, match); - return placeholder; - }); - - return { text, blocks }; -} - -function restoreCodeBlocks(text: string, blocks: Map): string { - let result = text; - for (const [placeholder, block] of blocks) { - result = result.replace(placeholder, block); - } - return result; -} - -// ─── Light Transformations ────────────────────────────────────────────────── - -function normalizeWhitespace(content: string): string { - // Collapse 3+ consecutive blank lines to 2 - let result = content.replace(/(\n\s*){3,}\n/g, "\n\n"); - // Trim trailing whitespace on every line - result = result.replace(/[ \t]+$/gm, ""); - return result; -} - -function removeMarkdownComments(content: string): string { - return content.replace(//g, ""); -} - -function removeHorizontalRules(content: string): string { - // Remove horizontal rules (---, ***, ___) that stand alone on a line - return content.replace(/^\s*[-*_]{3,}\s*$/gm, ""); -} - -function collapseEmptyListItems(content: string): string { - // Collapse repeated empty list items (- \n- \n- \n) into one - return content.replace(/(^[ \t]*[-*+]\s*$\n){2,}/gm, "$1"); -} - -function applyLightTransformations(content: string): { content: string; count: number } { - let count = 0; - let result = content; - - const after1 = normalizeWhitespace(result); - if (after1 !== result) count++; - result = after1; - - const after2 = removeMarkdownComments(result); - if (after2 !== result) count++; - result = after2; - - const after3 = removeHorizontalRules(result); - if (after3 !== result) count++; - result = after3; - - const after4 = collapseEmptyListItems(result); - if (after4 !== result) count++; - result = after4; - - return { content: result, count }; -} - -// ─── Moderate Transformations ─────────────────────────────────────────────── - -function abbreviateVerbosePhrases(content: string): { content: string; count: number } { - let count = 0; - let result = content; - - for (const [pattern, replacement] of VERBOSE_PHRASES) { - const after = result.replace(pattern, replacement); - if (after !== result) count++; - result = after; - } - - return { content: result, count }; -} - -function removeBoilerplateLines(content: string): string { - const lines = content.split("\n"); - const filtered = lines.filter((line) => { - const trimmed = line.trim(); - // Remove lines that are just N/A, (none), (empty), (not applicable) - if (/^(?:N\/A|\(none\)|\(empty\)|\(not applicable\))$/i.test(trimmed)) { - return false; - } - return true; - }); - return filtered.join("\n"); -} - -function deduplicateConsecutiveLines(content: string): string { - const lines = content.split("\n"); - const result: string[] = []; - - for (let i = 0; i < lines.length; i++) { - if (i === 0 || lines[i] !== lines[i - 1] || lines[i].trim() === "") { - result.push(lines[i]); - } - } - - return result.join("\n"); -} - -function collapseTableFormatting(content: string): string { - // Remove excessive padding in markdown table cells - // Matches table rows like | cell | cell | and collapses to | cell | cell | - return content.replace(/\|[ \t]{2,}([^|\n]*?)[ \t]{2,}\|/g, (_, cellContent) => { - return `| ${cellContent.trim()} |`; - }); -} - -function applyModerateTransformations(content: string): { content: string; count: number } { - let count = 0; - let result = content; - - const phraseResult = abbreviateVerbosePhrases(result); - count += phraseResult.count; - result = phraseResult.content; - - const after1 = removeBoilerplateLines(result); - if (after1 !== result) count++; - result = after1; - - const after2 = deduplicateConsecutiveLines(result); - if (after2 !== result) count++; - result = after2; - - const after3 = collapseTableFormatting(result); - if (after3 !== result) count++; - result = after3; - - return { content: result, count }; -} - -// ─── Aggressive Transformations ───────────────────────────────────────────── - -function removeMarkdownEmphasis(content: string): string { - // Bold: **text** or __text__ - let result = content.replace(/\*\*(.+?)\*\*/g, "$1"); - result = result.replace(/__(.+?)__/g, "$1"); - // Italic: *text* or _text_ (single, not inside words) - result = result.replace(/(? { - if (line.length <= 300) return line; - // Find a sentence boundary (. ! ?) near the 300 char mark - const truncateZone = line.slice(0, 300); - const lastSentenceEnd = Math.max( - truncateZone.lastIndexOf(". "), - truncateZone.lastIndexOf("! "), - truncateZone.lastIndexOf("? "), - ); - if (lastSentenceEnd > 150) { - return line.slice(0, lastSentenceEnd + 1); - } - // Fallback: cut at last space before 300 - const lastSpace = truncateZone.lastIndexOf(" "); - if (lastSpace > 150) { - return line.slice(0, lastSpace); - } - return truncateZone; - }); - return result.join("\n"); -} - -function removeBulletMarkers(content: string): string { - // Remove bullet markers: - , * , + , numbered (1. 2. etc) - return content.replace(/^[ \t]*(?:[-*+]|\d+\.)\s+/gm, ""); -} - -function removeBlockquoteMarkers(content: string): string { - return content.replace(/^[ \t]*>+\s?/gm, ""); -} - -function deduplicateStructuralPatterns(content: string): string { - // Deduplicate consecutive lines that match the same "Key: value" pattern - const lines = content.split("\n"); - const result: string[] = []; - const seen = new Set(); - let lastWasStructural = false; - - for (const line of lines) { - const trimmed = line.trim(); - // Detect structural patterns: "Key: value" - const structMatch = trimmed.match(/^(\w[\w\s]*?):\s+(.+)$/); - if (structMatch) { - if (seen.has(trimmed)) { - lastWasStructural = true; - continue; - } - seen.add(trimmed); - lastWasStructural = true; - } else { - // Reset seen set when structural block ends - if (!lastWasStructural || trimmed === "") { - seen.clear(); - } - lastWasStructural = false; - } - result.push(line); - } - - return result.join("\n"); -} - -function applyAggressiveTransformations( - content: string, - preserveHeadings: boolean, -): { content: string; count: number } { - let count = 0; - let result = content; - - const after1 = removeMarkdownEmphasis(result); - if (after1 !== result) count++; - result = after1; - - const after2 = removeMarkdownLinks(result); - if (after2 !== result) count++; - result = after2; - - const after3 = truncateLongLines(result); - if (after3 !== result) count++; - result = after3; - - const after4 = removeBulletMarkers(result); - if (after4 !== result) count++; - result = after4; - - const after5 = removeBlockquoteMarkers(result); - if (after5 !== result) count++; - result = after5; - - const after6 = deduplicateStructuralPatterns(result); - if (after6 !== result) count++; - result = after6; - - return { content: result, count }; -} - -// ─── Heading Preservation ─────────────────────────────────────────────────── - -interface ExtractedHeadings { - text: string; - headings: Map; -} - -function extractHeadings(content: string): ExtractedHeadings { - const headings = new Map(); - let counter = 0; - - const text = content.replace(/^(#{1,6}\s.+)$/gm, (match) => { - const placeholder = `\x00HEADING_${counter++}\x00`; - headings.set(placeholder, match); - return placeholder; - }); - - return { text, headings }; -} - -function restoreHeadings(text: string, headings: Map): string { - let result = text; - for (const [placeholder, heading] of headings) { - result = result.replace(placeholder, heading); - } - return result; -} - -// ─── Public API ───────────────────────────────────────────────────────────── - -/** - * Compress prompt content using deterministic text transformations. - */ -export function compressPrompt(content: string, options?: CompressionOptions): CompressionResult { - const level = options?.level ?? "moderate"; - const preserveHeadings = options?.preserveHeadings ?? true; - const preserveCodeBlocks = options?.preserveCodeBlocks ?? true; - - if (content === "") { - return { - content: "", - originalChars: 0, - compressedChars: 0, - savingsPercent: 0, - level, - transformationsApplied: 0, - }; - } - - const originalChars = content.length; - let working = content; - let totalTransformations = 0; - - // Extract code blocks if preserving - let codeBlocks: Map | null = null; - if (preserveCodeBlocks) { - const extracted = extractCodeBlocks(working); - working = extracted.text; - codeBlocks = extracted.blocks; - } - - // Extract headings if preserving - let headings: Map | null = null; - if (preserveHeadings) { - const extracted = extractHeadings(working); - working = extracted.text; - headings = extracted.headings; - } - - // Apply light transformations (always) - const lightResult = applyLightTransformations(working); - working = lightResult.content; - totalTransformations += lightResult.count; - - // Check target - if (options?.targetChars && getRestoredLength(working, codeBlocks, headings) <= options.targetChars) { - return buildResult(working, originalChars, level, totalTransformations, codeBlocks, headings); - } - - // Apply moderate transformations - if (level === "moderate" || level === "aggressive") { - const modResult = applyModerateTransformations(working); - working = modResult.content; - totalTransformations += modResult.count; - - if (options?.targetChars && getRestoredLength(working, codeBlocks, headings) <= options.targetChars) { - return buildResult(working, originalChars, level, totalTransformations, codeBlocks, headings); - } - } - - // Apply aggressive transformations - if (level === "aggressive") { - const aggResult = applyAggressiveTransformations(working, preserveHeadings); - working = aggResult.content; - totalTransformations += aggResult.count; - } - - return buildResult(working, originalChars, level, totalTransformations, codeBlocks, headings); -} - -/** - * Compress with a target size — applies progressively more aggressive - * compression until the target is reached or all transformations exhausted. - */ -export function compressToTarget(content: string, targetChars: number): CompressionResult { - if (content.length <= targetChars) { - return { - content, - originalChars: content.length, - compressedChars: content.length, - savingsPercent: 0, - level: "light", - transformationsApplied: 0, - }; - } - - const levels: CompressionLevel[] = ["light", "moderate", "aggressive"]; - - for (const level of levels) { - const result = compressPrompt(content, { level, targetChars }); - if (result.compressedChars <= targetChars) { - return result; - } - // If aggressive and still over target, return best effort - if (level === "aggressive") { - return result; - } - } - - // Unreachable, but satisfy TypeScript - return compressPrompt(content, { level: "aggressive" }); -} - -// ─── Helpers ──────────────────────────────────────────────────────────────── - -function getRestoredLength( - text: string, - codeBlocks: Map | null, - headings: Map | null, -): number { - let result = text; - if (headings) result = restoreHeadings(result, headings); - if (codeBlocks) result = restoreCodeBlocks(result, codeBlocks); - return result.length; -} - -function buildResult( - working: string, - originalChars: number, - level: CompressionLevel, - transformationsApplied: number, - codeBlocks: Map | null, - headings: Map | null, -): CompressionResult { - let content = working; - if (headings) content = restoreHeadings(content, headings); - if (codeBlocks) content = restoreCodeBlocks(content, codeBlocks); - - const compressedChars = content.length; - const savingsPercent = originalChars > 0 - ? Math.round(((originalChars - compressedChars) / originalChars) * 10000) / 100 - : 0; - - return { - content, - originalChars, - compressedChars, - savingsPercent, - level, - transformationsApplied, - }; -} diff --git a/src/resources/extensions/gsd/semantic-chunker.ts b/src/resources/extensions/gsd/semantic-chunker.ts deleted file mode 100644 index 41747dd89..000000000 --- a/src/resources/extensions/gsd/semantic-chunker.ts +++ /dev/null @@ -1,336 +0,0 @@ -// GSD Extension — Semantic Chunker with TF-IDF Relevance Scoring -// Splits code/text into semantic chunks and selects the most relevant ones for a given task. -// Pure TypeScript — no external dependencies. - -// ─── Types ────────────────────────────────────────────────────────────────── - -export interface Chunk { - content: string; - startLine: number; - endLine: number; - score: number; -} - -export interface ChunkResult { - chunks: Chunk[]; - totalChunks: number; - omittedChunks: number; - savingsPercent: number; -} - -interface ChunkOptions { - minLines?: number; - maxLines?: number; -} - -interface RelevanceOptions { - maxChunks?: number; - minChunkLines?: number; - maxChunkLines?: number; - minScore?: number; -} - -// ─── Constants ────────────────────────────────────────────────────────────── - -const CODE_BOUNDARY_RE = /^(export\s+)?(async\s+)?(function|class|interface|type|const|enum)\s/; - -const MARKDOWN_HEADING_RE = /^#{1,6}\s/; - -const STOP_WORDS = new Set([ - "the", "a", "an", "is", "are", "was", "were", "be", "to", "of", "in", - "for", "on", "with", "at", "by", "from", "this", "that", "it", "as", - "or", "and", "not", "but", "if", "do", "no", "so", "up", "its", "has", - "had", "get", "set", "can", "may", "all", "use", "new", "one", "two", - "also", "each", "than", "been", "into", "most", "only", "over", "such", - "how", "some", "any", "our", "his", "her", "out", "did", "let", "say", "she", -]); - -const DEFAULT_MIN_LINES = 3; -const DEFAULT_MAX_LINES = 80; -const DEFAULT_MAX_CHUNKS = 5; -const DEFAULT_MIN_SCORE = 0.1; - -// ─── Content Type Detection ───────────────────────────────────────────────── - -type ContentType = "code" | "markdown" | "text"; - -function detectContentType(lines: string[]): ContentType { - let codeSignals = 0; - let mdSignals = 0; - const sampleSize = Math.min(lines.length, 50); - - for (let i = 0; i < sampleSize; i++) { - const line = lines[i]; - if (CODE_BOUNDARY_RE.test(line) || /^\s*import\s/.test(line)) { - codeSignals++; - } - if (MARKDOWN_HEADING_RE.test(line)) { - mdSignals++; - } - } - - if (mdSignals >= 2 && mdSignals > codeSignals) return "markdown"; - if (codeSignals >= 2) return "code"; - return "text"; -} - -// ─── Tokenizer ────────────────────────────────────────────────────────────── - -function tokenize(text: string): string[] { - return text - .toLowerCase() - .split(/[\s\W]+/) - .filter((w) => w.length >= 2 && !STOP_WORDS.has(w)); -} - -// ─── splitIntoChunks ──────────────────────────────────────────────────────── - -export function splitIntoChunks( - content: string, - options?: ChunkOptions, -): Chunk[] { - if (!content || content.trim().length === 0) return []; - - const minLines = options?.minLines ?? DEFAULT_MIN_LINES; - const maxLines = options?.maxLines ?? DEFAULT_MAX_LINES; - const lines = content.split("\n"); - - if (lines.length === 0) return []; - - const contentType = detectContentType(lines); - let boundaries: number[]; - - switch (contentType) { - case "code": - boundaries = findCodeBoundaries(lines); - break; - case "markdown": - boundaries = findMarkdownBoundaries(lines); - break; - default: - boundaries = findTextBoundaries(lines); - break; - } - - // Always include 0 as first boundary - if (boundaries.length === 0 || boundaries[0] !== 0) { - boundaries.unshift(0); - } - - // Build raw chunks from boundaries - const rawChunks: Chunk[] = []; - for (let i = 0; i < boundaries.length; i++) { - const start = boundaries[i]; - const end = i + 1 < boundaries.length ? boundaries[i + 1] - 1 : lines.length - 1; - const chunkLines = lines.slice(start, end + 1); - rawChunks.push({ - content: chunkLines.join("\n"), - startLine: start + 1, // 1-based - endLine: end + 1, // 1-based - score: 0, - }); - } - - // Split oversized chunks at maxLines - const splitChunks: Chunk[] = []; - for (const chunk of rawChunks) { - const chunkLineCount = chunk.endLine - chunk.startLine + 1; - if (chunkLineCount <= maxLines) { - splitChunks.push(chunk); - } else { - const chunkLines = chunk.content.split("\n"); - for (let offset = 0; offset < chunkLines.length; offset += maxLines) { - const slice = chunkLines.slice(offset, offset + maxLines); - splitChunks.push({ - content: slice.join("\n"), - startLine: chunk.startLine + offset, - endLine: chunk.startLine + offset + slice.length - 1, - score: 0, - }); - } - } - } - - // Merge tiny chunks into predecessor - const merged: Chunk[] = []; - for (const chunk of splitChunks) { - const chunkLineCount = chunk.endLine - chunk.startLine + 1; - if (chunkLineCount < minLines && merged.length > 0) { - const prev = merged[merged.length - 1]; - prev.content += "\n" + chunk.content; - prev.endLine = chunk.endLine; - } else { - merged.push({ ...chunk }); - } - } - - return merged; -} - -function findCodeBoundaries(lines: string[]): number[] { - const boundaries: number[] = []; - for (let i = 0; i < lines.length; i++) { - if (CODE_BOUNDARY_RE.test(lines[i])) { - // Also consider a blank line before a boundary marker - if (i > 0 && lines[i - 1].trim() === "" && !boundaries.includes(i)) { - boundaries.push(i); - } else if (!boundaries.includes(i)) { - boundaries.push(i); - } - } - } - return boundaries; -} - -function findMarkdownBoundaries(lines: string[]): number[] { - const boundaries: number[] = []; - for (let i = 0; i < lines.length; i++) { - if (MARKDOWN_HEADING_RE.test(lines[i])) { - boundaries.push(i); - } - } - return boundaries; -} - -function findTextBoundaries(lines: string[]): number[] { - const boundaries: number[] = [0]; - for (let i = 1; i < lines.length; i++) { - if (lines[i - 1].trim() === "" && lines[i].trim() !== "") { - boundaries.push(i); - } - } - return boundaries; -} - -// ─── scoreChunks ──────────────────────────────────────────────────────────── - -export function scoreChunks(chunks: Chunk[], query: string): Chunk[] { - if (chunks.length === 0) return []; - - const queryTerms = tokenize(query); - if (queryTerms.length === 0) { - return chunks.map((c) => ({ ...c, score: 0 })); - } - - const totalChunks = chunks.length; - - // Pre-compute IDF for each query term - const termChunkCounts = new Map(); - const chunkTokenSets: Set[] = []; - - for (const chunk of chunks) { - const tokens = new Set(tokenize(chunk.content)); - chunkTokenSets.push(tokens); - for (const term of queryTerms) { - if (tokens.has(term)) { - termChunkCounts.set(term, (termChunkCounts.get(term) ?? 0) + 1); - } - } - } - - const idf = new Map(); - for (const term of queryTerms) { - const df = termChunkCounts.get(term) ?? 0; - idf.set(term, Math.log(1 + totalChunks / (1 + df))); - } - - // Score each chunk - const scored = chunks.map((chunk, idx) => { - const chunkTokens = tokenize(chunk.content); - const totalTerms = chunkTokens.length; - if (totalTerms === 0) return { ...chunk, score: 0 }; - - // Count term frequencies - const termFreq = new Map(); - for (const token of chunkTokens) { - termFreq.set(token, (termFreq.get(token) ?? 0) + 1); - } - - let score = 0; - for (const term of queryTerms) { - const tf = (termFreq.get(term) ?? 0) / totalTerms; - const termIdf = idf.get(term) ?? 0; - score += tf * termIdf; - } - - return { ...chunk, score }; - }); - - // Normalize to 0-1 - const maxScore = Math.max(...scored.map((c) => c.score)); - if (maxScore > 0) { - for (const chunk of scored) { - chunk.score = chunk.score / maxScore; - } - } - - return scored; -} - -// ─── chunkByRelevance ─────────────────────────────────────────────────────── - -export function chunkByRelevance( - content: string, - query: string, - options?: RelevanceOptions, -): ChunkResult { - const maxChunks = options?.maxChunks ?? DEFAULT_MAX_CHUNKS; - const minScore = options?.minScore ?? DEFAULT_MIN_SCORE; - const minLines = options?.minChunkLines ?? DEFAULT_MIN_LINES; - const maxLines = options?.maxChunkLines ?? DEFAULT_MAX_LINES; - - const rawChunks = splitIntoChunks(content, { minLines, maxLines }); - if (rawChunks.length === 0) { - return { chunks: [], totalChunks: 0, omittedChunks: 0, savingsPercent: 0 }; - } - - const scored = scoreChunks(rawChunks, query); - - // Filter by minScore and take top maxChunks by score - const qualifying = scored - .filter((c) => c.score >= minScore) - .sort((a, b) => b.score - a.score) - .slice(0, maxChunks); - - // Return in original document order (by startLine) - const selected = qualifying.sort((a, b) => a.startLine - b.startLine); - - const totalChars = content.length; - const selectedChars = selected.reduce((sum, c) => sum + c.content.length, 0); - const savingsPercent = totalChars > 0 - ? Math.round(((totalChars - selectedChars) / totalChars) * 100) - : 0; - - return { - chunks: selected, - totalChunks: rawChunks.length, - omittedChunks: rawChunks.length - selected.length, - savingsPercent: Math.max(0, savingsPercent), - }; -} - -// ─── formatChunks ─────────────────────────────────────────────────────────── - -export function formatChunks(result: ChunkResult, filePath: string): string { - if (result.chunks.length === 0) { - return `[${filePath}: empty or no relevant chunks]`; - } - - const parts: string[] = []; - let lastEndLine = 0; - - for (const chunk of result.chunks) { - // Show omission gap - if (lastEndLine > 0 && chunk.startLine > lastEndLine + 1) { - const gapLines = chunk.startLine - lastEndLine - 1; - parts.push(`[...${gapLines} lines omitted...]`); - } - - parts.push(`[Lines ${chunk.startLine}-${chunk.endLine}]`); - parts.push(chunk.content); - - lastEndLine = chunk.endLine; - } - - return parts.join("\n"); -} diff --git a/src/resources/extensions/gsd/summary-distiller.ts b/src/resources/extensions/gsd/summary-distiller.ts deleted file mode 100644 index 1aee5b203..000000000 --- a/src/resources/extensions/gsd/summary-distiller.ts +++ /dev/null @@ -1,258 +0,0 @@ -/** - * Summary distiller — extracts essential structured data from SUMMARY.md files, - * dropping verbose prose to save context budget. - */ - -export interface DistillationResult { - content: string; - summaryCount: number; - savingsPercent: number; - originalChars: number; - distilledChars: number; -} - -interface ParsedFrontmatter { - id: string; - provides: string[]; - requires: string[]; - key_files: string[]; - key_decisions: string[]; - patterns_established: string[]; -} - -interface DistilledEntry { - id: string; - oneLiner: string; - provides: string[]; - requires: string[]; - key_files: string[]; - key_decisions: string[]; - patterns: string[]; -} - -// ─── Frontmatter parsing ───────────────────────────────────────────────────── - -function parseFrontmatter(raw: string): ParsedFrontmatter { - const result: ParsedFrontmatter = { - id: "", - provides: [], - requires: [], - key_files: [], - key_decisions: [], - patterns_established: [], - }; - - // Extract frontmatter block between --- markers - const fmMatch = raw.match(/^---\r?\n([\s\S]*?)\r?\n---/); - if (!fmMatch) return result; - - const fmBlock = fmMatch[1]; - const lines = fmBlock.split(/\r?\n/); - - let currentKey: string | null = null; - - for (const line of lines) { - // Scalar value: key: value - const scalarMatch = line.match(/^(\w[\w_]*):\s*(.+)$/); - if (scalarMatch) { - const [, key, value] = scalarMatch; - currentKey = key; - setScalar(result, key, value.trim()); - continue; - } - - // Array-start key with empty value: key:\n or key: []\n - const arrayStartMatch = line.match(/^(\w[\w_]*):\s*(\[\])?\s*$/); - if (arrayStartMatch) { - currentKey = arrayStartMatch[1]; - continue; - } - - // Array item: - value - const itemMatch = line.match(/^\s+-\s+(.+)$/); - if (itemMatch && currentKey) { - pushItem(result, currentKey, itemMatch[1].trim()); - continue; - } - } - - return result; -} - -function setScalar(fm: ParsedFrontmatter, key: string, value: string): void { - if (key === "id") fm.id = value; -} - -function pushItem(fm: ParsedFrontmatter, key: string, value: string): void { - switch (key) { - case "provides": fm.provides.push(value); break; - case "requires": fm.requires.push(value); break; - case "key_files": fm.key_files.push(value); break; - case "key_decisions": fm.key_decisions.push(value); break; - case "patterns_established": fm.patterns_established.push(value); break; - } -} - -// ─── Body parsing ──────────────────────────────────────────────────────────── - -function extractTitleAndOneLiner(body: string): { id: string; oneLiner: string } { - const lines = body.split(/\r?\n/); - let titleId = ""; - let oneLiner = ""; - let foundTitle = false; - - for (const line of lines) { - const titleMatch = line.match(/^#\s+(\S+):\s*(.*)$/); - if (titleMatch && !foundTitle) { - titleId = titleMatch[1]; - // If the title line itself has text after "S01: ", use that as a fallback - if (titleMatch[2].trim()) { - oneLiner = titleMatch[2].trim(); - } - foundTitle = true; - continue; - } - - // First non-empty line after the title is the one-liner - if (foundTitle && !oneLiner && line.trim() && !line.startsWith("#")) { - oneLiner = line.trim(); - break; - } - } - - return { id: titleId, oneLiner }; -} - -function getBodyAfterFrontmatter(raw: string): string { - const fmMatch = raw.match(/^---\r?\n[\s\S]*?\r?\n---\r?\n?/); - if (fmMatch) { - return raw.slice(fmMatch[0].length); - } - return raw; -} - -// ─── Public API ────────────────────────────────────────────────────────────── - -/** - * Distill a single SUMMARY.md content string into a compact structured block. - */ -export function distillSingle(summary: string): string { - const fm = parseFrontmatter(summary); - const body = getBodyAfterFrontmatter(summary); - const { id: titleId, oneLiner } = extractTitleAndOneLiner(body); - - const id = fm.id || titleId || "???"; - - return formatEntry({ - id, - oneLiner, - provides: fm.provides, - requires: fm.requires, - key_files: fm.key_files, - key_decisions: fm.key_decisions, - patterns: fm.patterns_established, - }); -} - -function formatEntry(entry: DistilledEntry): string { - return formatEntryWithDropLevel(entry, 0); -} - -/** - * Format an entry, progressively dropping fields based on dropLevel: - * 0 = full output - * 1 = drop patterns - * 2 = drop patterns + key_decisions - * 3 = drop patterns + key_decisions + key_files - */ -function formatEntryWithDropLevel(entry: DistilledEntry, dropLevel: number): string { - const lines: string[] = []; - lines.push(`## ${entry.id}: ${entry.oneLiner}`); - - if (entry.provides.length > 0) { - lines.push(`provides: ${entry.provides.join(", ")}`); - } - if (entry.requires.length > 0) { - lines.push(`requires: ${entry.requires.join(", ")}`); - } - if (dropLevel < 3 && entry.key_files.length > 0) { - lines.push(`key_files: ${entry.key_files.join(", ")}`); - } - if (dropLevel < 2 && entry.key_decisions.length > 0) { - lines.push(`key_decisions: ${entry.key_decisions.join(", ")}`); - } - if (dropLevel < 1 && entry.patterns.length > 0) { - lines.push(`patterns: ${entry.patterns.join(", ")}`); - } - - return lines.join("\n"); -} - -/** - * Distill multiple SUMMARY.md contents into a budget-constrained output. - */ -export function distillSummaries(summaries: string[], budgetChars: number): DistillationResult { - const originalChars = summaries.reduce((sum, s) => sum + s.length, 0); - - if (summaries.length === 0) { - return { - content: "", - summaryCount: 0, - savingsPercent: 0, - originalChars: 0, - distilledChars: 0, - }; - } - - // Parse all entries up front - const entries: DistilledEntry[] = summaries.map((summary) => { - const fm = parseFrontmatter(summary); - const body = getBodyAfterFrontmatter(summary); - const { id: titleId, oneLiner } = extractTitleAndOneLiner(body); - return { - id: fm.id || titleId || "???", - oneLiner, - provides: fm.provides, - requires: fm.requires, - key_files: fm.key_files, - key_decisions: fm.key_decisions, - patterns: fm.patterns_established, - }; - }); - - // Try progressively more aggressive dropping until it fits - for (let dropLevel = 0; dropLevel <= 3; dropLevel++) { - const blocks = entries.map((e) => formatEntryWithDropLevel(e, dropLevel)); - const content = blocks.join("\n\n"); - if (content.length <= budgetChars) { - const distilledChars = content.length; - return { - content, - summaryCount: summaries.length, - savingsPercent: originalChars > 0 - ? Math.round((1 - distilledChars / originalChars) * 100) - : 0, - originalChars, - distilledChars, - }; - } - } - - // Even at max drop level it doesn't fit — truncate - const blocks = entries.map((e) => formatEntryWithDropLevel(e, 3)); - let content = blocks.join("\n\n"); - if (content.length > budgetChars) { - content = content.slice(0, Math.max(0, budgetChars - 15)) + "\n[...truncated]"; - } - - const distilledChars = content.length; - return { - content, - summaryCount: summaries.length, - savingsPercent: originalChars > 0 - ? Math.round((1 - distilledChars / originalChars) * 100) - : 0, - originalChars, - distilledChars, - }; -} diff --git a/src/resources/extensions/gsd/tests/context-compression.test.ts b/src/resources/extensions/gsd/tests/context-compression.test.ts deleted file mode 100644 index 4d0d4010d..000000000 --- a/src/resources/extensions/gsd/tests/context-compression.test.ts +++ /dev/null @@ -1,193 +0,0 @@ -/** - * Context Compression — unit tests for M004/S02. - * - * Verifies that prompt builders respect inlineLevel parameter by - * inspecting the auto-prompts.ts source for level-aware gating. - * Cannot call builders directly due to @gsd/pi-coding-agent import - * resolution — uses source-level structural verification instead. - */ - -import test from "node:test"; -import assert from "node:assert/strict"; -import { readFileSync } from "node:fs"; -import { join, dirname } from "node:path"; -import { fileURLToPath } from "node:url"; - -const __dirname = dirname(fileURLToPath(import.meta.url)); -const promptsSrc = readFileSync(join(__dirname, "..", "auto-prompts.ts"), "utf-8"); - -// ═══════════════════════════════════════════════════════════════════════════ -// inlineLevel Parameter Presence -// ═══════════════════════════════════════════════════════════════════════════ - -const BUILDERS_WITH_LEVEL = [ - "buildPlanMilestonePrompt", - "buildPlanSlicePrompt", - "buildExecuteTaskPrompt", - "buildCompleteSlicePrompt", - "buildCompleteMilestonePrompt", - "buildReassessRoadmapPrompt", -]; - -for (const builder of BUILDERS_WITH_LEVEL) { - test(`compression: ${builder} accepts inlineLevel parameter`, () => { - // Find the function signature - const sigRegex = new RegExp(`export async function ${builder}\\([^)]*level\\?: InlineLevel`); - assert.ok( - sigRegex.test(promptsSrc), - `${builder} should have level?: InlineLevel parameter`, - ); - }); -} - -// ═══════════════════════════════════════════════════════════════════════════ -// Default Level Resolution -// ═══════════════════════════════════════════════════════════════════════════ - -test("compression: builders default to resolveInlineLevel() when no level passed", () => { - const defaultPattern = /const inlineLevel = level \?\? resolveInlineLevel\(\)/g; - const matches = promptsSrc.match(defaultPattern); - assert.ok(matches, "should have resolveInlineLevel() fallback"); - assert.ok( - matches.length >= BUILDERS_WITH_LEVEL.length, - `should have ${BUILDERS_WITH_LEVEL.length} fallback instances, found ${matches?.length}`, - ); -}); - -// ═══════════════════════════════════════════════════════════════════════════ -// Minimal Level — Template Reduction -// ═══════════════════════════════════════════════════════════════════════════ - -test("compression: buildExecuteTaskPrompt minimal drops decisions template", () => { - // In the execute-task builder, minimal should only inline task-summary, not decisions - assert.ok( - promptsSrc.includes('inlineLevel === "minimal"') && - promptsSrc.includes('inlineTemplate("task-summary"'), - "execute-task should conditionally include decisions template based on level", - ); -}); - -test("compression: buildExecuteTaskPrompt minimal truncates prior summaries", () => { - assert.ok( - promptsSrc.includes('inlineLevel === "minimal" && priorSummaries.length > 1'), - "execute-task should limit prior summaries for minimal level", - ); -}); - -test("compression: buildExecuteTaskPrompt passes verificationBudget to loadPrompt (#707)", () => { - // The execute-task template declares {{verificationBudget}} — the builder must supply it - assert.ok( - promptsSrc.includes("verificationBudget"), - "buildExecuteTaskPrompt should pass verificationBudget in the loadPrompt vars object", - ); - // Verify it computes the budget from computeBudgets - assert.ok( - promptsSrc.includes("computeBudgets(contextWindow)"), - "buildExecuteTaskPrompt should compute budgets from the executor context window", - ); -}); - -test("compression: buildPlanMilestonePrompt minimal drops project/requirements/decisions files", () => { - // The plan-milestone builder should gate root file inlining on inlineLevel - assert.ok( - promptsSrc.includes('inlineLevel !== "minimal"') && - promptsSrc.includes("inlineProjectFromDb(base)"), - "plan-milestone should conditionally include project.md based on level", - ); -}); - -test("compression: buildPlanMilestonePrompt minimal drops extra templates", () => { - // Full inlines 5 templates, minimal should inline fewer - assert.ok( - promptsSrc.includes('if (inlineLevel === "full")') && - promptsSrc.includes('inlineTemplate("secrets-manifest"'), - "plan-milestone should only include secrets-manifest template at full level", - ); -}); - -// ═══════════════════════════════════════════════════════════════════════════ -// Complete-Slice Level Gating -// ═══════════════════════════════════════════════════════════════════════════ - -test("compression: buildCompleteSlicePrompt minimal drops requirements", () => { - // Find the complete-slice section and verify requirements gating - const completeSliceIdx = promptsSrc.indexOf("buildCompleteSlicePrompt"); - const nextBuilder = promptsSrc.indexOf("buildCompleteMilestonePrompt"); - const completeSliceBlock = promptsSrc.slice(completeSliceIdx, nextBuilder); - assert.ok( - completeSliceBlock.includes('inlineLevel !== "minimal"'), - "complete-slice should gate requirements inlining on level", - ); -}); - -test("compression: buildCompleteSlicePrompt minimal drops UAT template", () => { - const completeSliceIdx = promptsSrc.indexOf("buildCompleteSlicePrompt"); - const nextBuilder = promptsSrc.indexOf("buildCompleteMilestonePrompt"); - const completeSliceBlock = promptsSrc.slice(completeSliceIdx, nextBuilder); - assert.ok( - completeSliceBlock.includes('inlineLevel !== "minimal"') && - completeSliceBlock.includes('inlineTemplate("uat"'), - "complete-slice should conditionally include UAT template based on level", - ); -}); - -// ═══════════════════════════════════════════════════════════════════════════ -// Complete-Milestone Level Gating -// ═══════════════════════════════════════════════════════════════════════════ - -test("compression: buildCompleteMilestonePrompt minimal drops root GSD files", () => { - const completeMilestoneIdx = promptsSrc.indexOf("buildCompleteMilestonePrompt"); - const nextBuilder = promptsSrc.indexOf("buildReplanSlicePrompt"); - const block = promptsSrc.slice(completeMilestoneIdx, nextBuilder); - assert.ok( - block.includes('inlineLevel !== "minimal"') && - (block.includes('inlineGsdRootFile(base, "requirements.md"') || block.includes('inlineRequirementsFromDb(base')), - "complete-milestone should gate root file inlining on level", - ); -}); - -// ═══════════════════════════════════════════════════════════════════════════ -// Reassess-Roadmap Level Gating -// ═══════════════════════════════════════════════════════════════════════════ - -test("compression: buildReassessRoadmapPrompt minimal drops project/requirements/decisions", () => { - const reassessIdx = promptsSrc.indexOf("buildReassessRoadmapPrompt"); - const block = promptsSrc.slice(reassessIdx, reassessIdx + 1500); - assert.ok( - block.includes('inlineLevel !== "minimal"'), - "reassess-roadmap should gate file inlining on level", - ); -}); - -// ═══════════════════════════════════════════════════════════════════════════ -// Full Level — No Regression -// ═══════════════════════════════════════════════════════════════════════════ - -test("compression: full level preserves all templates and files (no regression)", () => { - // Verify the key template names are still present in the source - const expectedTemplates = [ - "roadmap", "decisions", "plan", "task-plan", "secrets-manifest", - "task-summary", "slice-summary", "uat", "milestone-summary", - ]; - for (const tpl of expectedTemplates) { - assert.ok( - promptsSrc.includes(`inlineTemplate("${tpl}"`), - `template "${tpl}" should still be present in auto-prompts.ts`, - ); - } -}); - -// ═══════════════════════════════════════════════════════════════════════════ -// Import Verification -// ═══════════════════════════════════════════════════════════════════════════ - -test("compression: auto-prompts.ts imports resolveInlineLevel and InlineLevel", () => { - assert.ok( - promptsSrc.includes("resolveInlineLevel"), - "should import resolveInlineLevel from preferences", - ); - assert.ok( - promptsSrc.includes("InlineLevel"), - "should import InlineLevel type from types", - ); -}); diff --git a/src/resources/extensions/gsd/tests/preferences.test.ts b/src/resources/extensions/gsd/tests/preferences.test.ts index 52080fbb8..2fae2652e 100644 --- a/src/resources/extensions/gsd/tests/preferences.test.ts +++ b/src/resources/extensions/gsd/tests/preferences.test.ts @@ -208,30 +208,25 @@ test("git fields comprehensive validation", () => { assert.equal(preferences.git?.isolation, "branch"); }); -test("auto_visualize, auto_report, compression_strategy, context_selection validate correctly", () => { +test("auto_visualize, auto_report, context_selection validate correctly", () => { const { preferences, errors } = validatePreferences({ auto_visualize: true, auto_report: false, - compression_strategy: "compress", context_selection: "smart", }); assert.equal(errors.length, 0); assert.equal(preferences.auto_visualize, true); assert.equal(preferences.auto_report, false); - assert.equal(preferences.compression_strategy, "compress"); assert.equal(preferences.context_selection, "smart"); }); -test("auto_visualize, auto_report, compression_strategy, context_selection reject invalid values", () => { +test("auto_visualize, auto_report, context_selection reject invalid values", () => { const { errors: e1 } = validatePreferences({ auto_visualize: "yes" as never }); assert.ok(e1.some(e => e.includes("auto_visualize"))); const { errors: e2 } = validatePreferences({ auto_report: 1 as never }); assert.ok(e2.some(e => e.includes("auto_report"))); - const { errors: e3 } = validatePreferences({ compression_strategy: "shrink" as never }); - assert.ok(e3.some(e => e.includes("compression_strategy"))); - const { errors: e4 } = validatePreferences({ context_selection: "partial" as never }); assert.ok(e4.some(e => e.includes("context_selection"))); }); diff --git a/src/resources/extensions/gsd/tests/prompt-compressor.test.ts b/src/resources/extensions/gsd/tests/prompt-compressor.test.ts deleted file mode 100644 index 36f99b4f8..000000000 --- a/src/resources/extensions/gsd/tests/prompt-compressor.test.ts +++ /dev/null @@ -1,529 +0,0 @@ -import test from "node:test"; -import assert from "node:assert/strict"; - -import { - compressPrompt, - compressToTarget, -} from "../prompt-compressor.js"; -import type { - CompressionLevel, - CompressionResult, - CompressionOptions, -} from "../prompt-compressor.js"; - -// ─── Test Fixtures ────────────────────────────────────────────────────────── - -const WHITESPACE_HEAVY = `# Section One - -Some content here. - - - -Another paragraph here. - - -Yet another paragraph. - - - -# Section Two - -More content.`; - -const MARKDOWN_COMMENTS = `# Title - - - -Some content here. - - - -More content.`; - -const HORIZONTAL_RULES = `# Section One - -Some content. - ---- - -# Section Two - -More content. - -*** - -# Section Three - -Final content.`; - -const VERBOSE_PROSE = `In order to implement this feature, it is important to note that the following -requirements must be met. Due to the fact that the system operates in real-time, -prior to deployment we need to verify all components. In addition to the main -module, a number of auxiliary services are required. In the event that a service -fails, subsequent to the failure, the system should recover. For the purpose of -monitoring, in accordance with our SLA, with regard to uptime, at this point in -time we achieve 99.9%. On the basis of recent data, in the case of peak traffic, -as mentioned previously, the system scales automatically.`; - -const BOILERPLATE_CONTENT = `# Requirements - -## Feature A -Must support pagination. - -## Feature B -N/A - -## Feature C -(none) - -## Feature D -(empty) - -## Feature E -(not applicable) - -## Feature F -Must handle errors gracefully.`; - -const DUPLICATE_LINES = `Status: active -Status: active -Status: active -Priority: high -Name: test project -Name: test project`; - -const EMPHASIS_CONTENT = `This is **bold text** and this is *italic text*. -Also __underline bold__ and _underline italic_. -Check [this link](https://example.com) and [another](https://test.org).`; - -const CODE_BLOCK_CONTENT = `# Setup Guide - -In order to configure the system, run the following command: - -\`\`\`typescript -const config = { - debug: true, - verbose: false, - timeout: 3000, -}; -\`\`\` - -Due to the fact that configuration is loaded at startup, prior to -running the application, verify the config file exists. - -\`\`\`bash -ls -la config.json -\`\`\` - -The following steps complete the setup.`; - -const HEADING_CONTENT = `# Main Title - -## Subsection A - -In order to do something, the following steps are needed. - -## Subsection B - -More content here with **emphasis** and [a link](https://example.com). - -### Sub-subsection - -Details here.`; - -const REALISTIC_GSD_CONTENT = `# Project: GSD Task Manager - - - -## Decisions - -| Decision ID | Title | Status | Date | -|---------------|------------------------------|------------|--------------| -| DEC-001 | Use TypeScript | Approved | 2024-01-15 | -| DEC-002 | Adopt monorepo | Approved | 2024-01-20 | -| DEC-003 | Use node:test | Approved | 2024-02-01 | - -## Requirements - -### Must-Have - -In order to support the core workflow, it is important to note that the following -requirements are non-negotiable. Due to the fact that the system must operate in -CI environments, prior to any release, all tests must pass. - -- The system must handle concurrent operations -- The system must handle concurrent operations -- Error recovery must be automatic -- Configuration must be file-based -- Configuration must be file-based - -### Nice-to-Have - -N/A - -### Out of Scope - -(none) - ---- - -## Implementation Notes - -> In accordance with our coding standards, all modules should follow -> the single responsibility principle. With regard to testing, a number of -> integration tests should supplement unit tests. - -For the purpose of maintaining code quality, at this point in time we require -100% branch coverage on critical paths. In the event that coverage drops below -the threshold, subsequent to the detection, the CI pipeline should fail. - -**Important**: The following constraints apply: -- *Memory usage* must stay under 512MB -- *CPU usage* must not exceed 80% sustained -- Response times under 100ms for the 95th percentile - -In addition to the above, the system should support plugin architecture. -As mentioned previously, this was decided in DEC-001. - ---- - -## Status - -Status: active -Status: active -Priority: high -Sprint: 14 -Sprint: 14 -Milestone: v2.1.0`; - -const LONG_LINE = "This is a very long line that goes on and on. It contains multiple sentences that discuss various topics. The purpose of this line is to test the truncation functionality. When lines exceed 300 characters, they should be truncated at a sentence boundary. This ensures that the compressed output remains readable. Additional text is added here to make sure we exceed the 300 character limit for testing purposes. Even more text follows to pad the line further."; - -const BLOCKQUOTE_CONTENT = `> This is a blockquote -> with multiple lines -> that should have markers removed. - -Normal paragraph here. - -> Another blockquote.`; - -const BULLET_LIST = `Some intro text: - -- First item in the list -- Second item in the list -* Third item with star -+ Fourth item with plus -1. Numbered item one -2. Numbered item two - -Closing text.`; - -// ─── Light Compression Tests ──────────────────────────────────────────────── - -test("light compression removes extra whitespace", () => { - const result = compressPrompt(WHITESPACE_HEAVY, { level: "light" }); - assert.ok(result.compressedChars < result.originalChars, "should reduce size"); - assert.ok(!result.content.includes(" \n"), "should not have trailing spaces"); - // Should not have 3+ consecutive blank lines - assert.ok(!result.content.match(/\n\s*\n\s*\n\s*\n/), "should not have 3+ blank lines"); - assert.equal(result.level, "light"); -}); - -test("light compression removes markdown comments", () => { - const result = compressPrompt(MARKDOWN_COMMENTS, { level: "light" }); - assert.ok(!result.content.includes(""), "should not contain comment end"); - assert.ok(result.content.includes("# Title"), "should preserve heading"); - assert.ok(result.content.includes("Some content here."), "should preserve normal content"); -}); - -test("light compression removes horizontal rules", () => { - const result = compressPrompt(HORIZONTAL_RULES, { level: "light" }); - assert.ok(!result.content.match(/^---$/m), "should not contain ---"); - assert.ok(!result.content.match(/^\*\*\*$/m), "should not contain ***"); - assert.ok(result.content.includes("# Section One"), "should preserve headings"); - assert.ok(result.content.includes("# Section Two"), "should preserve headings"); -}); - -test("light compression preserves code blocks", () => { - const result = compressPrompt(CODE_BLOCK_CONTENT, { level: "light" }); - assert.ok(result.content.includes("const config = {"), "should preserve code block content"); - assert.ok(result.content.includes("```typescript"), "should preserve code fence"); - assert.ok(result.content.includes("```bash"), "should preserve code fence"); -}); - -// ─── Moderate Compression Tests ───────────────────────────────────────────── - -test("moderate compression abbreviates verbose phrases", () => { - const result = compressPrompt(VERBOSE_PROSE, { level: "moderate" }); - assert.ok(result.content.includes("To implement"), "should abbreviate 'In order to'"); - assert.ok(result.content.includes("Because"), "should abbreviate 'Due to the fact that'"); - assert.ok(result.content.includes("Before deployment"), "should abbreviate 'Prior to'"); - assert.ok(result.content.includes("Also,"), "should abbreviate 'In addition to'"); - assert.ok(result.content.includes("Several"), "should abbreviate 'A number of'"); - assert.ok(result.content.includes("If"), "should abbreviate 'In the event that'"); - assert.ok(result.content.includes("After"), "should abbreviate 'Subsequent to'"); - assert.ok(!result.content.includes("For the purpose of"), "should abbreviate 'For the purpose of'"); - assert.ok(result.content.includes("Per"), "should abbreviate 'In accordance with'"); - assert.ok(result.content.includes("Re:"), "should abbreviate 'With regard to'"); - assert.ok(result.content.includes("Now"), "should abbreviate 'At this point in time'"); - assert.ok(result.content.includes("Based on"), "should abbreviate 'On the basis of'"); - assert.ok(result.content.includes("(see above)"), "should abbreviate 'As mentioned previously'"); - assert.ok(result.compressedChars < result.originalChars, "should reduce size"); -}); - -test("moderate compression deduplicates consecutive lines", () => { - const input = "Line one\nLine one\nLine one\nLine two\nLine three\nLine three"; - const result = compressPrompt(input, { level: "moderate" }); - const lines = result.content.split("\n").filter((l) => l.trim() !== ""); - // Count occurrences of "Line one" - const lineOneCount = lines.filter((l) => l === "Line one").length; - assert.equal(lineOneCount, 1, "should deduplicate 'Line one'"); - const lineThreeCount = lines.filter((l) => l === "Line three").length; - assert.equal(lineThreeCount, 1, "should deduplicate 'Line three'"); -}); - -test("moderate compression removes boilerplate", () => { - const result = compressPrompt(BOILERPLATE_CONTENT, { level: "moderate" }); - assert.ok(!result.content.match(/^\s*N\/A\s*$/m), "should remove N/A lines"); - assert.ok(!result.content.includes("(none)"), "should remove (none)"); - assert.ok(!result.content.includes("(empty)"), "should remove (empty)"); - assert.ok(!result.content.includes("(not applicable)"), "should remove (not applicable)"); - assert.ok(result.content.includes("Must support pagination"), "should keep real content"); - assert.ok(result.content.includes("Must handle errors"), "should keep real content"); -}); - -test("moderate compression collapses table formatting", () => { - const table = `| Name | Value | Status | -| foo | bar | active |`; - const result = compressPrompt(table, { level: "moderate" }); - // Should have reduced padding - assert.ok(result.compressedChars < result.originalChars, "should reduce table padding"); -}); - -// ─── Aggressive Compression Tests ─────────────────────────────────────────── - -test("aggressive compression removes emphasis and links", () => { - const result = compressPrompt(EMPHASIS_CONTENT, { level: "aggressive" }); - assert.ok(!result.content.includes("**"), "should remove bold markers"); - assert.ok(!result.content.includes("__"), "should remove underline bold markers"); - assert.ok(result.content.includes("bold text"), "should keep bold text content"); - assert.ok(result.content.includes("italic text"), "should keep italic text content"); - assert.ok(result.content.includes("this link"), "should keep link text"); - assert.ok(!result.content.includes("https://example.com"), "should remove link URLs"); - assert.ok(!result.content.includes("https://test.org"), "should remove link URLs"); -}); - -test("aggressive compression removes bullet markers", () => { - const result = compressPrompt(BULLET_LIST, { level: "aggressive" }); - assert.ok(!result.content.match(/^- /m), "should remove dash bullets"); - assert.ok(!result.content.match(/^\* /m), "should remove star bullets"); - assert.ok(!result.content.match(/^\+ /m), "should remove plus bullets"); - assert.ok(!result.content.match(/^\d+\. /m), "should remove numbered bullets"); - assert.ok(result.content.includes("First item"), "should keep bullet content"); - assert.ok(result.content.includes("Numbered item"), "should keep numbered content"); -}); - -test("aggressive compression removes blockquote markers", () => { - const result = compressPrompt(BLOCKQUOTE_CONTENT, { level: "aggressive" }); - assert.ok(!result.content.match(/^> /m), "should remove blockquote markers"); - assert.ok(result.content.includes("This is a blockquote"), "should keep blockquote content"); - assert.ok(result.content.includes("Normal paragraph"), "should keep normal content"); -}); - -test("aggressive compression truncates long lines", () => { - const result = compressPrompt(LONG_LINE, { level: "aggressive" }); - const lines = result.content.split("\n"); - for (const line of lines) { - assert.ok(line.length <= 300, `line should be <= 300 chars, got ${line.length}`); - } -}); - -test("aggressive compression deduplicates structural patterns", () => { - const result = compressPrompt(DUPLICATE_LINES, { level: "aggressive" }); - const lines = result.content.split("\n").filter((l) => l.trim() !== ""); - const statusCount = lines.filter((l) => l.includes("Status: active")).length; - assert.equal(statusCount, 1, "should keep only one Status: active"); - const nameCount = lines.filter((l) => l.includes("Name: test project")).length; - assert.equal(nameCount, 1, "should keep only one Name: test project"); -}); - -// ─── Preservation Tests ───────────────────────────────────────────────────── - -test("code block preservation protects code from compression", () => { - const result = compressPrompt(CODE_BLOCK_CONTENT, { - level: "aggressive", - preserveCodeBlocks: true, - }); - // Code blocks should be untouched - assert.ok(result.content.includes("const config = {"), "code block preserved"); - assert.ok(result.content.includes("debug: true,"), "code block details preserved"); - assert.ok(result.content.includes("ls -la config.json"), "bash code block preserved"); - // But surrounding prose should be compressed - assert.ok(!result.content.includes("In order to"), "prose should be compressed"); - assert.ok(!result.content.includes("Due to the fact that"), "prose should be compressed"); -}); - -test("code block preservation can be disabled", () => { - const result = compressPrompt(CODE_BLOCK_CONTENT, { - level: "aggressive", - preserveCodeBlocks: false, - }); - // Phrase abbreviation still works on surrounding text - assert.ok(result.compressedChars < result.originalChars, "should still compress"); -}); - -test("heading preservation keeps headings intact", () => { - const result = compressPrompt(HEADING_CONTENT, { - level: "aggressive", - preserveHeadings: true, - }); - assert.ok(result.content.includes("# Main Title"), "should preserve h1"); - assert.ok(result.content.includes("## Subsection A"), "should preserve h2"); - assert.ok(result.content.includes("## Subsection B"), "should preserve h2"); - assert.ok(result.content.includes("### Sub-subsection"), "should preserve h3"); -}); - -// ─── compressToTarget Tests ───────────────────────────────────────────────── - -test("compressToTarget tries progressively harder levels", () => { - // Set a target that light compression cannot reach - const lightResult = compressPrompt(REALISTIC_GSD_CONTENT, { level: "light" }); - const moderateResult = compressPrompt(REALISTIC_GSD_CONTENT, { level: "moderate" }); - - // Target between light and moderate results - const target = Math.floor((lightResult.compressedChars + moderateResult.compressedChars) / 2); - const result = compressToTarget(REALISTIC_GSD_CONTENT, target); - - // Should have used at least moderate - assert.ok( - result.level === "moderate" || result.level === "aggressive", - `should use moderate or aggressive, got ${result.level}`, - ); - assert.ok(result.compressedChars <= target, "should meet target"); -}); - -test("compressToTarget returns best effort when target unreachable", () => { - // Set an impossibly small target - const result = compressToTarget(REALISTIC_GSD_CONTENT, 10); - assert.equal(result.level, "aggressive", "should try aggressive as last resort"); - assert.ok(result.compressedChars > 10, "cannot reach impossibly small target"); - assert.ok( - result.compressedChars < REALISTIC_GSD_CONTENT.length, - "should still compress as much as possible", - ); -}); - -test("compressToTarget returns unchanged if already under target", () => { - const result = compressToTarget("short text", 1000); - assert.equal(result.content, "short text"); - assert.equal(result.savingsPercent, 0); - assert.equal(result.transformationsApplied, 0); -}); - -// ─── Realistic GSD Content Test ───────────────────────────────────────────── - -test("realistic GSD content compresses significantly", () => { - const result = compressPrompt(REALISTIC_GSD_CONTENT, { level: "aggressive" }); - - // Should achieve meaningful compression - assert.ok(result.savingsPercent > 15, `should achieve >15% savings, got ${result.savingsPercent}%`); - assert.ok(result.transformationsApplied > 3, "should apply multiple transformations"); - - // Key content preserved - assert.ok(result.content.includes("# Project: GSD Task Manager"), "title preserved"); - assert.ok(result.content.includes("DEC-001"), "decision IDs preserved"); - assert.ok(result.content.includes("TypeScript"), "decision content preserved"); - assert.ok(result.content.includes("## Decisions"), "section headings preserved"); - assert.ok(result.content.includes("## Requirements"), "section headings preserved"); - - // Comments removed - assert.ok(!result.content.includes("", - "", - "| # | When Context | Scope | Decision | Choice | Rationale | Revisable? |", - "|--------|----------------|-----------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|------------|", - ]; - for (const d of decisions) { - lines.push( - `| ${d.id.padEnd(6)} | ${d.when_context.padEnd(14)} | ${d.scope.padEnd(15)} | ${d.decision.padEnd(160)} | ${d.choice.padEnd(160)} | ${d.rationale.padEnd(160)} | ${d.revisable.padEnd(10)} |`, - ); - } - return lines.join("\n"); -} - -// --------------------------------------------------------------------------- -// Fixture: Markdown format for requirements (baseline) -// --------------------------------------------------------------------------- - -function formatRequirementsAsMarkdown( - requirements: ReturnType, -): string { - const lines: string[] = ["# Requirements", "", "## Active", ""]; - for (const r of requirements) { - lines.push(`### ${r.id} -- ${r.description}`); - lines.push(""); - lines.push(`- Class: ${r.class}`); - lines.push(`- Status: ${r.status}`); - lines.push(`- Why it matters: ${r.why}`); - lines.push(`- Primary owning slice: ${r.primary_owner}`); - lines.push(`- Validation: ${r.validation}`); - lines.push(""); - } - return lines.join("\n"); -} - -// --------------------------------------------------------------------------- -// Fixture: Realistic TypeScript code file (200+ lines, 8+ functions) -// --------------------------------------------------------------------------- - -const SAMPLE_CODE = `import { readFileSync, writeFileSync, existsSync } from "node:fs"; -import { join, resolve, dirname } from "node:path"; -import { createHash } from "node:crypto"; - -// ---- Types ---- - -interface Config { - basePath: string; - maxRetries: number; - timeout: number; - logLevel: "debug" | "info" | "warn" | "error"; - database: { - host: string; - port: number; - name: string; - poolSize: number; - }; -} - -interface User { - id: string; - email: string; - role: "admin" | "editor" | "viewer"; - createdAt: Date; - lastLogin: Date | null; -} - -interface AuthToken { - token: string; - userId: string; - expiresAt: Date; - scopes: string[]; -} - -interface LogEntry { - timestamp: Date; - level: string; - message: string; - context: Record; -} - -interface DatabaseConnection { - query(sql: string, params?: unknown[]): Promise; - execute(sql: string, params?: unknown[]): Promise<{ affectedRows: number }>; - close(): Promise; -} - -// ---- Config Module ---- - -export function loadConfig(path: string): Config { - if (!existsSync(path)) { - throw new Error(\`Config file not found: \${path}\`); - } - const raw = readFileSync(path, "utf-8"); - const parsed = JSON.parse(raw); - return validateConfig(parsed); -} - -export function validateConfig(config: unknown): Config { - if (typeof config !== "object" || config === null) { - throw new Error("Config must be a non-null object"); - } - const c = config as Record; - if (typeof c.basePath !== "string" || !c.basePath) { - throw new Error("Config.basePath must be a non-empty string"); - } - if (typeof c.maxRetries !== "number" || c.maxRetries < 0) { - throw new Error("Config.maxRetries must be a non-negative number"); - } - if (typeof c.timeout !== "number" || c.timeout <= 0) { - throw new Error("Config.timeout must be a positive number"); - } - return c as unknown as Config; -} - -export function mergeConfigs(base: Config, overrides: Partial): Config { - return { - ...base, - ...overrides, - database: { - ...base.database, - ...(overrides.database ?? {}), - }, - }; -} - -// ---- Database Module ---- - -export async function connectDatabase(config: Config): Promise { - const db = config.database; - const connectionString = \`\${db.host}:\${db.port}/\${db.name}\`; - let connected = false; - let attempts = 0; - - while (!connected && attempts < config.maxRetries) { - try { - attempts++; - // Simulated connection logic - connected = true; - } catch (err) { - if (attempts >= config.maxRetries) { - throw new Error(\`Failed to connect to \${connectionString} after \${attempts} attempts\`); - } - await new Promise((resolve) => setTimeout(resolve, 1000 * attempts)); - } - } - - return { - async query(sql: string, params?: unknown[]): Promise { - return []; - }, - async execute(sql: string, params?: unknown[]): Promise<{ affectedRows: number }> { - return { affectedRows: 0 }; - }, - async close(): Promise { - connected = false; - }, - }; -} - -export async function runMigrations(db: DatabaseConnection, migrationsDir: string): Promise { - const files = existsSync(migrationsDir) ? [] : []; - let applied = 0; - for (const file of files) { - const sql = readFileSync(join(migrationsDir, file), "utf-8"); - await db.execute(sql); - applied++; - } - return applied; -} - -// ---- Auth Module ---- - -export function hashPassword(password: string, salt: string): string { - return createHash("sha256") - .update(password + salt) - .digest("hex"); -} - -export function generateAuthToken(user: User, scopes: string[]): AuthToken { - const token = createHash("sha256") - .update(user.id + Date.now().toString() + Math.random().toString()) - .digest("hex"); - - return { - token, - userId: user.id, - expiresAt: new Date(Date.now() + 24 * 60 * 60 * 1000), - scopes, - }; -} - -export function validateAuthToken(token: AuthToken): boolean { - if (!token.token || token.token.length < 32) return false; - if (new Date() > token.expiresAt) return false; - if (!token.scopes || token.scopes.length === 0) return false; - return true; -} - -export function checkPermission(user: User, requiredRole: string): boolean { - const roleHierarchy: Record = { - viewer: 1, - editor: 2, - admin: 3, - }; - const userLevel = roleHierarchy[user.role] ?? 0; - const requiredLevel = roleHierarchy[requiredRole] ?? 999; - return userLevel >= requiredLevel; -} - -// ---- Logging Module ---- - -export function createLogger(config: Config) { - const levels: Record = { - debug: 0, - info: 1, - warn: 2, - error: 3, - }; - - const minLevel = levels[config.logLevel] ?? 1; - - return { - log(level: string, message: string, context: Record = {}): void { - if ((levels[level] ?? 0) < minLevel) return; - const entry: LogEntry = { - timestamp: new Date(), - level, - message, - context, - }; - console.error(JSON.stringify(entry)); - }, - debug(message: string, context?: Record): void { - this.log("debug", message, context); - }, - info(message: string, context?: Record): void { - this.log("info", message, context); - }, - warn(message: string, context?: Record): void { - this.log("warn", message, context); - }, - error(message: string, context?: Record): void { - this.log("error", message, context); - }, - }; -} - -// ---- Formatting Module ---- - -export function formatBytes(bytes: number): string { - if (bytes < 1024) return bytes + " B"; - if (bytes < 1024 * 1024) return (bytes / 1024).toFixed(1) + " KB"; - if (bytes < 1024 * 1024 * 1024) return (bytes / (1024 * 1024)).toFixed(1) + " MB"; - return (bytes / (1024 * 1024 * 1024)).toFixed(1) + " GB"; -} - -export function formatDuration(ms: number): string { - if (ms < 1000) return ms + "ms"; - if (ms < 60_000) return (ms / 1000).toFixed(1) + "s"; - const minutes = Math.floor(ms / 60_000); - const seconds = Math.floor((ms % 60_000) / 1000); - return minutes + "m " + seconds + "s"; -} - -export function truncateString(str: string, maxLen: number): string { - if (str.length <= maxLen) return str; - return str.slice(0, maxLen - 3) + "..."; -} - -// ---- Parsing Module ---- - -export function parseKeyValuePairs(input: string): Map { - const result = new Map(); - const lines = input.split("\\n"); - for (const line of lines) { - const idx = line.indexOf(":"); - if (idx > 0) { - const key = line.slice(0, idx).trim(); - const value = line.slice(idx + 1).trim(); - if (key && value) { - result.set(key, value); - } - } - } - return result; -} - -export function parseCSVLine(line: string): string[] { - const fields: string[] = []; - let current = ""; - let inQuotes = false; - for (const char of line) { - if (char === '"') { - inQuotes = !inQuotes; - } else if (char === "," && !inQuotes) { - fields.push(current.trim()); - current = ""; - } else { - current += char; - } - } - fields.push(current.trim()); - return fields; -} - -// ---- Utility Module ---- - -export function deepClone(obj: T): T { - return JSON.parse(JSON.stringify(obj)); -} - -export function debounce void>( - fn: T, - delayMs: number, -): (...args: Parameters) => void { - let timer: ReturnType | null = null; - return (...args: Parameters) => { - if (timer) clearTimeout(timer); - timer = setTimeout(() => fn(...args), delayMs); - }; -} - -export function retry( - fn: () => Promise, - maxAttempts: number, - delayMs: number, -): Promise { - return new Promise((resolve, reject) => { - let attempts = 0; - const attempt = async () => { - try { - attempts++; - const result = await fn(); - resolve(result); - } catch (err) { - if (attempts >= maxAttempts) { - reject(err); - } else { - setTimeout(attempt, delayMs); - } - } - }; - attempt(); - }); -} - -export function groupBy(items: T[], keyFn: (item: T) => string): Map { - const groups = new Map(); - for (const item of items) { - const key = keyFn(item); - const group = groups.get(key) ?? []; - group.push(item); - groups.set(key, group); - } - return groups; -} -`; - -// --------------------------------------------------------------------------- -// Fixture: Realistic SUMMARY.md contents (4 entries, 800-1200 chars each) -// --------------------------------------------------------------------------- - -function buildSummaries(): string[] { - return [ - `--- -id: S01 -provides: - - database-schema - - migration-engine - - connection-pool -requires: [] -key_files: - - src/db/schema.ts - - src/db/migrations/001-init.sql - - src/db/connection.ts - - src/db/pool.ts -key_decisions: - - D001 - - D004 -patterns_established: - - WAL-mode for all SQLite connections - - Migration files numbered sequentially - - Connection pool with 1 writer + N readers ---- - -# S01: Database Foundation - -This slice establishes the core database infrastructure used by all subsequent slices. -The SQLite database uses WAL mode for concurrent read access during background operations. - -## Implementation Details - -The schema defines tables for artifacts (decisions, requirements, tasks), metrics, -and session state. Each table includes created_at and updated_at timestamps with -automatic trigger-based updates. - -The migration engine supports forward-only migrations with checksum verification -to detect tampering. Each migration runs in a transaction with automatic rollback -on failure. - -## Testing Approach - -Integration tests use an in-memory SQLite database to avoid filesystem side effects. -Each test creates a fresh database, applies all migrations, and verifies the schema -matches expected structure. - -## Performance Characteristics - -Schema initialization takes approximately 5ms on modern hardware. Query latency -for typical operations (insert, select by ID, range scan) is under 1ms. The WAL -checkpoint runs automatically every 1000 pages or when the connection closes.`, - - `--- -id: S02 -provides: - - prompt-compressor - - token-counter - - context-budget -requires: - - database-schema -key_files: - - src/extensions/gsd/prompt-compressor.ts - - src/extensions/gsd/token-counter.ts - - src/extensions/gsd/context-budget.ts - - src/extensions/gsd/structured-data-formatter.ts -key_decisions: - - D002 - - D003 - - D005 -patterns_established: - - Deterministic compression with no LLM calls - - Three compression levels (light/moderate/aggressive) - - Provider-aware token estimation ---- - -# S02: Token Optimization Pipeline - -This slice implements the complete token optimization pipeline that reduces prompt -size while preserving semantic content. All transformations are deterministic and -require no external API calls. - -## Compression Strategy - -The pipeline applies transformations in order of increasing aggressiveness: -light (whitespace normalization, comment removal), moderate (phrase abbreviation, -boilerplate removal), and aggressive (emphasis removal, line truncation). - -Code blocks and markdown headings are preserved by default to maintain structural -readability for the LLM executor. - -## Budget Allocation - -Context budgets are computed proportionally from the executor model's context window. -Summaries receive 15%, inline context receives 40%, and verification sections receive -10%. The remaining 35% is reserved for the model's response generation. - -## Token Counting - -Token counts are estimated using provider-specific chars-per-token ratios: -Anthropic at 3.5, OpenAI at 4.0, Google at 4.0. When tiktoken is available, -exact counts replace estimates for OpenAI-compatible models.`, - - `--- -id: S03 -provides: - - semantic-chunker - - summary-distiller - - cache-optimizer -requires: - - prompt-compressor - - token-counter -key_files: - - src/extensions/gsd/semantic-chunker.ts - - src/extensions/gsd/summary-distiller.ts - - src/extensions/gsd/prompt-cache-optimizer.ts -key_decisions: - - D006 - - D007 -patterns_established: - - TF-IDF scoring for content relevance - - Progressive field dropping for budget compliance - - Static-first section ordering for cache efficiency ---- - -# S03: Advanced Context Selection - -This slice builds on the token optimization pipeline to provide intelligent content -selection and cache-aware prompt assembly. It includes semantic chunking for code -files, summary distillation for dependency context, and cache-optimized section ordering. - -## Semantic Chunking - -The chunker splits code files at semantic boundaries (function/class/interface -declarations) and scores each chunk against the task query using TF-IDF relevance. -Only the top-scoring chunks are included in the prompt, typically reducing code -context by 40-60%. - -## Summary Distillation - -SUMMARY.md files from dependency slices are distilled to their essential structured -data: provides, requires, key_files, and key_decisions. Verbose prose descriptions -are dropped to save context budget. Progressive field dropping ensures output fits -within any budget constraint. - -## Cache Optimization - -Prompt sections are classified as static (system prompt, templates), semi-static -(slice plan, decisions), or dynamic (task plan, file contents). Sections are reordered -to place static content first, maximizing the cacheable prefix length for both -Anthropic and OpenAI prompt caching strategies.`, - - `--- -id: S04 -provides: - - dispatch-pipeline - - task-routing - - verification-gate -requires: - - database-schema - - prompt-compressor - - semantic-chunker - - cache-optimizer -key_files: - - src/extensions/gsd/auto-dispatch.ts - - src/extensions/gsd/model-router.ts - - src/extensions/gsd/verification-gate.ts - - src/extensions/gsd/auto-supervisor.ts -key_decisions: - - D008 -patterns_established: - - Budget-aware dispatch with automatic compression - - Model routing based on task complexity - - Evidence-based verification before task completion ---- - -# S04: Dispatch Pipeline - -This slice implements the end-to-end dispatch pipeline that takes a task plan, -assembles an optimized prompt, routes it to the appropriate model, and verifies -the executor's output before marking the task complete. - -## Prompt Assembly - -The dispatch pipeline collects context from multiple sources: decisions and -requirements from the database, dependency summaries from prior slices, code -context from the workspace index, and task-specific instructions from the plan. -All content passes through the optimization pipeline before assembly. - -## Model Routing - -Tasks are routed to models based on complexity classification: simple tasks go -to smaller/faster models, complex tasks go to larger models with bigger context -windows. The router considers available context budget, estimated token usage, -and historical success rates for each model-task combination. - -## Verification - -Each completed task passes through a verification gate that checks for evidence -of completion: modified files, passing tests, and explicit verification commands -defined in the task plan. Tasks without sufficient evidence are flagged for -review rather than silently accepted.`, - ]; -} - -// --------------------------------------------------------------------------- -// Fixture: Verbose prompt content (5000+ chars) for compression benchmark -// --------------------------------------------------------------------------- - -function buildVerbosePrompt(): string { - return `# Executor Instructions - - - - - - ---- - -## Context and Background - - -In order to complete this task successfully, it is important to note that the system architecture follows a modular design pattern. The following sections describe the relevant context for your work. - -As mentioned previously, the database layer uses SQLite with WAL mode enabled. In addition to the database configuration, you should be aware of the caching strategy that has been implemented. - -Due to the fact that we need to maintain backward compatibility, all API changes must be additive. At this point in time, we do not support breaking changes to the public API surface. - -For the purpose of maintaining consistency, all new code should follow the established patterns documented in the architecture decision records. In the event that you encounter a conflict between patterns, prefer the most recent decision. - -With regard to testing, all new functionality must include unit tests with at least 80% branch coverage. Prior to submitting your changes, run the full test suite to verify no regressions. - -Subsequent to completing the implementation, update the SUMMARY.md file with any new patterns or decisions established during development. - - ---- - - -## Technical Requirements - -In accordance with the project standards, the implementation must satisfy the following requirements: - -(none) -N/A -(not applicable) -(empty) - -A number of performance constraints apply to this module. In the case of database operations, queries must complete within 10ms at the 95th percentile. On the basis of our load testing results, the system handles approximately 500 concurrent requests. - -In order to ensure proper error handling, all async functions must use try-catch blocks. In the event that an error occurs, it is important to note that the error should be logged before re-throwing. - -The following code patterns should be followed: - -\`\`\`typescript -// Always use strict null checks -interface Result { - data: T | null; - error: string | null; -} - -// Prefer explicit return types -export function processItem(item: unknown): Result { - if (!isValid(item)) { - return { data: null, error: "Invalid item format" }; - } - return { data: transform(item), error: null }; -} -\`\`\` - ---- - -## Dependencies - -- **Database module** (src/db/connection.ts): Provides connection pool management -- **Auth module** (src/auth/tokens.ts): Handles token validation and refresh -- **Logger** (src/utils/logger.ts): Structured logging with context propagation -- **Config module** (src/config/loader.ts): Configuration loading and validation - -> Note: The database module is currently being refactored as part of M002/S03. -> Use the stable API surface and avoid internal implementation details. -> In order to avoid breakage, do not import from internal paths. - ---- - -## Task Plan - -In order to implement the requested changes, you should follow these steps: - -1. Review the existing implementation in the target files -2. Implement the changes described in the task description -3. Write unit tests covering all new code paths -4. Update documentation if any public APIs change -5. Run the verification commands listed below - - - -## Carry-Forward Context - -In order to understand the current state of the codebase, it is important to note that the following decisions were made in prior slices: - -- In the event that a database connection fails, the system should retry with exponential backoff. Due to the fact that connection failures are transient, this approach works well. -- Due to the fact that we use SQLite, all write operations are serialized through a single writer connection. In order to prevent lock contention, the pool is configured with 1 writer and 4 readers. -- As mentioned previously, the token optimization pipeline processes content in three stages: light, moderate, and aggressive compression. In order to preserve semantic meaning, code blocks are excluded from compression. -- For the purpose of maintaining cache efficiency, static prompt sections are always placed before dynamic sections. In the event that sections are reordered, cache hit rates drop significantly. -- At this point in time, the system supports three providers: Anthropic, OpenAI, and Google. In order to add a new provider, implement the ProviderAdapter interface. -- In accordance with the security policy, all environment variables are filtered through an allowlist. For the purpose of preventing accidental exposure, unknown variables are redacted. -- With regard to the plugin system, plugins are loaded from the .gsd/plugins/ directory. Prior to loading, each plugin manifest is validated against the JSON schema. -- Subsequent to task completion, the verification gate checks for evidence of completion. In the case of missing evidence, the task is flagged for review. - -N/A -(none) -(not applicable) -(empty) - ---- - -## Verification Commands - -\`\`\`bash -npm run test -- --grep "database" -npm run lint -npm run build -\`\`\` - - -`; -} - -// ═══════════════════════════════════════════════════════════════════════════ -// Tests -// ═══════════════════════════════════════════════════════════════════════════ - -describe("Token Optimization Benchmark", () => { - // ----------------------------------------------------------------------- - // Test 1: Structured Data Savings - // ----------------------------------------------------------------------- - it("structured data savings benchmark", () => { - const decisions = buildDecisions(); - const requirements = buildRequirements(); - - const markdownDecisions = formatDecisionsAsMarkdownTable(decisions); - const compactDecisions = formatDecisionsCompact(decisions); - - const decisionSavings = measureSavings(compactDecisions, markdownDecisions); - - console.log( - ` Decisions compact: ${decisionSavings.toFixed(1)}% savings (${markdownDecisions.length} -> ${compactDecisions.length} chars)`, - ); - assert.ok( - decisionSavings > 15, - `Decisions savings should be >15%, got ${decisionSavings.toFixed(1)}%`, - ); - - const markdownReqs = formatRequirementsAsMarkdown(requirements); - const compactReqs = formatRequirementsCompact(requirements); - - const reqSavings = measureSavings(compactReqs, markdownReqs); - - console.log( - ` Requirements compact: ${reqSavings.toFixed(1)}% savings (${markdownReqs.length} -> ${compactReqs.length} chars)`, - ); - assert.ok( - reqSavings > 5, - `Requirements savings should be >5%, got ${reqSavings.toFixed(1)}%`, - ); - }); - - // ----------------------------------------------------------------------- - // Test 2: Prompt Compression - // ----------------------------------------------------------------------- - it("prompt compression benchmark", () => { - const verbose = buildVerbosePrompt(); - - const light = compressPrompt(verbose, { level: "light" }); - console.log( - ` Compression light: ${light.savingsPercent.toFixed(1)}% savings (${light.originalChars} -> ${light.compressedChars} chars, ${light.transformationsApplied} transforms)`, - ); - assert.ok( - light.savingsPercent > 5, - `Light compression should save >5%, got ${light.savingsPercent}%`, - ); - - const moderate = compressPrompt(verbose, { level: "moderate" }); - console.log( - ` Compression moderate: ${moderate.savingsPercent.toFixed(1)}% savings (${moderate.originalChars} -> ${moderate.compressedChars} chars, ${moderate.transformationsApplied} transforms)`, - ); - assert.ok( - moderate.savingsPercent > 10, - `Moderate compression should save >10%, got ${moderate.savingsPercent}%`, - ); - - const aggressive = compressPrompt(verbose, { level: "aggressive" }); - console.log( - ` Compression aggressive: ${aggressive.savingsPercent.toFixed(1)}% savings (${aggressive.originalChars} -> ${aggressive.compressedChars} chars, ${aggressive.transformationsApplied} transforms)`, - ); - assert.ok( - aggressive.savingsPercent > 15, - `Aggressive compression should save >15%, got ${aggressive.savingsPercent}%`, - ); - - // Verify code blocks are preserved - assert.ok( - aggressive.content.includes("interface Result"), - "Code blocks should be preserved through all compression levels", - ); - }); - - // ----------------------------------------------------------------------- - // Test 3: Semantic Chunking - // ----------------------------------------------------------------------- - it("semantic chunking benchmark", () => { - const query = "database connection config validation"; - const result = chunkByRelevance(SAMPLE_CODE, query, { - maxChunks: 5, - minScore: 0.05, - }); - - console.log( - ` Semantic chunking: ${result.totalChunks} total chunks, ${result.chunks.length} selected, ${result.omittedChunks} omitted`, - ); - console.log( - ` Chunking savings: ${result.savingsPercent}% of content omitted`, - ); - - assert.ok( - result.totalChunks >= 4, - `Should produce at least 4 chunks, got ${result.totalChunks}`, - ); - assert.ok( - result.savingsPercent > 40, - `Should omit >40% of content, got ${result.savingsPercent}%`, - ); - - // Verify that chunks relevant to the query score higher - const scores = result.chunks.map((c) => c.score); - const hasHighScorer = scores.some((s) => s > 0.5); - assert.ok(hasHighScorer, "At least one chunk should score above 0.5"); - - // Verify selected content contains query-relevant terms - const selectedText = result.chunks.map((c) => c.content).join("\n"); - const hasRelevantContent = - selectedText.includes("Config") || - selectedText.includes("config") || - selectedText.includes("database") || - selectedText.includes("connect") || - selectedText.includes("validate"); - assert.ok( - hasRelevantContent, - "Selected chunks should contain query-relevant content", - ); - }); - - // ----------------------------------------------------------------------- - // Test 4: Summary Distillation - // ----------------------------------------------------------------------- - it("summary distillation benchmark", () => { - const summaries = buildSummaries(); - const originalTotalChars = summaries.reduce((s, c) => s + c.length, 0); - - // Use a generous budget so we can measure natural distillation savings - const result = distillSummaries(summaries, 100_000); - - console.log( - ` Summary distillation: ${result.savingsPercent}% savings (${result.originalChars} -> ${result.distilledChars} chars, ${result.summaryCount} summaries)`, - ); - - assert.ok( - result.savingsPercent > 40, - `Summary distillation should save >40%, got ${result.savingsPercent}%`, - ); - assert.equal(result.summaryCount, 4, "Should process all 4 summaries"); - - // Verify key structured fields are preserved - assert.ok( - result.content.includes("provides:"), - "Distilled output should preserve 'provides' field", - ); - assert.ok( - result.content.includes("key_files:"), - "Distilled output should preserve 'key_files' field", - ); - assert.ok( - result.content.includes("key_decisions:"), - "Distilled output should preserve 'key_decisions' field", - ); - - // Verify slice IDs are preserved - assert.ok(result.content.includes("S01"), "Should preserve S01 reference"); - assert.ok(result.content.includes("S02"), "Should preserve S02 reference"); - assert.ok(result.content.includes("S03"), "Should preserve S03 reference"); - assert.ok(result.content.includes("S04"), "Should preserve S04 reference"); - }); - - // ----------------------------------------------------------------------- - // Test 5: Combined Pipeline - // ----------------------------------------------------------------------- - it("combined pipeline benchmark", () => { - const decisions = buildDecisions(); - const requirements = buildRequirements(); - const summaries = buildSummaries(); - const knowledgeFile = SAMPLE_CODE; - const carryForward = buildVerbosePrompt(); - - // --- Unoptimized baseline --- - const unoptDecisions = formatDecisionsAsMarkdownTable(decisions); - const unoptRequirements = formatRequirementsAsMarkdown(requirements); - const unoptSummaries = summaries.join("\n\n---\n\n"); - const unoptKnowledge = knowledgeFile; - const unoptCarry = carryForward; - - const unoptimizedTotal = - unoptDecisions.length + - unoptRequirements.length + - unoptSummaries.length + - unoptKnowledge.length + - unoptCarry.length; - - // --- Optimized pipeline --- - // 1. Compact format for decisions and requirements - const optDecisions = formatDecisionsCompact(decisions); - const optRequirements = formatRequirementsCompact(requirements); - - // 2. Distill summaries - const distilled = distillSummaries(summaries, 100_000); - - // 3. Chunk knowledge file - const chunked = chunkByRelevance(knowledgeFile, "database config validation", { - maxChunks: 5, - minScore: 0.05, - }); - const optKnowledge = chunked.chunks.map((c) => c.content).join("\n\n"); - - // 4. Compress carry-forward - const compressed = compressPrompt(carryForward, { level: "moderate" }); - - const optimizedTotal = - optDecisions.length + - optRequirements.length + - distilled.distilledChars + - optKnowledge.length + - compressed.compressedChars; - - const totalSavingsPercent = - ((unoptimizedTotal - optimizedTotal) / unoptimizedTotal) * 100; - - console.log( - ` Combined pipeline: ${totalSavingsPercent.toFixed(1)}% total savings (${unoptimizedTotal} -> ${optimizedTotal} chars)`, - ); - console.log( - ` Decisions: ${unoptDecisions.length} -> ${optDecisions.length} chars`, - ); - console.log( - ` Requirements: ${unoptRequirements.length} -> ${optRequirements.length} chars`, - ); - console.log( - ` Summaries: ${unoptSummaries.length} -> ${distilled.distilledChars} chars`, - ); - console.log( - ` Knowledge: ${unoptKnowledge.length} -> ${optKnowledge.length} chars`, - ); - console.log( - ` Carry-fwd: ${unoptCarry.length} -> ${compressed.compressedChars} chars`, - ); - - assert.ok( - totalSavingsPercent > 30, - `Combined pipeline should save >30%, got ${totalSavingsPercent.toFixed(1)}%`, - ); - }); - - // ----------------------------------------------------------------------- - // Test 6: Cache Efficiency Analysis - // ----------------------------------------------------------------------- - it("cache efficiency analysis", () => { - const sections_input = [ - section( - "system-prompt", - "You are a GSD executor agent. Follow the task plan precisely. Report evidence of completion. Do not deviate from the assigned scope. Always verify your work before reporting done.", - ), - section( - "template-executor", - "## Output Format\n\nProvide your response in the following structure:\n1. Analysis of the task requirements\n2. Implementation plan\n3. Code changes with file paths\n4. Verification evidence\n5. Summary of changes made\n\nDo not include preamble or meta-commentary.", - ), - section( - "slice-plan", - "## Slice S03: Advanced Context Selection\n\nTasks:\n- T01: Implement semantic chunker with TF-IDF scoring\n- T02: Build summary distiller with progressive dropping\n- T03: Create cache optimizer with section classification\n- T04: Write benchmark tests for all optimization modules\n- T05: Integration test for combined pipeline", - ), - section( - "decisions", - formatDecisionsCompact(buildDecisions()), - ), - section( - "requirements", - formatRequirementsCompact(buildRequirements()), - ), - section( - "task-plan", - "## T04: Write benchmark tests\n\nCreate comprehensive benchmark tests that measure token savings from each optimization module. Include realistic fixture data and conservative assertion targets.\n\nFiles: src/extensions/gsd/tests/token-optimization-benchmark.test.ts\nVerify: npm run test -- --grep benchmark", - ), - section( - "task-context", - "Current implementation status: all optimization modules are complete and passing unit tests. This task adds end-to-end validation.\n\nRecent changes:\n- prompt-compressor.ts: added aggressive level\n- semantic-chunker.ts: improved boundary detection\n- summary-distiller.ts: added progressive field dropping", - ), - ]; - - const optimized = optimizeForCaching(sections_input); - - console.log( - ` Cache efficiency: ${(optimized.cacheEfficiency * 100).toFixed(1)}% cacheable prefix (${optimized.cacheablePrefixChars} / ${optimized.totalChars} chars)`, - ); - console.log( - ` Static sections: ${optimized.sectionCounts.static}, Semi-static: ${optimized.sectionCounts["semi-static"]}, Dynamic: ${optimized.sectionCounts.dynamic}`, - ); - - assert.ok( - optimized.cacheEfficiency > 0.6, - `Cache efficiency should be >60%, got ${(optimized.cacheEfficiency * 100).toFixed(1)}%`, - ); - - const anthropicSavings = estimateCacheSavings(optimized, "anthropic"); - console.log( - ` Estimated Anthropic savings: ${(anthropicSavings * 100).toFixed(1)}%`, - ); - assert.ok( - anthropicSavings > 0.5, - `Anthropic cache savings should be >50%, got ${(anthropicSavings * 100).toFixed(1)}%`, - ); - - const openaiSavings = estimateCacheSavings(optimized, "openai"); - console.log( - ` Estimated OpenAI savings: ${(openaiSavings * 100).toFixed(1)}%`, - ); - assert.ok( - anthropicSavings > openaiSavings, - "Anthropic savings should exceed OpenAI savings (90% vs 50% discount)", - ); - }); - - // ----------------------------------------------------------------------- - // Test 7: Provider-Aware Budget Accuracy - // ----------------------------------------------------------------------- - it("provider-aware budget accuracy", () => { - const contextWindow = 200_000; - - const anthropicBudget = computeBudgets(contextWindow, "anthropic"); - const openaiBudget = computeBudgets(contextWindow, "openai"); - - const anthropicCharsPerToken = getCharsPerToken("anthropic"); - const openaiCharsPerToken = getCharsPerToken("openai"); - - console.log( - ` Anthropic: ${anthropicCharsPerToken} chars/token, inline budget: ${anthropicBudget.inlineContextBudgetChars} chars`, - ); - console.log( - ` OpenAI: ${openaiCharsPerToken} chars/token, inline budget: ${openaiBudget.inlineContextBudgetChars} chars`, - ); - - // OpenAI has higher chars-per-token (4.0 vs 3.5), so it gets more chars per budget - const charsDifference = - openaiBudget.inlineContextBudgetChars - - anthropicBudget.inlineContextBudgetChars; - const percentDifference = - (charsDifference / anthropicBudget.inlineContextBudgetChars) * 100; - - console.log( - ` OpenAI gets ${percentDifference.toFixed(1)}% more chars per budget unit (${charsDifference} chars difference)`, - ); - - // OpenAI should get ~14% more chars (4.0/3.5 = 1.143) - assert.ok( - percentDifference > 10, - `OpenAI should get >10% more chars, got ${percentDifference.toFixed(1)}%`, - ); - assert.ok( - percentDifference < 20, - `Difference should be <20%, got ${percentDifference.toFixed(1)}%`, - ); - - // Verify token estimates differ for the same content - const sampleContent = SAMPLE_CODE; - const anthropicTokens = estimateTokensForProvider(sampleContent, "anthropic"); - const openaiTokens = estimateTokensForProvider(sampleContent, "openai"); - - console.log( - ` Same content (${sampleContent.length} chars): Anthropic estimates ${anthropicTokens} tokens, OpenAI estimates ${openaiTokens} tokens`, - ); - - assert.ok( - anthropicTokens > openaiTokens, - "Anthropic should estimate more tokens (smaller chars-per-token ratio)", - ); - }); -}); diff --git a/src/resources/extensions/gsd/tests/token-optimization-prefs.test.ts b/src/resources/extensions/gsd/tests/token-optimization-prefs.test.ts deleted file mode 100644 index a093da5e1..000000000 --- a/src/resources/extensions/gsd/tests/token-optimization-prefs.test.ts +++ /dev/null @@ -1,164 +0,0 @@ -import { describe, it } from "node:test"; -import assert from "node:assert/strict"; - -// Test the type definitions exist and are correct -describe("token-optimization: types", () => { - it("CompressionStrategy accepts valid values", async () => { - const { } = await import("../types.js"); - // Type-level test — if this compiles, the types exist - const truncate: import("../types.js").CompressionStrategy = "truncate"; - const compress: import("../types.js").CompressionStrategy = "compress"; - assert.equal(truncate, "truncate"); - assert.equal(compress, "compress"); - }); - - it("ContextSelectionMode accepts valid values", async () => { - const full: import("../types.js").ContextSelectionMode = "full"; - const smart: import("../types.js").ContextSelectionMode = "smart"; - assert.equal(full, "full"); - assert.equal(smart, "smart"); - }); -}); - -// Test cache hit rate computation -describe("token-optimization: cache hit rate", () => { - it("computeCacheHitRate returns correct percentage", async () => { - const { computeCacheHitRate } = await import("../prompt-cache-optimizer.js"); - assert.equal(computeCacheHitRate({ cacheRead: 900, cacheWrite: 100, input: 100 }), 90); - assert.equal(computeCacheHitRate({ cacheRead: 0, cacheWrite: 0, input: 100 }), 0); - assert.equal(computeCacheHitRate({ cacheRead: 0, cacheWrite: 0, input: 0 }), 0); - assert.equal(computeCacheHitRate({ cacheRead: 500, cacheWrite: 0, input: 500 }), 50); - }); -}); - -// Test structured data savings -describe("token-optimization: structured data savings", () => { - it("compact decisions format is shorter than markdown table", async () => { - const { formatDecisionsCompact, measureSavings } = await import("../structured-data-formatter.js"); - const decisions = [ - { id: "D001", when_context: "M001/S01", scope: "architecture", decision: "Use SQLite for storage", choice: "WAL mode", rationale: "Built-in, no external deps", revisable: "yes" }, - { id: "D002", when_context: "M001/S02", scope: "testing", decision: "Unit test all parsers", choice: "node:test", rationale: "Fast, zero-dependency", revisable: "no" }, - ]; - const compact = formatDecisionsCompact(decisions); - // A realistic markdown table equivalent - const markdown = [ - "| # | When | Scope | Decision | Choice | Rationale | Revisable? |", - "|---|------|-------|----------|--------|-----------|------------|", - "| D001 | M001/S01 | architecture | Use SQLite for storage | WAL mode | Built-in, no external deps | yes |", - "| D002 | M001/S02 | testing | Unit test all parsers | node:test | Fast, zero-dependency | no |", - ].join("\n"); - const savings = measureSavings(compact, markdown); - assert.ok(savings > 10, `Expected >10% savings, got ${savings}%`); - }); - - it("compact requirements format drops low-value fields", async () => { - const { formatRequirementsCompact } = await import("../structured-data-formatter.js"); - const requirements = [{ - id: "R001", class: "functional", status: "active", - description: "API response time < 200ms", - why: "User experience", primary_owner: "S01", - validation: "Load test P99 < 200ms", - }]; - const compact = formatRequirementsCompact(requirements); - assert.ok(!compact.includes("source"), "Should not include source field"); - assert.ok(!compact.includes("supporting_slices"), "Should not include supporting_slices"); - assert.ok(compact.includes("R001"), "Should include requirement ID"); - }); -}); - -// Test compression levels -describe("token-optimization: prompt compression", () => { - it("light compression removes extra whitespace", async () => { - const { compressPrompt } = await import("../prompt-compressor.js"); - const input = "Line 1\n\n\n\n\nLine 2\n\n\n\nLine 3"; - const result = compressPrompt(input, { level: "light" }); - assert.ok(result.savingsPercent > 0, "Should have positive savings"); - assert.ok(!result.content.includes("\n\n\n"), "Should collapse multiple blank lines"); - }); - - it("moderate compression abbreviates verbose phrases", async () => { - const { compressPrompt } = await import("../prompt-compressor.js"); - const input = "In order to achieve this, it is important to note that the following steps are required."; - const result = compressPrompt(input, { level: "moderate" }); - assert.ok(result.compressedChars < result.originalChars, "Should be shorter"); - }); - - it("code blocks are preserved during compression", async () => { - const { compressPrompt } = await import("../prompt-compressor.js"); - const input = "In order to do this:\n\n```typescript\nconst x = 1;\n```\n\nIn order to verify:"; - const result = compressPrompt(input, { level: "aggressive" }); - assert.ok(result.content.includes("const x = 1;"), "Code block should be preserved"); - }); -}); - -// Test summary distillation -describe("token-optimization: summary distillation", () => { - it("distills summaries preserving key fields", async () => { - const { distillSummaries } = await import("../summary-distiller.js"); - const summary = `--- -id: S01 -provides: - - Core types -key_files: - - src/types.ts -key_decisions: - - D001 ---- - -# S01: Core Types - -Built the foundation type system. - -## What Happened - -Long prose about implementation details that should be dropped... -`; - const result = distillSummaries([summary], 5000); - assert.ok(result.savingsPercent > 0, "Should have savings"); - assert.ok(result.content.includes("Core types"), "Should preserve provides"); - assert.ok(result.content.includes("src/types.ts"), "Should preserve key_files"); - }); -}); - -// Test semantic chunker -describe("token-optimization: semantic chunking", () => { - it("chunks TypeScript code at function boundaries", async () => { - const { splitIntoChunks } = await import("../semantic-chunker.js"); - const code = `export function alpha() { - return 1; -} - -export function beta() { - return 2; -} - -export function gamma() { - return 3; -}`; - const chunks = splitIntoChunks(code); - assert.ok(chunks.length >= 2, `Expected >=2 chunks, got ${chunks.length}`); - }); - - it("scores chunks by relevance to query", async () => { - const { chunkByRelevance } = await import("../semantic-chunker.js"); - const code = `export function createUser(name: string) { - return { name, id: generateId() }; -} - -export function deleteDatabase() { - dropAllTables(); - clearCache(); -} - -export function updateUser(id: string, name: string) { - const user = findUser(id); - user.name = name; - return user; -}`; - const result = chunkByRelevance(code, "user creation and management", { maxChunks: 2 }); - // The user-related chunks should score higher - const content = result.chunks.map(c => c.content).join("\n"); - assert.ok(content.includes("createUser") || content.includes("updateUser"), - "Should include user-related chunks"); - }); -}); diff --git a/src/resources/extensions/gsd/types.ts b/src/resources/extensions/gsd/types.ts index 8c594369e..60f773388 100644 --- a/src/resources/extensions/gsd/types.ts +++ b/src/resources/extensions/gsd/types.ts @@ -423,7 +423,6 @@ export interface Requirement { // ─── Parallel Orchestration Types ──────────────────────────────────────── -export type CompressionStrategy = "truncate" | "compress"; export type ContextSelectionMode = "full" | "smart"; export type MergeStrategy = "per-slice" | "per-milestone";