refactor(gsd): remove prompt compression subsystem (~4,100 lines) (#1597)
Delete prompt-compressor, summary-distiller, and semantic-chunker modules plus all associated tests. Replace all compression/distillation/chunking call sites with section-boundary truncation via truncateAtSectionBoundary. Remove compression_strategy preference, validation, and documentation. Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
TÂCHES
GitHub
parent
e14eee14fe
commit
912dab1d81
19 changed files with 12 additions and 4093 deletions
|
|
@ -21,10 +21,7 @@ import type { GSDPreferences } from "./preferences.js";
|
|||
import { join } from "node:path";
|
||||
import { existsSync } from "node:fs";
|
||||
import { computeBudgets, resolveExecutorContextWindow } from "./context-budget.js";
|
||||
import { compressToTarget } from "./prompt-compressor.js";
|
||||
import { distillSummaries } from "./summary-distiller.js";
|
||||
import { formatDecisionsCompact, formatRequirementsCompact } from "./structured-data-formatter.js";
|
||||
import { chunkByRelevance, formatChunks } from "./semantic-chunker.js";
|
||||
|
||||
// ─── Executor Constraints ─────────────────────────────────────────────────────
|
||||
|
||||
|
|
@ -159,16 +156,10 @@ export async function inlineFileSmart(
|
|||
return `### ${label}\nSource: \`${relPath}\`\n\n${content.trim()}`;
|
||||
}
|
||||
|
||||
// Use semantic chunking for large files
|
||||
const result = chunkByRelevance(content, query, { maxChunks: 5, minScore: 0.05 });
|
||||
|
||||
// If chunking didn't save much (< 20%), just include full content
|
||||
if (result.savingsPercent < 20) {
|
||||
return `### ${label}\nSource: \`${relPath}\`\n\n${content.trim()}`;
|
||||
}
|
||||
|
||||
const formatted = formatChunks(result, relPath);
|
||||
return `### ${label} (${result.omittedChunks} sections omitted for relevance)\nSource: \`${relPath}\`\n\n${formatted}`;
|
||||
// For large files, truncate at section boundary
|
||||
const { truncateAtSectionBoundary } = await import("./context-budget.js");
|
||||
const truncated = truncateAtSectionBoundary(content, threshold).content;
|
||||
return `### ${label}\nSource: \`${relPath}\`\n\n${truncated}`;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -202,20 +193,6 @@ export async function inlineDependencySummaries(
|
|||
|
||||
const result = sections.join("\n\n");
|
||||
if (budgetChars !== undefined && result.length > budgetChars) {
|
||||
// For 3+ summaries, try distillation first (preserves more information)
|
||||
if (sections.length >= 3) {
|
||||
const rawSummaries = sections.map(s => {
|
||||
// Extract content after the header line
|
||||
const lines = s.split("\n");
|
||||
const contentStart = lines.findIndex(l => l.startsWith("Source:"));
|
||||
return contentStart >= 0 ? lines.slice(contentStart + 1).join("\n").trim() : s;
|
||||
});
|
||||
const distilled = distillSummaries(rawSummaries, budgetChars);
|
||||
if (distilled.content.length <= budgetChars) {
|
||||
return distilled.content;
|
||||
}
|
||||
}
|
||||
// Fall back to section-boundary truncation
|
||||
const { truncateAtSectionBoundary } = await import("./context-budget.js");
|
||||
return truncateAtSectionBoundary(result, budgetChars).content;
|
||||
}
|
||||
|
|
@ -900,15 +877,12 @@ export async function buildExecuteTaskPrompt(
|
|||
const budgets = computeBudgets(contextWindow);
|
||||
const verificationBudget = `~${Math.round(budgets.verificationBudgetChars / 1000)}K chars`;
|
||||
|
||||
// Compress carry-forward section when it exceeds 40% of inline context budget.
|
||||
// Only compress when compression_strategy is "compress" (budget/balanced profiles).
|
||||
// Truncate carry-forward section when it exceeds 40% of inline context budget.
|
||||
const carryForwardBudget = Math.floor(budgets.inlineContextBudgetChars * 0.4);
|
||||
let finalCarryForward = carryForwardSection;
|
||||
if (carryForwardSection.length > carryForwardBudget) {
|
||||
const { resolveCompressionStrategy } = await import("./preferences.js");
|
||||
if (resolveCompressionStrategy() === "compress") {
|
||||
finalCarryForward = compressToTarget(carryForwardSection, carryForwardBudget).content;
|
||||
}
|
||||
const { truncateAtSectionBoundary } = await import("./context-budget.js");
|
||||
finalCarryForward = truncateAtSectionBoundary(carryForwardSection, carryForwardBudget).content;
|
||||
}
|
||||
|
||||
return loadPrompt("execute-task", {
|
||||
|
|
|
|||
|
|
@ -745,7 +745,7 @@ export function serializePreferencesToFrontmatter(prefs: Record<string, unknown>
|
|||
"dynamic_routing", "token_profile", "phases", "parallel",
|
||||
"auto_visualize", "auto_report",
|
||||
"verification_commands", "verification_auto_fix", "verification_max_retries",
|
||||
"search_provider", "compression_strategy", "context_selection",
|
||||
"search_provider", "context_selection",
|
||||
];
|
||||
|
||||
const seen = new Set<string>();
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@
|
|||
*/
|
||||
|
||||
import { type TokenProvider, getCharsPerToken } from "./token-counter.js";
|
||||
import { compressToTarget } from "./prompt-compressor.js";
|
||||
|
||||
// ─── Budget ratio constants ──────────────────────────────────────────────────
|
||||
// Percentages of total context window allocated to each budget category.
|
||||
|
|
@ -202,22 +201,13 @@ export function resolveExecutorContextWindow(
|
|||
}
|
||||
|
||||
/**
|
||||
* Smart context reduction: compress first, then truncate if still over budget.
|
||||
* Returns the content within budget with maximum information preservation.
|
||||
* Reduce content to fit within budget using section-boundary truncation.
|
||||
*/
|
||||
export function reduceToFit(content: string, budgetChars: number): TruncationResult {
|
||||
if (!content || content.length <= budgetChars) {
|
||||
return { content, droppedSections: 0 };
|
||||
}
|
||||
|
||||
// Step 1: Try compression
|
||||
const compressed = compressToTarget(content, budgetChars);
|
||||
if (compressed.compressedChars <= budgetChars) {
|
||||
return { content: compressed.content, droppedSections: 0 };
|
||||
}
|
||||
|
||||
// Step 2: Truncate the compressed content at section boundaries
|
||||
return truncateAtSectionBoundary(compressed.content, budgetChars);
|
||||
return truncateAtSectionBoundary(content, budgetChars);
|
||||
}
|
||||
|
||||
// ─── Internal helpers ────────────────────────────────────────────────────────
|
||||
|
|
|
|||
|
|
@ -194,8 +194,6 @@ Setting `prefer_skills: []` does **not** disable skill discovery — it just mea
|
|||
|
||||
- `search_provider`: `"brave"`, `"tavily"`, `"ollama"`, `"native"`, or `"auto"` — selects the search backend for research phases. `"native"` forces Anthropic's built-in web search only; provider values force that backend and disable native search; `"auto"` uses the default heuristic. Default: `"auto"`.
|
||||
|
||||
- `compression_strategy`: `"truncate"` or `"compress"` — controls how context that exceeds the budget is reduced. `"truncate"` (default) drops sections from the end. `"compress"` applies heuristic compression before truncating, preserving more content at the cost of some fidelity. Default: `"truncate"`.
|
||||
|
||||
- `context_selection`: `"full"` or `"smart"` — controls how files are inlined into context. `"full"` inlines entire files; `"smart"` uses semantic chunking to include only the most relevant sections. Default is derived from `token_profile`.
|
||||
|
||||
- `parallel`: configures parallel orchestration for running multiple slices concurrently. Keys:
|
||||
|
|
|
|||
|
|
@ -295,18 +295,6 @@ export function resolveInlineLevel(): InlineLevel {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the compression strategy from the active token profile.
|
||||
* budget/balanced -> "compress", quality -> "truncate".
|
||||
* Explicit preference always wins.
|
||||
*/
|
||||
export function resolveCompressionStrategy(): import("./types.js").CompressionStrategy {
|
||||
const prefs = loadEffectiveGSDPreferences();
|
||||
if (prefs?.preferences.compression_strategy) return prefs.preferences.compression_strategy;
|
||||
const profile = resolveEffectiveProfile();
|
||||
return profile === "quality" ? "truncate" : "compress";
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the context selection mode from the active token profile.
|
||||
* budget -> "smart", balanced/quality -> "full".
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ import type {
|
|||
InlineLevel,
|
||||
PhaseSkipPreferences,
|
||||
ParallelConfig,
|
||||
CompressionStrategy,
|
||||
ContextSelectionMode,
|
||||
ReactiveExecutionConfig,
|
||||
} from "./types.js";
|
||||
|
|
@ -84,7 +83,6 @@ export const KNOWN_PREFERENCE_KEYS = new Set<string>([
|
|||
"verification_auto_fix",
|
||||
"verification_max_retries",
|
||||
"search_provider",
|
||||
"compression_strategy",
|
||||
"context_selection",
|
||||
"widget_mode",
|
||||
"reactive_execution",
|
||||
|
|
@ -211,8 +209,6 @@ export interface GSDPreferences {
|
|||
verification_max_retries?: number;
|
||||
/** Search provider preference. "brave"/"tavily"/"ollama" force that backend and disable native Anthropic search. "native" forces native only. "auto" = current default behavior. */
|
||||
search_provider?: "brave" | "tavily" | "ollama" | "native" | "auto";
|
||||
/** Compression strategy for context that exceeds budget. "truncate" (default) drops sections, "compress" applies heuristic compression first. */
|
||||
compression_strategy?: CompressionStrategy;
|
||||
/** Context selection mode for file inlining. "full" inlines entire files, "smart" uses semantic chunking. Default derived from token profile. */
|
||||
context_selection?: ContextSelectionMode;
|
||||
/** Default widget display mode for auto-mode dashboard. "full" | "small" | "min" | "off". Default: "full". */
|
||||
|
|
|
|||
|
|
@ -686,16 +686,6 @@ export function validatePreferences(preferences: GSDPreferences): {
|
|||
}
|
||||
}
|
||||
|
||||
// ─── Compression Strategy ───────────────────────────────────────────
|
||||
if (preferences.compression_strategy !== undefined) {
|
||||
const validStrategies = new Set(["truncate", "compress"]);
|
||||
if (typeof preferences.compression_strategy === "string" && validStrategies.has(preferences.compression_strategy)) {
|
||||
validated.compression_strategy = preferences.compression_strategy as GSDPreferences["compression_strategy"];
|
||||
} else {
|
||||
errors.push(`compression_strategy must be one of: truncate, compress`);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Context Selection ──────────────────────────────────────────────
|
||||
if (preferences.context_selection !== undefined) {
|
||||
const validModes = new Set(["full", "smart"]);
|
||||
|
|
|
|||
|
|
@ -77,7 +77,6 @@ export {
|
|||
resolveProfileDefaults,
|
||||
resolveEffectiveProfile,
|
||||
resolveInlineLevel,
|
||||
resolveCompressionStrategy,
|
||||
resolveContextSelection,
|
||||
resolveSearchProviderFromPreferences,
|
||||
} from "./preferences-models.js";
|
||||
|
|
@ -269,7 +268,6 @@ function mergePreferences(base: GSDPreferences, override: GSDPreferences): GSDPr
|
|||
verification_auto_fix: override.verification_auto_fix ?? base.verification_auto_fix,
|
||||
verification_max_retries: override.verification_max_retries ?? base.verification_max_retries,
|
||||
search_provider: override.search_provider ?? base.search_provider,
|
||||
compression_strategy: override.compression_strategy ?? base.compression_strategy,
|
||||
context_selection: override.context_selection ?? base.context_selection,
|
||||
auto_visualize: override.auto_visualize ?? base.auto_visualize,
|
||||
auto_report: override.auto_report ?? base.auto_report,
|
||||
|
|
|
|||
|
|
@ -1,508 +0,0 @@
|
|||
/**
|
||||
* Prompt Compressor — deterministic text compression for context reduction.
|
||||
*
|
||||
* Applies a series of lossless and near-lossless transformations to reduce
|
||||
* token count while preserving semantic meaning. No LLM calls, no external
|
||||
* dependencies. Sub-millisecond for typical prompt sizes.
|
||||
*
|
||||
* Compression techniques (applied in order):
|
||||
* 1. Redundant whitespace normalization
|
||||
* 2. Markdown formatting reduction (collapse verbose tables, lists)
|
||||
* 3. Common phrase abbreviation
|
||||
* 4. Repeated pattern deduplication
|
||||
* 5. Low-information content removal (empty sections, boilerplate)
|
||||
*/
|
||||
|
||||
export type CompressionLevel = "light" | "moderate" | "aggressive";
|
||||
|
||||
export interface CompressionResult {
|
||||
/** The compressed content */
|
||||
content: string;
|
||||
/** Original character count */
|
||||
originalChars: number;
|
||||
/** Compressed character count */
|
||||
compressedChars: number;
|
||||
/** Savings percentage (0-100) */
|
||||
savingsPercent: number;
|
||||
/** Which compression level was applied */
|
||||
level: CompressionLevel;
|
||||
/** Number of transformations applied */
|
||||
transformationsApplied: number;
|
||||
}
|
||||
|
||||
export interface CompressionOptions {
|
||||
/** Compression intensity. Default: "moderate" */
|
||||
level?: CompressionLevel;
|
||||
/** Preserve markdown headings (useful for section-boundary truncation). Default: true */
|
||||
preserveHeadings?: boolean;
|
||||
/** Preserve code blocks verbatim. Default: true */
|
||||
preserveCodeBlocks?: boolean;
|
||||
/** Target character count (compression stops when achieved). Default: no target */
|
||||
targetChars?: number;
|
||||
}
|
||||
|
||||
// ─── Phrase Abbreviation Map ────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Build a regex that matches a verbose phrase even when split across lines.
|
||||
* Whitespace between words is matched with \s+ to handle line wrapping.
|
||||
*/
|
||||
function phraseRegex(phrase: string): RegExp {
|
||||
const words = phrase.split(/\s+/);
|
||||
const pattern = `\\b${words.join("\\s+")}\\b`;
|
||||
return new RegExp(pattern, "gi");
|
||||
}
|
||||
|
||||
const VERBOSE_PHRASES: Array<[RegExp, string]> = [
|
||||
[phraseRegex("In order to"), "To"],
|
||||
[phraseRegex("It is important to note that"), "Note:"],
|
||||
[phraseRegex("As mentioned previously"), "(see above)"],
|
||||
[phraseRegex("The following"), "These"],
|
||||
[phraseRegex("In addition to"), "Also,"],
|
||||
[phraseRegex("Due to the fact that"), "Because"],
|
||||
[phraseRegex("At this point in time"), "Now"],
|
||||
[phraseRegex("For the purpose of"), "For"],
|
||||
[phraseRegex("In the event that"), "If"],
|
||||
[phraseRegex("With regard to"), "Re:"],
|
||||
[phraseRegex("Prior to"), "Before"],
|
||||
[phraseRegex("Subsequent to"), "After"],
|
||||
[phraseRegex("In accordance with"), "Per"],
|
||||
[phraseRegex("A number of"), "Several"],
|
||||
[phraseRegex("In the case of"), "For"],
|
||||
[phraseRegex("On the basis of"), "Based on"],
|
||||
];
|
||||
|
||||
// ─── Code Block Extraction ──────────────────────────────────────────────────
|
||||
|
||||
interface ExtractedBlocks {
|
||||
text: string;
|
||||
blocks: Map<string, string>;
|
||||
}
|
||||
|
||||
function extractCodeBlocks(content: string): ExtractedBlocks {
|
||||
const blocks = new Map<string, string>();
|
||||
let counter = 0;
|
||||
|
||||
const text = content.replace(/```[\s\S]*?```/g, (match) => {
|
||||
const placeholder = `\x00CODEBLOCK_${counter++}\x00`;
|
||||
blocks.set(placeholder, match);
|
||||
return placeholder;
|
||||
});
|
||||
|
||||
return { text, blocks };
|
||||
}
|
||||
|
||||
function restoreCodeBlocks(text: string, blocks: Map<string, string>): string {
|
||||
let result = text;
|
||||
for (const [placeholder, block] of blocks) {
|
||||
result = result.replace(placeholder, block);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// ─── Light Transformations ──────────────────────────────────────────────────
|
||||
|
||||
function normalizeWhitespace(content: string): string {
|
||||
// Collapse 3+ consecutive blank lines to 2
|
||||
let result = content.replace(/(\n\s*){3,}\n/g, "\n\n");
|
||||
// Trim trailing whitespace on every line
|
||||
result = result.replace(/[ \t]+$/gm, "");
|
||||
return result;
|
||||
}
|
||||
|
||||
function removeMarkdownComments(content: string): string {
|
||||
return content.replace(/<!--[\s\S]*?-->/g, "");
|
||||
}
|
||||
|
||||
function removeHorizontalRules(content: string): string {
|
||||
// Remove horizontal rules (---, ***, ___) that stand alone on a line
|
||||
return content.replace(/^\s*[-*_]{3,}\s*$/gm, "");
|
||||
}
|
||||
|
||||
function collapseEmptyListItems(content: string): string {
|
||||
// Collapse repeated empty list items (- \n- \n- \n) into one
|
||||
return content.replace(/(^[ \t]*[-*+]\s*$\n){2,}/gm, "$1");
|
||||
}
|
||||
|
||||
function applyLightTransformations(content: string): { content: string; count: number } {
|
||||
let count = 0;
|
||||
let result = content;
|
||||
|
||||
const after1 = normalizeWhitespace(result);
|
||||
if (after1 !== result) count++;
|
||||
result = after1;
|
||||
|
||||
const after2 = removeMarkdownComments(result);
|
||||
if (after2 !== result) count++;
|
||||
result = after2;
|
||||
|
||||
const after3 = removeHorizontalRules(result);
|
||||
if (after3 !== result) count++;
|
||||
result = after3;
|
||||
|
||||
const after4 = collapseEmptyListItems(result);
|
||||
if (after4 !== result) count++;
|
||||
result = after4;
|
||||
|
||||
return { content: result, count };
|
||||
}
|
||||
|
||||
// ─── Moderate Transformations ───────────────────────────────────────────────
|
||||
|
||||
function abbreviateVerbosePhrases(content: string): { content: string; count: number } {
|
||||
let count = 0;
|
||||
let result = content;
|
||||
|
||||
for (const [pattern, replacement] of VERBOSE_PHRASES) {
|
||||
const after = result.replace(pattern, replacement);
|
||||
if (after !== result) count++;
|
||||
result = after;
|
||||
}
|
||||
|
||||
return { content: result, count };
|
||||
}
|
||||
|
||||
function removeBoilerplateLines(content: string): string {
|
||||
const lines = content.split("\n");
|
||||
const filtered = lines.filter((line) => {
|
||||
const trimmed = line.trim();
|
||||
// Remove lines that are just N/A, (none), (empty), (not applicable)
|
||||
if (/^(?:N\/A|\(none\)|\(empty\)|\(not applicable\))$/i.test(trimmed)) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
return filtered.join("\n");
|
||||
}
|
||||
|
||||
function deduplicateConsecutiveLines(content: string): string {
|
||||
const lines = content.split("\n");
|
||||
const result: string[] = [];
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
if (i === 0 || lines[i] !== lines[i - 1] || lines[i].trim() === "") {
|
||||
result.push(lines[i]);
|
||||
}
|
||||
}
|
||||
|
||||
return result.join("\n");
|
||||
}
|
||||
|
||||
function collapseTableFormatting(content: string): string {
|
||||
// Remove excessive padding in markdown table cells
|
||||
// Matches table rows like | cell | cell | and collapses to | cell | cell |
|
||||
return content.replace(/\|[ \t]{2,}([^|\n]*?)[ \t]{2,}\|/g, (_, cellContent) => {
|
||||
return `| ${cellContent.trim()} |`;
|
||||
});
|
||||
}
|
||||
|
||||
function applyModerateTransformations(content: string): { content: string; count: number } {
|
||||
let count = 0;
|
||||
let result = content;
|
||||
|
||||
const phraseResult = abbreviateVerbosePhrases(result);
|
||||
count += phraseResult.count;
|
||||
result = phraseResult.content;
|
||||
|
||||
const after1 = removeBoilerplateLines(result);
|
||||
if (after1 !== result) count++;
|
||||
result = after1;
|
||||
|
||||
const after2 = deduplicateConsecutiveLines(result);
|
||||
if (after2 !== result) count++;
|
||||
result = after2;
|
||||
|
||||
const after3 = collapseTableFormatting(result);
|
||||
if (after3 !== result) count++;
|
||||
result = after3;
|
||||
|
||||
return { content: result, count };
|
||||
}
|
||||
|
||||
// ─── Aggressive Transformations ─────────────────────────────────────────────
|
||||
|
||||
function removeMarkdownEmphasis(content: string): string {
|
||||
// Bold: **text** or __text__
|
||||
let result = content.replace(/\*\*(.+?)\*\*/g, "$1");
|
||||
result = result.replace(/__(.+?)__/g, "$1");
|
||||
// Italic: *text* or _text_ (single, not inside words)
|
||||
result = result.replace(/(?<!\w)\*([^*\n]+?)\*(?!\w)/g, "$1");
|
||||
result = result.replace(/(?<!\w)_([^_\n]+?)_(?!\w)/g, "$1");
|
||||
return result;
|
||||
}
|
||||
|
||||
function removeMarkdownLinks(content: string): string {
|
||||
// [text](url) → text
|
||||
return content.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1");
|
||||
}
|
||||
|
||||
function truncateLongLines(content: string): string {
|
||||
const lines = content.split("\n");
|
||||
const result = lines.map((line) => {
|
||||
if (line.length <= 300) return line;
|
||||
// Find a sentence boundary (. ! ?) near the 300 char mark
|
||||
const truncateZone = line.slice(0, 300);
|
||||
const lastSentenceEnd = Math.max(
|
||||
truncateZone.lastIndexOf(". "),
|
||||
truncateZone.lastIndexOf("! "),
|
||||
truncateZone.lastIndexOf("? "),
|
||||
);
|
||||
if (lastSentenceEnd > 150) {
|
||||
return line.slice(0, lastSentenceEnd + 1);
|
||||
}
|
||||
// Fallback: cut at last space before 300
|
||||
const lastSpace = truncateZone.lastIndexOf(" ");
|
||||
if (lastSpace > 150) {
|
||||
return line.slice(0, lastSpace);
|
||||
}
|
||||
return truncateZone;
|
||||
});
|
||||
return result.join("\n");
|
||||
}
|
||||
|
||||
function removeBulletMarkers(content: string): string {
|
||||
// Remove bullet markers: - , * , + , numbered (1. 2. etc)
|
||||
return content.replace(/^[ \t]*(?:[-*+]|\d+\.)\s+/gm, "");
|
||||
}
|
||||
|
||||
function removeBlockquoteMarkers(content: string): string {
|
||||
return content.replace(/^[ \t]*>+\s?/gm, "");
|
||||
}
|
||||
|
||||
function deduplicateStructuralPatterns(content: string): string {
|
||||
// Deduplicate consecutive lines that match the same "Key: value" pattern
|
||||
const lines = content.split("\n");
|
||||
const result: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
let lastWasStructural = false;
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
// Detect structural patterns: "Key: value"
|
||||
const structMatch = trimmed.match(/^(\w[\w\s]*?):\s+(.+)$/);
|
||||
if (structMatch) {
|
||||
if (seen.has(trimmed)) {
|
||||
lastWasStructural = true;
|
||||
continue;
|
||||
}
|
||||
seen.add(trimmed);
|
||||
lastWasStructural = true;
|
||||
} else {
|
||||
// Reset seen set when structural block ends
|
||||
if (!lastWasStructural || trimmed === "") {
|
||||
seen.clear();
|
||||
}
|
||||
lastWasStructural = false;
|
||||
}
|
||||
result.push(line);
|
||||
}
|
||||
|
||||
return result.join("\n");
|
||||
}
|
||||
|
||||
function applyAggressiveTransformations(
|
||||
content: string,
|
||||
preserveHeadings: boolean,
|
||||
): { content: string; count: number } {
|
||||
let count = 0;
|
||||
let result = content;
|
||||
|
||||
const after1 = removeMarkdownEmphasis(result);
|
||||
if (after1 !== result) count++;
|
||||
result = after1;
|
||||
|
||||
const after2 = removeMarkdownLinks(result);
|
||||
if (after2 !== result) count++;
|
||||
result = after2;
|
||||
|
||||
const after3 = truncateLongLines(result);
|
||||
if (after3 !== result) count++;
|
||||
result = after3;
|
||||
|
||||
const after4 = removeBulletMarkers(result);
|
||||
if (after4 !== result) count++;
|
||||
result = after4;
|
||||
|
||||
const after5 = removeBlockquoteMarkers(result);
|
||||
if (after5 !== result) count++;
|
||||
result = after5;
|
||||
|
||||
const after6 = deduplicateStructuralPatterns(result);
|
||||
if (after6 !== result) count++;
|
||||
result = after6;
|
||||
|
||||
return { content: result, count };
|
||||
}
|
||||
|
||||
// ─── Heading Preservation ───────────────────────────────────────────────────
|
||||
|
||||
interface ExtractedHeadings {
|
||||
text: string;
|
||||
headings: Map<string, string>;
|
||||
}
|
||||
|
||||
function extractHeadings(content: string): ExtractedHeadings {
|
||||
const headings = new Map<string, string>();
|
||||
let counter = 0;
|
||||
|
||||
const text = content.replace(/^(#{1,6}\s.+)$/gm, (match) => {
|
||||
const placeholder = `\x00HEADING_${counter++}\x00`;
|
||||
headings.set(placeholder, match);
|
||||
return placeholder;
|
||||
});
|
||||
|
||||
return { text, headings };
|
||||
}
|
||||
|
||||
function restoreHeadings(text: string, headings: Map<string, string>): string {
|
||||
let result = text;
|
||||
for (const [placeholder, heading] of headings) {
|
||||
result = result.replace(placeholder, heading);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// ─── Public API ─────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Compress prompt content using deterministic text transformations.
|
||||
*/
|
||||
export function compressPrompt(content: string, options?: CompressionOptions): CompressionResult {
|
||||
const level = options?.level ?? "moderate";
|
||||
const preserveHeadings = options?.preserveHeadings ?? true;
|
||||
const preserveCodeBlocks = options?.preserveCodeBlocks ?? true;
|
||||
|
||||
if (content === "") {
|
||||
return {
|
||||
content: "",
|
||||
originalChars: 0,
|
||||
compressedChars: 0,
|
||||
savingsPercent: 0,
|
||||
level,
|
||||
transformationsApplied: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const originalChars = content.length;
|
||||
let working = content;
|
||||
let totalTransformations = 0;
|
||||
|
||||
// Extract code blocks if preserving
|
||||
let codeBlocks: Map<string, string> | null = null;
|
||||
if (preserveCodeBlocks) {
|
||||
const extracted = extractCodeBlocks(working);
|
||||
working = extracted.text;
|
||||
codeBlocks = extracted.blocks;
|
||||
}
|
||||
|
||||
// Extract headings if preserving
|
||||
let headings: Map<string, string> | null = null;
|
||||
if (preserveHeadings) {
|
||||
const extracted = extractHeadings(working);
|
||||
working = extracted.text;
|
||||
headings = extracted.headings;
|
||||
}
|
||||
|
||||
// Apply light transformations (always)
|
||||
const lightResult = applyLightTransformations(working);
|
||||
working = lightResult.content;
|
||||
totalTransformations += lightResult.count;
|
||||
|
||||
// Check target
|
||||
if (options?.targetChars && getRestoredLength(working, codeBlocks, headings) <= options.targetChars) {
|
||||
return buildResult(working, originalChars, level, totalTransformations, codeBlocks, headings);
|
||||
}
|
||||
|
||||
// Apply moderate transformations
|
||||
if (level === "moderate" || level === "aggressive") {
|
||||
const modResult = applyModerateTransformations(working);
|
||||
working = modResult.content;
|
||||
totalTransformations += modResult.count;
|
||||
|
||||
if (options?.targetChars && getRestoredLength(working, codeBlocks, headings) <= options.targetChars) {
|
||||
return buildResult(working, originalChars, level, totalTransformations, codeBlocks, headings);
|
||||
}
|
||||
}
|
||||
|
||||
// Apply aggressive transformations
|
||||
if (level === "aggressive") {
|
||||
const aggResult = applyAggressiveTransformations(working, preserveHeadings);
|
||||
working = aggResult.content;
|
||||
totalTransformations += aggResult.count;
|
||||
}
|
||||
|
||||
return buildResult(working, originalChars, level, totalTransformations, codeBlocks, headings);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress with a target size — applies progressively more aggressive
|
||||
* compression until the target is reached or all transformations exhausted.
|
||||
*/
|
||||
export function compressToTarget(content: string, targetChars: number): CompressionResult {
|
||||
if (content.length <= targetChars) {
|
||||
return {
|
||||
content,
|
||||
originalChars: content.length,
|
||||
compressedChars: content.length,
|
||||
savingsPercent: 0,
|
||||
level: "light",
|
||||
transformationsApplied: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const levels: CompressionLevel[] = ["light", "moderate", "aggressive"];
|
||||
|
||||
for (const level of levels) {
|
||||
const result = compressPrompt(content, { level, targetChars });
|
||||
if (result.compressedChars <= targetChars) {
|
||||
return result;
|
||||
}
|
||||
// If aggressive and still over target, return best effort
|
||||
if (level === "aggressive") {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
// Unreachable, but satisfy TypeScript
|
||||
return compressPrompt(content, { level: "aggressive" });
|
||||
}
|
||||
|
||||
// ─── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
function getRestoredLength(
|
||||
text: string,
|
||||
codeBlocks: Map<string, string> | null,
|
||||
headings: Map<string, string> | null,
|
||||
): number {
|
||||
let result = text;
|
||||
if (headings) result = restoreHeadings(result, headings);
|
||||
if (codeBlocks) result = restoreCodeBlocks(result, codeBlocks);
|
||||
return result.length;
|
||||
}
|
||||
|
||||
function buildResult(
|
||||
working: string,
|
||||
originalChars: number,
|
||||
level: CompressionLevel,
|
||||
transformationsApplied: number,
|
||||
codeBlocks: Map<string, string> | null,
|
||||
headings: Map<string, string> | null,
|
||||
): CompressionResult {
|
||||
let content = working;
|
||||
if (headings) content = restoreHeadings(content, headings);
|
||||
if (codeBlocks) content = restoreCodeBlocks(content, codeBlocks);
|
||||
|
||||
const compressedChars = content.length;
|
||||
const savingsPercent = originalChars > 0
|
||||
? Math.round(((originalChars - compressedChars) / originalChars) * 10000) / 100
|
||||
: 0;
|
||||
|
||||
return {
|
||||
content,
|
||||
originalChars,
|
||||
compressedChars,
|
||||
savingsPercent,
|
||||
level,
|
||||
transformationsApplied,
|
||||
};
|
||||
}
|
||||
|
|
@ -1,336 +0,0 @@
|
|||
// GSD Extension — Semantic Chunker with TF-IDF Relevance Scoring
|
||||
// Splits code/text into semantic chunks and selects the most relevant ones for a given task.
|
||||
// Pure TypeScript — no external dependencies.
|
||||
|
||||
// ─── Types ──────────────────────────────────────────────────────────────────
|
||||
|
||||
export interface Chunk {
|
||||
content: string;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
score: number;
|
||||
}
|
||||
|
||||
export interface ChunkResult {
|
||||
chunks: Chunk[];
|
||||
totalChunks: number;
|
||||
omittedChunks: number;
|
||||
savingsPercent: number;
|
||||
}
|
||||
|
||||
interface ChunkOptions {
|
||||
minLines?: number;
|
||||
maxLines?: number;
|
||||
}
|
||||
|
||||
interface RelevanceOptions {
|
||||
maxChunks?: number;
|
||||
minChunkLines?: number;
|
||||
maxChunkLines?: number;
|
||||
minScore?: number;
|
||||
}
|
||||
|
||||
// ─── Constants ──────────────────────────────────────────────────────────────
|
||||
|
||||
const CODE_BOUNDARY_RE = /^(export\s+)?(async\s+)?(function|class|interface|type|const|enum)\s/;
|
||||
|
||||
const MARKDOWN_HEADING_RE = /^#{1,6}\s/;
|
||||
|
||||
const STOP_WORDS = new Set([
|
||||
"the", "a", "an", "is", "are", "was", "were", "be", "to", "of", "in",
|
||||
"for", "on", "with", "at", "by", "from", "this", "that", "it", "as",
|
||||
"or", "and", "not", "but", "if", "do", "no", "so", "up", "its", "has",
|
||||
"had", "get", "set", "can", "may", "all", "use", "new", "one", "two",
|
||||
"also", "each", "than", "been", "into", "most", "only", "over", "such",
|
||||
"how", "some", "any", "our", "his", "her", "out", "did", "let", "say", "she",
|
||||
]);
|
||||
|
||||
const DEFAULT_MIN_LINES = 3;
|
||||
const DEFAULT_MAX_LINES = 80;
|
||||
const DEFAULT_MAX_CHUNKS = 5;
|
||||
const DEFAULT_MIN_SCORE = 0.1;
|
||||
|
||||
// ─── Content Type Detection ─────────────────────────────────────────────────
|
||||
|
||||
type ContentType = "code" | "markdown" | "text";
|
||||
|
||||
function detectContentType(lines: string[]): ContentType {
|
||||
let codeSignals = 0;
|
||||
let mdSignals = 0;
|
||||
const sampleSize = Math.min(lines.length, 50);
|
||||
|
||||
for (let i = 0; i < sampleSize; i++) {
|
||||
const line = lines[i];
|
||||
if (CODE_BOUNDARY_RE.test(line) || /^\s*import\s/.test(line)) {
|
||||
codeSignals++;
|
||||
}
|
||||
if (MARKDOWN_HEADING_RE.test(line)) {
|
||||
mdSignals++;
|
||||
}
|
||||
}
|
||||
|
||||
if (mdSignals >= 2 && mdSignals > codeSignals) return "markdown";
|
||||
if (codeSignals >= 2) return "code";
|
||||
return "text";
|
||||
}
|
||||
|
||||
// ─── Tokenizer ──────────────────────────────────────────────────────────────
|
||||
|
||||
function tokenize(text: string): string[] {
|
||||
return text
|
||||
.toLowerCase()
|
||||
.split(/[\s\W]+/)
|
||||
.filter((w) => w.length >= 2 && !STOP_WORDS.has(w));
|
||||
}
|
||||
|
||||
// ─── splitIntoChunks ────────────────────────────────────────────────────────
|
||||
|
||||
export function splitIntoChunks(
|
||||
content: string,
|
||||
options?: ChunkOptions,
|
||||
): Chunk[] {
|
||||
if (!content || content.trim().length === 0) return [];
|
||||
|
||||
const minLines = options?.minLines ?? DEFAULT_MIN_LINES;
|
||||
const maxLines = options?.maxLines ?? DEFAULT_MAX_LINES;
|
||||
const lines = content.split("\n");
|
||||
|
||||
if (lines.length === 0) return [];
|
||||
|
||||
const contentType = detectContentType(lines);
|
||||
let boundaries: number[];
|
||||
|
||||
switch (contentType) {
|
||||
case "code":
|
||||
boundaries = findCodeBoundaries(lines);
|
||||
break;
|
||||
case "markdown":
|
||||
boundaries = findMarkdownBoundaries(lines);
|
||||
break;
|
||||
default:
|
||||
boundaries = findTextBoundaries(lines);
|
||||
break;
|
||||
}
|
||||
|
||||
// Always include 0 as first boundary
|
||||
if (boundaries.length === 0 || boundaries[0] !== 0) {
|
||||
boundaries.unshift(0);
|
||||
}
|
||||
|
||||
// Build raw chunks from boundaries
|
||||
const rawChunks: Chunk[] = [];
|
||||
for (let i = 0; i < boundaries.length; i++) {
|
||||
const start = boundaries[i];
|
||||
const end = i + 1 < boundaries.length ? boundaries[i + 1] - 1 : lines.length - 1;
|
||||
const chunkLines = lines.slice(start, end + 1);
|
||||
rawChunks.push({
|
||||
content: chunkLines.join("\n"),
|
||||
startLine: start + 1, // 1-based
|
||||
endLine: end + 1, // 1-based
|
||||
score: 0,
|
||||
});
|
||||
}
|
||||
|
||||
// Split oversized chunks at maxLines
|
||||
const splitChunks: Chunk[] = [];
|
||||
for (const chunk of rawChunks) {
|
||||
const chunkLineCount = chunk.endLine - chunk.startLine + 1;
|
||||
if (chunkLineCount <= maxLines) {
|
||||
splitChunks.push(chunk);
|
||||
} else {
|
||||
const chunkLines = chunk.content.split("\n");
|
||||
for (let offset = 0; offset < chunkLines.length; offset += maxLines) {
|
||||
const slice = chunkLines.slice(offset, offset + maxLines);
|
||||
splitChunks.push({
|
||||
content: slice.join("\n"),
|
||||
startLine: chunk.startLine + offset,
|
||||
endLine: chunk.startLine + offset + slice.length - 1,
|
||||
score: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Merge tiny chunks into predecessor
|
||||
const merged: Chunk[] = [];
|
||||
for (const chunk of splitChunks) {
|
||||
const chunkLineCount = chunk.endLine - chunk.startLine + 1;
|
||||
if (chunkLineCount < minLines && merged.length > 0) {
|
||||
const prev = merged[merged.length - 1];
|
||||
prev.content += "\n" + chunk.content;
|
||||
prev.endLine = chunk.endLine;
|
||||
} else {
|
||||
merged.push({ ...chunk });
|
||||
}
|
||||
}
|
||||
|
||||
return merged;
|
||||
}
|
||||
|
||||
function findCodeBoundaries(lines: string[]): number[] {
|
||||
const boundaries: number[] = [];
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
if (CODE_BOUNDARY_RE.test(lines[i])) {
|
||||
// Also consider a blank line before a boundary marker
|
||||
if (i > 0 && lines[i - 1].trim() === "" && !boundaries.includes(i)) {
|
||||
boundaries.push(i);
|
||||
} else if (!boundaries.includes(i)) {
|
||||
boundaries.push(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
return boundaries;
|
||||
}
|
||||
|
||||
function findMarkdownBoundaries(lines: string[]): number[] {
|
||||
const boundaries: number[] = [];
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
if (MARKDOWN_HEADING_RE.test(lines[i])) {
|
||||
boundaries.push(i);
|
||||
}
|
||||
}
|
||||
return boundaries;
|
||||
}
|
||||
|
||||
function findTextBoundaries(lines: string[]): number[] {
|
||||
const boundaries: number[] = [0];
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
if (lines[i - 1].trim() === "" && lines[i].trim() !== "") {
|
||||
boundaries.push(i);
|
||||
}
|
||||
}
|
||||
return boundaries;
|
||||
}
|
||||
|
||||
// ─── scoreChunks ────────────────────────────────────────────────────────────
|
||||
|
||||
export function scoreChunks(chunks: Chunk[], query: string): Chunk[] {
|
||||
if (chunks.length === 0) return [];
|
||||
|
||||
const queryTerms = tokenize(query);
|
||||
if (queryTerms.length === 0) {
|
||||
return chunks.map((c) => ({ ...c, score: 0 }));
|
||||
}
|
||||
|
||||
const totalChunks = chunks.length;
|
||||
|
||||
// Pre-compute IDF for each query term
|
||||
const termChunkCounts = new Map<string, number>();
|
||||
const chunkTokenSets: Set<string>[] = [];
|
||||
|
||||
for (const chunk of chunks) {
|
||||
const tokens = new Set(tokenize(chunk.content));
|
||||
chunkTokenSets.push(tokens);
|
||||
for (const term of queryTerms) {
|
||||
if (tokens.has(term)) {
|
||||
termChunkCounts.set(term, (termChunkCounts.get(term) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const idf = new Map<string, number>();
|
||||
for (const term of queryTerms) {
|
||||
const df = termChunkCounts.get(term) ?? 0;
|
||||
idf.set(term, Math.log(1 + totalChunks / (1 + df)));
|
||||
}
|
||||
|
||||
// Score each chunk
|
||||
const scored = chunks.map((chunk, idx) => {
|
||||
const chunkTokens = tokenize(chunk.content);
|
||||
const totalTerms = chunkTokens.length;
|
||||
if (totalTerms === 0) return { ...chunk, score: 0 };
|
||||
|
||||
// Count term frequencies
|
||||
const termFreq = new Map<string, number>();
|
||||
for (const token of chunkTokens) {
|
||||
termFreq.set(token, (termFreq.get(token) ?? 0) + 1);
|
||||
}
|
||||
|
||||
let score = 0;
|
||||
for (const term of queryTerms) {
|
||||
const tf = (termFreq.get(term) ?? 0) / totalTerms;
|
||||
const termIdf = idf.get(term) ?? 0;
|
||||
score += tf * termIdf;
|
||||
}
|
||||
|
||||
return { ...chunk, score };
|
||||
});
|
||||
|
||||
// Normalize to 0-1
|
||||
const maxScore = Math.max(...scored.map((c) => c.score));
|
||||
if (maxScore > 0) {
|
||||
for (const chunk of scored) {
|
||||
chunk.score = chunk.score / maxScore;
|
||||
}
|
||||
}
|
||||
|
||||
return scored;
|
||||
}
|
||||
|
||||
// ─── chunkByRelevance ───────────────────────────────────────────────────────
|
||||
|
||||
export function chunkByRelevance(
|
||||
content: string,
|
||||
query: string,
|
||||
options?: RelevanceOptions,
|
||||
): ChunkResult {
|
||||
const maxChunks = options?.maxChunks ?? DEFAULT_MAX_CHUNKS;
|
||||
const minScore = options?.minScore ?? DEFAULT_MIN_SCORE;
|
||||
const minLines = options?.minChunkLines ?? DEFAULT_MIN_LINES;
|
||||
const maxLines = options?.maxChunkLines ?? DEFAULT_MAX_LINES;
|
||||
|
||||
const rawChunks = splitIntoChunks(content, { minLines, maxLines });
|
||||
if (rawChunks.length === 0) {
|
||||
return { chunks: [], totalChunks: 0, omittedChunks: 0, savingsPercent: 0 };
|
||||
}
|
||||
|
||||
const scored = scoreChunks(rawChunks, query);
|
||||
|
||||
// Filter by minScore and take top maxChunks by score
|
||||
const qualifying = scored
|
||||
.filter((c) => c.score >= minScore)
|
||||
.sort((a, b) => b.score - a.score)
|
||||
.slice(0, maxChunks);
|
||||
|
||||
// Return in original document order (by startLine)
|
||||
const selected = qualifying.sort((a, b) => a.startLine - b.startLine);
|
||||
|
||||
const totalChars = content.length;
|
||||
const selectedChars = selected.reduce((sum, c) => sum + c.content.length, 0);
|
||||
const savingsPercent = totalChars > 0
|
||||
? Math.round(((totalChars - selectedChars) / totalChars) * 100)
|
||||
: 0;
|
||||
|
||||
return {
|
||||
chunks: selected,
|
||||
totalChunks: rawChunks.length,
|
||||
omittedChunks: rawChunks.length - selected.length,
|
||||
savingsPercent: Math.max(0, savingsPercent),
|
||||
};
|
||||
}
|
||||
|
||||
// ─── formatChunks ───────────────────────────────────────────────────────────
|
||||
|
||||
export function formatChunks(result: ChunkResult, filePath: string): string {
|
||||
if (result.chunks.length === 0) {
|
||||
return `[${filePath}: empty or no relevant chunks]`;
|
||||
}
|
||||
|
||||
const parts: string[] = [];
|
||||
let lastEndLine = 0;
|
||||
|
||||
for (const chunk of result.chunks) {
|
||||
// Show omission gap
|
||||
if (lastEndLine > 0 && chunk.startLine > lastEndLine + 1) {
|
||||
const gapLines = chunk.startLine - lastEndLine - 1;
|
||||
parts.push(`[...${gapLines} lines omitted...]`);
|
||||
}
|
||||
|
||||
parts.push(`[Lines ${chunk.startLine}-${chunk.endLine}]`);
|
||||
parts.push(chunk.content);
|
||||
|
||||
lastEndLine = chunk.endLine;
|
||||
}
|
||||
|
||||
return parts.join("\n");
|
||||
}
|
||||
|
|
@ -1,258 +0,0 @@
|
|||
/**
|
||||
* Summary distiller — extracts essential structured data from SUMMARY.md files,
|
||||
* dropping verbose prose to save context budget.
|
||||
*/
|
||||
|
||||
export interface DistillationResult {
|
||||
content: string;
|
||||
summaryCount: number;
|
||||
savingsPercent: number;
|
||||
originalChars: number;
|
||||
distilledChars: number;
|
||||
}
|
||||
|
||||
interface ParsedFrontmatter {
|
||||
id: string;
|
||||
provides: string[];
|
||||
requires: string[];
|
||||
key_files: string[];
|
||||
key_decisions: string[];
|
||||
patterns_established: string[];
|
||||
}
|
||||
|
||||
interface DistilledEntry {
|
||||
id: string;
|
||||
oneLiner: string;
|
||||
provides: string[];
|
||||
requires: string[];
|
||||
key_files: string[];
|
||||
key_decisions: string[];
|
||||
patterns: string[];
|
||||
}
|
||||
|
||||
// ─── Frontmatter parsing ─────────────────────────────────────────────────────
|
||||
|
||||
function parseFrontmatter(raw: string): ParsedFrontmatter {
|
||||
const result: ParsedFrontmatter = {
|
||||
id: "",
|
||||
provides: [],
|
||||
requires: [],
|
||||
key_files: [],
|
||||
key_decisions: [],
|
||||
patterns_established: [],
|
||||
};
|
||||
|
||||
// Extract frontmatter block between --- markers
|
||||
const fmMatch = raw.match(/^---\r?\n([\s\S]*?)\r?\n---/);
|
||||
if (!fmMatch) return result;
|
||||
|
||||
const fmBlock = fmMatch[1];
|
||||
const lines = fmBlock.split(/\r?\n/);
|
||||
|
||||
let currentKey: string | null = null;
|
||||
|
||||
for (const line of lines) {
|
||||
// Scalar value: key: value
|
||||
const scalarMatch = line.match(/^(\w[\w_]*):\s*(.+)$/);
|
||||
if (scalarMatch) {
|
||||
const [, key, value] = scalarMatch;
|
||||
currentKey = key;
|
||||
setScalar(result, key, value.trim());
|
||||
continue;
|
||||
}
|
||||
|
||||
// Array-start key with empty value: key:\n or key: []\n
|
||||
const arrayStartMatch = line.match(/^(\w[\w_]*):\s*(\[\])?\s*$/);
|
||||
if (arrayStartMatch) {
|
||||
currentKey = arrayStartMatch[1];
|
||||
continue;
|
||||
}
|
||||
|
||||
// Array item: - value
|
||||
const itemMatch = line.match(/^\s+-\s+(.+)$/);
|
||||
if (itemMatch && currentKey) {
|
||||
pushItem(result, currentKey, itemMatch[1].trim());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function setScalar(fm: ParsedFrontmatter, key: string, value: string): void {
|
||||
if (key === "id") fm.id = value;
|
||||
}
|
||||
|
||||
function pushItem(fm: ParsedFrontmatter, key: string, value: string): void {
|
||||
switch (key) {
|
||||
case "provides": fm.provides.push(value); break;
|
||||
case "requires": fm.requires.push(value); break;
|
||||
case "key_files": fm.key_files.push(value); break;
|
||||
case "key_decisions": fm.key_decisions.push(value); break;
|
||||
case "patterns_established": fm.patterns_established.push(value); break;
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Body parsing ────────────────────────────────────────────────────────────
|
||||
|
||||
function extractTitleAndOneLiner(body: string): { id: string; oneLiner: string } {
|
||||
const lines = body.split(/\r?\n/);
|
||||
let titleId = "";
|
||||
let oneLiner = "";
|
||||
let foundTitle = false;
|
||||
|
||||
for (const line of lines) {
|
||||
const titleMatch = line.match(/^#\s+(\S+):\s*(.*)$/);
|
||||
if (titleMatch && !foundTitle) {
|
||||
titleId = titleMatch[1];
|
||||
// If the title line itself has text after "S01: ", use that as a fallback
|
||||
if (titleMatch[2].trim()) {
|
||||
oneLiner = titleMatch[2].trim();
|
||||
}
|
||||
foundTitle = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
// First non-empty line after the title is the one-liner
|
||||
if (foundTitle && !oneLiner && line.trim() && !line.startsWith("#")) {
|
||||
oneLiner = line.trim();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return { id: titleId, oneLiner };
|
||||
}
|
||||
|
||||
function getBodyAfterFrontmatter(raw: string): string {
|
||||
const fmMatch = raw.match(/^---\r?\n[\s\S]*?\r?\n---\r?\n?/);
|
||||
if (fmMatch) {
|
||||
return raw.slice(fmMatch[0].length);
|
||||
}
|
||||
return raw;
|
||||
}
|
||||
|
||||
// ─── Public API ──────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Distill a single SUMMARY.md content string into a compact structured block.
|
||||
*/
|
||||
export function distillSingle(summary: string): string {
|
||||
const fm = parseFrontmatter(summary);
|
||||
const body = getBodyAfterFrontmatter(summary);
|
||||
const { id: titleId, oneLiner } = extractTitleAndOneLiner(body);
|
||||
|
||||
const id = fm.id || titleId || "???";
|
||||
|
||||
return formatEntry({
|
||||
id,
|
||||
oneLiner,
|
||||
provides: fm.provides,
|
||||
requires: fm.requires,
|
||||
key_files: fm.key_files,
|
||||
key_decisions: fm.key_decisions,
|
||||
patterns: fm.patterns_established,
|
||||
});
|
||||
}
|
||||
|
||||
function formatEntry(entry: DistilledEntry): string {
|
||||
return formatEntryWithDropLevel(entry, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Format an entry, progressively dropping fields based on dropLevel:
|
||||
* 0 = full output
|
||||
* 1 = drop patterns
|
||||
* 2 = drop patterns + key_decisions
|
||||
* 3 = drop patterns + key_decisions + key_files
|
||||
*/
|
||||
function formatEntryWithDropLevel(entry: DistilledEntry, dropLevel: number): string {
|
||||
const lines: string[] = [];
|
||||
lines.push(`## ${entry.id}: ${entry.oneLiner}`);
|
||||
|
||||
if (entry.provides.length > 0) {
|
||||
lines.push(`provides: ${entry.provides.join(", ")}`);
|
||||
}
|
||||
if (entry.requires.length > 0) {
|
||||
lines.push(`requires: ${entry.requires.join(", ")}`);
|
||||
}
|
||||
if (dropLevel < 3 && entry.key_files.length > 0) {
|
||||
lines.push(`key_files: ${entry.key_files.join(", ")}`);
|
||||
}
|
||||
if (dropLevel < 2 && entry.key_decisions.length > 0) {
|
||||
lines.push(`key_decisions: ${entry.key_decisions.join(", ")}`);
|
||||
}
|
||||
if (dropLevel < 1 && entry.patterns.length > 0) {
|
||||
lines.push(`patterns: ${entry.patterns.join(", ")}`);
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Distill multiple SUMMARY.md contents into a budget-constrained output.
|
||||
*/
|
||||
export function distillSummaries(summaries: string[], budgetChars: number): DistillationResult {
|
||||
const originalChars = summaries.reduce((sum, s) => sum + s.length, 0);
|
||||
|
||||
if (summaries.length === 0) {
|
||||
return {
|
||||
content: "",
|
||||
summaryCount: 0,
|
||||
savingsPercent: 0,
|
||||
originalChars: 0,
|
||||
distilledChars: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// Parse all entries up front
|
||||
const entries: DistilledEntry[] = summaries.map((summary) => {
|
||||
const fm = parseFrontmatter(summary);
|
||||
const body = getBodyAfterFrontmatter(summary);
|
||||
const { id: titleId, oneLiner } = extractTitleAndOneLiner(body);
|
||||
return {
|
||||
id: fm.id || titleId || "???",
|
||||
oneLiner,
|
||||
provides: fm.provides,
|
||||
requires: fm.requires,
|
||||
key_files: fm.key_files,
|
||||
key_decisions: fm.key_decisions,
|
||||
patterns: fm.patterns_established,
|
||||
};
|
||||
});
|
||||
|
||||
// Try progressively more aggressive dropping until it fits
|
||||
for (let dropLevel = 0; dropLevel <= 3; dropLevel++) {
|
||||
const blocks = entries.map((e) => formatEntryWithDropLevel(e, dropLevel));
|
||||
const content = blocks.join("\n\n");
|
||||
if (content.length <= budgetChars) {
|
||||
const distilledChars = content.length;
|
||||
return {
|
||||
content,
|
||||
summaryCount: summaries.length,
|
||||
savingsPercent: originalChars > 0
|
||||
? Math.round((1 - distilledChars / originalChars) * 100)
|
||||
: 0,
|
||||
originalChars,
|
||||
distilledChars,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Even at max drop level it doesn't fit — truncate
|
||||
const blocks = entries.map((e) => formatEntryWithDropLevel(e, 3));
|
||||
let content = blocks.join("\n\n");
|
||||
if (content.length > budgetChars) {
|
||||
content = content.slice(0, Math.max(0, budgetChars - 15)) + "\n[...truncated]";
|
||||
}
|
||||
|
||||
const distilledChars = content.length;
|
||||
return {
|
||||
content,
|
||||
summaryCount: summaries.length,
|
||||
savingsPercent: originalChars > 0
|
||||
? Math.round((1 - distilledChars / originalChars) * 100)
|
||||
: 0,
|
||||
originalChars,
|
||||
distilledChars,
|
||||
};
|
||||
}
|
||||
|
|
@ -1,193 +0,0 @@
|
|||
/**
|
||||
* Context Compression — unit tests for M004/S02.
|
||||
*
|
||||
* Verifies that prompt builders respect inlineLevel parameter by
|
||||
* inspecting the auto-prompts.ts source for level-aware gating.
|
||||
* Cannot call builders directly due to @gsd/pi-coding-agent import
|
||||
* resolution — uses source-level structural verification instead.
|
||||
*/
|
||||
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join, dirname } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const promptsSrc = readFileSync(join(__dirname, "..", "auto-prompts.ts"), "utf-8");
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// inlineLevel Parameter Presence
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
const BUILDERS_WITH_LEVEL = [
|
||||
"buildPlanMilestonePrompt",
|
||||
"buildPlanSlicePrompt",
|
||||
"buildExecuteTaskPrompt",
|
||||
"buildCompleteSlicePrompt",
|
||||
"buildCompleteMilestonePrompt",
|
||||
"buildReassessRoadmapPrompt",
|
||||
];
|
||||
|
||||
for (const builder of BUILDERS_WITH_LEVEL) {
|
||||
test(`compression: ${builder} accepts inlineLevel parameter`, () => {
|
||||
// Find the function signature
|
||||
const sigRegex = new RegExp(`export async function ${builder}\\([^)]*level\\?: InlineLevel`);
|
||||
assert.ok(
|
||||
sigRegex.test(promptsSrc),
|
||||
`${builder} should have level?: InlineLevel parameter`,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Default Level Resolution
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
test("compression: builders default to resolveInlineLevel() when no level passed", () => {
|
||||
const defaultPattern = /const inlineLevel = level \?\? resolveInlineLevel\(\)/g;
|
||||
const matches = promptsSrc.match(defaultPattern);
|
||||
assert.ok(matches, "should have resolveInlineLevel() fallback");
|
||||
assert.ok(
|
||||
matches.length >= BUILDERS_WITH_LEVEL.length,
|
||||
`should have ${BUILDERS_WITH_LEVEL.length} fallback instances, found ${matches?.length}`,
|
||||
);
|
||||
});
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Minimal Level — Template Reduction
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
test("compression: buildExecuteTaskPrompt minimal drops decisions template", () => {
|
||||
// In the execute-task builder, minimal should only inline task-summary, not decisions
|
||||
assert.ok(
|
||||
promptsSrc.includes('inlineLevel === "minimal"') &&
|
||||
promptsSrc.includes('inlineTemplate("task-summary"'),
|
||||
"execute-task should conditionally include decisions template based on level",
|
||||
);
|
||||
});
|
||||
|
||||
test("compression: buildExecuteTaskPrompt minimal truncates prior summaries", () => {
|
||||
assert.ok(
|
||||
promptsSrc.includes('inlineLevel === "minimal" && priorSummaries.length > 1'),
|
||||
"execute-task should limit prior summaries for minimal level",
|
||||
);
|
||||
});
|
||||
|
||||
test("compression: buildExecuteTaskPrompt passes verificationBudget to loadPrompt (#707)", () => {
|
||||
// The execute-task template declares {{verificationBudget}} — the builder must supply it
|
||||
assert.ok(
|
||||
promptsSrc.includes("verificationBudget"),
|
||||
"buildExecuteTaskPrompt should pass verificationBudget in the loadPrompt vars object",
|
||||
);
|
||||
// Verify it computes the budget from computeBudgets
|
||||
assert.ok(
|
||||
promptsSrc.includes("computeBudgets(contextWindow)"),
|
||||
"buildExecuteTaskPrompt should compute budgets from the executor context window",
|
||||
);
|
||||
});
|
||||
|
||||
test("compression: buildPlanMilestonePrompt minimal drops project/requirements/decisions files", () => {
|
||||
// The plan-milestone builder should gate root file inlining on inlineLevel
|
||||
assert.ok(
|
||||
promptsSrc.includes('inlineLevel !== "minimal"') &&
|
||||
promptsSrc.includes("inlineProjectFromDb(base)"),
|
||||
"plan-milestone should conditionally include project.md based on level",
|
||||
);
|
||||
});
|
||||
|
||||
test("compression: buildPlanMilestonePrompt minimal drops extra templates", () => {
|
||||
// Full inlines 5 templates, minimal should inline fewer
|
||||
assert.ok(
|
||||
promptsSrc.includes('if (inlineLevel === "full")') &&
|
||||
promptsSrc.includes('inlineTemplate("secrets-manifest"'),
|
||||
"plan-milestone should only include secrets-manifest template at full level",
|
||||
);
|
||||
});
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Complete-Slice Level Gating
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
test("compression: buildCompleteSlicePrompt minimal drops requirements", () => {
|
||||
// Find the complete-slice section and verify requirements gating
|
||||
const completeSliceIdx = promptsSrc.indexOf("buildCompleteSlicePrompt");
|
||||
const nextBuilder = promptsSrc.indexOf("buildCompleteMilestonePrompt");
|
||||
const completeSliceBlock = promptsSrc.slice(completeSliceIdx, nextBuilder);
|
||||
assert.ok(
|
||||
completeSliceBlock.includes('inlineLevel !== "minimal"'),
|
||||
"complete-slice should gate requirements inlining on level",
|
||||
);
|
||||
});
|
||||
|
||||
test("compression: buildCompleteSlicePrompt minimal drops UAT template", () => {
|
||||
const completeSliceIdx = promptsSrc.indexOf("buildCompleteSlicePrompt");
|
||||
const nextBuilder = promptsSrc.indexOf("buildCompleteMilestonePrompt");
|
||||
const completeSliceBlock = promptsSrc.slice(completeSliceIdx, nextBuilder);
|
||||
assert.ok(
|
||||
completeSliceBlock.includes('inlineLevel !== "minimal"') &&
|
||||
completeSliceBlock.includes('inlineTemplate("uat"'),
|
||||
"complete-slice should conditionally include UAT template based on level",
|
||||
);
|
||||
});
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Complete-Milestone Level Gating
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
test("compression: buildCompleteMilestonePrompt minimal drops root GSD files", () => {
|
||||
const completeMilestoneIdx = promptsSrc.indexOf("buildCompleteMilestonePrompt");
|
||||
const nextBuilder = promptsSrc.indexOf("buildReplanSlicePrompt");
|
||||
const block = promptsSrc.slice(completeMilestoneIdx, nextBuilder);
|
||||
assert.ok(
|
||||
block.includes('inlineLevel !== "minimal"') &&
|
||||
(block.includes('inlineGsdRootFile(base, "requirements.md"') || block.includes('inlineRequirementsFromDb(base')),
|
||||
"complete-milestone should gate root file inlining on level",
|
||||
);
|
||||
});
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Reassess-Roadmap Level Gating
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
test("compression: buildReassessRoadmapPrompt minimal drops project/requirements/decisions", () => {
|
||||
const reassessIdx = promptsSrc.indexOf("buildReassessRoadmapPrompt");
|
||||
const block = promptsSrc.slice(reassessIdx, reassessIdx + 1500);
|
||||
assert.ok(
|
||||
block.includes('inlineLevel !== "minimal"'),
|
||||
"reassess-roadmap should gate file inlining on level",
|
||||
);
|
||||
});
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Full Level — No Regression
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
test("compression: full level preserves all templates and files (no regression)", () => {
|
||||
// Verify the key template names are still present in the source
|
||||
const expectedTemplates = [
|
||||
"roadmap", "decisions", "plan", "task-plan", "secrets-manifest",
|
||||
"task-summary", "slice-summary", "uat", "milestone-summary",
|
||||
];
|
||||
for (const tpl of expectedTemplates) {
|
||||
assert.ok(
|
||||
promptsSrc.includes(`inlineTemplate("${tpl}"`),
|
||||
`template "${tpl}" should still be present in auto-prompts.ts`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Import Verification
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
test("compression: auto-prompts.ts imports resolveInlineLevel and InlineLevel", () => {
|
||||
assert.ok(
|
||||
promptsSrc.includes("resolveInlineLevel"),
|
||||
"should import resolveInlineLevel from preferences",
|
||||
);
|
||||
assert.ok(
|
||||
promptsSrc.includes("InlineLevel"),
|
||||
"should import InlineLevel type from types",
|
||||
);
|
||||
});
|
||||
|
|
@ -208,30 +208,25 @@ test("git fields comprehensive validation", () => {
|
|||
assert.equal(preferences.git?.isolation, "branch");
|
||||
});
|
||||
|
||||
test("auto_visualize, auto_report, compression_strategy, context_selection validate correctly", () => {
|
||||
test("auto_visualize, auto_report, context_selection validate correctly", () => {
|
||||
const { preferences, errors } = validatePreferences({
|
||||
auto_visualize: true,
|
||||
auto_report: false,
|
||||
compression_strategy: "compress",
|
||||
context_selection: "smart",
|
||||
});
|
||||
assert.equal(errors.length, 0);
|
||||
assert.equal(preferences.auto_visualize, true);
|
||||
assert.equal(preferences.auto_report, false);
|
||||
assert.equal(preferences.compression_strategy, "compress");
|
||||
assert.equal(preferences.context_selection, "smart");
|
||||
});
|
||||
|
||||
test("auto_visualize, auto_report, compression_strategy, context_selection reject invalid values", () => {
|
||||
test("auto_visualize, auto_report, context_selection reject invalid values", () => {
|
||||
const { errors: e1 } = validatePreferences({ auto_visualize: "yes" as never });
|
||||
assert.ok(e1.some(e => e.includes("auto_visualize")));
|
||||
|
||||
const { errors: e2 } = validatePreferences({ auto_report: 1 as never });
|
||||
assert.ok(e2.some(e => e.includes("auto_report")));
|
||||
|
||||
const { errors: e3 } = validatePreferences({ compression_strategy: "shrink" as never });
|
||||
assert.ok(e3.some(e => e.includes("compression_strategy")));
|
||||
|
||||
const { errors: e4 } = validatePreferences({ context_selection: "partial" as never });
|
||||
assert.ok(e4.some(e => e.includes("context_selection")));
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,529 +0,0 @@
|
|||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import {
|
||||
compressPrompt,
|
||||
compressToTarget,
|
||||
} from "../prompt-compressor.js";
|
||||
import type {
|
||||
CompressionLevel,
|
||||
CompressionResult,
|
||||
CompressionOptions,
|
||||
} from "../prompt-compressor.js";
|
||||
|
||||
// ─── Test Fixtures ──────────────────────────────────────────────────────────
|
||||
|
||||
const WHITESPACE_HEAVY = `# Section One
|
||||
|
||||
Some content here.
|
||||
|
||||
|
||||
|
||||
Another paragraph here.
|
||||
|
||||
|
||||
Yet another paragraph.
|
||||
|
||||
|
||||
|
||||
# Section Two
|
||||
|
||||
More content.`;
|
||||
|
||||
const MARKDOWN_COMMENTS = `# Title
|
||||
|
||||
<!-- This is a comment that should be removed -->
|
||||
|
||||
Some content here.
|
||||
|
||||
<!-- Another
|
||||
multi-line
|
||||
comment -->
|
||||
|
||||
More content.`;
|
||||
|
||||
const HORIZONTAL_RULES = `# Section One
|
||||
|
||||
Some content.
|
||||
|
||||
---
|
||||
|
||||
# Section Two
|
||||
|
||||
More content.
|
||||
|
||||
***
|
||||
|
||||
# Section Three
|
||||
|
||||
Final content.`;
|
||||
|
||||
const VERBOSE_PROSE = `In order to implement this feature, it is important to note that the following
|
||||
requirements must be met. Due to the fact that the system operates in real-time,
|
||||
prior to deployment we need to verify all components. In addition to the main
|
||||
module, a number of auxiliary services are required. In the event that a service
|
||||
fails, subsequent to the failure, the system should recover. For the purpose of
|
||||
monitoring, in accordance with our SLA, with regard to uptime, at this point in
|
||||
time we achieve 99.9%. On the basis of recent data, in the case of peak traffic,
|
||||
as mentioned previously, the system scales automatically.`;
|
||||
|
||||
const BOILERPLATE_CONTENT = `# Requirements
|
||||
|
||||
## Feature A
|
||||
Must support pagination.
|
||||
|
||||
## Feature B
|
||||
N/A
|
||||
|
||||
## Feature C
|
||||
(none)
|
||||
|
||||
## Feature D
|
||||
(empty)
|
||||
|
||||
## Feature E
|
||||
(not applicable)
|
||||
|
||||
## Feature F
|
||||
Must handle errors gracefully.`;
|
||||
|
||||
const DUPLICATE_LINES = `Status: active
|
||||
Status: active
|
||||
Status: active
|
||||
Priority: high
|
||||
Name: test project
|
||||
Name: test project`;
|
||||
|
||||
const EMPHASIS_CONTENT = `This is **bold text** and this is *italic text*.
|
||||
Also __underline bold__ and _underline italic_.
|
||||
Check [this link](https://example.com) and [another](https://test.org).`;
|
||||
|
||||
const CODE_BLOCK_CONTENT = `# Setup Guide
|
||||
|
||||
In order to configure the system, run the following command:
|
||||
|
||||
\`\`\`typescript
|
||||
const config = {
|
||||
debug: true,
|
||||
verbose: false,
|
||||
timeout: 3000,
|
||||
};
|
||||
\`\`\`
|
||||
|
||||
Due to the fact that configuration is loaded at startup, prior to
|
||||
running the application, verify the config file exists.
|
||||
|
||||
\`\`\`bash
|
||||
ls -la config.json
|
||||
\`\`\`
|
||||
|
||||
The following steps complete the setup.`;
|
||||
|
||||
const HEADING_CONTENT = `# Main Title
|
||||
|
||||
## Subsection A
|
||||
|
||||
In order to do something, the following steps are needed.
|
||||
|
||||
## Subsection B
|
||||
|
||||
More content here with **emphasis** and [a link](https://example.com).
|
||||
|
||||
### Sub-subsection
|
||||
|
||||
Details here.`;
|
||||
|
||||
const REALISTIC_GSD_CONTENT = `# Project: GSD Task Manager
|
||||
|
||||
<!-- Generated by GSD v2.1.0 -->
|
||||
|
||||
## Decisions
|
||||
|
||||
| Decision ID | Title | Status | Date |
|
||||
|---------------|------------------------------|------------|--------------|
|
||||
| DEC-001 | Use TypeScript | Approved | 2024-01-15 |
|
||||
| DEC-002 | Adopt monorepo | Approved | 2024-01-20 |
|
||||
| DEC-003 | Use node:test | Approved | 2024-02-01 |
|
||||
|
||||
## Requirements
|
||||
|
||||
### Must-Have
|
||||
|
||||
In order to support the core workflow, it is important to note that the following
|
||||
requirements are non-negotiable. Due to the fact that the system must operate in
|
||||
CI environments, prior to any release, all tests must pass.
|
||||
|
||||
- The system must handle concurrent operations
|
||||
- The system must handle concurrent operations
|
||||
- Error recovery must be automatic
|
||||
- Configuration must be file-based
|
||||
- Configuration must be file-based
|
||||
|
||||
### Nice-to-Have
|
||||
|
||||
N/A
|
||||
|
||||
### Out of Scope
|
||||
|
||||
(none)
|
||||
|
||||
---
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
> In accordance with our coding standards, all modules should follow
|
||||
> the single responsibility principle. With regard to testing, a number of
|
||||
> integration tests should supplement unit tests.
|
||||
|
||||
For the purpose of maintaining code quality, at this point in time we require
|
||||
100% branch coverage on critical paths. In the event that coverage drops below
|
||||
the threshold, subsequent to the detection, the CI pipeline should fail.
|
||||
|
||||
**Important**: The following constraints apply:
|
||||
- *Memory usage* must stay under 512MB
|
||||
- *CPU usage* must not exceed 80% sustained
|
||||
- Response times under 100ms for the 95th percentile
|
||||
|
||||
In addition to the above, the system should support plugin architecture.
|
||||
As mentioned previously, this was decided in DEC-001.
|
||||
|
||||
---
|
||||
|
||||
## Status
|
||||
|
||||
Status: active
|
||||
Status: active
|
||||
Priority: high
|
||||
Sprint: 14
|
||||
Sprint: 14
|
||||
Milestone: v2.1.0`;
|
||||
|
||||
const LONG_LINE = "This is a very long line that goes on and on. It contains multiple sentences that discuss various topics. The purpose of this line is to test the truncation functionality. When lines exceed 300 characters, they should be truncated at a sentence boundary. This ensures that the compressed output remains readable. Additional text is added here to make sure we exceed the 300 character limit for testing purposes. Even more text follows to pad the line further.";
|
||||
|
||||
const BLOCKQUOTE_CONTENT = `> This is a blockquote
|
||||
> with multiple lines
|
||||
> that should have markers removed.
|
||||
|
||||
Normal paragraph here.
|
||||
|
||||
> Another blockquote.`;
|
||||
|
||||
const BULLET_LIST = `Some intro text:
|
||||
|
||||
- First item in the list
|
||||
- Second item in the list
|
||||
* Third item with star
|
||||
+ Fourth item with plus
|
||||
1. Numbered item one
|
||||
2. Numbered item two
|
||||
|
||||
Closing text.`;
|
||||
|
||||
// ─── Light Compression Tests ────────────────────────────────────────────────
|
||||
|
||||
test("light compression removes extra whitespace", () => {
|
||||
const result = compressPrompt(WHITESPACE_HEAVY, { level: "light" });
|
||||
assert.ok(result.compressedChars < result.originalChars, "should reduce size");
|
||||
assert.ok(!result.content.includes(" \n"), "should not have trailing spaces");
|
||||
// Should not have 3+ consecutive blank lines
|
||||
assert.ok(!result.content.match(/\n\s*\n\s*\n\s*\n/), "should not have 3+ blank lines");
|
||||
assert.equal(result.level, "light");
|
||||
});
|
||||
|
||||
test("light compression removes markdown comments", () => {
|
||||
const result = compressPrompt(MARKDOWN_COMMENTS, { level: "light" });
|
||||
assert.ok(!result.content.includes("<!--"), "should not contain comment start");
|
||||
assert.ok(!result.content.includes("-->"), "should not contain comment end");
|
||||
assert.ok(result.content.includes("# Title"), "should preserve heading");
|
||||
assert.ok(result.content.includes("Some content here."), "should preserve normal content");
|
||||
});
|
||||
|
||||
test("light compression removes horizontal rules", () => {
|
||||
const result = compressPrompt(HORIZONTAL_RULES, { level: "light" });
|
||||
assert.ok(!result.content.match(/^---$/m), "should not contain ---");
|
||||
assert.ok(!result.content.match(/^\*\*\*$/m), "should not contain ***");
|
||||
assert.ok(result.content.includes("# Section One"), "should preserve headings");
|
||||
assert.ok(result.content.includes("# Section Two"), "should preserve headings");
|
||||
});
|
||||
|
||||
test("light compression preserves code blocks", () => {
|
||||
const result = compressPrompt(CODE_BLOCK_CONTENT, { level: "light" });
|
||||
assert.ok(result.content.includes("const config = {"), "should preserve code block content");
|
||||
assert.ok(result.content.includes("```typescript"), "should preserve code fence");
|
||||
assert.ok(result.content.includes("```bash"), "should preserve code fence");
|
||||
});
|
||||
|
||||
// ─── Moderate Compression Tests ─────────────────────────────────────────────
|
||||
|
||||
test("moderate compression abbreviates verbose phrases", () => {
|
||||
const result = compressPrompt(VERBOSE_PROSE, { level: "moderate" });
|
||||
assert.ok(result.content.includes("To implement"), "should abbreviate 'In order to'");
|
||||
assert.ok(result.content.includes("Because"), "should abbreviate 'Due to the fact that'");
|
||||
assert.ok(result.content.includes("Before deployment"), "should abbreviate 'Prior to'");
|
||||
assert.ok(result.content.includes("Also,"), "should abbreviate 'In addition to'");
|
||||
assert.ok(result.content.includes("Several"), "should abbreviate 'A number of'");
|
||||
assert.ok(result.content.includes("If"), "should abbreviate 'In the event that'");
|
||||
assert.ok(result.content.includes("After"), "should abbreviate 'Subsequent to'");
|
||||
assert.ok(!result.content.includes("For the purpose of"), "should abbreviate 'For the purpose of'");
|
||||
assert.ok(result.content.includes("Per"), "should abbreviate 'In accordance with'");
|
||||
assert.ok(result.content.includes("Re:"), "should abbreviate 'With regard to'");
|
||||
assert.ok(result.content.includes("Now"), "should abbreviate 'At this point in time'");
|
||||
assert.ok(result.content.includes("Based on"), "should abbreviate 'On the basis of'");
|
||||
assert.ok(result.content.includes("(see above)"), "should abbreviate 'As mentioned previously'");
|
||||
assert.ok(result.compressedChars < result.originalChars, "should reduce size");
|
||||
});
|
||||
|
||||
test("moderate compression deduplicates consecutive lines", () => {
|
||||
const input = "Line one\nLine one\nLine one\nLine two\nLine three\nLine three";
|
||||
const result = compressPrompt(input, { level: "moderate" });
|
||||
const lines = result.content.split("\n").filter((l) => l.trim() !== "");
|
||||
// Count occurrences of "Line one"
|
||||
const lineOneCount = lines.filter((l) => l === "Line one").length;
|
||||
assert.equal(lineOneCount, 1, "should deduplicate 'Line one'");
|
||||
const lineThreeCount = lines.filter((l) => l === "Line three").length;
|
||||
assert.equal(lineThreeCount, 1, "should deduplicate 'Line three'");
|
||||
});
|
||||
|
||||
test("moderate compression removes boilerplate", () => {
|
||||
const result = compressPrompt(BOILERPLATE_CONTENT, { level: "moderate" });
|
||||
assert.ok(!result.content.match(/^\s*N\/A\s*$/m), "should remove N/A lines");
|
||||
assert.ok(!result.content.includes("(none)"), "should remove (none)");
|
||||
assert.ok(!result.content.includes("(empty)"), "should remove (empty)");
|
||||
assert.ok(!result.content.includes("(not applicable)"), "should remove (not applicable)");
|
||||
assert.ok(result.content.includes("Must support pagination"), "should keep real content");
|
||||
assert.ok(result.content.includes("Must handle errors"), "should keep real content");
|
||||
});
|
||||
|
||||
test("moderate compression collapses table formatting", () => {
|
||||
const table = `| Name | Value | Status |
|
||||
| foo | bar | active |`;
|
||||
const result = compressPrompt(table, { level: "moderate" });
|
||||
// Should have reduced padding
|
||||
assert.ok(result.compressedChars < result.originalChars, "should reduce table padding");
|
||||
});
|
||||
|
||||
// ─── Aggressive Compression Tests ───────────────────────────────────────────
|
||||
|
||||
test("aggressive compression removes emphasis and links", () => {
|
||||
const result = compressPrompt(EMPHASIS_CONTENT, { level: "aggressive" });
|
||||
assert.ok(!result.content.includes("**"), "should remove bold markers");
|
||||
assert.ok(!result.content.includes("__"), "should remove underline bold markers");
|
||||
assert.ok(result.content.includes("bold text"), "should keep bold text content");
|
||||
assert.ok(result.content.includes("italic text"), "should keep italic text content");
|
||||
assert.ok(result.content.includes("this link"), "should keep link text");
|
||||
assert.ok(!result.content.includes("https://example.com"), "should remove link URLs");
|
||||
assert.ok(!result.content.includes("https://test.org"), "should remove link URLs");
|
||||
});
|
||||
|
||||
test("aggressive compression removes bullet markers", () => {
|
||||
const result = compressPrompt(BULLET_LIST, { level: "aggressive" });
|
||||
assert.ok(!result.content.match(/^- /m), "should remove dash bullets");
|
||||
assert.ok(!result.content.match(/^\* /m), "should remove star bullets");
|
||||
assert.ok(!result.content.match(/^\+ /m), "should remove plus bullets");
|
||||
assert.ok(!result.content.match(/^\d+\. /m), "should remove numbered bullets");
|
||||
assert.ok(result.content.includes("First item"), "should keep bullet content");
|
||||
assert.ok(result.content.includes("Numbered item"), "should keep numbered content");
|
||||
});
|
||||
|
||||
test("aggressive compression removes blockquote markers", () => {
|
||||
const result = compressPrompt(BLOCKQUOTE_CONTENT, { level: "aggressive" });
|
||||
assert.ok(!result.content.match(/^> /m), "should remove blockquote markers");
|
||||
assert.ok(result.content.includes("This is a blockquote"), "should keep blockquote content");
|
||||
assert.ok(result.content.includes("Normal paragraph"), "should keep normal content");
|
||||
});
|
||||
|
||||
test("aggressive compression truncates long lines", () => {
|
||||
const result = compressPrompt(LONG_LINE, { level: "aggressive" });
|
||||
const lines = result.content.split("\n");
|
||||
for (const line of lines) {
|
||||
assert.ok(line.length <= 300, `line should be <= 300 chars, got ${line.length}`);
|
||||
}
|
||||
});
|
||||
|
||||
test("aggressive compression deduplicates structural patterns", () => {
|
||||
const result = compressPrompt(DUPLICATE_LINES, { level: "aggressive" });
|
||||
const lines = result.content.split("\n").filter((l) => l.trim() !== "");
|
||||
const statusCount = lines.filter((l) => l.includes("Status: active")).length;
|
||||
assert.equal(statusCount, 1, "should keep only one Status: active");
|
||||
const nameCount = lines.filter((l) => l.includes("Name: test project")).length;
|
||||
assert.equal(nameCount, 1, "should keep only one Name: test project");
|
||||
});
|
||||
|
||||
// ─── Preservation Tests ─────────────────────────────────────────────────────
|
||||
|
||||
test("code block preservation protects code from compression", () => {
|
||||
const result = compressPrompt(CODE_BLOCK_CONTENT, {
|
||||
level: "aggressive",
|
||||
preserveCodeBlocks: true,
|
||||
});
|
||||
// Code blocks should be untouched
|
||||
assert.ok(result.content.includes("const config = {"), "code block preserved");
|
||||
assert.ok(result.content.includes("debug: true,"), "code block details preserved");
|
||||
assert.ok(result.content.includes("ls -la config.json"), "bash code block preserved");
|
||||
// But surrounding prose should be compressed
|
||||
assert.ok(!result.content.includes("In order to"), "prose should be compressed");
|
||||
assert.ok(!result.content.includes("Due to the fact that"), "prose should be compressed");
|
||||
});
|
||||
|
||||
test("code block preservation can be disabled", () => {
|
||||
const result = compressPrompt(CODE_BLOCK_CONTENT, {
|
||||
level: "aggressive",
|
||||
preserveCodeBlocks: false,
|
||||
});
|
||||
// Phrase abbreviation still works on surrounding text
|
||||
assert.ok(result.compressedChars < result.originalChars, "should still compress");
|
||||
});
|
||||
|
||||
test("heading preservation keeps headings intact", () => {
|
||||
const result = compressPrompt(HEADING_CONTENT, {
|
||||
level: "aggressive",
|
||||
preserveHeadings: true,
|
||||
});
|
||||
assert.ok(result.content.includes("# Main Title"), "should preserve h1");
|
||||
assert.ok(result.content.includes("## Subsection A"), "should preserve h2");
|
||||
assert.ok(result.content.includes("## Subsection B"), "should preserve h2");
|
||||
assert.ok(result.content.includes("### Sub-subsection"), "should preserve h3");
|
||||
});
|
||||
|
||||
// ─── compressToTarget Tests ─────────────────────────────────────────────────
|
||||
|
||||
test("compressToTarget tries progressively harder levels", () => {
|
||||
// Set a target that light compression cannot reach
|
||||
const lightResult = compressPrompt(REALISTIC_GSD_CONTENT, { level: "light" });
|
||||
const moderateResult = compressPrompt(REALISTIC_GSD_CONTENT, { level: "moderate" });
|
||||
|
||||
// Target between light and moderate results
|
||||
const target = Math.floor((lightResult.compressedChars + moderateResult.compressedChars) / 2);
|
||||
const result = compressToTarget(REALISTIC_GSD_CONTENT, target);
|
||||
|
||||
// Should have used at least moderate
|
||||
assert.ok(
|
||||
result.level === "moderate" || result.level === "aggressive",
|
||||
`should use moderate or aggressive, got ${result.level}`,
|
||||
);
|
||||
assert.ok(result.compressedChars <= target, "should meet target");
|
||||
});
|
||||
|
||||
test("compressToTarget returns best effort when target unreachable", () => {
|
||||
// Set an impossibly small target
|
||||
const result = compressToTarget(REALISTIC_GSD_CONTENT, 10);
|
||||
assert.equal(result.level, "aggressive", "should try aggressive as last resort");
|
||||
assert.ok(result.compressedChars > 10, "cannot reach impossibly small target");
|
||||
assert.ok(
|
||||
result.compressedChars < REALISTIC_GSD_CONTENT.length,
|
||||
"should still compress as much as possible",
|
||||
);
|
||||
});
|
||||
|
||||
test("compressToTarget returns unchanged if already under target", () => {
|
||||
const result = compressToTarget("short text", 1000);
|
||||
assert.equal(result.content, "short text");
|
||||
assert.equal(result.savingsPercent, 0);
|
||||
assert.equal(result.transformationsApplied, 0);
|
||||
});
|
||||
|
||||
// ─── Realistic GSD Content Test ─────────────────────────────────────────────
|
||||
|
||||
test("realistic GSD content compresses significantly", () => {
|
||||
const result = compressPrompt(REALISTIC_GSD_CONTENT, { level: "aggressive" });
|
||||
|
||||
// Should achieve meaningful compression
|
||||
assert.ok(result.savingsPercent > 15, `should achieve >15% savings, got ${result.savingsPercent}%`);
|
||||
assert.ok(result.transformationsApplied > 3, "should apply multiple transformations");
|
||||
|
||||
// Key content preserved
|
||||
assert.ok(result.content.includes("# Project: GSD Task Manager"), "title preserved");
|
||||
assert.ok(result.content.includes("DEC-001"), "decision IDs preserved");
|
||||
assert.ok(result.content.includes("TypeScript"), "decision content preserved");
|
||||
assert.ok(result.content.includes("## Decisions"), "section headings preserved");
|
||||
assert.ok(result.content.includes("## Requirements"), "section headings preserved");
|
||||
|
||||
// Comments removed
|
||||
assert.ok(!result.content.includes("<!--"), "comments removed");
|
||||
|
||||
// Verbose phrases abbreviated
|
||||
assert.ok(!result.content.includes("In order to"), "verbose phrases compressed");
|
||||
assert.ok(!result.content.includes("Due to the fact that"), "verbose phrases compressed");
|
||||
|
||||
// Boilerplate removed
|
||||
assert.ok(!result.content.match(/^\s*N\/A\s*$/m), "N/A removed");
|
||||
assert.ok(!result.content.includes("(none)"), "(none) removed");
|
||||
});
|
||||
|
||||
// ─── Accuracy and Edge Cases ────────────────────────────────────────────────
|
||||
|
||||
test("savingsPercent is accurate", () => {
|
||||
const result = compressPrompt(VERBOSE_PROSE, { level: "moderate" });
|
||||
const expectedPercent =
|
||||
Math.round(((result.originalChars - result.compressedChars) / result.originalChars) * 10000) / 100;
|
||||
assert.equal(result.savingsPercent, expectedPercent, "savings percent should be accurate");
|
||||
});
|
||||
|
||||
test("empty input returns empty output", () => {
|
||||
const result = compressPrompt("", { level: "aggressive" });
|
||||
assert.equal(result.content, "");
|
||||
assert.equal(result.originalChars, 0);
|
||||
assert.equal(result.compressedChars, 0);
|
||||
assert.equal(result.savingsPercent, 0);
|
||||
assert.equal(result.transformationsApplied, 0);
|
||||
});
|
||||
|
||||
test("already-compressed content is idempotent at same level", () => {
|
||||
const first = compressPrompt(VERBOSE_PROSE, { level: "moderate" });
|
||||
const second = compressPrompt(first.content, { level: "moderate" });
|
||||
|
||||
assert.equal(first.content, second.content, "double compression should produce same result");
|
||||
});
|
||||
|
||||
test("content with only code blocks is unchanged", () => {
|
||||
const codeOnly = "```typescript\nconst x = 1;\nconst y = 2;\n```";
|
||||
const result = compressPrompt(codeOnly, {
|
||||
level: "aggressive",
|
||||
preserveCodeBlocks: true,
|
||||
});
|
||||
assert.equal(result.content, codeOnly, "code-only content should be unchanged");
|
||||
});
|
||||
|
||||
test("compression result contains correct metadata", () => {
|
||||
const result = compressPrompt(VERBOSE_PROSE, { level: "moderate" });
|
||||
assert.equal(result.originalChars, VERBOSE_PROSE.length);
|
||||
assert.equal(result.compressedChars, result.content.length);
|
||||
assert.equal(result.level, "moderate");
|
||||
assert.ok(result.transformationsApplied > 0, "should report transformations");
|
||||
assert.ok(result.savingsPercent > 0, "should have positive savings");
|
||||
assert.ok(result.savingsPercent < 100, "savings should be less than 100%");
|
||||
});
|
||||
|
||||
test("light compression with defaults", () => {
|
||||
// Test that default options work (moderate level, preserve headings/code)
|
||||
const result = compressPrompt(REALISTIC_GSD_CONTENT);
|
||||
assert.equal(result.level, "moderate", "default level should be moderate");
|
||||
assert.ok(result.content.includes("# Project:"), "headings preserved by default");
|
||||
assert.ok(result.compressedChars < result.originalChars, "should compress");
|
||||
});
|
||||
|
||||
test("multiple code blocks are all preserved", () => {
|
||||
const multiCode = `Some text.
|
||||
|
||||
\`\`\`js
|
||||
function a() { return 1; }
|
||||
\`\`\`
|
||||
|
||||
Middle text with **emphasis**.
|
||||
|
||||
\`\`\`python
|
||||
def b():
|
||||
return 2
|
||||
\`\`\`
|
||||
|
||||
End text.`;
|
||||
|
||||
const result = compressPrompt(multiCode, {
|
||||
level: "aggressive",
|
||||
preserveCodeBlocks: true,
|
||||
});
|
||||
assert.ok(result.content.includes("function a()"), "first code block preserved");
|
||||
assert.ok(result.content.includes("def b():"), "second code block preserved");
|
||||
assert.ok(result.content.includes("emphasis"), "emphasis text kept (markers removed)");
|
||||
assert.ok(!result.content.includes("**emphasis**"), "emphasis markers removed");
|
||||
});
|
||||
|
|
@ -1,426 +0,0 @@
|
|||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import {
|
||||
splitIntoChunks,
|
||||
scoreChunks,
|
||||
chunkByRelevance,
|
||||
formatChunks,
|
||||
} from "../semantic-chunker.js";
|
||||
import type { Chunk, ChunkResult } from "../semantic-chunker.js";
|
||||
|
||||
// ─── Test Fixtures ──────────────────────────────────────────────────────────
|
||||
|
||||
const TYPESCRIPT_CODE = `import { readFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
export interface Config {
|
||||
name: string;
|
||||
debug: boolean;
|
||||
}
|
||||
|
||||
export function loadConfig(path: string): Config {
|
||||
const raw = readFileSync(path, "utf-8");
|
||||
return JSON.parse(raw);
|
||||
}
|
||||
|
||||
export async function saveConfig(path: string, config: Config): Promise<void> {
|
||||
const data = JSON.stringify(config, null, 2);
|
||||
await writeFile(path, data, "utf-8");
|
||||
}
|
||||
|
||||
export class ConfigManager {
|
||||
private config: Config;
|
||||
|
||||
constructor(private path: string) {
|
||||
this.config = loadConfig(path);
|
||||
}
|
||||
|
||||
get(key: keyof Config) {
|
||||
return this.config[key];
|
||||
}
|
||||
|
||||
set(key: keyof Config, value: Config[keyof Config]) {
|
||||
this.config[key] = value;
|
||||
}
|
||||
|
||||
save() {
|
||||
return saveConfig(this.path, this.config);
|
||||
}
|
||||
}
|
||||
|
||||
const DEFAULT_CONFIG: Config = {
|
||||
name: "default",
|
||||
debug: false,
|
||||
};`;
|
||||
|
||||
const MARKDOWN_CONTENT = `# Project Overview
|
||||
|
||||
This project provides a task management system.
|
||||
|
||||
## Installation
|
||||
|
||||
Run the following command:
|
||||
|
||||
\`\`\`bash
|
||||
npm install gsd
|
||||
\`\`\`
|
||||
|
||||
## Usage
|
||||
|
||||
Import the module and initialize:
|
||||
|
||||
\`\`\`typescript
|
||||
import { gsd } from "gsd";
|
||||
gsd.init();
|
||||
\`\`\`
|
||||
|
||||
## API Reference
|
||||
|
||||
### init()
|
||||
|
||||
Initializes the system.
|
||||
|
||||
### run(task: string)
|
||||
|
||||
Runs a specified task.
|
||||
|
||||
## Contributing
|
||||
|
||||
Please read CONTRIBUTING.md before submitting PRs.`;
|
||||
|
||||
const PLAIN_TEXT = `The quick brown fox jumps over the lazy dog. This is a sample paragraph
|
||||
that tests plain text chunking behavior.
|
||||
|
||||
Another paragraph begins here. It contains different content that should
|
||||
be separated from the first paragraph by a blank line.
|
||||
|
||||
A third paragraph with more text. This should form its own chunk when
|
||||
processed by the text boundary detection.
|
||||
|
||||
Final paragraph wrapping up the test content.`;
|
||||
|
||||
// ─── splitIntoChunks — TypeScript Code ──────────────────────────────────────
|
||||
|
||||
test("splitIntoChunks splits TypeScript code at function/class/export boundaries", () => {
|
||||
const chunks = splitIntoChunks(TYPESCRIPT_CODE);
|
||||
assert.ok(chunks.length > 1, `Expected multiple chunks, got ${chunks.length}`);
|
||||
|
||||
// Should find boundaries at export interface, export function, export class, const
|
||||
const contents = chunks.map((c) => c.content);
|
||||
const hasInterface = contents.some((c) => c.includes("export interface Config"));
|
||||
const hasLoadConfig = contents.some((c) => c.includes("export function loadConfig"));
|
||||
const hasClass = contents.some((c) => c.includes("export class ConfigManager"));
|
||||
assert.ok(hasInterface, "Should have a chunk containing the interface");
|
||||
assert.ok(hasLoadConfig, "Should have a chunk containing loadConfig");
|
||||
assert.ok(hasClass, "Should have a chunk containing ConfigManager");
|
||||
});
|
||||
|
||||
test("splitIntoChunks preserves all content across chunks", () => {
|
||||
const chunks = splitIntoChunks(TYPESCRIPT_CODE);
|
||||
const reassembled = chunks.map((c) => c.content).join("\n");
|
||||
assert.equal(reassembled, TYPESCRIPT_CODE);
|
||||
});
|
||||
|
||||
test("splitIntoChunks assigns correct line numbers", () => {
|
||||
const chunks = splitIntoChunks(TYPESCRIPT_CODE);
|
||||
// First chunk starts at line 1
|
||||
assert.equal(chunks[0].startLine, 1);
|
||||
// Last chunk ends at total line count
|
||||
const totalLines = TYPESCRIPT_CODE.split("\n").length;
|
||||
assert.equal(chunks[chunks.length - 1].endLine, totalLines);
|
||||
// Chunks should be contiguous
|
||||
for (let i = 1; i < chunks.length; i++) {
|
||||
assert.equal(chunks[i].startLine, chunks[i - 1].endLine + 1,
|
||||
`Chunk ${i} should start right after chunk ${i - 1}`);
|
||||
}
|
||||
});
|
||||
|
||||
// ─── splitIntoChunks — Markdown ─────────────────────────────────────────────
|
||||
|
||||
test("splitIntoChunks splits markdown at heading boundaries", () => {
|
||||
const chunks = splitIntoChunks(MARKDOWN_CONTENT);
|
||||
assert.ok(chunks.length > 1, `Expected multiple chunks, got ${chunks.length}`);
|
||||
|
||||
const contents = chunks.map((c) => c.content);
|
||||
const hasOverview = contents.some((c) => c.includes("# Project Overview"));
|
||||
const hasInstallation = contents.some((c) => c.includes("## Installation"));
|
||||
const hasApi = contents.some((c) => c.includes("## API Reference"));
|
||||
assert.ok(hasOverview, "Should have overview chunk");
|
||||
assert.ok(hasInstallation, "Should have installation chunk");
|
||||
assert.ok(hasApi, "Should have API reference chunk");
|
||||
});
|
||||
|
||||
// ─── splitIntoChunks — Plain Text ───────────────────────────────────────────
|
||||
|
||||
test("splitIntoChunks splits plain text at paragraph boundaries", () => {
|
||||
const chunks = splitIntoChunks(PLAIN_TEXT);
|
||||
assert.ok(chunks.length >= 2, `Expected multiple chunks, got ${chunks.length}`);
|
||||
});
|
||||
|
||||
// ─── splitIntoChunks — Edge Cases ───────────────────────────────────────────
|
||||
|
||||
test("splitIntoChunks returns empty array for empty content", () => {
|
||||
assert.deepEqual(splitIntoChunks(""), []);
|
||||
assert.deepEqual(splitIntoChunks(" "), []);
|
||||
});
|
||||
|
||||
test("splitIntoChunks handles single-line content", () => {
|
||||
const chunks = splitIntoChunks("const x = 1;");
|
||||
assert.equal(chunks.length, 1);
|
||||
assert.equal(chunks[0].content, "const x = 1;");
|
||||
assert.equal(chunks[0].startLine, 1);
|
||||
assert.equal(chunks[0].endLine, 1);
|
||||
});
|
||||
|
||||
test("splitIntoChunks merges tiny chunks below minLines into predecessor", () => {
|
||||
const content = `export function foo() {
|
||||
return 1;
|
||||
}
|
||||
|
||||
export function bar() {
|
||||
return 2;
|
||||
}
|
||||
|
||||
export function baz() {
|
||||
return 3;
|
||||
}
|
||||
|
||||
const x = 1;`;
|
||||
|
||||
// With high minLines, tiny chunks get merged
|
||||
const chunks = splitIntoChunks(content, { minLines: 5, maxLines: 80 });
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
const lineCount = chunks[i].endLine - chunks[i].startLine + 1;
|
||||
// First chunk may be smaller, but subsequent ones should be >= minLines or merged
|
||||
if (i > 0) {
|
||||
assert.ok(lineCount >= 3, `Chunk ${i} has only ${lineCount} lines`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("splitIntoChunks respects maxLines by splitting oversized chunks", () => {
|
||||
// Build a long function
|
||||
const longLines = ["export function longFunc() {"];
|
||||
for (let i = 0; i < 100; i++) {
|
||||
longLines.push(` const v${i} = ${i};`);
|
||||
}
|
||||
longLines.push("}");
|
||||
const content = longLines.join("\n");
|
||||
|
||||
const chunks = splitIntoChunks(content, { minLines: 1, maxLines: 30 });
|
||||
for (const chunk of chunks) {
|
||||
const lineCount = chunk.endLine - chunk.startLine + 1;
|
||||
assert.ok(lineCount <= 30, `Chunk has ${lineCount} lines, exceeding maxLines=30`);
|
||||
}
|
||||
});
|
||||
|
||||
// ─── scoreChunks ────────────────────────────────────────────────────────────
|
||||
|
||||
test("scoreChunks scores chunk with query terms higher than chunk without", () => {
|
||||
const chunks: Chunk[] = [
|
||||
{ content: "function loadConfig reads configuration from disk", startLine: 1, endLine: 1, score: 0 },
|
||||
{ content: "function saveData writes data to database storage", startLine: 2, endLine: 2, score: 0 },
|
||||
];
|
||||
|
||||
const scored = scoreChunks(chunks, "loadConfig configuration disk");
|
||||
const configChunk = scored.find((c) => c.content.includes("loadConfig"))!;
|
||||
const dataChunk = scored.find((c) => c.content.includes("saveData"))!;
|
||||
assert.ok(configChunk.score > dataChunk.score,
|
||||
`Config chunk (${configChunk.score}) should score higher than data chunk (${dataChunk.score})`);
|
||||
});
|
||||
|
||||
test("scoreChunks normalizes scores between 0 and 1", () => {
|
||||
const chunks: Chunk[] = [
|
||||
{ content: "alpha beta gamma delta", startLine: 1, endLine: 1, score: 0 },
|
||||
{ content: "epsilon zeta eta theta", startLine: 2, endLine: 2, score: 0 },
|
||||
];
|
||||
|
||||
const scored = scoreChunks(chunks, "alpha gamma");
|
||||
for (const chunk of scored) {
|
||||
assert.ok(chunk.score >= 0 && chunk.score <= 1,
|
||||
`Score ${chunk.score} should be between 0 and 1`);
|
||||
}
|
||||
// At least one chunk should have score 1 (the max)
|
||||
assert.ok(scored.some((c) => c.score === 1), "Max scoring chunk should be normalized to 1");
|
||||
});
|
||||
|
||||
test("scoreChunks returns all zero scores when no query terms match", () => {
|
||||
const chunks: Chunk[] = [
|
||||
{ content: "alpha beta gamma", startLine: 1, endLine: 1, score: 0 },
|
||||
{ content: "delta epsilon zeta", startLine: 2, endLine: 2, score: 0 },
|
||||
];
|
||||
|
||||
const scored = scoreChunks(chunks, "xxxxxxxxx yyyyyyyyy");
|
||||
for (const chunk of scored) {
|
||||
assert.equal(chunk.score, 0, "Non-matching chunks should have score 0");
|
||||
}
|
||||
});
|
||||
|
||||
test("scoreChunks handles empty query gracefully", () => {
|
||||
const chunks: Chunk[] = [
|
||||
{ content: "some content here", startLine: 1, endLine: 1, score: 0 },
|
||||
];
|
||||
const scored = scoreChunks(chunks, "");
|
||||
assert.equal(scored[0].score, 0);
|
||||
});
|
||||
|
||||
test("scoreChunks handles empty chunks array", () => {
|
||||
const scored = scoreChunks([], "some query");
|
||||
assert.deepEqual(scored, []);
|
||||
});
|
||||
|
||||
test("scoreChunks filters stop words from query", () => {
|
||||
const chunks: Chunk[] = [
|
||||
{ content: "the configuration module handles loading", startLine: 1, endLine: 1, score: 0 },
|
||||
{ content: "database connection pool management system", startLine: 2, endLine: 2, score: 0 },
|
||||
];
|
||||
|
||||
// "the" and "is" are stop words; "configuration" should be the only scoring term
|
||||
const scored = scoreChunks(chunks, "the configuration is");
|
||||
const configChunk = scored.find((c) => c.content.includes("configuration"))!;
|
||||
const dbChunk = scored.find((c) => c.content.includes("database"))!;
|
||||
assert.ok(configChunk.score > dbChunk.score);
|
||||
});
|
||||
|
||||
// ─── chunkByRelevance ───────────────────────────────────────────────────────
|
||||
|
||||
test("chunkByRelevance selects top-scoring chunks up to maxChunks", () => {
|
||||
const result = chunkByRelevance(TYPESCRIPT_CODE, "ConfigManager save config", {
|
||||
maxChunks: 2,
|
||||
minScore: 0,
|
||||
});
|
||||
|
||||
assert.ok(result.chunks.length <= 2, `Expected at most 2 chunks, got ${result.chunks.length}`);
|
||||
assert.ok(result.totalChunks > 2, "Total chunks should be more than selected");
|
||||
assert.ok(result.omittedChunks > 0, "Should have omitted chunks");
|
||||
});
|
||||
|
||||
test("chunkByRelevance returns chunks in original document order", () => {
|
||||
const result = chunkByRelevance(TYPESCRIPT_CODE, "Config loadConfig saveConfig", {
|
||||
maxChunks: 10,
|
||||
minScore: 0,
|
||||
});
|
||||
|
||||
for (let i = 1; i < result.chunks.length; i++) {
|
||||
assert.ok(result.chunks[i].startLine > result.chunks[i - 1].startLine,
|
||||
"Chunks should be in ascending line order");
|
||||
}
|
||||
});
|
||||
|
||||
test("chunkByRelevance respects minScore filtering", () => {
|
||||
const result = chunkByRelevance(TYPESCRIPT_CODE, "ConfigManager", {
|
||||
maxChunks: 10,
|
||||
minScore: 0.5,
|
||||
});
|
||||
|
||||
for (const chunk of result.chunks) {
|
||||
assert.ok(chunk.score >= 0.5,
|
||||
`Chunk score ${chunk.score} should be >= minScore 0.5`);
|
||||
}
|
||||
});
|
||||
|
||||
test("chunkByRelevance calculates savings percent", () => {
|
||||
const result = chunkByRelevance(TYPESCRIPT_CODE, "ConfigManager", {
|
||||
maxChunks: 1,
|
||||
minScore: 0,
|
||||
});
|
||||
|
||||
assert.ok(result.savingsPercent >= 0 && result.savingsPercent <= 100,
|
||||
`Savings ${result.savingsPercent}% should be between 0 and 100`);
|
||||
if (result.omittedChunks > 0) {
|
||||
assert.ok(result.savingsPercent > 0, "Should have positive savings when chunks are omitted");
|
||||
}
|
||||
});
|
||||
|
||||
test("chunkByRelevance handles empty content", () => {
|
||||
const result = chunkByRelevance("", "query");
|
||||
assert.deepEqual(result.chunks, []);
|
||||
assert.equal(result.totalChunks, 0);
|
||||
assert.equal(result.omittedChunks, 0);
|
||||
assert.equal(result.savingsPercent, 0);
|
||||
});
|
||||
|
||||
test("chunkByRelevance uses default options when none provided", () => {
|
||||
const result = chunkByRelevance(TYPESCRIPT_CODE, "Config");
|
||||
assert.ok(result.chunks.length <= 5, "Default maxChunks should be 5");
|
||||
});
|
||||
|
||||
// ─── formatChunks ───────────────────────────────────────────────────────────
|
||||
|
||||
test("formatChunks produces line range markers", () => {
|
||||
const result: ChunkResult = {
|
||||
chunks: [
|
||||
{ content: "line one\nline two", startLine: 1, endLine: 2, score: 1 },
|
||||
{ content: "line ten\nline eleven", startLine: 10, endLine: 11, score: 0.5 },
|
||||
],
|
||||
totalChunks: 5,
|
||||
omittedChunks: 3,
|
||||
savingsPercent: 60,
|
||||
};
|
||||
|
||||
const formatted = formatChunks(result, "src/config.ts");
|
||||
assert.ok(formatted.includes("[Lines 1-2]"), "Should include first line range");
|
||||
assert.ok(formatted.includes("[Lines 10-11]"), "Should include second line range");
|
||||
assert.ok(formatted.includes("line one\nline two"), "Should include first chunk content");
|
||||
assert.ok(formatted.includes("line ten\nline eleven"), "Should include second chunk content");
|
||||
});
|
||||
|
||||
test("formatChunks shows omission indicators between non-contiguous chunks", () => {
|
||||
const result: ChunkResult = {
|
||||
chunks: [
|
||||
{ content: "first chunk", startLine: 1, endLine: 5, score: 1 },
|
||||
{ content: "second chunk", startLine: 81, endLine: 90, score: 0.5 },
|
||||
],
|
||||
totalChunks: 4,
|
||||
omittedChunks: 2,
|
||||
savingsPercent: 50,
|
||||
};
|
||||
|
||||
const formatted = formatChunks(result, "src/main.ts");
|
||||
assert.ok(formatted.includes("[...75 lines omitted...]"),
|
||||
`Expected omission marker, got:\n${formatted}`);
|
||||
});
|
||||
|
||||
test("formatChunks handles empty result", () => {
|
||||
const result: ChunkResult = {
|
||||
chunks: [],
|
||||
totalChunks: 0,
|
||||
omittedChunks: 0,
|
||||
savingsPercent: 0,
|
||||
};
|
||||
|
||||
const formatted = formatChunks(result, "empty.ts");
|
||||
assert.ok(formatted.includes("empty.ts"), "Should mention the file path");
|
||||
});
|
||||
|
||||
test("formatChunks does not show omission for contiguous chunks", () => {
|
||||
const result: ChunkResult = {
|
||||
chunks: [
|
||||
{ content: "chunk one", startLine: 1, endLine: 5, score: 1 },
|
||||
{ content: "chunk two", startLine: 6, endLine: 10, score: 0.8 },
|
||||
],
|
||||
totalChunks: 2,
|
||||
omittedChunks: 0,
|
||||
savingsPercent: 0,
|
||||
};
|
||||
|
||||
const formatted = formatChunks(result, "src/test.ts");
|
||||
assert.ok(!formatted.includes("omitted"), "Contiguous chunks should not show omission");
|
||||
});
|
||||
|
||||
// ─── inlineFileSmart integration tests ─────────────────────────────────────
|
||||
|
||||
// These test the formatChunks function in the context of how it'll be used
|
||||
test("formatChunks includes file path in line range headers", () => {
|
||||
const result = chunkByRelevance(
|
||||
"export function foo() {}\n\nexport function bar() {}\n\nexport function baz() {}",
|
||||
"foo function",
|
||||
{ maxChunks: 1 },
|
||||
);
|
||||
const formatted = formatChunks(result, "src/utils.ts");
|
||||
assert.ok(
|
||||
formatted.includes("src/utils.ts") || formatted.includes("[Lines"),
|
||||
"Formatted output should include file path or line range markers",
|
||||
);
|
||||
});
|
||||
|
|
@ -1,323 +0,0 @@
|
|||
/**
|
||||
* Tests for summary-distiller.ts — the summary distillation module.
|
||||
* Verifies frontmatter extraction, compact formatting, budget enforcement,
|
||||
* and progressive field dropping.
|
||||
*/
|
||||
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { distillSingle, distillSummaries } from "../summary-distiller.js";
|
||||
|
||||
// ─── Fixtures ────────────────────────────────────────────────────────────────
|
||||
|
||||
const REALISTIC_SUMMARY = `---
|
||||
id: S01
|
||||
parent: M001
|
||||
milestone: M001
|
||||
provides:
|
||||
- Core type definitions
|
||||
- File I/O utilities
|
||||
requires: []
|
||||
affects:
|
||||
- All downstream slices
|
||||
key_files:
|
||||
- src/types.ts
|
||||
- src/files.ts
|
||||
- src/paths.ts
|
||||
key_decisions:
|
||||
- D001
|
||||
- D003
|
||||
patterns_established:
|
||||
- Pure function modules
|
||||
- Dependency injection via parameters
|
||||
drill_down_paths:
|
||||
- src/types.ts for interface contracts
|
||||
observability_surfaces:
|
||||
- Unit test coverage > 90%
|
||||
duration: 45m
|
||||
verification_result: pass
|
||||
completed_at: 2025-03-15T10:00:00Z
|
||||
blocker_discovered: false
|
||||
---
|
||||
|
||||
# S01: Core Type Definitions and File I/O
|
||||
|
||||
Foundation types and file operations for the GSD extension.
|
||||
|
||||
## What Happened
|
||||
|
||||
Implemented 12 core interfaces spanning roadmap parsing, slice plans, summaries,
|
||||
and continuation state. Added file I/O utilities for reading, parsing, and writing
|
||||
GSD artifact files. Established the path resolution module for computing absolute
|
||||
and relative paths to milestone, slice, and task artifacts.
|
||||
|
||||
## Deviations
|
||||
|
||||
Minor deviation from plan: added \`filesModified\` field to Summary interface that
|
||||
was not in the original design, based on the realization that tracking modified
|
||||
files in summaries enables better diff-context prioritization.
|
||||
|
||||
## Files Modified
|
||||
|
||||
- \`src/types.ts\` — 12 interfaces, 4 type aliases
|
||||
- \`src/files.ts\` — 8 parser functions, 3 writer functions
|
||||
- \`src/paths.ts\` — 14 path resolver functions
|
||||
`;
|
||||
|
||||
const SECOND_SUMMARY = `---
|
||||
id: S02
|
||||
parent: M001
|
||||
milestone: M001
|
||||
provides:
|
||||
- Roadmap parser
|
||||
- Slice dependency resolver
|
||||
requires:
|
||||
- Core type definitions
|
||||
key_files:
|
||||
- src/roadmap.ts
|
||||
- src/deps.ts
|
||||
key_decisions:
|
||||
- D004
|
||||
patterns_established:
|
||||
- DAG-based ordering
|
||||
drill_down_paths:
|
||||
- src/deps.ts for topological sort
|
||||
duration: 30m
|
||||
verification_result: pass
|
||||
completed_at: 2025-03-15T11:00:00Z
|
||||
---
|
||||
|
||||
# S02: Roadmap Parser and Dependency Resolution
|
||||
|
||||
Built the roadmap parser and DAG-based dependency resolver.
|
||||
|
||||
## What Happened
|
||||
|
||||
Created a Markdown-based roadmap parser that extracts slice metadata from
|
||||
structured headings and bullet lists. Implemented a topological sort for
|
||||
resolving slice execution order based on declared dependencies.
|
||||
|
||||
## Files Modified
|
||||
|
||||
- \`src/roadmap.ts\` — parser with regex-based extraction
|
||||
- \`src/deps.ts\` — DAG builder and topological sort
|
||||
`;
|
||||
|
||||
const NO_FRONTMATTER = `# S99: Quick Fix
|
||||
|
||||
A quick patch with no frontmatter at all.
|
||||
|
||||
## What Happened
|
||||
|
||||
Fixed a typo.
|
||||
`;
|
||||
|
||||
const EMPTY_ARRAYS_SUMMARY = `---
|
||||
id: S03
|
||||
provides: []
|
||||
requires: []
|
||||
key_files: []
|
||||
key_decisions: []
|
||||
patterns_established: []
|
||||
---
|
||||
|
||||
# S03: Empty Slice
|
||||
|
||||
Nothing to provide or require.
|
||||
`;
|
||||
|
||||
// ─── distillSingle ──────────────────────────────────────────────────────────
|
||||
|
||||
describe("summary-distiller: distillSingle", () => {
|
||||
it("extracts frontmatter fields from a realistic summary", () => {
|
||||
const result = distillSingle(REALISTIC_SUMMARY);
|
||||
assert.ok(result.includes("## S01:"), "should include the id header");
|
||||
assert.ok(result.includes("provides: Core type definitions, File I/O utilities"),
|
||||
"should list provides");
|
||||
assert.ok(result.includes("key_files: src/types.ts, src/files.ts, src/paths.ts"),
|
||||
"should list key_files");
|
||||
assert.ok(result.includes("key_decisions: D001, D003"),
|
||||
"should list key_decisions");
|
||||
assert.ok(result.includes("patterns: Pure function modules, Dependency injection via parameters"),
|
||||
"should list patterns");
|
||||
});
|
||||
|
||||
it("extracts the one-liner from the title line", () => {
|
||||
const result = distillSingle(REALISTIC_SUMMARY);
|
||||
// The title line "# S01: Core Type Definitions and File I/O" provides the one-liner
|
||||
assert.ok(
|
||||
result.includes("Core Type Definitions and File I/O"),
|
||||
"should include one-liner from title",
|
||||
);
|
||||
});
|
||||
|
||||
it("falls back to first paragraph when title has no inline text", () => {
|
||||
const summary = `---
|
||||
id: S10
|
||||
provides:
|
||||
- Widget API
|
||||
---
|
||||
|
||||
# S10:
|
||||
|
||||
Widget API for rendering dashboard components.
|
||||
|
||||
## What Happened
|
||||
|
||||
Built the widget system.
|
||||
`;
|
||||
const result = distillSingle(summary);
|
||||
assert.ok(
|
||||
result.includes("Widget API for rendering"),
|
||||
"should use first paragraph as one-liner when title text is empty",
|
||||
);
|
||||
});
|
||||
|
||||
it("drops verbose prose sections", () => {
|
||||
const result = distillSingle(REALISTIC_SUMMARY);
|
||||
assert.ok(!result.includes("What Happened"), "should not include What Happened heading");
|
||||
assert.ok(!result.includes("Implemented 12 core"), "should not include prose body");
|
||||
assert.ok(!result.includes("Deviations"), "should not include Deviations");
|
||||
assert.ok(!result.includes("filesModified"), "should not include deviation details");
|
||||
assert.ok(!result.includes("drill_down_paths"), "should not include drill_down_paths label");
|
||||
assert.ok(!result.includes("duration"), "should not include duration");
|
||||
assert.ok(!result.includes("verification_result"), "should not include verification_result");
|
||||
assert.ok(!result.includes("completed_at"), "should not include completed_at");
|
||||
});
|
||||
|
||||
it("handles array fields in provides/requires", () => {
|
||||
const result = distillSingle(SECOND_SUMMARY);
|
||||
assert.ok(result.includes("provides: Roadmap parser, Slice dependency resolver"),
|
||||
"should join provides array");
|
||||
assert.ok(result.includes("requires: Core type definitions"),
|
||||
"should join requires array");
|
||||
});
|
||||
|
||||
it("omits empty requires when none declared", () => {
|
||||
const result = distillSingle(REALISTIC_SUMMARY);
|
||||
assert.ok(!result.includes("requires:"), "should omit requires when empty");
|
||||
});
|
||||
|
||||
it("handles missing frontmatter gracefully", () => {
|
||||
const result = distillSingle(NO_FRONTMATTER);
|
||||
assert.ok(result.includes("## S99:"), "should extract id from title");
|
||||
assert.ok(result.includes("Quick Fix"), "should include title text");
|
||||
});
|
||||
|
||||
it("handles empty array frontmatter fields", () => {
|
||||
const result = distillSingle(EMPTY_ARRAYS_SUMMARY);
|
||||
assert.ok(result.includes("## S03:"), "should have the id");
|
||||
assert.ok(!result.includes("provides:"), "should omit empty provides");
|
||||
assert.ok(!result.includes("requires:"), "should omit empty requires");
|
||||
assert.ok(!result.includes("key_files:"), "should omit empty key_files");
|
||||
assert.ok(!result.includes("key_decisions:"), "should omit empty key_decisions");
|
||||
assert.ok(!result.includes("patterns:"), "should omit empty patterns");
|
||||
});
|
||||
|
||||
it("produces significantly shorter output than input", () => {
|
||||
const result = distillSingle(REALISTIC_SUMMARY);
|
||||
assert.ok(
|
||||
result.length < REALISTIC_SUMMARY.length * 0.5,
|
||||
`distilled (${result.length}) should be <50% of original (${REALISTIC_SUMMARY.length})`,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── distillSummaries ────────────────────────────────────────────────────────
|
||||
|
||||
describe("summary-distiller: distillSummaries", () => {
|
||||
it("combines multiple summaries into structured blocks", () => {
|
||||
const result = distillSummaries([REALISTIC_SUMMARY, SECOND_SUMMARY], 10_000);
|
||||
assert.equal(result.summaryCount, 2);
|
||||
assert.ok(result.content.includes("## S01:"), "should include first summary");
|
||||
assert.ok(result.content.includes("## S02:"), "should include second summary");
|
||||
});
|
||||
|
||||
it("reports positive savings percentage", () => {
|
||||
const result = distillSummaries([REALISTIC_SUMMARY, SECOND_SUMMARY], 10_000);
|
||||
assert.ok(result.savingsPercent > 0, `savings should be positive, got ${result.savingsPercent}%`);
|
||||
assert.ok(result.distilledChars < result.originalChars,
|
||||
"distilled chars should be less than original");
|
||||
});
|
||||
|
||||
it("fits content within budgetChars when budget is generous", () => {
|
||||
const result = distillSummaries([REALISTIC_SUMMARY, SECOND_SUMMARY], 10_000);
|
||||
assert.ok(
|
||||
result.content.length <= 10_000,
|
||||
`content length ${result.content.length} should be within budget 10000`,
|
||||
);
|
||||
assert.ok(!result.content.includes("[...truncated]"), "should not truncate with generous budget");
|
||||
});
|
||||
|
||||
it("enforces budget with truncation when needed", () => {
|
||||
const result = distillSummaries([REALISTIC_SUMMARY, SECOND_SUMMARY], 200);
|
||||
assert.ok(
|
||||
result.content.length <= 215, // allow some slack for truncation marker
|
||||
`content length ${result.content.length} should be near budget 200`,
|
||||
);
|
||||
assert.ok(result.content.includes("[...truncated]"), "should include truncation marker");
|
||||
});
|
||||
|
||||
it("progressively drops fields when budget is tight", () => {
|
||||
// With a budget that can fit the header lines but not all fields,
|
||||
// patterns should be dropped first, then key_decisions, then key_files
|
||||
const full = distillSummaries([REALISTIC_SUMMARY], 100_000);
|
||||
assert.ok(full.content.includes("patterns:"), "full output should have patterns");
|
||||
|
||||
// Find a budget that forces dropping patterns but keeps key_decisions
|
||||
const withoutPatterns = full.content.replace(/patterns:.*$/m, "").length;
|
||||
const withPatterns = full.content.length;
|
||||
|
||||
if (withPatterns > withoutPatterns) {
|
||||
const tightBudget = withoutPatterns + 5;
|
||||
const tight = distillSummaries([REALISTIC_SUMMARY], tightBudget);
|
||||
assert.ok(!tight.content.includes("patterns:"),
|
||||
"tight budget should drop patterns first");
|
||||
assert.ok(tight.content.includes("key_decisions:"),
|
||||
"tight budget should still have key_decisions");
|
||||
}
|
||||
});
|
||||
|
||||
it("handles a single summary", () => {
|
||||
const result = distillSummaries([REALISTIC_SUMMARY], 10_000);
|
||||
assert.equal(result.summaryCount, 1);
|
||||
assert.ok(result.content.includes("## S01:"), "should include the single summary");
|
||||
});
|
||||
|
||||
it("handles empty input array", () => {
|
||||
const result = distillSummaries([], 10_000);
|
||||
assert.equal(result.summaryCount, 0);
|
||||
assert.equal(result.content, "");
|
||||
assert.equal(result.savingsPercent, 0);
|
||||
assert.equal(result.originalChars, 0);
|
||||
assert.equal(result.distilledChars, 0);
|
||||
});
|
||||
|
||||
it("handles malformed content gracefully", () => {
|
||||
const malformed = "this is not a valid summary at all\nno frontmatter\nno headings";
|
||||
const result = distillSummaries([malformed], 10_000);
|
||||
assert.equal(result.summaryCount, 1);
|
||||
// Should not throw, should produce some output
|
||||
assert.ok(result.content.length > 0, "should produce output even for malformed input");
|
||||
});
|
||||
|
||||
it("handles very tight budget (100 chars) with truncation", () => {
|
||||
const result = distillSummaries([REALISTIC_SUMMARY, SECOND_SUMMARY], 100);
|
||||
assert.ok(
|
||||
result.content.length <= 115, // small slack for marker
|
||||
`content (${result.content.length}) should be near budget 100`,
|
||||
);
|
||||
assert.ok(result.content.includes("[...truncated]"), "should truncate at very tight budget");
|
||||
assert.ok(result.savingsPercent > 80, `savings should be very high, got ${result.savingsPercent}%`);
|
||||
});
|
||||
|
||||
it("tracks original and distilled character counts accurately", () => {
|
||||
const summaries = [REALISTIC_SUMMARY, SECOND_SUMMARY];
|
||||
const totalOriginal = summaries.reduce((s, c) => s + c.length, 0);
|
||||
const result = distillSummaries(summaries, 10_000);
|
||||
assert.equal(result.originalChars, totalOriginal, "originalChars should match input total");
|
||||
assert.equal(result.distilledChars, result.content.length,
|
||||
"distilledChars should match content length");
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,164 +0,0 @@
|
|||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
// Test the type definitions exist and are correct
|
||||
describe("token-optimization: types", () => {
|
||||
it("CompressionStrategy accepts valid values", async () => {
|
||||
const { } = await import("../types.js");
|
||||
// Type-level test — if this compiles, the types exist
|
||||
const truncate: import("../types.js").CompressionStrategy = "truncate";
|
||||
const compress: import("../types.js").CompressionStrategy = "compress";
|
||||
assert.equal(truncate, "truncate");
|
||||
assert.equal(compress, "compress");
|
||||
});
|
||||
|
||||
it("ContextSelectionMode accepts valid values", async () => {
|
||||
const full: import("../types.js").ContextSelectionMode = "full";
|
||||
const smart: import("../types.js").ContextSelectionMode = "smart";
|
||||
assert.equal(full, "full");
|
||||
assert.equal(smart, "smart");
|
||||
});
|
||||
});
|
||||
|
||||
// Test cache hit rate computation
|
||||
describe("token-optimization: cache hit rate", () => {
|
||||
it("computeCacheHitRate returns correct percentage", async () => {
|
||||
const { computeCacheHitRate } = await import("../prompt-cache-optimizer.js");
|
||||
assert.equal(computeCacheHitRate({ cacheRead: 900, cacheWrite: 100, input: 100 }), 90);
|
||||
assert.equal(computeCacheHitRate({ cacheRead: 0, cacheWrite: 0, input: 100 }), 0);
|
||||
assert.equal(computeCacheHitRate({ cacheRead: 0, cacheWrite: 0, input: 0 }), 0);
|
||||
assert.equal(computeCacheHitRate({ cacheRead: 500, cacheWrite: 0, input: 500 }), 50);
|
||||
});
|
||||
});
|
||||
|
||||
// Test structured data savings
|
||||
describe("token-optimization: structured data savings", () => {
|
||||
it("compact decisions format is shorter than markdown table", async () => {
|
||||
const { formatDecisionsCompact, measureSavings } = await import("../structured-data-formatter.js");
|
||||
const decisions = [
|
||||
{ id: "D001", when_context: "M001/S01", scope: "architecture", decision: "Use SQLite for storage", choice: "WAL mode", rationale: "Built-in, no external deps", revisable: "yes" },
|
||||
{ id: "D002", when_context: "M001/S02", scope: "testing", decision: "Unit test all parsers", choice: "node:test", rationale: "Fast, zero-dependency", revisable: "no" },
|
||||
];
|
||||
const compact = formatDecisionsCompact(decisions);
|
||||
// A realistic markdown table equivalent
|
||||
const markdown = [
|
||||
"| # | When | Scope | Decision | Choice | Rationale | Revisable? |",
|
||||
"|---|------|-------|----------|--------|-----------|------------|",
|
||||
"| D001 | M001/S01 | architecture | Use SQLite for storage | WAL mode | Built-in, no external deps | yes |",
|
||||
"| D002 | M001/S02 | testing | Unit test all parsers | node:test | Fast, zero-dependency | no |",
|
||||
].join("\n");
|
||||
const savings = measureSavings(compact, markdown);
|
||||
assert.ok(savings > 10, `Expected >10% savings, got ${savings}%`);
|
||||
});
|
||||
|
||||
it("compact requirements format drops low-value fields", async () => {
|
||||
const { formatRequirementsCompact } = await import("../structured-data-formatter.js");
|
||||
const requirements = [{
|
||||
id: "R001", class: "functional", status: "active",
|
||||
description: "API response time < 200ms",
|
||||
why: "User experience", primary_owner: "S01",
|
||||
validation: "Load test P99 < 200ms",
|
||||
}];
|
||||
const compact = formatRequirementsCompact(requirements);
|
||||
assert.ok(!compact.includes("source"), "Should not include source field");
|
||||
assert.ok(!compact.includes("supporting_slices"), "Should not include supporting_slices");
|
||||
assert.ok(compact.includes("R001"), "Should include requirement ID");
|
||||
});
|
||||
});
|
||||
|
||||
// Test compression levels
|
||||
describe("token-optimization: prompt compression", () => {
|
||||
it("light compression removes extra whitespace", async () => {
|
||||
const { compressPrompt } = await import("../prompt-compressor.js");
|
||||
const input = "Line 1\n\n\n\n\nLine 2\n\n\n\nLine 3";
|
||||
const result = compressPrompt(input, { level: "light" });
|
||||
assert.ok(result.savingsPercent > 0, "Should have positive savings");
|
||||
assert.ok(!result.content.includes("\n\n\n"), "Should collapse multiple blank lines");
|
||||
});
|
||||
|
||||
it("moderate compression abbreviates verbose phrases", async () => {
|
||||
const { compressPrompt } = await import("../prompt-compressor.js");
|
||||
const input = "In order to achieve this, it is important to note that the following steps are required.";
|
||||
const result = compressPrompt(input, { level: "moderate" });
|
||||
assert.ok(result.compressedChars < result.originalChars, "Should be shorter");
|
||||
});
|
||||
|
||||
it("code blocks are preserved during compression", async () => {
|
||||
const { compressPrompt } = await import("../prompt-compressor.js");
|
||||
const input = "In order to do this:\n\n```typescript\nconst x = 1;\n```\n\nIn order to verify:";
|
||||
const result = compressPrompt(input, { level: "aggressive" });
|
||||
assert.ok(result.content.includes("const x = 1;"), "Code block should be preserved");
|
||||
});
|
||||
});
|
||||
|
||||
// Test summary distillation
|
||||
describe("token-optimization: summary distillation", () => {
|
||||
it("distills summaries preserving key fields", async () => {
|
||||
const { distillSummaries } = await import("../summary-distiller.js");
|
||||
const summary = `---
|
||||
id: S01
|
||||
provides:
|
||||
- Core types
|
||||
key_files:
|
||||
- src/types.ts
|
||||
key_decisions:
|
||||
- D001
|
||||
---
|
||||
|
||||
# S01: Core Types
|
||||
|
||||
Built the foundation type system.
|
||||
|
||||
## What Happened
|
||||
|
||||
Long prose about implementation details that should be dropped...
|
||||
`;
|
||||
const result = distillSummaries([summary], 5000);
|
||||
assert.ok(result.savingsPercent > 0, "Should have savings");
|
||||
assert.ok(result.content.includes("Core types"), "Should preserve provides");
|
||||
assert.ok(result.content.includes("src/types.ts"), "Should preserve key_files");
|
||||
});
|
||||
});
|
||||
|
||||
// Test semantic chunker
|
||||
describe("token-optimization: semantic chunking", () => {
|
||||
it("chunks TypeScript code at function boundaries", async () => {
|
||||
const { splitIntoChunks } = await import("../semantic-chunker.js");
|
||||
const code = `export function alpha() {
|
||||
return 1;
|
||||
}
|
||||
|
||||
export function beta() {
|
||||
return 2;
|
||||
}
|
||||
|
||||
export function gamma() {
|
||||
return 3;
|
||||
}`;
|
||||
const chunks = splitIntoChunks(code);
|
||||
assert.ok(chunks.length >= 2, `Expected >=2 chunks, got ${chunks.length}`);
|
||||
});
|
||||
|
||||
it("scores chunks by relevance to query", async () => {
|
||||
const { chunkByRelevance } = await import("../semantic-chunker.js");
|
||||
const code = `export function createUser(name: string) {
|
||||
return { name, id: generateId() };
|
||||
}
|
||||
|
||||
export function deleteDatabase() {
|
||||
dropAllTables();
|
||||
clearCache();
|
||||
}
|
||||
|
||||
export function updateUser(id: string, name: string) {
|
||||
const user = findUser(id);
|
||||
user.name = name;
|
||||
return user;
|
||||
}`;
|
||||
const result = chunkByRelevance(code, "user creation and management", { maxChunks: 2 });
|
||||
// The user-related chunks should score higher
|
||||
const content = result.chunks.map(c => c.content).join("\n");
|
||||
assert.ok(content.includes("createUser") || content.includes("updateUser"),
|
||||
"Should include user-related chunks");
|
||||
});
|
||||
});
|
||||
|
|
@ -423,7 +423,6 @@ export interface Requirement {
|
|||
|
||||
// ─── Parallel Orchestration Types ────────────────────────────────────────
|
||||
|
||||
export type CompressionStrategy = "truncate" | "compress";
|
||||
export type ContextSelectionMode = "full" | "smart";
|
||||
|
||||
export type MergeStrategy = "per-slice" | "per-milestone";
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue