fix: show verbose prompt traces

This commit is contained in:
Mikael Hugo 2026-05-06 06:45:15 +02:00
parent a95e2947df
commit 42c651d106
5 changed files with 166 additions and 3 deletions

View file

@ -559,6 +559,36 @@ export function formatHeadlessHeartbeat(ctx: HeadlessHeartbeatContext): string {
return `${c.dim}${tag("headless")}still running ${formatDuration(ctx.elapsedMs)}; quiet ${formatDuration(ctx.quietMs)}; last=${lastEvent}; events=${ctx.totalEvents}; tools=${ctx.toolCallCount}${activity}${openTools}${unit}${model}${c.reset}`;
}
/**
* Format a capped prompt preview for verbose headless dogfooding.
*
* Purpose: make the actual instruction payload visible when diagnosing
* autonomous quality without flooding stderr with full context files.
*/
export function formatPromptTraceLines(
customType: string,
content: string,
sessionFile: string,
options: { maxChars?: number; maxLines?: number } = {},
): string[] {
const maxChars = Math.max(0, options.maxChars ?? 2400);
const maxLines = Math.max(1, options.maxLines ?? 80);
const clipped = content.slice(0, maxChars);
const rawLines = clipped.split(/\r?\n/).slice(0, maxLines);
const lines = [
`${tag("prompt")}${customType || "custom"} instructions (${content.length} chars) session=${sessionFile}`,
`${tag("prompt")}--- preview ---`,
...rawLines.map((line) => `${tag("prompt")}${line}`),
];
if (
content.length > clipped.length ||
clipped.split(/\r?\n/).length > maxLines
) {
lines.push(`${tag("prompt")}... truncated`);
}
return lines;
}
// ---------------------------------------------------------------------------
// Phase Label Parser
// ---------------------------------------------------------------------------

View file

@ -62,6 +62,7 @@ import type { ExtensionUIRequest, ProgressContext } from "./headless-ui.js";
import {
formatHeadlessHeartbeat,
formatProgress,
formatPromptTraceLines,
formatTextEnd,
formatTextLine,
formatTextStart,
@ -824,6 +825,7 @@ async function runHeadlessOnce(
| undefined;
let thinkingBuffer = "";
let assistantTextBuffer = "";
const promptTraceSessionFiles = new Set<string>();
// Drop only adjacent identical formatProgress output. A widget that
// re-emits the same setStatus on every LLM call would otherwise print
// the same line N times in a row. Two different lines still both show;
@ -1064,6 +1066,57 @@ async function runHeadlessOnce(
return shown;
}
function resolvePromptTracePreviewChars(): number {
const raw = process.env.SF_HEADLESS_PROMPT_TRACE_CHARS?.trim();
if (!raw) return 2400;
const parsed = Number.parseInt(raw, 10);
if (!Number.isFinite(parsed) || parsed < 0) return 2400;
return Math.min(parsed, 12_000);
}
function findSessionPromptTrace(
sessionFile: string,
): { customType: string; content: string } | null {
let text = "";
try {
text = readFileSync(sessionFile, "utf-8");
} catch {
return null;
}
for (const line of text.split(/\r?\n/)) {
if (!line.trim()) continue;
try {
const entry = JSON.parse(line) as Record<string, unknown>;
if (entry.type !== "custom_message") continue;
const content = entry.content;
if (typeof content !== "string" || !content.trim()) continue;
return {
customType: String(entry.customType ?? "custom"),
content,
};
} catch {}
}
return null;
}
function maybeWritePromptTrace(eventObj: Record<string, unknown>): void {
if (!options.verbose) return;
if (process.env.SF_HEADLESS_PROMPT_TRACE === "0") return;
const sessionFile = String(eventObj.sessionFile ?? "");
if (!sessionFile || promptTraceSessionFiles.has(sessionFile)) return;
const trace = findSessionPromptTrace(sessionFile);
if (!trace) return;
promptTraceSessionFiles.add(sessionFile);
for (const line of formatPromptTraceLines(
trace.customType,
trace.content,
sessionFile,
{ maxChars: resolvePromptTracePreviewChars() },
)) {
writeHeadlessLine(line);
}
}
// Client started flag — replaces old stdinWriter null-check
let clientStarted = false;
// Adapter for AnswerInjector — wraps client.sendUIResponse in a writeToStdin-compatible callback
@ -1181,6 +1234,7 @@ async function runHeadlessOnce(
client.onEvent((event) => {
const eventObj = event as unknown as Record<string, unknown>;
trackEvent(eventObj);
maybeWritePromptTrace(eventObj);
const eventType = String(eventObj.type ?? "");
if (eventType === "tool_execution_start") {

View file

@ -43,12 +43,13 @@ function extractHeadingText(line) {
}
/**
* Classify a heading by matching against known roles.
* Uses substring matching so headings like "## UNIT: Execute Task T1.1" don't match
* but "## Inlined Task Plan" does. Unknown headings default to "dynamic".
* Uses exact matching so inlined artifact headings such as
* `## Requirements Advanced` stay inside their original prompt position instead
* of being mistaken for the top-level `## Requirements` context block.
*/
function classifyHeading(heading) {
for (const [key, role] of Object.entries(HEADING_ROLES)) {
if (heading === key || heading.startsWith(key)) {
if (heading === key) {
return role;
}
}

View file

@ -0,0 +1,61 @@
import assert from "node:assert/strict";
import { test } from "vitest";
import { reorderForCaching } from "../prompt-ordering.js";
test("reorderForCaching_when_inlined_slice_summary_has_requirements_advanced_keeps_it_after_mission", () => {
const prompt = [
"# Milestone Validation",
"",
"## Working Directory",
"/repo",
"",
"## Mission",
"Dispatch reviewers.",
"",
"## Context",
"Inlined below.",
"",
"## Inlined Context",
"### S01 Summary",
"# S01",
"",
"## Requirements Advanced",
"- R1",
"",
"## Requirements Validated",
"None.",
].join("\n");
const reordered = reorderForCaching(prompt);
assert.ok(
reordered.indexOf("## Mission") <
reordered.indexOf("## Requirements Advanced"),
);
assert.ok(
reordered.indexOf("## Context") <
reordered.indexOf("## Requirements Advanced"),
);
});
test("reorderForCaching_when_top_level_requirements_exists_still_hoists_exact_requirements_block", () => {
const prompt = [
"# Execute",
"",
"## Mission",
"Do work.",
"",
"## Requirements",
"- R1",
"",
"## Verification",
"Run tests.",
].join("\n");
const reordered = reorderForCaching(prompt);
assert.ok(
reordered.indexOf("## Requirements") < reordered.indexOf("## Mission"),
);
});

View file

@ -5,6 +5,7 @@ import {
formatCostLine,
formatHeadlessHeartbeat,
formatProgress,
formatPromptTraceLines,
formatThinkingLine,
summarizeToolArgs,
} from "../headless-ui.js";
@ -71,6 +72,22 @@ describe("formatProgress", () => {
});
});
describe("formatPromptTraceLines", () => {
it("shows capped prompt preview with session path", () => {
const lines = formatPromptTraceLines(
"sf-auto",
"# Mission\nDo the work\nKeep going",
"/tmp/session.jsonl",
{ maxChars: 17, maxLines: 3 },
);
assert.ok(lines[0].includes("sf-auto instructions"));
assert.ok(lines[0].includes("/tmp/session.jsonl"));
assert.ok(lines.some((line) => line.includes("# Mission")));
assert.ok(lines.at(-1)?.includes("truncated"));
});
});
describe("tool_execution_start", () => {
it("shows tool name and summarized args in verbose mode", () => {
const result = formatProgress(