fix: show verbose prompt traces
This commit is contained in:
parent
a95e2947df
commit
42c651d106
5 changed files with 166 additions and 3 deletions
|
|
@ -559,6 +559,36 @@ export function formatHeadlessHeartbeat(ctx: HeadlessHeartbeatContext): string {
|
|||
return `${c.dim}${tag("headless")}still running ${formatDuration(ctx.elapsedMs)}; quiet ${formatDuration(ctx.quietMs)}; last=${lastEvent}; events=${ctx.totalEvents}; tools=${ctx.toolCallCount}${activity}${openTools}${unit}${model}${c.reset}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a capped prompt preview for verbose headless dogfooding.
|
||||
*
|
||||
* Purpose: make the actual instruction payload visible when diagnosing
|
||||
* autonomous quality without flooding stderr with full context files.
|
||||
*/
|
||||
export function formatPromptTraceLines(
|
||||
customType: string,
|
||||
content: string,
|
||||
sessionFile: string,
|
||||
options: { maxChars?: number; maxLines?: number } = {},
|
||||
): string[] {
|
||||
const maxChars = Math.max(0, options.maxChars ?? 2400);
|
||||
const maxLines = Math.max(1, options.maxLines ?? 80);
|
||||
const clipped = content.slice(0, maxChars);
|
||||
const rawLines = clipped.split(/\r?\n/).slice(0, maxLines);
|
||||
const lines = [
|
||||
`${tag("prompt")}${customType || "custom"} instructions (${content.length} chars) session=${sessionFile}`,
|
||||
`${tag("prompt")}--- preview ---`,
|
||||
...rawLines.map((line) => `${tag("prompt")}${line}`),
|
||||
];
|
||||
if (
|
||||
content.length > clipped.length ||
|
||||
clipped.split(/\r?\n/).length > maxLines
|
||||
) {
|
||||
lines.push(`${tag("prompt")}... truncated`);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Phase Label Parser
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -62,6 +62,7 @@ import type { ExtensionUIRequest, ProgressContext } from "./headless-ui.js";
|
|||
import {
|
||||
formatHeadlessHeartbeat,
|
||||
formatProgress,
|
||||
formatPromptTraceLines,
|
||||
formatTextEnd,
|
||||
formatTextLine,
|
||||
formatTextStart,
|
||||
|
|
@ -824,6 +825,7 @@ async function runHeadlessOnce(
|
|||
| undefined;
|
||||
let thinkingBuffer = "";
|
||||
let assistantTextBuffer = "";
|
||||
const promptTraceSessionFiles = new Set<string>();
|
||||
// Drop only adjacent identical formatProgress output. A widget that
|
||||
// re-emits the same setStatus on every LLM call would otherwise print
|
||||
// the same line N times in a row. Two different lines still both show;
|
||||
|
|
@ -1064,6 +1066,57 @@ async function runHeadlessOnce(
|
|||
return shown;
|
||||
}
|
||||
|
||||
function resolvePromptTracePreviewChars(): number {
|
||||
const raw = process.env.SF_HEADLESS_PROMPT_TRACE_CHARS?.trim();
|
||||
if (!raw) return 2400;
|
||||
const parsed = Number.parseInt(raw, 10);
|
||||
if (!Number.isFinite(parsed) || parsed < 0) return 2400;
|
||||
return Math.min(parsed, 12_000);
|
||||
}
|
||||
|
||||
function findSessionPromptTrace(
|
||||
sessionFile: string,
|
||||
): { customType: string; content: string } | null {
|
||||
let text = "";
|
||||
try {
|
||||
text = readFileSync(sessionFile, "utf-8");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
for (const line of text.split(/\r?\n/)) {
|
||||
if (!line.trim()) continue;
|
||||
try {
|
||||
const entry = JSON.parse(line) as Record<string, unknown>;
|
||||
if (entry.type !== "custom_message") continue;
|
||||
const content = entry.content;
|
||||
if (typeof content !== "string" || !content.trim()) continue;
|
||||
return {
|
||||
customType: String(entry.customType ?? "custom"),
|
||||
content,
|
||||
};
|
||||
} catch {}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function maybeWritePromptTrace(eventObj: Record<string, unknown>): void {
|
||||
if (!options.verbose) return;
|
||||
if (process.env.SF_HEADLESS_PROMPT_TRACE === "0") return;
|
||||
const sessionFile = String(eventObj.sessionFile ?? "");
|
||||
if (!sessionFile || promptTraceSessionFiles.has(sessionFile)) return;
|
||||
const trace = findSessionPromptTrace(sessionFile);
|
||||
if (!trace) return;
|
||||
promptTraceSessionFiles.add(sessionFile);
|
||||
for (const line of formatPromptTraceLines(
|
||||
trace.customType,
|
||||
trace.content,
|
||||
sessionFile,
|
||||
{ maxChars: resolvePromptTracePreviewChars() },
|
||||
)) {
|
||||
writeHeadlessLine(line);
|
||||
}
|
||||
}
|
||||
|
||||
// Client started flag — replaces old stdinWriter null-check
|
||||
let clientStarted = false;
|
||||
// Adapter for AnswerInjector — wraps client.sendUIResponse in a writeToStdin-compatible callback
|
||||
|
|
@ -1181,6 +1234,7 @@ async function runHeadlessOnce(
|
|||
client.onEvent((event) => {
|
||||
const eventObj = event as unknown as Record<string, unknown>;
|
||||
trackEvent(eventObj);
|
||||
maybeWritePromptTrace(eventObj);
|
||||
|
||||
const eventType = String(eventObj.type ?? "");
|
||||
if (eventType === "tool_execution_start") {
|
||||
|
|
|
|||
|
|
@ -43,12 +43,13 @@ function extractHeadingText(line) {
|
|||
}
|
||||
/**
|
||||
* Classify a heading by matching against known roles.
|
||||
* Uses substring matching so headings like "## UNIT: Execute Task T1.1" don't match
|
||||
* but "## Inlined Task Plan" does. Unknown headings default to "dynamic".
|
||||
* Uses exact matching so inlined artifact headings such as
|
||||
* `## Requirements Advanced` stay inside their original prompt position instead
|
||||
* of being mistaken for the top-level `## Requirements` context block.
|
||||
*/
|
||||
function classifyHeading(heading) {
|
||||
for (const [key, role] of Object.entries(HEADING_ROLES)) {
|
||||
if (heading === key || heading.startsWith(key)) {
|
||||
if (heading === key) {
|
||||
return role;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
61
src/resources/extensions/sf/tests/prompt-ordering.test.mjs
Normal file
61
src/resources/extensions/sf/tests/prompt-ordering.test.mjs
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { test } from "vitest";
|
||||
|
||||
import { reorderForCaching } from "../prompt-ordering.js";
|
||||
|
||||
test("reorderForCaching_when_inlined_slice_summary_has_requirements_advanced_keeps_it_after_mission", () => {
|
||||
const prompt = [
|
||||
"# Milestone Validation",
|
||||
"",
|
||||
"## Working Directory",
|
||||
"/repo",
|
||||
"",
|
||||
"## Mission",
|
||||
"Dispatch reviewers.",
|
||||
"",
|
||||
"## Context",
|
||||
"Inlined below.",
|
||||
"",
|
||||
"## Inlined Context",
|
||||
"### S01 Summary",
|
||||
"# S01",
|
||||
"",
|
||||
"## Requirements Advanced",
|
||||
"- R1",
|
||||
"",
|
||||
"## Requirements Validated",
|
||||
"None.",
|
||||
].join("\n");
|
||||
|
||||
const reordered = reorderForCaching(prompt);
|
||||
|
||||
assert.ok(
|
||||
reordered.indexOf("## Mission") <
|
||||
reordered.indexOf("## Requirements Advanced"),
|
||||
);
|
||||
assert.ok(
|
||||
reordered.indexOf("## Context") <
|
||||
reordered.indexOf("## Requirements Advanced"),
|
||||
);
|
||||
});
|
||||
|
||||
test("reorderForCaching_when_top_level_requirements_exists_still_hoists_exact_requirements_block", () => {
|
||||
const prompt = [
|
||||
"# Execute",
|
||||
"",
|
||||
"## Mission",
|
||||
"Do work.",
|
||||
"",
|
||||
"## Requirements",
|
||||
"- R1",
|
||||
"",
|
||||
"## Verification",
|
||||
"Run tests.",
|
||||
].join("\n");
|
||||
|
||||
const reordered = reorderForCaching(prompt);
|
||||
|
||||
assert.ok(
|
||||
reordered.indexOf("## Requirements") < reordered.indexOf("## Mission"),
|
||||
);
|
||||
});
|
||||
|
|
@ -5,6 +5,7 @@ import {
|
|||
formatCostLine,
|
||||
formatHeadlessHeartbeat,
|
||||
formatProgress,
|
||||
formatPromptTraceLines,
|
||||
formatThinkingLine,
|
||||
summarizeToolArgs,
|
||||
} from "../headless-ui.js";
|
||||
|
|
@ -71,6 +72,22 @@ describe("formatProgress", () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe("formatPromptTraceLines", () => {
|
||||
it("shows capped prompt preview with session path", () => {
|
||||
const lines = formatPromptTraceLines(
|
||||
"sf-auto",
|
||||
"# Mission\nDo the work\nKeep going",
|
||||
"/tmp/session.jsonl",
|
||||
{ maxChars: 17, maxLines: 3 },
|
||||
);
|
||||
|
||||
assert.ok(lines[0].includes("sf-auto instructions"));
|
||||
assert.ok(lines[0].includes("/tmp/session.jsonl"));
|
||||
assert.ok(lines.some((line) => line.includes("# Mission")));
|
||||
assert.ok(lines.at(-1)?.includes("truncated"));
|
||||
});
|
||||
});
|
||||
|
||||
describe("tool_execution_start", () => {
|
||||
it("shows tool name and summarized args in verbose mode", () => {
|
||||
const result = formatProgress(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue