diff --git a/src/resources/extensions/claude-code-cli/stream-adapter.ts b/src/resources/extensions/claude-code-cli/stream-adapter.ts index d8d3e35f5..4187fc4e3 100644 --- a/src/resources/extensions/claude-code-cli/stream-adapter.ts +++ b/src/resources/extensions/claude-code-cli/stream-adapter.ts @@ -96,6 +96,31 @@ interface ParsedTextInputField { secure: boolean; } +interface SDKInputImageBlock { + type: "image"; + source: { + type: "base64"; + media_type: string; + data: string; + }; +} + +interface SDKInputTextBlock { + type: "text"; + text: string; +} + +type SDKInputUserContentBlock = SDKInputImageBlock | SDKInputTextBlock; + +interface SDKInputUserMessage { + type: "user"; + message: { + role: "user"; + content: SDKInputUserContentBlock[]; + }; + parent_tool_use_id: null; +} + const OTHER_OPTION_LABEL = "None of the above"; const SENSITIVE_FIELD_PATTERN = /(password|passphrase|secret|token|api[_\s-]*key|private[_\s-]*key|credential)/i; @@ -222,6 +247,74 @@ export function buildPromptFromContext(context: Context): string { return parts.join("\n\n"); } +function stripDataUriPrefix(value: string): string { + const commaIndex = value.indexOf(","); + if (value.startsWith("data:") && commaIndex !== -1) { + return value.slice(commaIndex + 1); + } + return value; +} + +function inferMimeTypeFromDataUri(value: string): string | null { + const match = /^data:([^;,]+);base64,/.exec(value); + return match?.[1] ?? null; +} + +export function extractImageBlocksFromContext(context: Context): SDKInputImageBlock[] { + const imageBlocks: SDKInputImageBlock[] = []; + + for (const msg of context.messages) { + if (msg.role !== "user" || !Array.isArray(msg.content)) continue; + for (const part of msg.content) { + if (!part || typeof part !== "object") continue; + const block = part as { type?: unknown; data?: unknown; mimeType?: unknown }; + if (block.type !== "image" || typeof block.data !== "string") continue; + + const mimeType = + typeof block.mimeType === "string" && block.mimeType.length > 0 + ? block.mimeType + : inferMimeTypeFromDataUri(block.data); + if (!mimeType) continue; + + imageBlocks.push({ + type: "image", + source: { + type: "base64", + media_type: mimeType, + data: stripDataUriPrefix(block.data), + }, + }); + } + } + + return imageBlocks; +} + +export function buildSdkQueryPrompt( + context: Context, + textPrompt: string = buildPromptFromContext(context), +): string | AsyncIterable { + const imageBlocks = extractImageBlocksFromContext(context); + if (imageBlocks.length === 0) { + return textPrompt; + } + + const content: SDKInputUserContentBlock[] = [...imageBlocks]; + if (textPrompt) { + content.push({ type: "text", text: textPrompt }); + } + + const sdkMessage: SDKInputUserMessage = { + type: "user", + message: { role: "user", content }, + parent_tool_use_id: null, + }; + + return (async function* () { + yield sdkMessage; + })(); +} + // --------------------------------------------------------------------------- // Error helper // --------------------------------------------------------------------------- @@ -821,6 +914,7 @@ async function pumpSdkMessages( } const prompt = buildPromptFromContext(context); + const queryPrompt = buildSdkQueryPrompt(context, prompt); const permissionMode = await resolveClaudePermissionMode(); const sdkOpts = buildSdkOptions( modelId, @@ -836,7 +930,7 @@ async function pumpSdkMessages( ); const queryResult = sdk.query({ - prompt, + prompt: queryPrompt, options: { ...sdkOpts, abortController: controller, diff --git a/src/resources/extensions/claude-code-cli/tests/stream-adapter.test.ts b/src/resources/extensions/claude-code-cli/tests/stream-adapter.test.ts index a600852a4..0a4256665 100644 --- a/src/resources/extensions/claude-code-cli/tests/stream-adapter.test.ts +++ b/src/resources/extensions/claude-code-cli/tests/stream-adapter.test.ts @@ -10,8 +10,10 @@ import { mergePendingToolCalls, resolveClaudePermissionMode, buildPromptFromContext, + buildSdkQueryPrompt, buildSdkOptions, createClaudeCodeElicitationHandler, + extractImageBlocksFromContext, extractToolResultsFromSdkUserMessage, getClaudeLookupCommand, parseAskUserQuestionsElicitation, @@ -167,6 +169,92 @@ describe("stream-adapter — full context prompt (#2859)", () => { }); }); +describe("stream-adapter — image prompt forwarding (#4183)", () => { + test("extractImageBlocksFromContext maps user image parts to Anthropic base64 image blocks", () => { + const context: Context = { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "look" }, + { + type: "image", + data: "data:image/png;base64,abc123", + mimeType: "image/png", + }, + ], + } as Message, + ], + }; + + const imageBlocks = extractImageBlocksFromContext(context); + assert.deepEqual(imageBlocks, [ + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: "abc123", + }, + }, + ]); + }); + + test("buildSdkQueryPrompt returns plain string when no images exist in context", () => { + const context: Context = { + messages: [{ role: "user", content: "hello" } as Message], + }; + const textPrompt = buildPromptFromContext(context); + + const prompt = buildSdkQueryPrompt(context, textPrompt); + assert.equal(typeof prompt, "string"); + assert.equal(prompt, textPrompt); + }); + + test("buildSdkQueryPrompt wraps images and prompt text in an SDK user message iterable", async () => { + const context: Context = { + messages: [ + { + role: "user", + content: [ + { type: "image", data: "ZmFrZQ==", mimeType: "image/jpeg" }, + { type: "text", text: "What is in this image?" }, + ], + } as Message, + ], + }; + const textPrompt = buildPromptFromContext(context); + + const prompt = buildSdkQueryPrompt(context, textPrompt); + assert.notEqual(typeof prompt, "string"); + assert.ok(prompt && typeof (prompt as any)[Symbol.asyncIterator] === "function"); + + const messages: any[] = []; + for await (const item of prompt as AsyncIterable) { + messages.push(item); + } + assert.equal(messages.length, 1); + assert.deepEqual(messages[0], { + type: "user", + message: { + role: "user", + content: [ + { + type: "image", + source: { + type: "base64", + media_type: "image/jpeg", + data: "ZmFrZQ==", + }, + }, + { type: "text", text: textPrompt }, + ], + }, + parent_tool_use_id: null, + }); + }); +}); + // --------------------------------------------------------------------------- // Bug #4102 — transcript fabrication regression tests // ---------------------------------------------------------------------------