From 5f660bf3ce8b6829821acefab48ba96abf3b23c2 Mon Sep 17 00:00:00 2001 From: Tom Boucher Date: Mon, 30 Mar 2026 16:40:35 -0400 Subject: [PATCH] fix: recover from many-image dimension overflow by stripping older images (#3075) When a session accumulates many images (screenshots, file reads), the Anthropic API enforces a 2000px dimension limit for "many-image requests" and returns a 400 error. Previously this error was not classified as retryable, causing the session to get permanently stuck in an error loop with no recovery path. This adds automatic recovery: detect the specific "image dimensions exceed max allowed size for many-image requests" error, strip older images from the conversation history (keeping the 5 most recent), and auto-retry. Also handles manual retry (continue/retry) by downsizing before retrying. Closes #2874 Co-authored-by: Claude Opus 4.6 --- .../pi-coding-agent/src/core/agent-session.ts | 39 ++- .../src/core/image-overflow-recovery.test.ts | 228 ++++++++++++++++++ .../src/core/image-overflow-recovery.ts | 118 +++++++++ .../controllers/chat-controller.ts | 7 + 4 files changed, 391 insertions(+), 1 deletion(-) create mode 100644 packages/pi-coding-agent/src/core/image-overflow-recovery.test.ts create mode 100644 packages/pi-coding-agent/src/core/image-overflow-recovery.ts diff --git a/packages/pi-coding-agent/src/core/agent-session.ts b/packages/pi-coding-agent/src/core/agent-session.ts index fb84b9209..fe5e1f853 100644 --- a/packages/pi-coding-agent/src/core/agent-session.ts +++ b/packages/pi-coding-agent/src/core/agent-session.ts @@ -72,6 +72,7 @@ import type { ModelRegistry } from "./model-registry.js"; import { expandPromptTemplate, type PromptTemplate } from "./prompt-templates.js"; import type { ResourceExtensionPaths, ResourceLoader } from "./resource-loader.js"; import { RetryHandler } from "./retry-handler.js"; +import { isImageDimensionError, downsizeConversationImages } from "./image-overflow-recovery.js"; import type { BranchSummaryEntry, SessionManager } from "./session-manager.js"; import { getLatestCompactionEntry } from "./session-manager.js"; import type { SettingsManager } from "./settings-manager.js"; @@ -136,7 +137,8 @@ export type AgentSessionEvent = | { type: "auto_retry_end"; success: boolean; attempt: number; finalError?: string } | { type: "fallback_provider_switch"; from: string; to: string; reason: string } | { type: "fallback_provider_restored"; provider: string; reason: string } - | { type: "fallback_chain_exhausted"; reason: string }; + | { type: "fallback_chain_exhausted"; reason: string } + | { type: "image_overflow_recovery"; strippedCount: number; imageCount: number }; /** Listener function for agent session events */ export type AgentSessionEventListener = (event: AgentSessionEvent) => void; @@ -487,6 +489,36 @@ export class AgentSession { if (didRetry) return; // Retry was initiated, don't proceed to compaction } + // Check for image dimension overflow (many-image 400 error). + // When a session accumulates many images, the API rejects requests + // whose images exceed the many-image dimension limit. Strip older + // images from the conversation and auto-retry. (#2874) + if ( + msg.stopReason === "error" && + isImageDimensionError(msg.errorMessage) + ) { + const messages = this.agent.state.messages; + const result = downsizeConversationImages(messages as Message[]); + if (result.processed) { + // Remove the trailing error assistant message, then replace + if (messages.length > 0 && messages[messages.length - 1].role === "assistant") { + this.agent.replaceMessages(messages.slice(0, -1)); + } + + this._emit({ + type: "image_overflow_recovery", + strippedCount: result.strippedCount, + imageCount: result.imageCount, + }); + + // Auto-retry after downsizing + setTimeout(() => { + this.agent.continue().catch(() => {}); + }, 0); + return; + } + } + await this._compactionOrchestrator.checkCompaction(msg); } } @@ -1986,6 +2018,11 @@ export class AgentSession { const messages = this.agent.state.messages; const last = messages[messages.length - 1]; if (last?.role === "assistant" && (last as AssistantMessage).stopReason === "error") { + // If the error was an image dimension overflow, downsize images + // before retrying so the retry doesn't hit the same error (#2874) + if (isImageDimensionError((last as AssistantMessage).errorMessage)) { + downsizeConversationImages(messages as Message[]); + } this.agent.replaceMessages(messages.slice(0, -1)); this.agent.continue().catch((err) => { runner.emitError({ diff --git a/packages/pi-coding-agent/src/core/image-overflow-recovery.test.ts b/packages/pi-coding-agent/src/core/image-overflow-recovery.test.ts new file mode 100644 index 000000000..de075c280 --- /dev/null +++ b/packages/pi-coding-agent/src/core/image-overflow-recovery.test.ts @@ -0,0 +1,228 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { + isImageDimensionError, + MANY_IMAGE_MAX_DIMENSION, + downsizeConversationImages, +} from "./image-overflow-recovery.js"; +import type { Message } from "@gsd/pi-ai"; + +// ─── isImageDimensionError ──────────────────────────────────────────────────── + +describe("isImageDimensionError", () => { + it("returns true for Anthropic many-image dimension error", () => { + const errorMessage = + 'Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"messages.125.content.38.image.source.base64.data: At least one of the image dimensions exceed max allowed size for many-image requests: 2000 pixels"}}'; + assert.equal(isImageDimensionError(errorMessage), true); + }); + + it("returns true for bare dimension exceed message", () => { + const errorMessage = + "image dimensions exceed max allowed size for many-image requests: 2000 pixels"; + assert.equal(isImageDimensionError(errorMessage), true); + }); + + it("returns false for unrelated 400 error", () => { + const errorMessage = + 'Error: 400 {"type":"error","error":{"type":"invalid_request_error","message":"max_tokens: 4096 > 2048"}}'; + assert.equal(isImageDimensionError(errorMessage), false); + }); + + it("returns false for rate limit error", () => { + assert.equal(isImageDimensionError("429 rate limit exceeded"), false); + }); + + it("returns false for empty string", () => { + assert.equal(isImageDimensionError(""), false); + }); + + it("returns false for undefined", () => { + assert.equal(isImageDimensionError(undefined), false); + }); +}); + +// ─── MANY_IMAGE_MAX_DIMENSION ───────────────────────────────────────────────── + +describe("MANY_IMAGE_MAX_DIMENSION", () => { + it("is less than 2000 (the API-enforced limit)", () => { + assert.ok(MANY_IMAGE_MAX_DIMENSION < 2000); + }); + + it("is a positive integer", () => { + assert.ok(MANY_IMAGE_MAX_DIMENSION > 0); + assert.equal(MANY_IMAGE_MAX_DIMENSION, Math.floor(MANY_IMAGE_MAX_DIMENSION)); + }); +}); + +// ─── helpers ────────────────────────────────────────────────────────────────── + +function makeUserMsg(content: Message["content"] & any): Message { + return { role: "user", content, timestamp: Date.now() } as Message; +} + +function makeAssistantMsg(text: string): Message { + return { + role: "assistant", + content: [{ type: "text", text }], + api: "anthropic-messages", + provider: "anthropic", + model: "claude-opus-4-6", + usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, + stopReason: "stop", + timestamp: Date.now(), + } as Message; +} + +function makeToolResultMsg(images: number): Message { + const content: any[] = []; + for (let i = 0; i < images; i++) { + content.push({ type: "image", data: `img${i}`, mimeType: "image/png" }); + } + return { + role: "toolResult", + toolCallId: `tc${Math.random()}`, + toolName: "screenshot", + content, + isError: false, + timestamp: Date.now(), + } as Message; +} + +// ─── downsizeConversationImages ─────────────────────────────────────────────── + +describe("downsizeConversationImages", () => { + it("counts images in user and toolResult messages", () => { + const messages: Message[] = [ + makeUserMsg([ + { type: "image", data: "img1", mimeType: "image/png" }, + { type: "image", data: "img2", mimeType: "image/png" }, + ]), + makeAssistantMsg("I see them"), + makeToolResultMsg(1), + ]; + + const result = downsizeConversationImages(messages); + assert.equal(result.imageCount, 3); + }); + + it("returns processed=false when no images present", () => { + const messages: Message[] = [ + makeUserMsg("just text"), + makeAssistantMsg("reply"), + ]; + + const result = downsizeConversationImages(messages); + assert.equal(result.imageCount, 0); + assert.equal(result.processed, false); + }); + + it("returns processed=false when image count <= RECENT_IMAGES_TO_KEEP", () => { + const messages: Message[] = [ + makeUserMsg([ + { type: "image", data: "img1", mimeType: "image/png" }, + ]), + makeAssistantMsg("got it"), + ]; + + const result = downsizeConversationImages(messages); + assert.equal(result.imageCount, 1); + assert.equal(result.processed, false); + }); + + it("strips older images when many images present, preserves recent ones", () => { + const messages: Message[] = []; + for (let i = 0; i < 25; i++) { + messages.push( + makeUserMsg([ + { type: "text", text: `message ${i}` }, + { type: "image", data: `img${i}`, mimeType: "image/png" }, + ]), + ); + messages.push(makeAssistantMsg(`reply ${i}`)); + } + + const result = downsizeConversationImages(messages); + assert.ok(result.processed); + assert.equal(result.imageCount, 25); + assert.equal(result.strippedCount, 20); // 25 - 5 recent + + // Count remaining images + let remainingImages = 0; + for (const msg of messages) { + if (msg.role === "assistant") continue; + if (typeof msg.content === "string") continue; + const arr = msg.content as any[]; + for (const block of arr) { + if (block.type === "image") remainingImages++; + } + } + assert.equal(remainingImages, 5, "Should keep exactly 5 most recent images"); + + // The 5 most recent user messages (indices 40,42,44,46,48) should have images + for (let i = 20; i < 25; i++) { + const userMsg = messages[i * 2]; // user messages at even indices + const arr = userMsg.content as any[]; + const hasImage = arr.some((c: any) => c.type === "image"); + assert.ok(hasImage, `Recent message ${i} should retain its image`); + } + }); + + it("adds text placeholder when stripping an image", () => { + const messages: Message[] = []; + for (let i = 0; i < 10; i++) { + messages.push( + makeUserMsg([ + { type: "image", data: `img${i}`, mimeType: "image/jpeg" }, + ]), + ); + messages.push(makeAssistantMsg(`reply ${i}`)); + } + + downsizeConversationImages(messages); + + // First message's image should have been replaced with text + const firstMsg = messages[0]; + const arr = firstMsg.content as any[]; + const placeholder = arr.find( + (c: any) => c.type === "text" && c.text.includes("[image removed"), + ); + assert.ok(placeholder, "Stripped image should be replaced with text placeholder"); + assert.ok( + placeholder.text.includes("image/jpeg"), + "Placeholder should mention original mime type", + ); + }); + + it("handles toolResult messages with images", () => { + const messages: Message[] = []; + for (let i = 0; i < 10; i++) { + messages.push(makeToolResultMsg(1)); + messages.push(makeAssistantMsg(`reply ${i}`)); + } + + const result = downsizeConversationImages(messages); + assert.equal(result.imageCount, 10); + assert.equal(result.strippedCount, 5); + assert.ok(result.processed); + }); + + it("handles mixed user and toolResult images", () => { + const messages: Message[] = []; + for (let i = 0; i < 8; i++) { + messages.push( + makeUserMsg([ + { type: "text", text: `check ${i}` }, + { type: "image", data: `uimg${i}`, mimeType: "image/png" }, + ]), + ); + messages.push(makeAssistantMsg(`processing ${i}`)); + messages.push(makeToolResultMsg(1)); + messages.push(makeAssistantMsg(`done ${i}`)); + } + + const result = downsizeConversationImages(messages); + // 8 user images + 8 tool result images = 16 total + assert.equal(result.imageCount, 16); + assert.equal(result.strippedCount, 11); // 16 - 5 recent + }); +}); diff --git a/packages/pi-coding-agent/src/core/image-overflow-recovery.ts b/packages/pi-coding-agent/src/core/image-overflow-recovery.ts new file mode 100644 index 000000000..3573514e4 --- /dev/null +++ b/packages/pi-coding-agent/src/core/image-overflow-recovery.ts @@ -0,0 +1,118 @@ +/** + * Image overflow recovery for many-image sessions. + * + * When a conversation accumulates many images (screenshots, file reads, etc.), + * the Anthropic API enforces a stricter per-image dimension limit (2000px) for + * "many-image requests." This module detects the resulting 400 error and + * recovers by stripping older images from the conversation history, preserving + * the most recent ones to maintain session continuity. + * + * @see https://github.com/gsd-build/gsd-2/issues/2874 + */ + +import type { Message, ImageContent, TextContent } from "@gsd/pi-ai"; + +/** + * Maximum image dimension (px) that the Anthropic API allows in many-image + * requests. Images at or above this size in a large conversation will be + * rejected with a 400 error. We use 1568 as the safe ceiling (Anthropic's + * recommended max for multi-image requests). + */ +export const MANY_IMAGE_MAX_DIMENSION = 1568; + +/** + * Number of recent images to preserve when stripping old images. + * Keeps the most recent screenshots/images so the model retains visual context + * for the current task. + */ +const RECENT_IMAGES_TO_KEEP = 5; + +/** + * Regex matching the Anthropic API error for oversized images in many-image requests. + */ +const IMAGE_DIMENSION_ERROR_RE = + /image.dimensions?.exceed.*max.*allowed.*size.*many.image/i; + +/** + * Detect whether an error message is the Anthropic "image dimensions exceed max + * allowed size for many-image requests" 400 error. + */ +export function isImageDimensionError(errorMessage: string | undefined | null): boolean { + if (!errorMessage) return false; + return IMAGE_DIMENSION_ERROR_RE.test(errorMessage); +} + +export interface DownsizeResult { + /** Total number of images found in the conversation */ + imageCount: number; + /** Whether any images were stripped */ + processed: boolean; + /** Number of images that were stripped */ + strippedCount: number; +} + +/** + * Strip older images from conversation messages to recover from many-image + * dimension errors. Preserves the N most recent images and replaces older ones + * with a text placeholder. + * + * Mutates messages in place (same pattern as replaceMessages/compaction). + * + * Accepts Message[] (the LLM message union) so it works with both + * agent.state.messages and session entries. + */ +export function downsizeConversationImages(messages: Message[]): DownsizeResult { + // First pass: collect all image locations (message index + content index) + const imageLocations: Array<{ msgIdx: number; contentIdx: number }> = []; + + for (let msgIdx = 0; msgIdx < messages.length; msgIdx++) { + const msg = messages[msgIdx]; + if (msg.role === "assistant") continue; + + // UserMessage can have string content; ToolResultMessage always has array + if (msg.role === "user" && typeof msg.content === "string") continue; + + const contentArr = msg.content as (TextContent | ImageContent)[]; + if (!Array.isArray(contentArr)) continue; + + for (let contentIdx = 0; contentIdx < contentArr.length; contentIdx++) { + if (contentArr[contentIdx].type === "image") { + imageLocations.push({ msgIdx, contentIdx }); + } + } + } + + const imageCount = imageLocations.length; + if (imageCount === 0) { + return { imageCount: 0, processed: false, strippedCount: 0 }; + } + + // Determine which images to strip (all except the N most recent) + const stripCount = Math.max(0, imageCount - RECENT_IMAGES_TO_KEEP); + if (stripCount === 0) { + return { imageCount, processed: false, strippedCount: 0 }; + } + + const toStrip = imageLocations.slice(0, stripCount); + + // Second pass: replace stripped images with text placeholder. + // Process in reverse order to maintain content indices. + for (let i = toStrip.length - 1; i >= 0; i--) { + const { msgIdx, contentIdx } = toStrip[i]; + const msg = messages[msgIdx]; + if (msg.role === "assistant") continue; + if (msg.role === "user" && typeof msg.content === "string") continue; + + const contentArr = msg.content as (TextContent | ImageContent)[]; + const imageBlock = contentArr[contentIdx] as ImageContent; + const mimeType = imageBlock.mimeType || "image/unknown"; + + // Replace the image block with a text placeholder + (contentArr as any[])[contentIdx] = { + type: "text", + text: `[image removed to reduce context size — was ${mimeType}]`, + } as TextContent; + } + + return { imageCount, processed: true, strippedCount: stripCount }; +} diff --git a/packages/pi-coding-agent/src/modes/interactive/controllers/chat-controller.ts b/packages/pi-coding-agent/src/modes/interactive/controllers/chat-controller.ts index ebe9231ed..aeb2be064 100644 --- a/packages/pi-coding-agent/src/modes/interactive/controllers/chat-controller.ts +++ b/packages/pi-coding-agent/src/modes/interactive/controllers/chat-controller.ts @@ -337,5 +337,12 @@ export async function handleAgentEvent(host: InteractiveModeStateHost & { host.showError(event.reason); host.ui.requestRender(); break; + + case "image_overflow_recovery": + host.showStatus( + `Removed ${event.strippedCount} older image(s) to comply with API limits. Retrying...`, + ); + host.ui.requestRender(); + break; } }