diff --git a/packages/pi-coding-agent/src/modes/interactive/components/tool-execution.ts b/packages/pi-coding-agent/src/modes/interactive/components/tool-execution.ts index efb4a2f3c..b64755fbe 100644 --- a/packages/pi-coding-agent/src/modes/interactive/components/tool-execution.ts +++ b/packages/pi-coding-agent/src/modes/interactive/components/tool-execution.ts @@ -3,7 +3,6 @@ import { Box, Container, getCapabilities, - getImageDimensions, Image, imageFallback, Spacer, @@ -491,6 +490,10 @@ export class ToolExecutionComponent extends Container { { fallbackColor: (s: string) => theme.fg("toolOutput", s) }, { maxWidthCells: 60 }, ); + imageComponent.setOnDimensionsResolved(() => { + this.updateDisplay(); + this.ui.requestRender(); + }); this.imageComponents.push(imageComponent); this.addChild(imageComponent); } @@ -601,8 +604,7 @@ export class ToolExecutionComponent extends Container { if (imageBlocks.length > 0 && (!caps.images || !this.showImages)) { const imageIndicators = imageBlocks .map((img: any) => { - const dims = img.data ? (getImageDimensions(img.data, img.mimeType) ?? undefined) : undefined; - return imageFallback(img.mimeType, dims); + return imageFallback(img.mimeType); }) .join("\n"); output = output ? `${output}\n${imageIndicators}` : imageIndicators; diff --git a/packages/pi-coding-agent/src/utils/clipboard-image.ts b/packages/pi-coding-agent/src/utils/clipboard-image.ts index bc3ebb9f3..8e2d7f997 100644 --- a/packages/pi-coding-agent/src/utils/clipboard-image.ts +++ b/packages/pi-coding-agent/src/utils/clipboard-image.ts @@ -1,7 +1,7 @@ import { spawnSync } from "child_process"; import { readImageFromClipboard as nativeReadImage } from "@gsd/native/clipboard"; -import { loadPhoton } from "./photon.js"; +import { ImageFormat, parseImage } from "@gsd/native/image"; export type ClipboardImage = { bytes: Uint8Array; @@ -60,22 +60,14 @@ function isSupportedImageMimeType(mimeType: string): boolean { } /** - * Convert unsupported image formats to PNG using Photon. - * Returns null if conversion is unavailable or fails. + * Convert unsupported image formats to PNG using the native Rust image module. + * Returns null if conversion fails. */ async function convertToPng(bytes: Uint8Array): Promise { - const photon = await loadPhoton(); - if (!photon) { - return null; - } - try { - const image = photon.PhotonImage.new_from_byteslice(bytes); - try { - return image.get_bytes(); - } finally { - image.free(); - } + const image = await parseImage(bytes); + const pngBytes = await image.encode(ImageFormat.PNG, 100); + return new Uint8Array(pngBytes); } catch { return null; } diff --git a/packages/pi-coding-agent/src/utils/image-convert.ts b/packages/pi-coding-agent/src/utils/image-convert.ts index 74198c630..3c1857ae9 100644 --- a/packages/pi-coding-agent/src/utils/image-convert.ts +++ b/packages/pi-coding-agent/src/utils/image-convert.ts @@ -1,4 +1,4 @@ -import { loadPhoton } from "./photon.js"; +import { ImageFormat, parseImage } from "@gsd/native/image"; /** * Convert image to PNG format for terminal display. @@ -13,24 +13,14 @@ export async function convertToPng( return { data: base64Data, mimeType }; } - const photon = await loadPhoton(); - if (!photon) { - // Photon not available, can't convert - return null; - } - try { const bytes = new Uint8Array(Buffer.from(base64Data, "base64")); - const image = photon.PhotonImage.new_from_byteslice(bytes); - try { - const pngBuffer = image.get_bytes(); - return { - data: Buffer.from(pngBuffer).toString("base64"), - mimeType: "image/png", - }; - } finally { - image.free(); - } + const image = await parseImage(bytes); + const pngBytes = await image.encode(ImageFormat.PNG, 100); + return { + data: Buffer.from(new Uint8Array(pngBytes)).toString("base64"), + mimeType: "image/png", + }; } catch { // Conversion failed return null; diff --git a/packages/pi-coding-agent/src/utils/image-resize.ts b/packages/pi-coding-agent/src/utils/image-resize.ts index 87047f2f9..813ddef4b 100644 --- a/packages/pi-coding-agent/src/utils/image-resize.ts +++ b/packages/pi-coding-agent/src/utils/image-resize.ts @@ -1,5 +1,6 @@ import type { ImageContent } from "@gsd/pi-ai"; -import { loadPhoton } from "./photon.js"; +import { ImageFormat, parseImage, SamplingFilter } from "@gsd/native/image"; +import type { NativeImageHandle } from "@gsd/native/image"; export interface ImageResizeOptions { maxWidth?: number; // Default: 2000 @@ -40,8 +41,7 @@ function pickSmaller( * Resize an image to fit within the specified max dimensions and file size. * Returns the original image if it already fits within the limits. * - * Uses Photon (Rust/WASM) for image processing. If Photon is not available, - * returns the original image unchanged. + * Uses the native Rust image module (N-API) for image processing. * * Strategy for staying under maxBytes: * 1. First resize to maxWidth/maxHeight @@ -53,9 +53,11 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption const opts = { ...DEFAULT_OPTIONS, ...options }; const inputBuffer = Buffer.from(img.data, "base64"); - const photon = await loadPhoton(); - if (!photon) { - // Photon not available, return original image + let image: NativeImageHandle; + try { + image = await parseImage(new Uint8Array(inputBuffer)); + } catch { + // Failed to decode image return { data: img.data, mimeType: img.mimeType, @@ -67,12 +69,9 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption }; } - let image: ReturnType | undefined; try { - image = photon.PhotonImage.new_from_byteslice(new Uint8Array(inputBuffer)); - - const originalWidth = image.get_width(); - const originalHeight = image.get_height(); + const originalWidth = image.width; + const originalHeight = image.height; const format = img.mimeType?.split("/")[1] ?? "png"; // Check if already within all limits (dimensions AND size) @@ -103,24 +102,23 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption } // Helper to resize and encode in both formats, returning the smaller one - function tryBothFormats( + async function tryBothFormats( width: number, height: number, jpegQuality: number, - ): { buffer: Uint8Array; mimeType: string } { - const resized = photon!.resize(image!, width, height, photon!.SamplingFilter.Lanczos3); + ): Promise<{ buffer: Uint8Array; mimeType: string }> { + const resized = await image.resize(width, height, SamplingFilter.Lanczos3); - try { - const pngBuffer = resized.get_bytes(); - const jpegBuffer = resized.get_bytes_jpeg(jpegQuality); + const pngBytes = await resized.encode(ImageFormat.PNG, 100); + const jpegBytes = await resized.encode(ImageFormat.JPEG, jpegQuality); - return pickSmaller( - { buffer: pngBuffer, mimeType: "image/png" }, - { buffer: jpegBuffer, mimeType: "image/jpeg" }, - ); - } finally { - resized.free(); - } + const pngBuffer = new Uint8Array(pngBytes); + const jpegBuffer = new Uint8Array(jpegBytes); + + return pickSmaller( + { buffer: pngBuffer, mimeType: "image/png" }, + { buffer: jpegBuffer, mimeType: "image/jpeg" }, + ); } // Try to produce an image under maxBytes @@ -132,7 +130,7 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption let finalHeight = targetHeight; // First attempt: resize to target dimensions, try both formats - best = tryBothFormats(targetWidth, targetHeight, opts.jpegQuality); + best = await tryBothFormats(targetWidth, targetHeight, opts.jpegQuality); if (best.buffer.length <= opts.maxBytes) { return { @@ -148,7 +146,7 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption // Still too large - try JPEG with decreasing quality for (const quality of qualitySteps) { - best = tryBothFormats(targetWidth, targetHeight, quality); + best = await tryBothFormats(targetWidth, targetHeight, quality); if (best.buffer.length <= opts.maxBytes) { return { @@ -173,7 +171,7 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption } for (const quality of qualitySteps) { - best = tryBothFormats(finalWidth, finalHeight, quality); + best = await tryBothFormats(finalWidth, finalHeight, quality); if (best.buffer.length <= opts.maxBytes) { return { @@ -200,7 +198,7 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption wasResized: true, }; } catch { - // Failed to load image + // Failed to process image return { data: img.data, mimeType: img.mimeType, @@ -210,10 +208,6 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption height: 0, wasResized: false, }; - } finally { - if (image) { - image.free(); - } } } diff --git a/packages/pi-tui/src/components/image.ts b/packages/pi-tui/src/components/image.ts index ca76cddde..c789a0a5b 100644 --- a/packages/pi-tui/src/components/image.ts +++ b/packages/pi-tui/src/components/image.ts @@ -26,6 +26,8 @@ export class Image implements Component { private theme: ImageTheme; private options: ImageOptions; private imageId?: number; + private dimensionsResolved = false; + private onDimensionsResolved?: () => void; private cachedLines?: string[]; private cachedWidth?: number; @@ -41,8 +43,28 @@ export class Image implements Component { this.mimeType = mimeType; this.theme = theme; this.options = options; - this.dimensions = dimensions || getImageDimensions(base64Data, mimeType) || { widthPx: 800, heightPx: 600 }; + this.dimensions = dimensions || { widthPx: 800, heightPx: 600 }; + this.dimensionsResolved = !!dimensions; this.imageId = options.imageId; + + if (!dimensions) { + getImageDimensions(base64Data).then((dims) => { + if (dims) { + this.dimensions = dims; + this.dimensionsResolved = true; + this.invalidate(); + this.onDimensionsResolved?.(); + } + }); + } + } + + /** + * Register a callback invoked when async dimension parsing completes. + * Useful for triggering a re-render after the Image updates its layout. + */ + setOnDimensionsResolved(cb: () => void): void { + this.onDimensionsResolved = cb; } /** Get the Kitty image ID used by this image (if any). */ diff --git a/packages/pi-tui/src/index.ts b/packages/pi-tui/src/index.ts index 5e8c90ab6..8f8ee69a0 100644 --- a/packages/pi-tui/src/index.ts +++ b/packages/pi-tui/src/index.ts @@ -62,11 +62,7 @@ export { encodeKitty, getCapabilities, getCellDimensions, - getGifDimensions, getImageDimensions, - getJpegDimensions, - getPngDimensions, - getWebpDimensions, type ImageDimensions, type ImageProtocol, type ImageRenderOptions, diff --git a/packages/pi-tui/src/terminal-image.ts b/packages/pi-tui/src/terminal-image.ts index e706fedcf..7e219ca99 100644 --- a/packages/pi-tui/src/terminal-image.ts +++ b/packages/pi-tui/src/terminal-image.ts @@ -199,147 +199,21 @@ export function calculateImageRows( return Math.max(1, rows); } -export function getPngDimensions(base64Data: string): ImageDimensions | null { +/** + * Parse image dimensions using the native Rust image module. + * Auto-detects format from byte content (PNG, JPEG, GIF, WebP). + */ +export async function getImageDimensions(base64Data: string): Promise { + const { parseImage: parse } = await import("@gsd/native/image"); try { - const buffer = Buffer.from(base64Data, "base64"); - - if (buffer.length < 24) { - return null; - } - - if (buffer[0] !== 0x89 || buffer[1] !== 0x50 || buffer[2] !== 0x4e || buffer[3] !== 0x47) { - return null; - } - - const width = buffer.readUInt32BE(16); - const height = buffer.readUInt32BE(20); - - return { widthPx: width, heightPx: height }; + const bytes = new Uint8Array(Buffer.from(base64Data, "base64")); + const handle = await parse(bytes); + return { widthPx: handle.width, heightPx: handle.height }; } catch { return null; } } -export function getJpegDimensions(base64Data: string): ImageDimensions | null { - try { - const buffer = Buffer.from(base64Data, "base64"); - - if (buffer.length < 2) { - return null; - } - - if (buffer[0] !== 0xff || buffer[1] !== 0xd8) { - return null; - } - - let offset = 2; - while (offset < buffer.length - 9) { - if (buffer[offset] !== 0xff) { - offset++; - continue; - } - - const marker = buffer[offset + 1]; - - if (marker >= 0xc0 && marker <= 0xc2) { - const height = buffer.readUInt16BE(offset + 5); - const width = buffer.readUInt16BE(offset + 7); - return { widthPx: width, heightPx: height }; - } - - if (offset + 3 >= buffer.length) { - return null; - } - const length = buffer.readUInt16BE(offset + 2); - if (length < 2) { - return null; - } - offset += 2 + length; - } - - return null; - } catch { - return null; - } -} - -export function getGifDimensions(base64Data: string): ImageDimensions | null { - try { - const buffer = Buffer.from(base64Data, "base64"); - - if (buffer.length < 10) { - return null; - } - - const sig = buffer.slice(0, 6).toString("ascii"); - if (sig !== "GIF87a" && sig !== "GIF89a") { - return null; - } - - const width = buffer.readUInt16LE(6); - const height = buffer.readUInt16LE(8); - - return { widthPx: width, heightPx: height }; - } catch { - return null; - } -} - -export function getWebpDimensions(base64Data: string): ImageDimensions | null { - try { - const buffer = Buffer.from(base64Data, "base64"); - - if (buffer.length < 30) { - return null; - } - - const riff = buffer.slice(0, 4).toString("ascii"); - const webp = buffer.slice(8, 12).toString("ascii"); - if (riff !== "RIFF" || webp !== "WEBP") { - return null; - } - - const chunk = buffer.slice(12, 16).toString("ascii"); - if (chunk === "VP8 ") { - if (buffer.length < 30) return null; - const width = buffer.readUInt16LE(26) & 0x3fff; - const height = buffer.readUInt16LE(28) & 0x3fff; - return { widthPx: width, heightPx: height }; - } else if (chunk === "VP8L") { - if (buffer.length < 25) return null; - const bits = buffer.readUInt32LE(21); - const width = (bits & 0x3fff) + 1; - const height = ((bits >> 14) & 0x3fff) + 1; - return { widthPx: width, heightPx: height }; - } else if (chunk === "VP8X") { - if (buffer.length < 30) return null; - const width = (buffer[24] | (buffer[25] << 8) | (buffer[26] << 16)) + 1; - const height = (buffer[27] | (buffer[28] << 8) | (buffer[29] << 16)) + 1; - return { widthPx: width, heightPx: height }; - } - - return null; - } catch { - return null; - } -} - -export function getImageDimensions(base64Data: string, mimeType: string): ImageDimensions | null { - if (mimeType === "image/png") { - return getPngDimensions(base64Data); - } - if (mimeType === "image/jpeg") { - return getJpegDimensions(base64Data); - } - if (mimeType === "image/gif") { - return getGifDimensions(base64Data); - } - if (mimeType === "image/webp") { - return getWebpDimensions(base64Data); - } - return null; -} - export function renderImage( base64Data: string, imageDimensions: ImageDimensions,