singularity/singularity-forge
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

feat: wire native Rust image module into image processing pipeline

Browse source 
Replace manual binary header parsing (PNG/JPEG/GIF/WebP) in terminal-image.ts
with the native @gsd/native/image module, and replace photon-node (WASM) with
native N-API calls for image resize and format conversion.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Lex Christopherson 2026-03-13 13:39:06 -06:00
parent 77bbbc19a9
commit ec7d6eee4c
7 changed files with 76 additions and 206 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
packages/pi-coding-agent/src/modes/interactive/components/tool-execution.ts
View file

@ -3,7 +3,6 @@ import {
Box,
Container,
getCapabilities,
getImageDimensions,
Image,
imageFallback,
Spacer,
@ -491,6 +490,10 @@ export class ToolExecutionComponent extends Container {
{ fallbackColor: (s: string) => theme.fg("toolOutput", s) },
{ maxWidthCells: 60 },
);
imageComponent.setOnDimensionsResolved(() => {
this.updateDisplay();
this.ui.requestRender();
});
this.imageComponents.push(imageComponent);
this.addChild(imageComponent);
}
@ -601,8 +604,7 @@ export class ToolExecutionComponent extends Container {
if (imageBlocks.length > 0 && (!caps.images || !this.showImages)) {
const imageIndicators = imageBlocks
.map((img: any) => {
const dims = img.data ? (getImageDimensions(img.data, img.mimeType) ?? undefined) : undefined;
return imageFallback(img.mimeType, dims);
return imageFallback(img.mimeType);
})
.join("\n");
output = output ? `${output}\n${imageIndicators}` : imageIndicators;

20
packages/pi-coding-agent/src/utils/clipboard-image.ts
View file

@ -1,7 +1,7 @@
import { spawnSync } from "child_process";
import { readImageFromClipboard as nativeReadImage } from "@gsd/native/clipboard";
import { loadPhoton } from "./photon.js";
import { ImageFormat, parseImage } from "@gsd/native/image";
export type ClipboardImage = {
bytes: Uint8Array;
@ -60,22 +60,14 @@ function isSupportedImageMimeType(mimeType: string): boolean {
}
/**
* Convert unsupported image formats to PNG using Photon.
* Returns null if conversion is unavailable or fails.
* Convert unsupported image formats to PNG using the native Rust image module.
* Returns null if conversion fails.
*/
async function convertToPng(bytes: Uint8Array): Promise<Uint8Array | null> {
const photon = await loadPhoton();
if (!photon) {
return null;
}
try {
const image = photon.PhotonImage.new_from_byteslice(bytes);
try {
return image.get_bytes();
} finally {
image.free();
}
const image = await parseImage(bytes);
const pngBytes = await image.encode(ImageFormat.PNG, 100);
return new Uint8Array(pngBytes);
} catch {
return null;
}

24
packages/pi-coding-agent/src/utils/image-convert.ts
View file

@ -1,4 +1,4 @@
import { loadPhoton } from "./photon.js";
import { ImageFormat, parseImage } from "@gsd/native/image";
/**
* Convert image to PNG format for terminal display.
@ -13,24 +13,14 @@ export async function convertToPng(
return { data: base64Data, mimeType };
}
const photon = await loadPhoton();
if (!photon) {
// Photon not available, can't convert
return null;
}
try {
const bytes = new Uint8Array(Buffer.from(base64Data, "base64"));
const image = photon.PhotonImage.new_from_byteslice(bytes);
try {
const pngBuffer = image.get_bytes();
return {
data: Buffer.from(pngBuffer).toString("base64"),
mimeType: "image/png",
};
} finally {
image.free();
}
const image = await parseImage(bytes);
const pngBytes = await image.encode(ImageFormat.PNG, 100);
return {
data: Buffer.from(new Uint8Array(pngBytes)).toString("base64"),
mimeType: "image/png",
};
} catch {
// Conversion failed
return null;

58
packages/pi-coding-agent/src/utils/image-resize.ts
View file

@ -1,5 +1,6 @@
import type { ImageContent } from "@gsd/pi-ai";
import { loadPhoton } from "./photon.js";
import { ImageFormat, parseImage, SamplingFilter } from "@gsd/native/image";
import type { NativeImageHandle } from "@gsd/native/image";
export interface ImageResizeOptions {
maxWidth?: number; // Default: 2000
@ -40,8 +41,7 @@ function pickSmaller(
* Resize an image to fit within the specified max dimensions and file size.
* Returns the original image if it already fits within the limits.
*
* Uses Photon (Rust/WASM) for image processing. If Photon is not available,
* returns the original image unchanged.
* Uses the native Rust image module (N-API) for image processing.
*
* Strategy for staying under maxBytes:
* 1. First resize to maxWidth/maxHeight
@ -53,9 +53,11 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption
const opts = { ...DEFAULT_OPTIONS, ...options };
const inputBuffer = Buffer.from(img.data, "base64");
const photon = await loadPhoton();
if (!photon) {
// Photon not available, return original image
let image: NativeImageHandle;
try {
image = await parseImage(new Uint8Array(inputBuffer));
} catch {
// Failed to decode image
return {
data: img.data,
mimeType: img.mimeType,
@ -67,12 +69,9 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption
};
}
let image: ReturnType<typeof photon.PhotonImage.new_from_byteslice> | undefined;
try {
image = photon.PhotonImage.new_from_byteslice(new Uint8Array(inputBuffer));
const originalWidth = image.get_width();
const originalHeight = image.get_height();
const originalWidth = image.width;
const originalHeight = image.height;
const format = img.mimeType?.split("/")[1] ?? "png";
// Check if already within all limits (dimensions AND size)
@ -103,24 +102,23 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption
}
// Helper to resize and encode in both formats, returning the smaller one
function tryBothFormats(
async function tryBothFormats(
width: number,
height: number,
jpegQuality: number,
): { buffer: Uint8Array; mimeType: string } {
const resized = photon!.resize(image!, width, height, photon!.SamplingFilter.Lanczos3);
): Promise<{ buffer: Uint8Array; mimeType: string }> {
const resized = await image.resize(width, height, SamplingFilter.Lanczos3);
try {
const pngBuffer = resized.get_bytes();
const jpegBuffer = resized.get_bytes_jpeg(jpegQuality);
const pngBytes = await resized.encode(ImageFormat.PNG, 100);
const jpegBytes = await resized.encode(ImageFormat.JPEG, jpegQuality);
return pickSmaller(
{ buffer: pngBuffer, mimeType: "image/png" },
{ buffer: jpegBuffer, mimeType: "image/jpeg" },
);
} finally {
resized.free();
}
const pngBuffer = new Uint8Array(pngBytes);
const jpegBuffer = new Uint8Array(jpegBytes);
return pickSmaller(
{ buffer: pngBuffer, mimeType: "image/png" },
{ buffer: jpegBuffer, mimeType: "image/jpeg" },
);
}
// Try to produce an image under maxBytes
@ -132,7 +130,7 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption
let finalHeight = targetHeight;
// First attempt: resize to target dimensions, try both formats
best = tryBothFormats(targetWidth, targetHeight, opts.jpegQuality);
best = await tryBothFormats(targetWidth, targetHeight, opts.jpegQuality);
if (best.buffer.length <= opts.maxBytes) {
return {
@ -148,7 +146,7 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption
// Still too large - try JPEG with decreasing quality
for (const quality of qualitySteps) {
best = tryBothFormats(targetWidth, targetHeight, quality);
best = await tryBothFormats(targetWidth, targetHeight, quality);
if (best.buffer.length <= opts.maxBytes) {
return {
@ -173,7 +171,7 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption
}
for (const quality of qualitySteps) {
best = tryBothFormats(finalWidth, finalHeight, quality);
best = await tryBothFormats(finalWidth, finalHeight, quality);
if (best.buffer.length <= opts.maxBytes) {
return {
@ -200,7 +198,7 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption
wasResized: true,
};
} catch {
// Failed to load image
// Failed to process image
return {
data: img.data,
mimeType: img.mimeType,
@ -210,10 +208,6 @@ export async function resizeImage(img: ImageContent, options?: ImageResizeOption
height: 0,
wasResized: false,
};
} finally {
if (image) {
image.free();
}
}
}

24
packages/pi-tui/src/components/image.ts
View file

@ -26,6 +26,8 @@ export class Image implements Component {
private theme: ImageTheme;
private options: ImageOptions;
private imageId?: number;
private dimensionsResolved = false;
private onDimensionsResolved?: () => void;
private cachedLines?: string[];
private cachedWidth?: number;
@ -41,8 +43,28 @@ export class Image implements Component {
this.mimeType = mimeType;
this.theme = theme;
this.options = options;
this.dimensions = dimensions || getImageDimensions(base64Data, mimeType) || { widthPx: 800, heightPx: 600 };
this.dimensions = dimensions || { widthPx: 800, heightPx: 600 };
this.dimensionsResolved = !!dimensions;
this.imageId = options.imageId;
if (!dimensions) {
getImageDimensions(base64Data).then((dims) => {
if (dims) {
this.dimensions = dims;
this.dimensionsResolved = true;
this.invalidate();
this.onDimensionsResolved?.();
}
});
}
}
/**
* Register a callback invoked when async dimension parsing completes.
* Useful for triggering a re-render after the Image updates its layout.
*/
setOnDimensionsResolved(cb: () => void): void {
this.onDimensionsResolved = cb;
}
/** Get the Kitty image ID used by this image (if any). */

4
packages/pi-tui/src/index.ts
View file

@ -62,11 +62,7 @@ export {
encodeKitty,
getCapabilities,
getCellDimensions,
getGifDimensions,
getImageDimensions,
getJpegDimensions,
getPngDimensions,
getWebpDimensions,
type ImageDimensions,
type ImageProtocol,
type ImageRenderOptions,

144
packages/pi-tui/src/terminal-image.ts
View file

@ -199,147 +199,21 @@ export function calculateImageRows(
return Math.max(1, rows);
}
export function getPngDimensions(base64Data: string): ImageDimensions | null {
/**
* Parse image dimensions using the native Rust image module.
* Auto-detects format from byte content (PNG, JPEG, GIF, WebP).
*/
export async function getImageDimensions(base64Data: string): Promise<ImageDimensions | null> {
const { parseImage: parse } = await import("@gsd/native/image");
try {
const buffer = Buffer.from(base64Data, "base64");
if (buffer.length < 24) {
return null;
}
if (buffer[0] !== 0x89 || buffer[1] !== 0x50 || buffer[2] !== 0x4e || buffer[3] !== 0x47) {
return null;
}
const width = buffer.readUInt32BE(16);
const height = buffer.readUInt32BE(20);
return { widthPx: width, heightPx: height };
const bytes = new Uint8Array(Buffer.from(base64Data, "base64"));
const handle = await parse(bytes);
return { widthPx: handle.width, heightPx: handle.height };
} catch {
return null;
}
}
export function getJpegDimensions(base64Data: string): ImageDimensions | null {
try {
const buffer = Buffer.from(base64Data, "base64");
if (buffer.length < 2) {
return null;
}
if (buffer[0] !== 0xff || buffer[1] !== 0xd8) {
return null;
}
let offset = 2;
while (offset < buffer.length - 9) {
if (buffer[offset] !== 0xff) {
offset++;
continue;
}
const marker = buffer[offset + 1];
if (marker >= 0xc0 && marker <= 0xc2) {
const height = buffer.readUInt16BE(offset + 5);
const width = buffer.readUInt16BE(offset + 7);
return { widthPx: width, heightPx: height };
}
if (offset + 3 >= buffer.length) {
return null;
}
const length = buffer.readUInt16BE(offset + 2);
if (length < 2) {
return null;
}
offset += 2 + length;
}
return null;
} catch {
return null;
}
}
export function getGifDimensions(base64Data: string): ImageDimensions | null {
try {
const buffer = Buffer.from(base64Data, "base64");
if (buffer.length < 10) {
return null;
}
const sig = buffer.slice(0, 6).toString("ascii");
if (sig !== "GIF87a" && sig !== "GIF89a") {
return null;
}
const width = buffer.readUInt16LE(6);
const height = buffer.readUInt16LE(8);
return { widthPx: width, heightPx: height };
} catch {
return null;
}
}
export function getWebpDimensions(base64Data: string): ImageDimensions | null {
try {
const buffer = Buffer.from(base64Data, "base64");
if (buffer.length < 30) {
return null;
}
const riff = buffer.slice(0, 4).toString("ascii");
const webp = buffer.slice(8, 12).toString("ascii");
if (riff !== "RIFF" || webp !== "WEBP") {
return null;
}
const chunk = buffer.slice(12, 16).toString("ascii");
if (chunk === "VP8 ") {
if (buffer.length < 30) return null;
const width = buffer.readUInt16LE(26) & 0x3fff;
const height = buffer.readUInt16LE(28) & 0x3fff;
return { widthPx: width, heightPx: height };
} else if (chunk === "VP8L") {
if (buffer.length < 25) return null;
const bits = buffer.readUInt32LE(21);
const width = (bits & 0x3fff) + 1;
const height = ((bits >> 14) & 0x3fff) + 1;
return { widthPx: width, heightPx: height };
} else if (chunk === "VP8X") {
if (buffer.length < 30) return null;
const width = (buffer[24] | (buffer[25] << 8) | (buffer[26] << 16)) + 1;
const height = (buffer[27] | (buffer[28] << 8) | (buffer[29] << 16)) + 1;
return { widthPx: width, heightPx: height };
}
return null;
} catch {
return null;
}
}
export function getImageDimensions(base64Data: string, mimeType: string): ImageDimensions | null {
if (mimeType === "image/png") {
return getPngDimensions(base64Data);
}
if (mimeType === "image/jpeg") {
return getJpegDimensions(base64Data);
}
if (mimeType === "image/gif") {
return getGifDimensions(base64Data);
}
if (mimeType === "image/webp") {
return getWebpDimensions(base64Data);
}
return null;
}
export function renderImage(
base64Data: string,
imageDimensions: ImageDimensions,