diff --git a/.gsd/REQUIREMENTS.md b/.gsd/REQUIREMENTS.md index 78eb866dc..763787523 100644 --- a/.gsd/REQUIREMENTS.md +++ b/.gsd/REQUIREMENTS.md @@ -6,24 +6,24 @@ This file is the explicit capability and coverage contract for the project. ### R020 — Sharp-based screenshot resizing - Class: core-capability -- Status: active +- Status: validated - Description: constrainScreenshot uses the sharp Node library for image resizing instead of bouncing buffers through page canvas context. Faster, no page dependency. - Why it matters: The current approach sends full screenshot buffer to the page as base64, creates an Image, draws to canvas, exports, then sends back. This is slow and fragile (depends on working canvas context). - Source: user - Primary owning slice: M002/S03 - Supporting slices: M002/S01 -- Validation: unmapped +- Validation: constrainScreenshot uses sharp(buffer).metadata() and sharp(buffer).resize(). Zero page.evaluate calls in capture.ts. sharp added to root dependencies and extension peerDependencies. Build passes. - Notes: sharp added as a dependency. API: sharp(buffer).resize(w, h, { fit: 'inside' }).jpeg({ quality }).toBuffer() ### R021 — Opt-in screenshots on navigate - Class: core-capability -- Status: active +- Status: validated - Description: browser_navigate does not capture or return a screenshot by default. An explicit parameter (e.g. screenshot: true) opts in to screenshot capture. - Why it matters: The current always-inline screenshot is a large base64 payload in every navigation response. For many verifications the compact page summary + diff is sufficient. Significant token savings. - Source: user - Primary owning slice: M002/S03 - Supporting slices: none -- Validation: unmapped +- Validation: browser_navigate has screenshot: Type.Optional(Type.Boolean({ default: false })) parameter. Screenshot capture gated with if (params.screenshot). browser_reload unchanged. Build passes. - Notes: Default is off. The agent can still use browser_screenshot explicitly when visual verification is needed. ### R022 — Form analysis tool (browser_analyze_form) @@ -341,8 +341,8 @@ This file is the explicit capability and coverage contract for the project. | R017 | core-capability | validated | M002/S02 | M002/S01 | postActionSummary eliminated, countOpenDialogs removed from ToolDeps, consolidated capture pattern | | R018 | core-capability | validated | M002/S02 | none | explicit includeBodyText true/false per tool signal level, classification in D017 | | R019 | core-capability | validated | M002/S02 | none | zero_mutation_shortcut settle reason, combined readSettleState poll, 60ms/30ms thresholds in D019 | -| R020 | core-capability | active | M002/S03 | M002/S01 | unmapped | -| R021 | core-capability | active | M002/S03 | none | unmapped | +| R020 | core-capability | validated | M002/S03 | M002/S01 | sharp-based constrainScreenshot, zero page.evaluate in capture.ts, build passes | +| R021 | core-capability | validated | M002/S03 | none | screenshot param default false, capture gated, browser_reload unchanged, build passes | | R022 | core-capability | active | M002/S04 | M002/S01 | unmapped | | R023 | core-capability | active | M002/S04 | M002/S01 | unmapped | | R024 | core-capability | active | M002/S05 | M002/S01 | unmapped | @@ -353,8 +353,8 @@ This file is the explicit capability and coverage contract for the project. ## Coverage Summary -- Active requirements: 7 -- Validated requirements: 15 +- Active requirements: 5 +- Validated requirements: 17 - Deferred requirements: 3 - Out of scope: 3 -- Unmapped active requirements: 7 +- Unmapped active requirements: 5 diff --git a/.gsd/STATE.md b/.gsd/STATE.md index 3c6f15cc4..6633fd1b5 100644 --- a/.gsd/STATE.md +++ b/.gsd/STATE.md @@ -1,7 +1,7 @@ # GSD State **Active Milestone:** M002 — Browser Tools Performance & Intelligence -**Active Slice:** S03 — Screenshot pipeline +**Active Slice:** S04 — Form intelligence **Phase:** planning **Requirements Status:** 7 active · 15 validated · 3 deferred · 3 out of scope @@ -16,4 +16,4 @@ - None ## Next Action -Plan slice S03 (Screenshot pipeline). +Plan slice S04 (Form intelligence). diff --git a/.gsd/milestones/M002/M002-ROADMAP.md b/.gsd/milestones/M002/M002-ROADMAP.md index dee536e26..dddb7153d 100644 --- a/.gsd/milestones/M002/M002-ROADMAP.md +++ b/.gsd/milestones/M002/M002-ROADMAP.md @@ -64,7 +64,7 @@ This milestone is complete only when all are true: - [x] **S02: Action pipeline performance** `risk:medium` `depends:[S01]` > After this: captureCompactPageState and postActionSummary are consolidated into fewer evaluate calls per action; settleAfterActionAdaptive short-circuits on zero-mutation actions; low-signal actions (scroll, hover, Tab) skip body text capture — verified by build success and behavioral spot-check. -- [ ] **S03: Screenshot pipeline** `risk:low` `depends:[S01]` +- [x] **S03: Screenshot pipeline** `risk:low` `depends:[S01]` > After this: constrainScreenshot uses sharp instead of canvas; browser_navigate returns no screenshot by default with an explicit parameter to opt in — verified by build success and running browser_navigate to confirm no screenshot in response. - [ ] **S04: Form intelligence** `risk:medium` `depends:[S01]` diff --git a/.gsd/milestones/M002/slices/S03/S03-PLAN.md b/.gsd/milestones/M002/slices/S03/S03-PLAN.md new file mode 100644 index 000000000..c9f1464aa --- /dev/null +++ b/.gsd/milestones/M002/slices/S03/S03-PLAN.md @@ -0,0 +1,40 @@ +# S03: Screenshot pipeline + +**Goal:** `constrainScreenshot` uses sharp instead of canvas; `browser_navigate` returns no screenshot by default. +**Demo:** Build passes, `constrainScreenshot` calls sharp for dimension check and resize (no `page.evaluate`), `browser_navigate` omits screenshot unless `screenshot: true` is passed. + +## Must-Haves + +- `constrainScreenshot` uses `sharp(buffer).metadata()` for dimensions and `sharp(buffer).resize().jpeg()/png().toBuffer()` for resizing — no `page.evaluate` call +- Images already within MAX_SCREENSHOT_DIM bounds are returned unchanged (no re-encoding) +- JPEG output uses the `quality` parameter; PNG output uses lossless `.png()` (no quality param) +- `constrainScreenshot` keeps its existing `(page, buffer, mimeType, quality)` signature for backward compatibility +- `browser_navigate` has a `screenshot` parameter (default: `false`) gating screenshot capture +- `browser_reload` screenshot behavior is unchanged +- `captureErrorScreenshot` works with the new `constrainScreenshot` +- sharp added to root `package.json` dependencies and extension `peerDependencies` + +## Verification + +- `node -e "require('sharp')"` — sharp is installed and loadable +- `npx tsc --noEmit` or equivalent build check passes +- Grep verification: `grep -c "page.evaluate" src/resources/extensions/browser-tools/capture.ts` returns 0 +- Grep verification: `grep "screenshot.*boolean" src/resources/extensions/browser-tools/tools/navigation.ts` finds the parameter +- Grep verification: `grep "default.*false\|screenshot.*false" src/resources/extensions/browser-tools/tools/navigation.ts` confirms default is false +- Extension loads via jiti and all 43 tools register + +## Tasks + +- [x] **T01: Replace constrainScreenshot with sharp and make navigate screenshots opt-in** `est:30m` + - Why: Delivers both R020 (sharp-based resizing) and R021 (opt-in navigate screenshots) — the two requirements this slice owns + - Files: `package.json`, `src/resources/extensions/browser-tools/package.json`, `src/resources/extensions/browser-tools/capture.ts`, `src/resources/extensions/browser-tools/tools/navigation.ts` + - Do: (1) Add sharp to root `package.json` dependencies and extension `peerDependencies`, run install. (2) Rewrite `constrainScreenshot` internals: use `sharp(buffer).metadata()` for width/height, return buffer unchanged if within bounds, otherwise `sharp(buffer).resize(MAX, MAX, { fit: 'inside' }).jpeg({ quality }).toBuffer()` for JPEG or `.png().toBuffer()` for PNG. Keep the `page` parameter unused. (3) Add `screenshot?: boolean` parameter (default: false) to `browser_navigate`, gate the screenshot capture block on it. Update the tool description. (4) Verify build, grep checks, extension load. + - Verify: Build passes; `grep -c "page.evaluate" capture.ts` returns 0; extension loads with 43 tools; navigate tool schema includes `screenshot` boolean parameter + - Done when: sharp handles all screenshot resizing with no page dependency; navigate returns no screenshot by default + +## Files Likely Touched + +- `package.json` +- `src/resources/extensions/browser-tools/package.json` +- `src/resources/extensions/browser-tools/capture.ts` +- `src/resources/extensions/browser-tools/tools/navigation.ts` diff --git a/.gsd/milestones/M002/slices/S03/tasks/T01-PLAN.md b/.gsd/milestones/M002/slices/S03/tasks/T01-PLAN.md new file mode 100644 index 000000000..380b7d1d8 --- /dev/null +++ b/.gsd/milestones/M002/slices/S03/tasks/T01-PLAN.md @@ -0,0 +1,61 @@ +--- +estimated_steps: 4 +estimated_files: 4 +--- + +# T01: Replace constrainScreenshot with sharp and make navigate screenshots opt-in + +**Slice:** S03 — Screenshot pipeline +**Milestone:** M002 + +## Description + +Two contained changes delivering R020 and R021. Replace `constrainScreenshot`'s manual JPEG/PNG header parsing and canvas-based resizing with sharp's `metadata()` and `resize()` APIs. Add an opt-in `screenshot` boolean parameter to `browser_navigate` (default false) so screenshots are only captured when explicitly requested. + +## Steps + +1. Add `sharp` to root `package.json` dependencies and to `src/resources/extensions/browser-tools/package.json` peerDependencies. Run `npm install`. +2. Rewrite `constrainScreenshot` in `capture.ts`: + - Add `import sharp from "sharp"` at top + - Replace manual header parsing with `const { width, height } = await sharp(buffer).metadata()` + - Early-return original buffer if `width <= MAX_SCREENSHOT_DIM && height <= MAX_SCREENSHOT_DIM` + - For JPEG: `return Buffer.from(await sharp(buffer).resize(MAX_SCREENSHOT_DIM, MAX_SCREENSHOT_DIM, { fit: 'inside' }).jpeg({ quality }).toBuffer())` + - For PNG: `return Buffer.from(await sharp(buffer).resize(MAX_SCREENSHOT_DIM, MAX_SCREENSHOT_DIM, { fit: 'inside' }).png().toBuffer())` + - Keep `page: Page` as first parameter (unused) — signature stability per D008 constraints +3. In `navigation.ts`, modify `browser_navigate`: + - Add `screenshot: Type.Optional(Type.Boolean({ description: "Capture and return a screenshot (default: false)", default: false }))` to parameters + - Gate the `screenshotContent` block with `if (params.screenshot)` + - Update the tool description to mention screenshots are opt-in +4. Verify: build passes, grep checks confirm no `page.evaluate` in capture.ts, extension loads with 43 tools via jiti + +## Must-Haves + +- [ ] `constrainScreenshot` uses sharp — zero `page.evaluate` calls in capture.ts +- [ ] Images within bounds returned unchanged (no re-encoding) +- [ ] JPEG uses quality param; PNG uses lossless `.png()` +- [ ] `(page, buffer, mimeType, quality)` signature preserved +- [ ] `browser_navigate` screenshot parameter defaults to false +- [ ] `browser_reload` screenshot behavior unchanged +- [ ] Build passes and extension loads with 43 tools + +## Verification + +- `npm install` succeeds with sharp +- `grep -c "page.evaluate" src/resources/extensions/browser-tools/capture.ts` returns 0 +- `grep "screenshot.*Type.Boolean\|screenshot.*boolean" src/resources/extensions/browser-tools/tools/navigation.ts` finds the parameter +- Build/typecheck passes +- Extension loads via jiti: 43 tools registered + +## Inputs + +- `src/resources/extensions/browser-tools/capture.ts` — current `constrainScreenshot` with manual header parsing and canvas resizing (lines 126-182) +- `src/resources/extensions/browser-tools/tools/navigation.ts` — current `browser_navigate` with always-on screenshot (lines 56-61) +- `src/resources/extensions/browser-tools/state.ts` — ToolDeps interface with `constrainScreenshot` signature (line ~342) +- S01 summary — module structure, import patterns, ToolDeps contract + +## Expected Output + +- `package.json` — sharp added to dependencies +- `src/resources/extensions/browser-tools/package.json` — sharp added to peerDependencies +- `src/resources/extensions/browser-tools/capture.ts` — `constrainScreenshot` rewritten with sharp, zero `page.evaluate` calls +- `src/resources/extensions/browser-tools/tools/navigation.ts` — `browser_navigate` has `screenshot` parameter (default false), gated screenshot block, updated description diff --git a/package-lock.json b/package-lock.json index ee29e3018..7e2ec40f3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,7 +25,8 @@ "@gsd/pi-coding-agent": "*", "@gsd/pi-tui": "*", "picocolors": "^1.1.1", - "playwright": "^1.58.2" + "playwright": "^1.58.2", + "sharp": "^0.34.5" }, "bin": { "gsd": "dist/loader.js", @@ -792,6 +793,16 @@ "sisteransi": "^1.0.5" } }, + "node_modules/@emnapi/runtime": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.9.0.tgz", + "integrity": "sha512-QN75eB0IH2ywSpRpNddCRfQIhmJYBCJ1x5Lb3IscKAL8bMnVAKnRg8dCoXbHzVLLH7P38N2Z3mtulB7W0J0FKw==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@google/genai": { "version": "1.45.0", "resolved": "https://registry.npmjs.org/@google/genai/-/genai-1.45.0.tgz", @@ -831,6 +842,471 @@ "resolved": "packages/pi-tui", "link": true }, + "node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", + "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", + "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", + "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", + "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", + "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", + "cpu": [ + "arm" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", + "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", + "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", + "cpu": [ + "ppc64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", + "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", + "cpu": [ + "riscv64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", + "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", + "cpu": [ + "s390x" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", + "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", + "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", + "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", + "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", + "cpu": [ + "arm" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", + "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", + "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", + "cpu": [ + "ppc64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", + "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", + "cpu": [ + "riscv64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", + "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", + "cpu": [ + "s390x" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", + "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", + "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", + "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", + "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.7.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", + "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", + "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", + "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@mariozechner/clipboard": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/@mariozechner/clipboard/-/clipboard-0.3.2.tgz", @@ -2150,6 +2626,15 @@ "node": ">= 14" } }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, "node_modules/diff": { "version": "8.0.3", "resolved": "https://registry.npmjs.org/diff/-/diff-8.0.3.tgz", @@ -3160,6 +3645,62 @@ ], "license": "MIT" }, + "node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sharp": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", + "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.0.0", + "detect-libc": "^2.1.2", + "semver": "^7.7.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.34.5", + "@img/sharp-darwin-x64": "0.34.5", + "@img/sharp-libvips-darwin-arm64": "1.2.4", + "@img/sharp-libvips-darwin-x64": "1.2.4", + "@img/sharp-libvips-linux-arm": "1.2.4", + "@img/sharp-libvips-linux-arm64": "1.2.4", + "@img/sharp-libvips-linux-ppc64": "1.2.4", + "@img/sharp-libvips-linux-riscv64": "1.2.4", + "@img/sharp-libvips-linux-s390x": "1.2.4", + "@img/sharp-libvips-linux-x64": "1.2.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", + "@img/sharp-libvips-linuxmusl-x64": "1.2.4", + "@img/sharp-linux-arm": "0.34.5", + "@img/sharp-linux-arm64": "0.34.5", + "@img/sharp-linux-ppc64": "0.34.5", + "@img/sharp-linux-riscv64": "0.34.5", + "@img/sharp-linux-s390x": "0.34.5", + "@img/sharp-linux-x64": "0.34.5", + "@img/sharp-linuxmusl-arm64": "0.34.5", + "@img/sharp-linuxmusl-x64": "0.34.5", + "@img/sharp-wasm32": "0.34.5", + "@img/sharp-win32-arm64": "0.34.5", + "@img/sharp-win32-ia32": "0.34.5", + "@img/sharp-win32-x64": "0.34.5" + } + }, "node_modules/signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", diff --git a/package.json b/package.json index ed820cd39..511df11aa 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,8 @@ "@gsd/pi-coding-agent": "*", "@gsd/pi-tui": "*", "picocolors": "^1.1.1", - "playwright": "^1.58.2" + "playwright": "^1.58.2", + "sharp": "^0.34.5" }, "bundleDependencies": [ "@gsd/pi-agent-core", diff --git a/src/resources/extensions/browser-tools/capture.ts b/src/resources/extensions/browser-tools/capture.ts index 7ec307368..895ffc13b 100644 --- a/src/resources/extensions/browser-tools/capture.ts +++ b/src/resources/extensions/browser-tools/capture.ts @@ -6,6 +6,7 @@ */ import type { Frame, Page } from "playwright"; +import sharp from "sharp"; import type { CompactPageState, CompactSelectorState } from "./state.js"; import { formatCompactStateSummary } from "./utils.js"; @@ -120,61 +121,35 @@ export async function postActionSummary(p: Page, target?: Page | Frame): Promise /** * If either dimension of the image buffer exceeds MAX_SCREENSHOT_DIM, - * downscale proportionally using the browser's canvas (zero dependencies). - * Returns the original buffer unchanged if already within limits. + * downscale proportionally using sharp. Returns the original buffer + * unchanged if already within limits. + * + * `page` parameter is retained for ToolDeps signature stability (D008) + * but is no longer used — all processing is server-side via sharp. */ export async function constrainScreenshot( - page: Page, + _page: Page, buffer: Buffer, mimeType: string, quality: number, ): Promise { - let width: number; - let height: number; + const { width, height } = await sharp(buffer).metadata(); - if (mimeType === "image/png") { - width = buffer.readUInt32BE(16); - height = buffer.readUInt32BE(20); - } else { - width = 0; - height = 0; - for (let i = 0; i < buffer.length - 8; i++) { - if (buffer[i] === 0xff && (buffer[i + 1] === 0xc0 || buffer[i + 1] === 0xc2)) { - height = buffer.readUInt16BE(i + 5); - width = buffer.readUInt16BE(i + 7); - break; - } - } - } - - if (width <= MAX_SCREENSHOT_DIM && height <= MAX_SCREENSHOT_DIM) { + if ( + width !== undefined && + height !== undefined && + width <= MAX_SCREENSHOT_DIM && + height <= MAX_SCREENSHOT_DIM + ) { return buffer; } - const b64 = buffer.toString("base64"); - const result = await page.evaluate( - async ({ b64, mime, maxDim, q }) => { - const img = new Image(); - await new Promise((resolve, reject) => { - img.onload = () => resolve(); - img.onerror = reject; - img.src = `data:${mime};base64,${b64}`; - }); - const scale = Math.min(maxDim / img.width, maxDim / img.height); - const w = Math.round(img.width * scale); - const h = Math.round(img.height * scale); - const canvas = document.createElement("canvas"); - canvas.width = w; - canvas.height = h; - const ctx = canvas.getContext("2d")!; - ctx.drawImage(img, 0, 0, w, h); - return canvas.toDataURL(mime, q / 100); - }, - { b64, mime: mimeType, maxDim: MAX_SCREENSHOT_DIM, q: quality }, - ); + const resizer = sharp(buffer).resize(MAX_SCREENSHOT_DIM, MAX_SCREENSHOT_DIM, { fit: "inside" }); - const resizedB64 = result.split(",")[1]; - return Buffer.from(resizedB64, "base64"); + if (mimeType === "image/png") { + return Buffer.from(await resizer.png().toBuffer()); + } + return Buffer.from(await resizer.jpeg({ quality }).toBuffer()); } /** Capture a JPEG screenshot for error debugging. Returns base64 or null. */ diff --git a/src/resources/extensions/browser-tools/package.json b/src/resources/extensions/browser-tools/package.json index 17849cbb4..e37964d3d 100644 --- a/src/resources/extensions/browser-tools/package.json +++ b/src/resources/extensions/browser-tools/package.json @@ -10,11 +10,15 @@ "extensions": ["./index.ts"] }, "peerDependencies": { - "playwright": ">=1.40.0" + "playwright": ">=1.40.0", + "sharp": ">=0.33.0" }, "peerDependenciesMeta": { "playwright": { "optional": true + }, + "sharp": { + "optional": true } } } diff --git a/src/resources/extensions/browser-tools/tools/navigation.ts b/src/resources/extensions/browser-tools/tools/navigation.ts index 376e37692..9910ec423 100644 --- a/src/resources/extensions/browser-tools/tools/navigation.ts +++ b/src/resources/extensions/browser-tools/tools/navigation.ts @@ -17,9 +17,10 @@ export function registerNavigationTools(pi: ExtensionAPI, deps: ToolDeps): void name: "browser_navigate", label: "Browser Navigate", description: - "Open the browser (if not already open) and navigate to a URL. Waits for network idle. Returns page title and current URL. Use ONLY for visually verifying locally-running web apps (e.g. http://localhost:3000). Do NOT use for documentation sites, GitHub, search results, or any external URL — use web_search instead.", + "Open the browser (if not already open) and navigate to a URL. Waits for network idle. Returns page title and current URL. Use ONLY for visually verifying locally-running web apps (e.g. http://localhost:3000). Do NOT use for documentation sites, GitHub, search results, or any external URL — use web_search instead. Screenshots are only captured when the `screenshot` parameter is set to true.", parameters: Type.Object({ url: Type.String({ description: "URL to navigate to, e.g. http://localhost:3000" }), + screenshot: Type.Optional(Type.Boolean({ description: "Capture and return a screenshot (default: false)", default: false })), }), async execute(_toolCallId, params, _signal, _onUpdate, _ctx) { @@ -54,11 +55,13 @@ export function registerNavigationTools(pi: ExtensionAPI, deps: ToolDeps): void }); let screenshotContent: any[] = []; - try { - let buf = await p.screenshot({ type: "jpeg", quality: 80, scale: "css" }); - buf = await deps.constrainScreenshot(p, buf, "image/jpeg", 80); - screenshotContent = [{ type: "image", data: buf.toString("base64"), mimeType: "image/jpeg" }]; - } catch {} + if (params.screenshot) { + try { + let buf = await p.screenshot({ type: "jpeg", quality: 80, scale: "css" }); + buf = await deps.constrainScreenshot(p, buf, "image/jpeg", 80); + screenshotContent = [{ type: "image", data: buf.toString("base64"), mimeType: "image/jpeg" }]; + } catch {} + } return { content: [