diff --git a/docs/dev/ADR-005-multi-model-provider-tool-strategy.md b/docs/dev/ADR-005-multi-model-provider-tool-strategy.md new file mode 100644 index 000000000..bdf00706a --- /dev/null +++ b/docs/dev/ADR-005-multi-model-provider-tool-strategy.md @@ -0,0 +1,67 @@ +# ADR-005: Multi-Model, Multi-Provider, and Tool Strategy + +**Status:** Accepted +**Date:** 2026-03-27 +**Deciders:** Jeremy McSpadden +**Related:** ADR-004 (capability-aware model routing), ADR-003 (pipeline simplification), [Issue #2790](https://github.com/gsd-build/gsd-2/issues/2790) + +## Context + +PR #2755 lands capability-aware model routing (ADR-004), extending the router from a one-dimensional complexity-tier system to a two-dimensional system that scores models across 7 capability dimensions. GSD can now intelligently pick the best model for a task from a heterogeneous pool. + +But model selection is only one piece of the multi-model puzzle. The system faces structural gaps as users configure diverse provider pools: + +1. **Tool compatibility is assumed, not verified** — Every registered tool is sent to every model regardless of provider capabilities. +2. **No tool-aware model routing** — ADR-004 scores 7 capability dimensions but none encode whether a model can actually use the tools a task requires. +3. **Provider failover loses context fidelity** — Cross-provider switches silently degrade conversation quality (thinking blocks dropped, tool IDs remapped). +4. **Tool availability is static across a session** — The same tools are presented regardless of the selected model's capabilities. +5. **No provider capability registry** — Provider quirks are scattered across `*-shared.ts` files. + +## Decision + +Introduce a provider capability registry and tool compatibility layer that integrates with ADR-004's capability-aware model router. + +### Design Principles + +1. **Layered on ADR-004, not replacing it.** Capability scoring remains primary. This adds tool compatibility as a hard constraint. +2. **Hard constraints filter; soft scores rank.** Tool support is binary — it filters the eligible set before scoring. +3. **Provider knowledge is declarative, not scattered.** Provider capabilities move to an explicit registry. +4. **Tool sets adapt to model capabilities.** Active tool set adjusts when the router selects a different model. +5. **Graceful degradation preserved.** Unknown providers get full tool access — same as today. + +### Implementation Phases + +1. **Phase 1:** Provider Capabilities Registry (`packages/pi-ai/src/providers/provider-capabilities.ts`) +2. **Phase 2:** Tool Compatibility Metadata (extend `ToolDefinition` with `compatibility` field) +3. **Phase 3:** Tool-compatibility filter in routing pipeline + `ProviderSwitchReport` in `transform-messages.ts` +4. **Phase 4:** `adjustToolSet` extension hook + +## Consequences + +### Positive +- Eliminates silent tool failures when routing to incompatible providers +- Makes cross-provider routing safe by default +- Provider knowledge becomes queryable (registry vs scattered code) +- Cross-provider context loss becomes visible via `ProviderSwitchReport` + +### Negative +- More metadata to maintain (provider capabilities, tool compatibility) +- Tool filtering adds a pipeline step (sub-millisecond, O(models × tools)) +- Risk of over-filtering (mitigated: opt-in per tool, permissive defaults) + +### Neutral +- Existing behavior unchanged without metadata +- ADR-004 scoring is unmodified +- Provider implementations simplify over time as registry replaces scattered workarounds + +## Appendix: Architecture Reference + +| File | Role | +|------|------| +| `packages/pi-ai/src/providers/register-builtins.ts` | Provider registration | +| `packages/pi-ai/src/providers/*-shared.ts` | Provider-specific handling | +| `packages/pi-ai/src/providers/transform-messages.ts` | Cross-provider normalization | +| `packages/pi-ai/src/types.ts` | Core types | +| `packages/pi-coding-agent/src/core/extensions/types.ts` | ToolDefinition, ExtensionAPI | +| `src/resources/extensions/gsd/model-router.ts` | Capability scoring (ADR-004) | +| `src/resources/extensions/gsd/auto-model-selection.ts` | Model selection orchestration | diff --git a/packages/pi-ai/src/index.ts b/packages/pi-ai/src/index.ts index c8d9e1e8c..8b81cc22e 100644 --- a/packages/pi-ai/src/index.ts +++ b/packages/pi-ai/src/index.ts @@ -12,7 +12,10 @@ export * from "./providers/google-vertex.js"; export * from "./providers/mistral.js"; export * from "./providers/openai-completions.js"; export * from "./providers/openai-responses.js"; +export * from "./providers/provider-capabilities.js"; export * from "./providers/register-builtins.js"; +export type { ProviderSwitchReport } from "./providers/transform-messages.js"; +export { createEmptyReport, hasTransformations, transformMessagesWithReport } from "./providers/transform-messages.js"; export * from "./stream.js"; export * from "./types.js"; export * from "./utils/event-stream.js"; diff --git a/packages/pi-ai/src/providers/amazon-bedrock.ts b/packages/pi-ai/src/providers/amazon-bedrock.ts index 52b42b4d1..473c90d15 100644 --- a/packages/pi-ai/src/providers/amazon-bedrock.ts +++ b/packages/pi-ai/src/providers/amazon-bedrock.ts @@ -43,7 +43,7 @@ import { AssistantMessageEventStream } from "../utils/event-stream.js"; import { parseStreamingJson } from "../utils/json-parse.js"; import { sanitizeSurrogates } from "../utils/sanitize-unicode.js"; import { adjustMaxTokensForThinking, buildBaseOptions, clampReasoning } from "./simple-options.js"; -import { transformMessages } from "./transform-messages.js"; +import { transformMessagesWithReport } from "./transform-messages.js"; export interface BedrockOptions extends StreamOptions { region?: string; @@ -487,7 +487,7 @@ function convertMessages( cacheRetention: CacheRetention, ): Message[] { const result: Message[] = []; - const transformedMessages = transformMessages(context.messages, model, normalizeToolCallId); + const transformedMessages = transformMessagesWithReport(context.messages, model, normalizeToolCallId, "bedrock-converse-stream"); for (let i = 0; i < transformedMessages.length; i++) { const m = transformedMessages[i]; diff --git a/packages/pi-ai/src/providers/anthropic-shared.ts b/packages/pi-ai/src/providers/anthropic-shared.ts index 098f50721..4b9a57ea4 100644 --- a/packages/pi-ai/src/providers/anthropic-shared.ts +++ b/packages/pi-ai/src/providers/anthropic-shared.ts @@ -33,7 +33,7 @@ import type { AssistantMessageEventStream } from "../utils/event-stream.js"; import { parseStreamingJson } from "../utils/json-parse.js"; import { hasXmlParameterTags, repairToolJson } from "../utils/repair-tool-json.js"; import { sanitizeSurrogates } from "../utils/sanitize-unicode.js"; -import { transformMessages } from "./transform-messages.js"; +import { transformMessagesWithReport } from "./transform-messages.js"; export type AnthropicEffort = "low" | "medium" | "high" | "max"; @@ -235,7 +235,7 @@ export function convertMessages( ): MessageParam[] { const params: MessageParam[] = []; - const transformedMessages = transformMessages(messages, model, normalizeToolCallId); + const transformedMessages = transformMessagesWithReport(messages, model, normalizeToolCallId, "anthropic-messages"); for (let i = 0; i < transformedMessages.length; i++) { const msg = transformedMessages[i]; diff --git a/packages/pi-ai/src/providers/google-shared.ts b/packages/pi-ai/src/providers/google-shared.ts index e6a31771f..7984bdd4b 100644 --- a/packages/pi-ai/src/providers/google-shared.ts +++ b/packages/pi-ai/src/providers/google-shared.ts @@ -5,7 +5,7 @@ import { type Content, FinishReason, FunctionCallingConfigMode, type Part } from "@google/genai"; import type { Context, ImageContent, Model, StopReason, TextContent, Tool } from "../types.js"; import { sanitizeSurrogates } from "../utils/sanitize-unicode.js"; -import { transformMessages } from "./transform-messages.js"; +import { transformMessagesWithReport } from "./transform-messages.js"; type GoogleApiType = "google-generative-ai" | "google-gemini-cli" | "google-vertex"; @@ -80,7 +80,7 @@ export function convertMessages(model: Model, contex return id.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 64); }; - const transformedMessages = transformMessages(context.messages, model, normalizeToolCallId); + const transformedMessages = transformMessagesWithReport(context.messages, model, normalizeToolCallId, "google-generative-ai"); for (const msg of transformedMessages) { if (msg.role === "user") { diff --git a/packages/pi-ai/src/providers/mistral.ts b/packages/pi-ai/src/providers/mistral.ts index 7c9b54b91..0a6a28e5c 100644 --- a/packages/pi-ai/src/providers/mistral.ts +++ b/packages/pi-ai/src/providers/mistral.ts @@ -39,7 +39,7 @@ import { shortHash } from "../utils/hash.js"; import { parseStreamingJson } from "../utils/json-parse.js"; import { sanitizeSurrogates } from "../utils/sanitize-unicode.js"; import { buildBaseOptions, clampReasoning } from "./simple-options.js"; -import { transformMessages } from "./transform-messages.js"; +import { transformMessagesWithReport } from "./transform-messages.js"; const MISTRAL_TOOL_CALL_ID_LENGTH = 9; const MAX_MISTRAL_ERROR_BODY_CHARS = 4000; @@ -79,7 +79,7 @@ export const streamMistral: StreamFunction<"mistral-conversations", MistralOptio }); const normalizeMistralToolCallId = createMistralToolCallIdNormalizer(); - const transformedMessages = transformMessages(context.messages, model, (id) => normalizeMistralToolCallId(id)); + const transformedMessages = transformMessagesWithReport(context.messages, model, (id) => normalizeMistralToolCallId(id), "mistral-conversations"); let payload = buildChatPayload(model, context, transformedMessages, options); const nextPayload = await options?.onPayload?.(payload, model); diff --git a/packages/pi-ai/src/providers/openai-completions.ts b/packages/pi-ai/src/providers/openai-completions.ts index 4d6e1a3cf..137e0efaf 100644 --- a/packages/pi-ai/src/providers/openai-completions.ts +++ b/packages/pi-ai/src/providers/openai-completions.ts @@ -39,7 +39,7 @@ import { finalizeStream, handleStreamError, } from "./openai-shared.js"; -import { transformMessages } from "./transform-messages.js"; +import { transformMessagesWithReport } from "./transform-messages.js"; /** * Check if conversation messages contain tool calls or tool results. @@ -441,7 +441,7 @@ export function convertMessages( return id; }; - const transformedMessages = transformMessages(context.messages, model, (id) => normalizeToolCallId(id)); + const transformedMessages = transformMessagesWithReport(context.messages, model, (id) => normalizeToolCallId(id), "openai-completions"); if (context.systemPrompt) { const useDeveloperRole = model.reasoning && compat.supportsDeveloperRole; diff --git a/packages/pi-ai/src/providers/openai-responses-shared.ts b/packages/pi-ai/src/providers/openai-responses-shared.ts index 10ac5ee1b..8227dcff5 100644 --- a/packages/pi-ai/src/providers/openai-responses-shared.ts +++ b/packages/pi-ai/src/providers/openai-responses-shared.ts @@ -30,7 +30,7 @@ import type { AssistantMessageEventStream } from "../utils/event-stream.js"; import { shortHash } from "../utils/hash.js"; import { parseStreamingJson } from "../utils/json-parse.js"; import { sanitizeSurrogates } from "../utils/sanitize-unicode.js"; -import { transformMessages } from "./transform-messages.js"; +import { transformMessagesWithReport } from "./transform-messages.js"; // ============================================================================= // Utilities @@ -108,7 +108,7 @@ export function convertResponsesMessages( return `${normalizedCallId}|${normalizedItemId}`; }; - const transformedMessages = transformMessages(context.messages, model, normalizeToolCallId); + const transformedMessages = transformMessagesWithReport(context.messages, model, normalizeToolCallId, "openai-responses"); const includeSystemPrompt = options?.includeSystemPrompt ?? true; if (includeSystemPrompt && context.systemPrompt) { diff --git a/packages/pi-ai/src/providers/provider-capabilities.test.ts b/packages/pi-ai/src/providers/provider-capabilities.test.ts new file mode 100644 index 000000000..7b8728975 --- /dev/null +++ b/packages/pi-ai/src/providers/provider-capabilities.test.ts @@ -0,0 +1,174 @@ +// GSD-2 — Provider Capabilities Registry Tests (ADR-005 Phase 1) +import { describe, test } from "node:test"; +import assert from "node:assert/strict"; + +import { + PROVIDER_CAPABILITIES, + getProviderCapabilities, + getUnsupportedFeatures, + mergeCapabilityOverrides, + getRegisteredApis, +} from "./provider-capabilities.js"; + +// ─── Registry Completeness ────────────────────────────────────────────────── + +describe("PROVIDER_CAPABILITIES registry", () => { + const EXPECTED_APIS = [ + "anthropic-messages", + "anthropic-vertex", + "openai-responses", + "azure-openai-responses", + "openai-codex-responses", + "openai-completions", + "google-generative-ai", + "google-gemini-cli", + "google-vertex", + "mistral-conversations", + "bedrock-converse-stream", + "ollama-chat", + ]; + + test("covers all expected API providers", () => { + for (const api of EXPECTED_APIS) { + assert.ok( + PROVIDER_CAPABILITIES[api], + `Missing capability entry for API: ${api}`, + ); + } + }); + + test("getRegisteredApis returns all entries", () => { + const registered = getRegisteredApis(); + for (const api of EXPECTED_APIS) { + assert.ok(registered.includes(api), `getRegisteredApis missing: ${api}`); + } + }); + + test("all entries have required fields", () => { + for (const [api, caps] of Object.entries(PROVIDER_CAPABILITIES)) { + assert.equal(typeof caps.toolCalling, "boolean", `${api}.toolCalling`); + assert.equal(typeof caps.maxTools, "number", `${api}.maxTools`); + assert.equal(typeof caps.imageToolResults, "boolean", `${api}.imageToolResults`); + assert.equal(typeof caps.structuredOutput, "boolean", `${api}.structuredOutput`); + assert.ok(caps.toolCallIdFormat, `${api}.toolCallIdFormat`); + assert.equal(typeof caps.toolCallIdFormat.maxLength, "number", `${api}.toolCallIdFormat.maxLength`); + assert.ok(caps.toolCallIdFormat.allowedChars instanceof RegExp, `${api}.toolCallIdFormat.allowedChars`); + assert.ok( + ["full", "text-only", "none"].includes(caps.thinkingPersistence), + `${api}.thinkingPersistence is "${caps.thinkingPersistence}"`, + ); + assert.ok(Array.isArray(caps.unsupportedSchemaFeatures), `${api}.unsupportedSchemaFeatures`); + } + }); +}); + +// ─── Provider-specific Values ─────────────────────────────────────────────── + +describe("provider-specific capabilities", () => { + test("Anthropic supports full thinking persistence", () => { + assert.equal(PROVIDER_CAPABILITIES["anthropic-messages"].thinkingPersistence, "full"); + }); + + test("Anthropic supports image tool results", () => { + assert.equal(PROVIDER_CAPABILITIES["anthropic-messages"].imageToolResults, true); + }); + + test("Anthropic tool call ID is 64 chars max", () => { + assert.equal(PROVIDER_CAPABILITIES["anthropic-messages"].toolCallIdFormat.maxLength, 64); + }); + + test("Mistral tool call ID is 9 chars max", () => { + assert.equal(PROVIDER_CAPABILITIES["mistral-conversations"].toolCallIdFormat.maxLength, 9); + }); + + test("Mistral has no thinking persistence", () => { + assert.equal(PROVIDER_CAPABILITIES["mistral-conversations"].thinkingPersistence, "none"); + }); + + test("Google does not support patternProperties", () => { + assert.ok( + PROVIDER_CAPABILITIES["google-generative-ai"].unsupportedSchemaFeatures.includes("patternProperties"), + ); + }); + + test("Google does not support const", () => { + assert.ok( + PROVIDER_CAPABILITIES["google-generative-ai"].unsupportedSchemaFeatures.includes("const"), + ); + }); + + test("OpenAI Responses does not support image tool results", () => { + assert.equal(PROVIDER_CAPABILITIES["openai-responses"].imageToolResults, false); + }); + + test("OpenAI Responses has text-only thinking persistence", () => { + assert.equal(PROVIDER_CAPABILITIES["openai-responses"].thinkingPersistence, "text-only"); + }); +}); + +// ─── getProviderCapabilities ──────────────────────────────────────────────── + +describe("getProviderCapabilities", () => { + test("returns known provider capabilities", () => { + const caps = getProviderCapabilities("anthropic-messages"); + assert.equal(caps.toolCalling, true); + assert.equal(caps.thinkingPersistence, "full"); + }); + + test("returns permissive defaults for unknown providers", () => { + const caps = getProviderCapabilities("unknown-provider-xyz"); + assert.equal(caps.toolCalling, true); + assert.equal(caps.imageToolResults, true); + assert.deepEqual(caps.unsupportedSchemaFeatures, []); + }); +}); + +// ─── getUnsupportedFeatures ───────────────────────────────────────────────── + +describe("getUnsupportedFeatures", () => { + test("returns unsupported features for Google", () => { + const unsupported = getUnsupportedFeatures("google-generative-ai", ["patternProperties", "const"]); + assert.deepEqual(unsupported, ["patternProperties", "const"]); + }); + + test("returns empty for Anthropic with any features", () => { + const unsupported = getUnsupportedFeatures("anthropic-messages", ["patternProperties", "const"]); + assert.deepEqual(unsupported, []); + }); + + test("returns empty for unknown provider", () => { + const unsupported = getUnsupportedFeatures("unknown-xyz", ["patternProperties"]); + assert.deepEqual(unsupported, []); + }); +}); + +// ─── mergeCapabilityOverrides ─────────────────────────────────────────────── + +describe("mergeCapabilityOverrides", () => { + test("overrides individual fields", () => { + const merged = mergeCapabilityOverrides("openai-responses", { + imageToolResults: true, + }); + assert.equal(merged.imageToolResults, true); + // Non-overridden fields preserved + assert.equal(merged.toolCalling, true); + assert.equal(merged.thinkingPersistence, "text-only"); + }); + + test("deep-merges toolCallIdFormat", () => { + const merged = mergeCapabilityOverrides("anthropic-messages", { + toolCallIdFormat: { maxLength: 128 }, + }); + assert.equal(merged.toolCallIdFormat.maxLength, 128); + // allowedChars preserved from base + assert.ok(merged.toolCallIdFormat.allowedChars instanceof RegExp); + }); + + test("uses permissive defaults for unknown provider", () => { + const merged = mergeCapabilityOverrides("unknown-xyz", { + imageToolResults: false, + }); + assert.equal(merged.imageToolResults, false); + assert.equal(merged.toolCalling, true); // from default + }); +}); diff --git a/packages/pi-ai/src/providers/provider-capabilities.ts b/packages/pi-ai/src/providers/provider-capabilities.ts new file mode 100644 index 000000000..b49a1f319 --- /dev/null +++ b/packages/pi-ai/src/providers/provider-capabilities.ts @@ -0,0 +1,215 @@ +// GSD-2 — Provider Capabilities Registry (ADR-005 Phase 1) +// Declarative registry of what each API provider supports, consolidating +// scattered knowledge from *-shared.ts files into a queryable data structure. + +import type { Api } from "../types.js"; + +// ─── Types ────────────────────────────────────────────────────────────────── + +/** + * Declarative capability profile for an API provider. + * Used by the model router to filter incompatible models and by the tool + * system to adjust tool sets per provider. + */ +export interface ProviderCapabilities { + /** Whether models from this provider support tool/function calling */ + toolCalling: boolean; + /** Maximum number of tools the provider handles well (0 = unlimited) */ + maxTools: number; + /** Whether tool results can contain images */ + imageToolResults: boolean; + /** Whether the provider supports structured JSON output */ + structuredOutput: boolean; + /** Tool call ID format constraints */ + toolCallIdFormat: { + maxLength: number; + allowedChars: RegExp; + }; + /** Whether thinking/reasoning blocks are preserved cross-turn */ + thinkingPersistence: "full" | "text-only" | "none"; + /** Schema features NOT supported (tools using these get filtered) */ + unsupportedSchemaFeatures: string[]; +} + +// ─── Registry ─────────────────────────────────────────────────────────────── + +/** + * Built-in provider capability profiles. + * + * Sources (consolidated from scattered *-shared.ts files): + * - anthropic-shared.ts: normalizeToolCallId (64-char, [a-zA-Z0-9_-]) + * - openai-responses-shared.ts: ID normalization (64-char, fc_ prefix), image-in-tool-result workaround + * - google-shared.ts: sanitizeSchemaForGoogle (patternProperties, const), requiresToolCallId + * - mistral.ts: MISTRAL_TOOL_CALL_ID_LENGTH = 9 + * - amazon-bedrock.ts: normalizeToolCallId (64-char, [a-zA-Z0-9_-]) + */ +export const PROVIDER_CAPABILITIES: Record = { + "anthropic-messages": { + toolCalling: true, + maxTools: 0, + imageToolResults: true, + structuredOutput: true, + toolCallIdFormat: { maxLength: 64, allowedChars: /^[a-zA-Z0-9_-]+$/ }, + thinkingPersistence: "full", + unsupportedSchemaFeatures: [], + }, + "anthropic-vertex": { + toolCalling: true, + maxTools: 0, + imageToolResults: true, + structuredOutput: true, + toolCallIdFormat: { maxLength: 64, allowedChars: /^[a-zA-Z0-9_-]+$/ }, + thinkingPersistence: "full", + unsupportedSchemaFeatures: [], + }, + "openai-responses": { + toolCalling: true, + maxTools: 0, + imageToolResults: false, // images sent as separate user message, not in tool result + structuredOutput: true, + toolCallIdFormat: { maxLength: 512, allowedChars: /^.+$/ }, + thinkingPersistence: "text-only", + unsupportedSchemaFeatures: [], + }, + "azure-openai-responses": { + toolCalling: true, + maxTools: 0, + imageToolResults: false, + structuredOutput: true, + toolCallIdFormat: { maxLength: 512, allowedChars: /^.+$/ }, + thinkingPersistence: "text-only", + unsupportedSchemaFeatures: [], + }, + "openai-codex-responses": { + toolCalling: true, + maxTools: 0, + imageToolResults: false, + structuredOutput: true, + toolCallIdFormat: { maxLength: 64, allowedChars: /^[a-zA-Z0-9_-]+$/ }, + thinkingPersistence: "text-only", + unsupportedSchemaFeatures: [], + }, + "openai-completions": { + toolCalling: true, + maxTools: 0, + imageToolResults: false, + structuredOutput: true, + toolCallIdFormat: { maxLength: 64, allowedChars: /^[a-zA-Z0-9_-]+$/ }, + thinkingPersistence: "text-only", + unsupportedSchemaFeatures: [], + }, + "google-generative-ai": { + toolCalling: true, + maxTools: 0, + imageToolResults: true, + structuredOutput: true, + toolCallIdFormat: { maxLength: 64, allowedChars: /^[a-zA-Z0-9_-]+$/ }, + thinkingPersistence: "text-only", + unsupportedSchemaFeatures: ["patternProperties", "const"], + }, + "google-gemini-cli": { + toolCalling: true, + maxTools: 0, + imageToolResults: true, + structuredOutput: true, + toolCallIdFormat: { maxLength: 64, allowedChars: /^[a-zA-Z0-9_-]+$/ }, + thinkingPersistence: "text-only", + unsupportedSchemaFeatures: ["patternProperties", "const"], + }, + "google-vertex": { + toolCalling: true, + maxTools: 0, + imageToolResults: true, + structuredOutput: true, + toolCallIdFormat: { maxLength: 64, allowedChars: /^[a-zA-Z0-9_-]+$/ }, + thinkingPersistence: "text-only", + unsupportedSchemaFeatures: ["patternProperties", "const"], + }, + "mistral-conversations": { + toolCalling: true, + maxTools: 0, + imageToolResults: false, + structuredOutput: true, + toolCallIdFormat: { maxLength: 9, allowedChars: /^[a-zA-Z0-9]+$/ }, + thinkingPersistence: "none", + unsupportedSchemaFeatures: [], + }, + "bedrock-converse-stream": { + toolCalling: true, + maxTools: 0, + imageToolResults: true, // Bedrock supports image content blocks in tool results + structuredOutput: true, + toolCallIdFormat: { maxLength: 64, allowedChars: /^[a-zA-Z0-9_-]+$/ }, + thinkingPersistence: "text-only", + unsupportedSchemaFeatures: [], + }, + "ollama-chat": { + toolCalling: true, + maxTools: 0, + imageToolResults: false, + structuredOutput: false, + toolCallIdFormat: { maxLength: 64, allowedChars: /^[a-zA-Z0-9_-]+$/ }, + thinkingPersistence: "none", + unsupportedSchemaFeatures: [], + }, +}; + +// ─── Default (permissive) profile for unknown providers ───────────────────── + +const DEFAULT_CAPABILITIES: ProviderCapabilities = { + toolCalling: true, + maxTools: 0, + imageToolResults: true, + structuredOutput: true, + toolCallIdFormat: { maxLength: 512, allowedChars: /^.+$/ }, + thinkingPersistence: "text-only", + unsupportedSchemaFeatures: [], +}; + +// ─── Public API ───────────────────────────────────────────────────────────── + +/** + * Get capabilities for a provider API. Returns a permissive default for + * unknown providers (preserving existing behavior per ADR-005 principle 5). + */ +export function getProviderCapabilities(api: string): ProviderCapabilities { + return PROVIDER_CAPABILITIES[api] ?? DEFAULT_CAPABILITIES; +} + +/** + * Check if a provider supports all required schema features. + * Returns the list of unsupported features (empty if all supported). + */ +export function getUnsupportedFeatures(api: string, requiredFeatures: string[]): string[] { + const caps = getProviderCapabilities(api); + return requiredFeatures.filter(f => caps.unsupportedSchemaFeatures.includes(f)); +} + +/** + * Deep-merge user-provided capability overrides with built-in defaults. + * Partial overrides merge with the built-in profile for the given API. + */ +export function mergeCapabilityOverrides( + api: string, + overrides: Partial> & { + toolCallIdFormat?: Partial; + }, +): ProviderCapabilities { + const base = getProviderCapabilities(api); + return { + ...base, + ...overrides, + toolCallIdFormat: overrides.toolCallIdFormat + ? { ...base.toolCallIdFormat, ...overrides.toolCallIdFormat } + : base.toolCallIdFormat, + }; +} + +/** + * Get all registered API names in the capability registry. + * Used by lint rules to verify all providers in register-builtins.ts + * have corresponding capability entries. + */ +export function getRegisteredApis(): string[] { + return Object.keys(PROVIDER_CAPABILITIES); +} diff --git a/packages/pi-ai/src/providers/transform-messages-report.test.ts b/packages/pi-ai/src/providers/transform-messages-report.test.ts new file mode 100644 index 000000000..85ae585ba --- /dev/null +++ b/packages/pi-ai/src/providers/transform-messages-report.test.ts @@ -0,0 +1,189 @@ +// GSD-2 — ProviderSwitchReport Tests (ADR-005 Phase 3) +import { describe, test } from "node:test"; +import assert from "node:assert/strict"; + +import { transformMessages, createEmptyReport, hasTransformations } from "./transform-messages.js"; +import type { ProviderSwitchReport } from "./transform-messages.js"; +import type { Message, Model, AssistantMessage, ToolCall } from "../types.js"; + +// ─── Helpers ──────────────────────────────────────────────────────────────── + +function makeModel(overrides: Partial> = {}): Model { + return { + id: "claude-sonnet-4-6", + name: "Claude Sonnet 4.6", + api: "anthropic-messages", + provider: "anthropic", + baseUrl: "", + reasoning: false, + input: ["text", "image"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 200000, + maxTokens: 8192, + ...overrides, + } as Model; +} + +function makeAssistantMsg(overrides: Partial = {}): AssistantMessage { + return { + role: "assistant", + content: [], + api: "anthropic-messages", + provider: "anthropic", + model: "claude-sonnet-4-6", + usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, + stopReason: "stop", + timestamp: Date.now(), + ...overrides, + }; +} + +// ─── createEmptyReport / hasTransformations ───────────────────────────────── + +describe("createEmptyReport", () => { + test("creates report with zero counters", () => { + const report = createEmptyReport("anthropic-messages", "openai-responses"); + assert.equal(report.fromApi, "anthropic-messages"); + assert.equal(report.toApi, "openai-responses"); + assert.equal(report.thinkingBlocksDropped, 0); + assert.equal(report.thinkingBlocksDowngraded, 0); + assert.equal(report.toolCallIdsRemapped, 0); + assert.equal(report.syntheticToolResultsInserted, 0); + assert.equal(report.thoughtSignaturesDropped, 0); + }); +}); + +describe("hasTransformations", () => { + test("returns false for empty report", () => { + const report = createEmptyReport("a", "b"); + assert.equal(hasTransformations(report), false); + }); + + test("returns true when any counter is non-zero", () => { + const report = createEmptyReport("a", "b"); + report.thinkingBlocksDropped = 1; + assert.equal(hasTransformations(report), true); + }); +}); + +// ─── Report Tracking in transformMessages ─────────────────────────────────── + +describe("transformMessages with report tracking", () => { + test("tracks thinking blocks dropped for redacted cross-model", () => { + const model = makeModel({ id: "gpt-5", api: "openai-responses", provider: "openai" }); + const messages: Message[] = [ + makeAssistantMsg({ + content: [ + { type: "thinking", thinking: "", redacted: true }, + { type: "text", text: "Hello" }, + ], + }), + ]; + const report = createEmptyReport("anthropic-messages", "openai-responses"); + transformMessages(messages, model, undefined, report); + assert.equal(report.thinkingBlocksDropped, 1); + }); + + test("tracks thinking blocks downgraded to plain text", () => { + const model = makeModel({ id: "gpt-5", api: "openai-responses", provider: "openai" }); + const messages: Message[] = [ + makeAssistantMsg({ + content: [ + { type: "thinking", thinking: "Let me think about this..." }, + { type: "text", text: "Here is my answer" }, + ], + }), + ]; + const report = createEmptyReport("anthropic-messages", "openai-responses"); + transformMessages(messages, model, undefined, report); + assert.equal(report.thinkingBlocksDowngraded, 1); + }); + + test("tracks tool call IDs remapped", () => { + const model = makeModel({ id: "claude-sonnet-4-6", api: "anthropic-messages", provider: "anthropic" }); + const toolCall: ToolCall = { + type: "toolCall", + id: "original-long-id-that-needs-normalization|with-special-chars", + name: "bash", + arguments: { command: "ls" }, + }; + const messages: Message[] = [ + makeAssistantMsg({ + provider: "openai", + api: "openai-responses", + model: "gpt-5", + content: [toolCall], + }), + ]; + const normalizer = (id: string) => id.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 64); + const report = createEmptyReport("openai-responses", "anthropic-messages"); + transformMessages(messages, model, normalizer, report); + assert.equal(report.toolCallIdsRemapped, 1); + }); + + test("tracks thought signatures dropped", () => { + const model = makeModel({ id: "claude-sonnet-4-6", api: "anthropic-messages", provider: "anthropic" }); + const toolCall: ToolCall = { + type: "toolCall", + id: "tc_001", + name: "bash", + arguments: { command: "ls" }, + thoughtSignature: "some-opaque-signature", + }; + const messages: Message[] = [ + makeAssistantMsg({ + provider: "google", + api: "google-generative-ai", + model: "gemini-2.5-pro", + content: [toolCall], + }), + ]; + const report = createEmptyReport("google-generative-ai", "anthropic-messages"); + transformMessages(messages, model, undefined, report); + assert.equal(report.thoughtSignaturesDropped, 1); + }); + + test("tracks synthetic tool results inserted", () => { + const model = makeModel(); + const toolCall: ToolCall = { + type: "toolCall", + id: "tc_orphan", + name: "bash", + arguments: { command: "ls" }, + }; + // Assistant message with tool call followed by another assistant (no tool result) + const messages: Message[] = [ + makeAssistantMsg({ content: [toolCall, { type: "text", text: "Using bash" }] }), + makeAssistantMsg({ content: [{ type: "text", text: "Next message" }] }), + ]; + const report = createEmptyReport("anthropic-messages", "anthropic-messages"); + transformMessages(messages, model, undefined, report); + assert.equal(report.syntheticToolResultsInserted, 1); + }); + + test("does not count transformations for same-model messages", () => { + const model = makeModel(); + const messages: Message[] = [ + makeAssistantMsg({ + content: [ + { type: "thinking", thinking: "Let me think..." }, + { type: "text", text: "Answer" }, + ], + }), + ]; + const report = createEmptyReport("anthropic-messages", "anthropic-messages"); + transformMessages(messages, model, undefined, report); + assert.equal(report.thinkingBlocksDowngraded, 0); + assert.equal(report.thinkingBlocksDropped, 0); + }); + + test("works without report parameter (backward compatible)", () => { + const model = makeModel(); + const messages: Message[] = [ + makeAssistantMsg({ content: [{ type: "text", text: "Hello" }] }), + ]; + // Should not throw + const result = transformMessages(messages, model); + assert.ok(Array.isArray(result)); + }); +}); diff --git a/packages/pi-ai/src/providers/transform-messages.ts b/packages/pi-ai/src/providers/transform-messages.ts index f61f08037..bcfd5234a 100644 --- a/packages/pi-ai/src/providers/transform-messages.ts +++ b/packages/pi-ai/src/providers/transform-messages.ts @@ -1,5 +1,87 @@ import type { Api, AssistantMessage, Message, Model, ToolCall, ToolResultMessage } from "../types.js"; +/** + * Report of context transformations during a cross-provider switch (ADR-005 Phase 3). + * Tracks what was lost or downgraded when replaying conversation history to a different provider. + */ +export interface ProviderSwitchReport { + /** API of the messages being transformed from */ + fromApi: string; + /** API of the target model */ + toApi: string; + /** Number of thinking blocks completely dropped (redacted/encrypted, cross-model) */ + thinkingBlocksDropped: number; + /** Number of thinking blocks downgraded from structured to plain text */ + thinkingBlocksDowngraded: number; + /** Number of tool call IDs that were remapped/normalized */ + toolCallIdsRemapped: number; + /** Number of synthetic tool results inserted for orphaned tool calls */ + syntheticToolResultsInserted: number; + /** Number of thought signatures dropped (Google-specific opaque context) */ + thoughtSignaturesDropped: number; +} + +/** + * Create an empty provider switch report. + */ +export function createEmptyReport(fromApi: string, toApi: string): ProviderSwitchReport { + return { + fromApi, + toApi, + thinkingBlocksDropped: 0, + thinkingBlocksDowngraded: 0, + toolCallIdsRemapped: 0, + syntheticToolResultsInserted: 0, + thoughtSignaturesDropped: 0, + }; +} + +/** + * Check if a provider switch report has any non-zero transformations. + */ +export function hasTransformations(report: ProviderSwitchReport): boolean { + return ( + report.thinkingBlocksDropped > 0 || + report.thinkingBlocksDowngraded > 0 || + report.toolCallIdsRemapped > 0 || + report.syntheticToolResultsInserted > 0 || + report.thoughtSignaturesDropped > 0 + ); +} + +/** + * Create a report, run transformMessages, and log if non-empty. + * Convenience wrapper for provider adapters (ADR-005). + */ +export function transformMessagesWithReport( + messages: Message[], + model: Model, + normalizeToolCallId?: (id: string, model: Model, source: AssistantMessage) => string, + sourceApi?: string, +): Message[] { + const report = createEmptyReport(sourceApi ?? "unknown", model.api); + const result = transformMessages(messages, model, normalizeToolCallId, report); + if (hasTransformations(report)) { + logProviderSwitchReport(report); + } + return result; +} + +/** Log a non-empty ProviderSwitchReport as a debug-level warning. */ +function logProviderSwitchReport(report: ProviderSwitchReport): void { + const parts: string[] = [`Provider switch ${report.fromApi} → ${report.toApi}:`]; + if (report.thinkingBlocksDropped > 0) parts.push(`${report.thinkingBlocksDropped} thinking blocks dropped`); + if (report.thinkingBlocksDowngraded > 0) parts.push(`${report.thinkingBlocksDowngraded} thinking blocks downgraded`); + if (report.toolCallIdsRemapped > 0) parts.push(`${report.toolCallIdsRemapped} tool call IDs remapped`); + if (report.syntheticToolResultsInserted > 0) parts.push(`${report.syntheticToolResultsInserted} synthetic tool results inserted`); + if (report.thoughtSignaturesDropped > 0) parts.push(`${report.thoughtSignaturesDropped} thought signatures dropped`); + // Use process.stderr for debug output — this is observable in verbose/debug modes + // without polluting stdout which may be used for structured output (RPC/MCP). + if (process.env.GSD_VERBOSE === "1" || process.env.PI_VERBOSE === "1") { + process.stderr.write(`[provider-switch] ${parts.join(", ")}\n`); + } +} + /** * Normalize tool call ID for cross-provider compatibility. * OpenAI Responses API generates IDs that are 450+ chars with special characters like `|`. @@ -9,6 +91,7 @@ export function transformMessages( messages: Message[], model: Model, normalizeToolCallId?: (id: string, model: Model, source: AssistantMessage) => string, + report?: ProviderSwitchReport, ): Message[] { // Build a map of original tool call IDs to normalized IDs const toolCallIdMap = new Map(); @@ -42,14 +125,20 @@ export function transformMessages( // Redacted thinking is opaque encrypted content, only valid for the same model. // Drop it for cross-model to avoid API errors. if (block.redacted) { + if (!isSameModel && report) report.thinkingBlocksDropped++; return isSameModel ? block : []; } // For same model: keep thinking blocks with signatures (needed for replay) // even if the thinking text is empty (OpenAI encrypted reasoning) if (isSameModel && block.thinkingSignature) return block; // Skip empty thinking blocks, convert others to plain text - if (!block.thinking || block.thinking.trim() === "") return []; + if (!block.thinking || block.thinking.trim() === "") { + if (!isSameModel && report) report.thinkingBlocksDropped++; + return []; + } if (isSameModel) return block; + // Downgrade: structured thinking → plain text + if (report) report.thinkingBlocksDowngraded++; return { type: "text" as const, text: block.thinking, @@ -71,6 +160,7 @@ export function transformMessages( if (!isSameModel && toolCall.thoughtSignature) { normalizedToolCall = { ...toolCall }; delete (normalizedToolCall as { thoughtSignature?: string }).thoughtSignature; + if (report) report.thoughtSignaturesDropped++; } if (!isSameModel && normalizeToolCallId) { @@ -78,6 +168,7 @@ export function transformMessages( if (normalizedId !== toolCall.id) { toolCallIdMap.set(toolCall.id, normalizedId); normalizedToolCall = { ...normalizedToolCall, id: normalizedId }; + if (report) report.toolCallIdsRemapped++; } } @@ -117,6 +208,7 @@ export function transformMessages( isError: true, timestamp: Date.now(), } as ToolResultMessage); + if (report) report.syntheticToolResultsInserted++; } } pendingToolCalls = []; @@ -157,6 +249,7 @@ export function transformMessages( isError: true, timestamp: Date.now(), } as ToolResultMessage); + if (report) report.syntheticToolResultsInserted++; } } pendingToolCalls = []; diff --git a/packages/pi-coding-agent/src/core/extensions/index.ts b/packages/pi-coding-agent/src/core/extensions/index.ts index 70525095a..0438d364b 100644 --- a/packages/pi-coding-agent/src/core/extensions/index.ts +++ b/packages/pi-coding-agent/src/core/extensions/index.ts @@ -43,6 +43,9 @@ export type { BeforeProviderRequestEventResult, // Context CompactOptions, + // Events - Adjust Tool Set (ADR-005) + AdjustToolSetEvent, + AdjustToolSetResult, // Events - Agent ContextEvent, // Event Results @@ -135,6 +138,7 @@ export type { ToolCallEvent, ToolCallEventResult, // Tools + ToolCompatibility, ToolDefinition, // Events - Tool Execution ToolExecutionEndEvent, diff --git a/packages/pi-coding-agent/src/core/extensions/loader.ts b/packages/pi-coding-agent/src/core/extensions/loader.ts index 7e25c837d..016f05448 100644 --- a/packages/pi-coding-agent/src/core/extensions/loader.ts +++ b/packages/pi-coding-agent/src/core/extensions/loader.ts @@ -38,6 +38,7 @@ import type { ExecOptions } from "../exec.js"; import { execCommand } from "../exec.js"; import { getUntrustedExtensionPaths } from "./project-trust.js"; export { isProjectTrusted, trustProject, getUntrustedExtensionPaths } from "./project-trust.js"; +import { registerToolCompatibility } from "../tools/tool-compatibility-registry.js"; import type { Extension, ExtensionAPI, @@ -428,8 +429,9 @@ export function createExtensionRuntime(): ExtensionRuntime { unregisterProvider: (name) => { runtime.pendingProviderRegistrations = runtime.pendingProviderRegistrations.filter((r) => r.name !== name); }, - // Stub replaced by ExtensionRunner at construction time via bindEmitMethods(). + // Stubs replaced by ExtensionRunner at construction time via bindEmitMethods(). emitBeforeModelSelect: async () => undefined, + emitAdjustToolSet: async () => undefined, }; return runtime; @@ -459,6 +461,10 @@ function createExtensionAPI( definition: tool, extensionPath: extension.path, }); + // ADR-005: auto-register tool compatibility metadata + if (tool.compatibility) { + registerToolCompatibility(tool.name, tool.compatibility); + } runtime.refreshTools(); }, @@ -585,6 +591,10 @@ function createExtensionAPI( return runtime.emitBeforeModelSelect(event); }, + async emitAdjustToolSet(event: Omit): Promise { + return runtime.emitAdjustToolSet(event); + }, + events: eventBus, } as ExtensionAPI; diff --git a/packages/pi-coding-agent/src/core/extensions/runner.ts b/packages/pi-coding-agent/src/core/extensions/runner.ts index 048ad534c..0b0f6114b 100644 --- a/packages/pi-coding-agent/src/core/extensions/runner.ts +++ b/packages/pi-coding-agent/src/core/extensions/runner.ts @@ -11,6 +11,8 @@ import type { KeyAction, KeybindingsConfig } from "../keybindings.js"; import type { ModelRegistry } from "../model-registry.js"; import type { SessionManager } from "../session-manager.js"; import type { + AdjustToolSetEvent, + AdjustToolSetResult, BeforeAgentStartEvent, BeforeAgentStartEventResult, BeforeModelSelectEvent, @@ -234,6 +236,7 @@ export class ExtensionRunner { this.modelRegistry = modelRegistry; // Bind emit methods into the shared runtime so createExtensionAPI can delegate to them. this.runtime.emitBeforeModelSelect = (event) => this.emitBeforeModelSelect(event); + this.runtime.emitAdjustToolSet = (event) => this.emitAdjustToolSet(event); } bindCore(actions: ExtensionActions, contextActions: ExtensionContextActions): void { @@ -713,6 +716,21 @@ export class ExtensionRunner { return result; } + async emitAdjustToolSet(event: Omit): Promise { + let result: AdjustToolSetResult | undefined; + await this.invokeHandlers("adjust_tool_set", () => ({ + type: "adjust_tool_set" as const, + ...event, + } satisfies AdjustToolSetEvent), (handlerResult) => { + if (handlerResult) { + result = handlerResult as AdjustToolSetResult; + return { done: true }; // first override wins + } + return { done: false }; + }); + return result; + } + async emitBeforeAgentStart( prompt: string, images: ImageContent[] | undefined, diff --git a/packages/pi-coding-agent/src/core/extensions/types.ts b/packages/pi-coding-agent/src/core/extensions/types.ts index f4c153992..a1aad0dc9 100644 --- a/packages/pi-coding-agent/src/core/extensions/types.ts +++ b/packages/pi-coding-agent/src/core/extensions/types.ts @@ -331,6 +331,19 @@ export interface ToolRenderResultOptions { isPartial: boolean; } +/** + * Tool compatibility metadata for provider-aware tool filtering (ADR-005 Phase 2). + * Tools without compatibility metadata are assumed universally compatible. + */ +export interface ToolCompatibility { + /** Tool produces image content in results (filtered for providers without imageToolResults) */ + producesImages?: boolean; + /** Tool requires schema features that some providers don't support (e.g., ["patternProperties"]) */ + schemaFeatures?: string[]; + /** Tool is effective only with models above a minimum capability threshold */ + minCapabilityTier?: "light" | "standard" | "heavy"; +} + /** * Tool definition for registerTool(). */ @@ -347,6 +360,8 @@ export interface ToolDefinition): void; on(event: "input", handler: ExtensionHandler): void; on(event: "before_model_select", handler: ExtensionHandler): void; + on(event: "adjust_tool_set", handler: ExtensionHandler): void; // ========================================================================= // Event Emission (for host extensions that orchestrate model selection) @@ -1077,6 +1117,9 @@ export interface ExtensionAPI { /** Emit before_model_select event. Returns override model ID or undefined. */ emitBeforeModelSelect(event: Omit): Promise; + /** Emit adjust_tool_set event (ADR-005). Returns override tool names or undefined. */ + emitAdjustToolSet(event: Omit): Promise; + // ========================================================================= // Tool Registration // ========================================================================= @@ -1395,6 +1438,8 @@ export interface ExtensionRuntimeState { unregisterProvider: (name: string) => void; /** Emit before_model_select event to all registered handlers. Bound by ExtensionRunner. */ emitBeforeModelSelect: (event: Omit) => Promise; + /** Emit adjust_tool_set event to all registered handlers. Bound by ExtensionRunner (ADR-005). */ + emitAdjustToolSet: (event: Omit) => Promise; } /** diff --git a/packages/pi-coding-agent/src/core/tools/index.ts b/packages/pi-coding-agent/src/core/tools/index.ts index d54ac2a9c..90a5a524c 100644 --- a/packages/pi-coding-agent/src/core/tools/index.ts +++ b/packages/pi-coding-agent/src/core/tools/index.ts @@ -112,6 +112,13 @@ export { lspTool, } from "../lsp/index.js"; export type { LspServerStatus } from "../lsp/client.js"; +export { + registerToolCompatibility, + getToolCompatibility, + getAllToolCompatibility, + registerMcpToolCompatibility, + resetToolCompatibilityRegistry, +} from "./tool-compatibility-registry.js"; import type { AgentTool } from "@gsd/pi-agent-core"; import { type BashToolOptions, bashTool, createBashTool } from "./bash.js"; diff --git a/packages/pi-coding-agent/src/core/tools/tool-compatibility-registry.ts b/packages/pi-coding-agent/src/core/tools/tool-compatibility-registry.ts new file mode 100644 index 000000000..9e5bea3b5 --- /dev/null +++ b/packages/pi-coding-agent/src/core/tools/tool-compatibility-registry.ts @@ -0,0 +1,83 @@ +// GSD-2 — Tool Compatibility Registry (ADR-005 Phase 2) +// Maps tool names to their provider compatibility metadata. +// Used by the model router to filter tools incompatible with the selected provider. + +import type { ToolCompatibility } from "../extensions/types.js"; + +// ─── Registry State ───────────────────────────────────────────────────────── + +const registry = new Map(); + +// ─── Built-in Tool Compatibility (universally compatible) ─────────────────── +// Built-in tools (bash, read, write, edit, grep, find, ls) produce text-only +// results and use standard JSON Schema — compatible with all providers. + +const BUILTIN_TOOLS: Record = { + bash: {}, + read: {}, + write: {}, + edit: {}, + grep: {}, + find: {}, + ls: {}, + lsp: {}, + hashline_edit: {}, + hashline_read: {}, +}; + +// Pre-populate registry with built-in tools +for (const [name, compat] of Object.entries(BUILTIN_TOOLS)) { + registry.set(name, compat); +} + +// ─── MCP Tool Defaults ───────────────────────────────────────────────────── +// MCP tools may use complex schemas. Default to cautious compatibility. + +const MCP_TOOL_DEFAULTS: ToolCompatibility = { + schemaFeatures: ["patternProperties"], +}; + +// ─── Public API ───────────────────────────────────────────────────────────── + +/** + * Register compatibility metadata for a tool. + * Called automatically by registerTool() for extension tools that include + * compatibility metadata in their ToolDefinition. + */ +export function registerToolCompatibility(toolName: string, compatibility: ToolCompatibility): void { + registry.set(toolName, compatibility); +} + +/** + * Get compatibility metadata for a tool. + * Returns undefined for unknown tools (treated as universally compatible + * per ADR-005 principle: "fail open, don't restrict without data"). + */ +export function getToolCompatibility(toolName: string): ToolCompatibility | undefined { + return registry.get(toolName); +} + +/** + * Get all registered tool compatibility entries. + */ +export function getAllToolCompatibility(): ReadonlyMap { + return registry; +} + +/** + * Register an MCP tool with default cautious compatibility. + * MCP tools may use complex schemas that some providers don't support. + */ +export function registerMcpToolCompatibility(toolName: string, overrides?: Partial): void { + registry.set(toolName, { ...MCP_TOOL_DEFAULTS, ...overrides }); +} + +/** + * Clear all non-builtin entries (for testing). + */ +export function resetToolCompatibilityRegistry(): void { + registry.clear(); + for (const [name, compat] of Object.entries(BUILTIN_TOOLS)) { + registry.set(name, compat); + } +} diff --git a/packages/pi-coding-agent/src/index.ts b/packages/pi-coding-agent/src/index.ts index 86686caf0..ab7de8bac 100644 --- a/packages/pi-coding-agent/src/index.ts +++ b/packages/pi-coding-agent/src/index.ts @@ -49,6 +49,8 @@ export { export { createEventBus, type EventBus, type EventBusController } from "./core/event-bus.js"; // Extension system export type { + AdjustToolSetEvent, + AdjustToolSetResult, AgentEndEvent, AgentStartEvent, AgentToolResult, @@ -118,6 +120,7 @@ export type { SlashCommandSource, TerminalInputHandler, ToolCallEvent, + ToolCompatibility, ToolDefinition, ToolInfo, SortResult, @@ -310,6 +313,12 @@ export { type HashlineReadToolDetails, type HashlineReadToolInput, type HashlineReadToolOptions, + // Tool compatibility registry (ADR-005) + registerToolCompatibility, + getToolCompatibility, + getAllToolCompatibility, + registerMcpToolCompatibility, + resetToolCompatibilityRegistry, } from "./core/tools/index.js"; // Main entry point export { main } from "./main.js"; diff --git a/src/resources/extensions/gsd/auto-model-selection.ts b/src/resources/extensions/gsd/auto-model-selection.ts index a8f0d5db4..1097964f2 100644 --- a/src/resources/extensions/gsd/auto-model-selection.ts +++ b/src/resources/extensions/gsd/auto-model-selection.ts @@ -5,12 +5,13 @@ */ import type { Api, Model } from "@gsd/pi-ai"; +import { getProviderCapabilities } from "@gsd/pi-ai"; import type { ExtensionAPI, ExtensionContext } from "@gsd/pi-coding-agent"; import type { GSDPreferences } from "./preferences.js"; import { resolveModelWithFallbacksForUnit, resolveDynamicRoutingConfig } from "./preferences.js"; import type { ComplexityTier } from "./complexity-classifier.js"; import { classifyUnitComplexity, tierLabel } from "./complexity-classifier.js"; -import { resolveModelForComplexity, escalateTier, getEligibleModels, loadCapabilityOverrides } from "./model-router.js"; +import { resolveModelForComplexity, escalateTier, getEligibleModels, loadCapabilityOverrides, adjustToolSet, filterToolsForProvider } from "./model-router.js"; import { getLedger, getProjectTotals } from "./metrics.js"; import { unitPhaseLabel } from "./auto-dashboard.js"; @@ -244,12 +245,45 @@ export async function selectAndApplyModel( const ok = await pi.setModel(model, { persist: false }); if (ok) { appliedModel = model; + + // ADR-005: Adjust active tool set for the selected model's provider capabilities. + // Hard-filter incompatible tools, then let extensions override via adjust_tool_set hook. + const activeToolNames = pi.getActiveTools(); + const { toolNames: compatibleTools, removedTools } = adjustToolSet(activeToolNames, model.api); + let finalToolNames = compatibleTools; + + // Fire adjust_tool_set hook — extensions can override the filtered tool set + if (routingConfig.hooks !== false) { + const hookResult = await pi.emitAdjustToolSet({ + selectedModelApi: model.api, + selectedModelProvider: model.provider, + selectedModelId: model.id, + activeToolNames, + filteredTools: removedTools, + }); + if (hookResult?.toolNames) { + finalToolNames = hookResult.toolNames; + } + } + + // Apply the filtered tool set if any tools were removed + if (removedTools.length > 0 || finalToolNames.length !== activeToolNames.length) { + pi.setActiveTools(finalToolNames); + } + if (verbose) { const fallbackNote = modelId === effectiveModelConfig.primary ? "" : ` (fallback from ${effectiveModelConfig.primary})`; const phase = unitPhaseLabel(unitType); ctx.ui.notify(`Model [${phase}]${routingTierLabel}: ${model.provider}/${model.id}${fallbackNote}`, "info"); + // ADR-005: Report tools filtered due to provider incompatibility + if (removedTools.length > 0) { + ctx.ui.notify( + `Tool compatibility: ${removedTools.length} tools filtered for ${model.api} — ${removedTools.join(", ")}`, + "info", + ); + } } break; } else { diff --git a/src/resources/extensions/gsd/bootstrap/register-hooks.ts b/src/resources/extensions/gsd/bootstrap/register-hooks.ts index 4bb105f71..6c88de385 100644 --- a/src/resources/extensions/gsd/bootstrap/register-hooks.ts +++ b/src/resources/extensions/gsd/bootstrap/register-hooks.ts @@ -446,4 +446,12 @@ export function registerHooks(pi: ExtensionAPI): void { // Default: no override — let capability scoring handle selection return undefined; }); + + // Tool set adaptation hook (ADR-005 Phase 4) + // Extensions can override tool set after model selection by returning { toolNames: [...] } + // Return undefined to let the built-in provider compatibility filtering proceed. + pi.on("adjust_tool_set", async (_event) => { + // Default: no override — let provider capability filtering handle tool set + return undefined; + }); } diff --git a/src/resources/extensions/gsd/model-router.ts b/src/resources/extensions/gsd/model-router.ts index 17ff1c70a..cc915877a 100644 --- a/src/resources/extensions/gsd/model-router.ts +++ b/src/resources/extensions/gsd/model-router.ts @@ -5,6 +5,9 @@ import type { ComplexityTier, ClassificationResult, TaskMetadata } from "./complexity-classifier.js"; import { tierOrdinal } from "./complexity-classifier.js"; import type { ResolvedModelConfig } from "./preferences.js"; +import { getProviderCapabilities, type ProviderCapabilities } from "@gsd/pi-ai"; +import { getToolCompatibility, getAllToolCompatibility } from "@gsd/pi-coding-agent"; +import type { ToolCompatibility } from "@gsd/pi-coding-agent"; // ─── Types ─────────────────────────────────────────────────────────────────── @@ -37,6 +40,8 @@ export interface RoutingDecision { selectionMethod: "tier-only" | "capability-scored"; /** Capability scores per eligible model (capability-scored path only) */ capabilityScores?: Record; + /** Tools filtered out due to provider incompatibility (ADR-005) */ + filteredTools?: string[]; /** Task requirement vector used for scoring */ taskRequirements?: Partial>; } @@ -536,3 +541,71 @@ function getModelCost(modelId: string): number { // Unknown cost — assume expensive to avoid routing to unknown cheap models return 999; } + +// ─── Tool Compatibility Filter (ADR-005 Phase 3) ─────────────────────────── + +/** + * Check if a tool is compatible with a provider's capabilities. + * Returns true if the tool can be used with the provider. + */ +export function isToolCompatibleWithProvider( + toolName: string, + providerCaps: ProviderCapabilities, +): boolean { + const compat = getToolCompatibility(toolName); + if (!compat) return true; // no metadata = always compatible + + // Hard filter: provider doesn't support image tool results + if (compat.producesImages && !providerCaps.imageToolResults) return false; + + // Hard filter: tool uses schema features provider doesn't support + if (compat.schemaFeatures?.some(f => providerCaps.unsupportedSchemaFeatures.includes(f))) { + return false; + } + + return true; +} + +/** + * Filter a list of tool names to only those compatible with a provider. + * Used by the routing pipeline to adjust tool sets when switching providers. + */ +export function filterToolsForProvider( + toolNames: string[], + providerApi: string, +): { compatible: string[]; filtered: string[] } { + const providerCaps = getProviderCapabilities(providerApi); + + // Provider doesn't support tool calling at all + if (!providerCaps.toolCalling) { + return { compatible: [], filtered: toolNames }; + } + + const compatible: string[] = []; + const filtered: string[] = []; + + for (const name of toolNames) { + if (isToolCompatibleWithProvider(name, providerCaps)) { + compatible.push(name); + } else { + filtered.push(name); + } + } + + return { compatible, filtered }; +} + +/** + * Adjust the active tool set for a selected model's provider capabilities. + * Returns tool names that should be active — removes incompatible tools. + * + * This is a hard filter only — it removes tools that would fail at the + * provider level. It does NOT remove tools based on soft heuristics. + */ +export function adjustToolSet( + activeToolNames: string[], + selectedModelApi: string, +): { toolNames: string[]; removedTools: string[] } { + const { compatible, filtered } = filterToolsForProvider(activeToolNames, selectedModelApi); + return { toolNames: compatible, removedTools: filtered }; +} diff --git a/src/resources/extensions/gsd/tests/tool-compatibility.test.ts b/src/resources/extensions/gsd/tests/tool-compatibility.test.ts new file mode 100644 index 000000000..6b533bf63 --- /dev/null +++ b/src/resources/extensions/gsd/tests/tool-compatibility.test.ts @@ -0,0 +1,199 @@ +// GSD-2 — Tool Compatibility + Model Router Tool Filtering Tests (ADR-005 Phases 2-3) +import { describe, test, beforeEach } from "node:test"; +import assert from "node:assert/strict"; + +import { + registerToolCompatibility, + getToolCompatibility, + getAllToolCompatibility, + registerMcpToolCompatibility, + resetToolCompatibilityRegistry, +} from "@gsd/pi-coding-agent"; + +import { + isToolCompatibleWithProvider, + filterToolsForProvider, + adjustToolSet, +} from "../model-router.js"; + +import { + getProviderCapabilities, +} from "@gsd/pi-ai"; + +// ─── Tool Compatibility Registry ──────────────────────────────────────────── + +describe("tool compatibility registry", () => { + beforeEach(() => { + resetToolCompatibilityRegistry(); + }); + + test("built-in tools are pre-registered", () => { + const builtins = ["bash", "read", "write", "edit", "grep", "find", "ls", "lsp"]; + for (const name of builtins) { + const compat = getToolCompatibility(name); + assert.ok(compat !== undefined, `${name} should be pre-registered`); + } + }); + + test("unknown tool returns undefined", () => { + assert.equal(getToolCompatibility("nonexistent_tool_xyz"), undefined); + }); + + test("registerToolCompatibility stores and retrieves metadata", () => { + registerToolCompatibility("screenshot_tool", { + producesImages: true, + minCapabilityTier: "standard", + }); + const compat = getToolCompatibility("screenshot_tool"); + assert.ok(compat); + assert.equal(compat.producesImages, true); + assert.equal(compat.minCapabilityTier, "standard"); + }); + + test("registerMcpToolCompatibility sets default schema features", () => { + registerMcpToolCompatibility("mcp__test__tool"); + const compat = getToolCompatibility("mcp__test__tool"); + assert.ok(compat); + assert.ok(compat.schemaFeatures?.includes("patternProperties")); + }); + + test("registerMcpToolCompatibility allows overrides", () => { + registerMcpToolCompatibility("mcp__test__override", { producesImages: true }); + const compat = getToolCompatibility("mcp__test__override"); + assert.ok(compat); + assert.equal(compat.producesImages, true); + assert.ok(compat.schemaFeatures?.includes("patternProperties")); + }); + + test("getAllToolCompatibility returns all entries", () => { + const all = getAllToolCompatibility(); + assert.ok(all.size >= 10); // at least built-in tools + assert.ok(all.has("bash")); + assert.ok(all.has("read")); + }); + + test("resetToolCompatibilityRegistry clears custom entries but keeps builtins", () => { + registerToolCompatibility("custom_tool", { producesImages: true }); + assert.ok(getToolCompatibility("custom_tool")); + resetToolCompatibilityRegistry(); + assert.equal(getToolCompatibility("custom_tool"), undefined); + assert.ok(getToolCompatibility("bash")); // built-in preserved + }); +}); + +// ─── isToolCompatibleWithProvider ─────────────────────────────────────────── + +describe("isToolCompatibleWithProvider", () => { + beforeEach(() => { + resetToolCompatibilityRegistry(); + }); + + test("tool without compatibility metadata is always compatible", () => { + const caps = getProviderCapabilities("anthropic-messages"); + assert.equal(isToolCompatibleWithProvider("unknown_tool", caps), true); + }); + + test("built-in tools are compatible with all providers", () => { + const providers = ["anthropic-messages", "openai-responses", "google-generative-ai", "mistral-conversations"]; + const tools = ["bash", "read", "write", "edit"]; + for (const api of providers) { + const caps = getProviderCapabilities(api); + for (const tool of tools) { + assert.equal( + isToolCompatibleWithProvider(tool, caps), + true, + `${tool} should be compatible with ${api}`, + ); + } + } + }); + + test("image-producing tool filtered for providers without image support", () => { + registerToolCompatibility("screenshot", { producesImages: true }); + const openaiCaps = getProviderCapabilities("openai-responses"); + assert.equal(isToolCompatibleWithProvider("screenshot", openaiCaps), false); + + const anthropicCaps = getProviderCapabilities("anthropic-messages"); + assert.equal(isToolCompatibleWithProvider("screenshot", anthropicCaps), true); + }); + + test("tool with unsupported schema features filtered for Google", () => { + registerToolCompatibility("complex_schema_tool", { + schemaFeatures: ["patternProperties"], + }); + const googleCaps = getProviderCapabilities("google-generative-ai"); + assert.equal(isToolCompatibleWithProvider("complex_schema_tool", googleCaps), false); + + const anthropicCaps = getProviderCapabilities("anthropic-messages"); + assert.equal(isToolCompatibleWithProvider("complex_schema_tool", anthropicCaps), true); + }); +}); + +// ─── filterToolsForProvider ───────────────────────────────────────────────── + +describe("filterToolsForProvider", () => { + beforeEach(() => { + resetToolCompatibilityRegistry(); + }); + + test("all built-in tools pass for any provider", () => { + const toolNames = ["bash", "read", "write", "edit", "grep", "find", "ls"]; + const { compatible, filtered } = filterToolsForProvider(toolNames, "mistral-conversations"); + assert.deepEqual(compatible, toolNames); + assert.deepEqual(filtered, []); + }); + + test("image tool filtered for OpenAI Responses", () => { + registerToolCompatibility("browser_screenshot", { producesImages: true }); + const toolNames = ["bash", "read", "browser_screenshot"]; + const { compatible, filtered } = filterToolsForProvider(toolNames, "openai-responses"); + assert.deepEqual(compatible, ["bash", "read"]); + assert.deepEqual(filtered, ["browser_screenshot"]); + }); + + test("MCP tool with patternProperties filtered for Google", () => { + registerMcpToolCompatibility("mcp__repowise__search"); + const toolNames = ["bash", "read", "mcp__repowise__search"]; + const { compatible, filtered } = filterToolsForProvider(toolNames, "google-generative-ai"); + assert.deepEqual(compatible, ["bash", "read"]); + assert.deepEqual(filtered, ["mcp__repowise__search"]); + }); + + test("unknown provider passes all tools (permissive default)", () => { + registerToolCompatibility("image_tool", { producesImages: true }); + registerMcpToolCompatibility("mcp_tool"); + const toolNames = ["bash", "image_tool", "mcp_tool"]; + const { compatible, filtered } = filterToolsForProvider(toolNames, "unknown-provider-xyz"); + assert.deepEqual(compatible, toolNames); + assert.deepEqual(filtered, []); + }); +}); + +// ─── adjustToolSet ────────────────────────────────────────────────────────── + +describe("adjustToolSet", () => { + beforeEach(() => { + resetToolCompatibilityRegistry(); + }); + + test("returns all tools for Anthropic (most permissive)", () => { + registerToolCompatibility("screenshot", { producesImages: true }); + const toolNames = ["bash", "read", "screenshot"]; + const { toolNames: result, removedTools } = adjustToolSet(toolNames, "anthropic-messages"); + assert.deepEqual(result, toolNames); + assert.deepEqual(removedTools, []); + }); + + test("removes incompatible tools and reports them", () => { + registerToolCompatibility("screenshot", { producesImages: true }); + registerMcpToolCompatibility("mcp_complex"); + const toolNames = ["bash", "read", "screenshot", "mcp_complex"]; + const { toolNames: result, removedTools } = adjustToolSet(toolNames, "google-generative-ai"); + // Google supports images but not patternProperties + assert.ok(result.includes("bash")); + assert.ok(result.includes("read")); + assert.ok(result.includes("screenshot")); // Google supports images + assert.ok(!result.includes("mcp_complex")); // patternProperties not supported + assert.deepEqual(removedTools, ["mcp_complex"]); + }); +});