From a342868068c3b3264473f3afca97bdc96a789aaa Mon Sep 17 00:00:00 2001 From: Mikael Hugo Date: Thu, 14 May 2026 06:48:19 +0200 Subject: [PATCH] feat(packages): extract @singularity-forge/openai-codex-provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirrors the @singularity-forge/google-gemini-cli-provider package layout for the codex CLI integration boundary. The new package owns: - CodexAppServerClient (the JSON-RPC subprocess client; previously packages/ai/src/providers/codex-app-server-client.ts, no pi-ai internal coupling) - snapshotCodexCliAccount / discoverCodexCliModels (reads ~/.codex/models_cache.json with visibility=list ∧ supported_in_api filter; previously inline in src/resources/extensions/sf/openai-codex-catalog.js) openai-codex-responses.ts (the stream-shaping provider) intentionally stays in @singularity-forge/ai because it depends on pi-ai stream-event internals and is not reusable outside the provider — same scope as google-gemini-cli.ts vs google-gemini-cli-provider. The SF extension's openai-codex-catalog.js is now a thin SF-side cache writer that delegates to discoverCodexCliModels, mirroring how gemini-catalog.js delegates to discoverGeminiCliModels. readCodexAvailableModels became async to match the dynamic-import path; tests updated. Closes sf-mp4u5fcz-wh6ac9 (with documented AC2 narrowing — see resolution). Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/ai/package.json | 1 + .../src/providers/openai-codex-responses.ts | 2 +- packages/openai-codex-provider/package.json | 20 ++++ .../src}/codex-app-server-client.ts | 0 packages/openai-codex-provider/src/index.ts | 32 +++++++ .../openai-codex-provider/src/snapshot.ts | 92 +++++++++++++++++++ packages/openai-codex-provider/tsconfig.json | 26 ++++++ .../extensions/sf/openai-codex-catalog.js | 87 ++++++------------ .../sf/tests/openai-codex-catalog.test.mjs | 54 +++++++---- 9 files changed, 236 insertions(+), 78 deletions(-) create mode 100644 packages/openai-codex-provider/package.json rename packages/{ai/src/providers => openai-codex-provider/src}/codex-app-server-client.ts (100%) create mode 100644 packages/openai-codex-provider/src/index.ts create mode 100644 packages/openai-codex-provider/src/snapshot.ts create mode 100644 packages/openai-codex-provider/tsconfig.json diff --git a/packages/ai/package.json b/packages/ai/package.json index 234f4a358..1aaaaf41b 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -30,6 +30,7 @@ "@google/genai": "^2.0.1", "@mistralai/mistralai": "^2.2.1", "@singularity-forge/google-gemini-cli-provider": "^2.75.3", + "@singularity-forge/openai-codex-provider": "^2.75.3", "@sinclair/typebox": "^0.34.49", "ajv": "^8.20.0", "ajv-formats": "^3.0.1", diff --git a/packages/ai/src/providers/openai-codex-responses.ts b/packages/ai/src/providers/openai-codex-responses.ts index 4bc5df923..45a29e49d 100644 --- a/packages/ai/src/providers/openai-codex-responses.ts +++ b/packages/ai/src/providers/openai-codex-responses.ts @@ -16,7 +16,7 @@ import { parseStreamingJson } from "../utils/json-parse.js"; import { type CodexAppServerNotification, getCodexAppServerClient, -} from "./codex-app-server-client.js"; +} from "@singularity-forge/openai-codex-provider"; import { convertResponsesMessages } from "./openai-responses-shared.js"; import { buildBaseOptions, diff --git a/packages/openai-codex-provider/package.json b/packages/openai-codex-provider/package.json new file mode 100644 index 000000000..2ad08aa5a --- /dev/null +++ b/packages/openai-codex-provider/package.json @@ -0,0 +1,20 @@ +{ + "name": "@singularity-forge/openai-codex-provider", + "version": "2.75.3", + "description": "OpenAI Codex CLI app-server transport helper for SF providers", + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "tsgo -p tsconfig.json" + }, + "engines": { + "node": ">=26.1.0" + } +} diff --git a/packages/ai/src/providers/codex-app-server-client.ts b/packages/openai-codex-provider/src/codex-app-server-client.ts similarity index 100% rename from packages/ai/src/providers/codex-app-server-client.ts rename to packages/openai-codex-provider/src/codex-app-server-client.ts diff --git a/packages/openai-codex-provider/src/index.ts b/packages/openai-codex-provider/src/index.ts new file mode 100644 index 000000000..fa2fc63fd --- /dev/null +++ b/packages/openai-codex-provider/src/index.ts @@ -0,0 +1,32 @@ +/** + * OpenAI Codex CLI transport helper. + * + * Purpose: keep the codex CLI app-server JSON-RPC client and account/catalog + * snapshot in a dedicated workspace package so provider code can depend on + * one small helper instead of embedding the upstream subprocess integration + * inline (mirrors @singularity-forge/google-gemini-cli-provider). + * + * Scope: this package owns the codex-CLI integration boundary — the + * subprocess client and the local cache/snapshot reader. Stream shaping + * (turning Codex events into pi-ai assistant messages) intentionally stays + * inside @singularity-forge/ai because it depends on pi-ai's stream-event + * internals and is not reusable outside the provider. + * + * Consumer: `@singularity-forge/ai` openai-codex-responses provider for + * the JSON-RPC client; SF extension catalog reader + future usage UI for + * snapshot/discover. + */ +export { + CodexAppServerClient, + clearCodexAppServerClient, + getCodexAppServerClient, + type CodexAppServerClientOptions, + type CodexAppServerNotification, + type CodexAppServerNotificationHandler, +} from "./codex-app-server-client.js"; + +export { + type CodexCliAccountSnapshot, + discoverCodexCliModels, + snapshotCodexCliAccount, +} from "./snapshot.js"; diff --git a/packages/openai-codex-provider/src/snapshot.ts b/packages/openai-codex-provider/src/snapshot.ts new file mode 100644 index 000000000..2eaff2c3c --- /dev/null +++ b/packages/openai-codex-provider/src/snapshot.ts @@ -0,0 +1,92 @@ +/** + * OpenAI Codex CLI account snapshot + model discovery. + * + * Why a file read and not an API call: the codex CLI maintains + * ~/.codex/models_cache.json itself (with fetched_at / etag for refresh + * tracking) as part of its normal operation. That cache is the authoritative + * "what models can THIS ChatGPT account actually serve" record — distinct + * from the static catalog in models.generated.ts which can carry phantom + * entries like `gpt-5-codex` that 400 with "model is not supported when + * using Codex with a ChatGPT account." + * + * Asymmetry vs gemini-cli (which calls setupUser + retrieveUserQuota over + * the wire): codex CLI caches locally; gemini-cli does not. Each provider + * gets the cheapest reliable discovery path. + * + * Consumer: SF extension's openai-codex-catalog.js (via discoverCodexCliModels); + * future usage UI consumers via snapshotCodexCliAccount. + */ +import { existsSync, readFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; + +export interface CodexCliAccountSnapshot { + /** Slug ids the account can actually serve (visibility=list ∧ supported_in_api). */ + modelIds: string[]; + /** Path of the cache file we read. */ + cachePath: string; + /** When the codex CLI last wrote the cache (ISO string), if available. */ + fetchedAt?: string; +} + +function codexHome(): string { + return process.env.CODEX_HOME ?? join(homedir(), ".codex"); +} + +function codexCachePath(): string { + return join(codexHome(), "models_cache.json"); +} + +/** + * Read the codex CLI's local cache and return a snapshot of the slugs the + * account can actually serve. Returns null when the cache is missing, + * malformed, or the codex CLI has not been initialized. + * + * Filters: visibility === "list" AND supported_in_api === true. + * + * Why both filters: visibility=hide is for codex-internal models like + * codex-auto-review (not user-facing). supported_in_api=false catches + * preview slugs that exist in the cache but reject API requests for the + * current account tier (observed live with gpt-5.3-codex-spark). + */ +export function snapshotCodexCliAccount(): CodexCliAccountSnapshot | null { + const path = codexCachePath(); + if (!existsSync(path)) return null; + try { + const cache = JSON.parse(readFileSync(path, "utf-8")) as { + models?: unknown; + fetched_at?: string; + }; + const models = Array.isArray(cache?.models) ? cache.models : null; + if (!models) return null; + const slugs = models + .filter( + (m): m is { slug: string } => + !!m && + typeof m === "object" && + (m as Record).visibility === "list" && + (m as Record).supported_in_api === true && + typeof (m as Record).slug === "string" && + ((m as Record).slug as string).length > 0, + ) + .map((m) => m.slug); + if (slugs.length === 0) return null; + return { + modelIds: slugs, + cachePath: path, + fetchedAt: + typeof cache.fetched_at === "string" ? cache.fetched_at : undefined, + }; + } catch { + return null; + } +} + +/** + * Convenience wrapper that returns just the model IDs (matching the + * discoverGeminiCliModels signature for symmetry with that package). + */ +export function discoverCodexCliModels(): string[] | null { + const snap = snapshotCodexCliAccount(); + return snap ? snap.modelIds : null; +} diff --git a/packages/openai-codex-provider/tsconfig.json b/packages/openai-codex-provider/tsconfig.json new file mode 100644 index 000000000..e22f0f518 --- /dev/null +++ b/packages/openai-codex-provider/tsconfig.json @@ -0,0 +1,26 @@ +{ + "compilerOptions": { + "target": "ES2024", + "module": "Node16", + "lib": ["ES2024"], + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "incremental": true, + "forceConsistentCasingInFileNames": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "inlineSources": true, + "inlineSourceMap": false, + "moduleResolution": "Node16", + "resolveJsonModule": true, + "allowImportingTsExtensions": false, + "useDefineForClassFields": false, + "types": ["node"], + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*.ts"], + "exclude": ["node_modules", "dist", "**/*.d.ts", "src/**/*.d.ts"] +} diff --git a/src/resources/extensions/sf/openai-codex-catalog.js b/src/resources/extensions/sf/openai-codex-catalog.js index 655247fd3..cd86cb6c0 100644 --- a/src/resources/extensions/sf/openai-codex-catalog.js +++ b/src/resources/extensions/sf/openai-codex-catalog.js @@ -1,39 +1,22 @@ /** - * openai-codex-catalog.js — read codex CLI's own models_cache.json. + * openai-codex-catalog.js — SF-side cache writer for codex-cli model discovery. * - * Why a file read and not an API call: the codex CLI maintains - * ~/.codex/models_cache.json itself (with fetched_at / etag for refresh - * tracking) as part of its normal operation. That cache is the authoritative - * "what models can THIS ChatGPT account actually serve" record — distinct - * from the static catalog in models.generated.ts which can carry phantom - * entries like `gpt-5-codex` that 400 with "model is not supported when - * using Codex with a ChatGPT account." + * The actual cache reader (~/.codex/models_cache.json parsing + visibility + + * supported_in_api filtering) lives in the dedicated + * @singularity-forge/openai-codex-provider package. This module only handles + * the SF-specific concerns: where on disk to mirror the result, how often to + * refresh, and the session_start lifecycle hook. * - * SF mirrors the visible+supported subset into - * .sf/runtime/model-catalog/openai-codex.json so getKnownModelIds and the - * model picker pick it up transparently — same shape as the generic - * model-catalog-cache and gemini-catalog modules. - * - * Asymmetry vs gemini-cli (which calls setupUser + retrieveUserQuota over - * the wire): codex CLI caches locally; gemini-cli does not. Each provider - * gets the cheapest reliable discovery path. + * Cache file shape stays compatible with model-catalog-cache.getKnownModelIds + * so consumers read both transparently. */ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { homedir } from "node:os"; import { join } from "node:path"; import { sfRuntimeRoot } from "./paths.js"; const PROVIDER_ID = "openai-codex"; const CATALOG_TTL_MS = 6 * 60 * 60 * 1000; -function codexHome() { - return process.env.CODEX_HOME ?? join(homedir(), ".codex"); -} - -function codexCachePath() { - return join(codexHome(), "models_cache.json"); -} - function sfCacheFilePath(basePath) { return join( sfRuntimeRoot(basePath), @@ -73,36 +56,19 @@ function writeSfCache(basePath, modelIds) { } /** - * Read the codex CLI's local cache and return the slugs the user's account - * can actually serve. Returns null when the cache is missing, malformed, or - * the codex CLI has not been initialized. + * Read the codex CLI's local model cache and return the slugs the user's + * account can actually serve. Returns null when unavailable. * - * Filters: visibility === "list" AND supported_in_api === true. - * - * Why both filters: visibility=hide is for codex-internal models like - * codex-auto-review (not user-facing). supported_in_api=false catches - * preview slugs that exist in the cache but reject API requests for the - * current account tier (observed live with gpt-5.3-codex-spark). + * Thin compatibility wrapper around discoverCodexCliModels — kept exported + * for any direct consumer that wants the raw list without writing the SF + * cache. */ -export function readCodexAvailableModels() { +export async function readCodexAvailableModels() { try { - const path = codexCachePath(); - if (!existsSync(path)) return null; - const cache = JSON.parse(readFileSync(path, "utf-8")); - const models = Array.isArray(cache?.models) ? cache.models : null; - if (!models) return null; - const slugs = models - .filter( - (m) => - m && - typeof m === "object" && - m.visibility === "list" && - m.supported_in_api === true && - typeof m.slug === "string" && - m.slug.length > 0, - ) - .map((m) => m.slug); - return slugs.length > 0 ? slugs : null; + const { discoverCodexCliModels } = await import( + "@singularity-forge/openai-codex-provider" + ); + return discoverCodexCliModels(); } catch { return null; } @@ -114,8 +80,8 @@ export function readCodexAvailableModels() { * * Consumer: scheduleOpenaiCodexCatalogRefresh during session_start. */ -export function refreshOpenaiCodexCatalog(basePath) { - const modelIds = readCodexAvailableModels(); +export async function refreshOpenaiCodexCatalog(basePath) { + const modelIds = await readCodexAvailableModels(); if (!modelIds || modelIds.length === 0) return null; writeSfCache(basePath, modelIds); return modelIds; @@ -123,15 +89,16 @@ export function refreshOpenaiCodexCatalog(basePath) { /** * Fire-and-forget background refresh. Skipped if the SF cache is fresh. - * Synchronous — reading a local file is cheap; no setImmediate needed. * * Consumer: bootstrap/register-hooks.js session_start hook. */ export function scheduleOpenaiCodexCatalogRefresh(basePath) { if (isSfCacheFresh(basePath)) return; - try { - refreshOpenaiCodexCatalog(basePath); - } catch { - // Per-provider failure is silently swallowed. - } + setImmediate(async () => { + try { + await refreshOpenaiCodexCatalog(basePath); + } catch { + // Per-provider failure is silently swallowed. + } + }); } diff --git a/src/resources/extensions/sf/tests/openai-codex-catalog.test.mjs b/src/resources/extensions/sf/tests/openai-codex-catalog.test.mjs index 656f0084e..33b1c1f74 100644 --- a/src/resources/extensions/sf/tests/openai-codex-catalog.test.mjs +++ b/src/resources/extensions/sf/tests/openai-codex-catalog.test.mjs @@ -72,22 +72,22 @@ const REAL_SHAPE_CACHE = { }; describe("readCodexAvailableModels", () => { - test("returns null when ~/.codex/models_cache.json is missing", () => { + test("returns null when ~/.codex/models_cache.json is missing", async () => { makeCodexHome(null); - expect(readCodexAvailableModels()).toBe(null); + expect(await readCodexAvailableModels()).toBe(null); }); - test("returns null on malformed cache", () => { + test("returns null on malformed cache", async () => { const dir = mkdtempSync(join(tmpdir(), "codex-bad-")); tmpDirs.push(dir); writeFileSync(join(dir, "models_cache.json"), "{not json"); process.env.CODEX_HOME = dir; - expect(readCodexAvailableModels()).toBe(null); + expect(await readCodexAvailableModels()).toBe(null); }); - test("filters by visibility=list AND supported_in_api=true", () => { + test("filters by visibility=list AND supported_in_api=true", async () => { makeCodexHome(REAL_SHAPE_CACHE); - const ids = readCodexAvailableModels(); + const ids = await readCodexAvailableModels(); expect(ids).toEqual([ "gpt-5.5", "gpt-5.4", @@ -100,7 +100,7 @@ describe("readCodexAvailableModels", () => { expect(ids).not.toContain("gpt-5.3-codex-spark"); }); - test("returns null when no models pass the filter", () => { + test("returns null when no models pass the filter", async () => { makeCodexHome({ fetched_at: "2026-05-14T00:00:00Z", models: [ @@ -108,10 +108,10 @@ describe("readCodexAvailableModels", () => { { slug: "preview", visibility: "list", supported_in_api: false }, ], }); - expect(readCodexAvailableModels()).toBe(null); + expect(await readCodexAvailableModels()).toBe(null); }); - test("ignores entries with missing/invalid slug", () => { + test("ignores entries with missing/invalid slug", async () => { makeCodexHome({ models: [ { visibility: "list", supported_in_api: true }, @@ -119,16 +119,16 @@ describe("readCodexAvailableModels", () => { { slug: "gpt-5.5", visibility: "list", supported_in_api: true }, ], }); - expect(readCodexAvailableModels()).toEqual(["gpt-5.5"]); + expect(await readCodexAvailableModels()).toEqual(["gpt-5.5"]); }); }); describe("refreshOpenaiCodexCatalog", () => { - test("writes the SF cache from the codex CLI cache", () => { + test("writes the SF cache from the codex CLI cache", async () => { const project = makeProject(); makeCodexHome(REAL_SHAPE_CACHE); - const result = refreshOpenaiCodexCatalog(project); + const result = await refreshOpenaiCodexCatalog(project); expect(result).toEqual([ "gpt-5.5", "gpt-5.4", @@ -149,15 +149,32 @@ describe("refreshOpenaiCodexCatalog", () => { expect(new Date(cache.fetchedAt).toString()).not.toBe("Invalid Date"); }); - test("returns null and writes nothing when codex cache is missing", () => { + test("returns null and writes nothing when codex cache is missing", async () => { const project = makeProject(); makeCodexHome(null); - expect(refreshOpenaiCodexCatalog(project)).toBe(null); + expect(await refreshOpenaiCodexCatalog(project)).toBe(null); }); }); +// Wait for the setImmediate + dynamic import + async refresh chain to +// complete — short polling rather than fixed delay so the test stays +// fast on CI. +async function waitForCacheWrite(cachePath, timeoutMs = 2000) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + const cache = JSON.parse(readFileSync(cachePath, "utf-8")); + if (cache?.modelIds) return cache; + } catch { + // not yet + } + await new Promise((r) => setImmediate(r)); + } + throw new Error(`SF cache file ${cachePath} not written within ${timeoutMs}ms`); +} + describe("scheduleOpenaiCodexCatalogRefresh", () => { - test("populates SF cache on first call", () => { + test("populates SF cache on first call", async () => { const project = makeProject(); makeCodexHome(REAL_SHAPE_CACHE); @@ -169,11 +186,11 @@ describe("scheduleOpenaiCodexCatalogRefresh", () => { "model-catalog", "openai-codex.json", ); - const cache = JSON.parse(readFileSync(cachePath, "utf-8")); + const cache = await waitForCacheWrite(cachePath); expect(cache.modelIds).toContain("gpt-5.5"); }); - test("skips refresh when SF cache is fresh", () => { + test("skips refresh when SF cache is fresh", async () => { const project = makeProject(); makeCodexHome(REAL_SHAPE_CACHE); // Pre-seed a fresh cache with a deliberately different model list @@ -188,6 +205,9 @@ describe("scheduleOpenaiCodexCatalogRefresh", () => { ); scheduleOpenaiCodexCatalogRefresh(project); + // Give any spurious setImmediate chain a chance to run. + await new Promise((r) => setImmediate(r)); + await new Promise((r) => setImmediate(r)); const cache = JSON.parse( readFileSync(