From 6101bd5ce3181b42dcf20ef8b59967aadfeff3e6 Mon Sep 17 00:00:00 2001 From: ngoding sendiri Date: Sat, 1 Aug 2026 18:24:02 +0700 Subject: [PATCH 01/15] feat(collectors): AI-generate custom gateway plugins from a manifest Adds a gateway generator that lets a free-tier LLM (Gemini flash-lite by default, OpenRouter :free fallback) produce custom gateway plugins for providers whose APIs do not match the OpenAI-compatible shape. How it works: - Register a gateway in packages/collectors/manifest.json (url, auth, secrets, optional static sample). - pnpm gen-gateway probes the endpoint (or uses the sample), captures a redacted fixture, summarizes the response shape, and asks the LLM for a complete CustomGateway plugin. Output is structurally validated (importable, correct id, has collect(), no forbidden patterns) and retried up to 3 times. - heal mode (--heal) regenerates a plugin from the existing fixture when a sample changes or a mapping breaks. - .github/workflows/gateway-ai.yml runs the generator on demand and opens a review PR; it also validates the manifest on PRs that touch it. Includes the first generated example: cohere.ts (written by the LLM from a static sample), its fixture, a generic harness test that validates generated plugins against their fixtures, and the COHERE_API_KEY secret registration. --- .github/workflows/gateway-ai.yml | 142 ++++++++++ packages/collectors/manifest.json | 46 ++++ packages/collectors/package.json | 1 + .../src/__tests__/generated-gateways.test.ts | 69 +++++ .../collectors/src/core/gateway-secrets.ts | 1 + .../src/gateway-gen/__tests__/probe.test.ts | 47 ++++ packages/collectors/src/gateway-gen/heal.ts | 44 ++++ packages/collectors/src/gateway-gen/index.ts | 93 +++++++ packages/collectors/src/gateway-gen/llm.ts | 82 ++++++ .../collectors/src/gateway-gen/manifest.ts | 78 ++++++ packages/collectors/src/gateway-gen/probe.ts | 242 ++++++++++++++++++ .../collectors/src/gateway-gen/prompts.ts | 157 ++++++++++++ packages/collectors/src/gateway-gen/write.ts | 128 +++++++++ .../src/gateways/__fixtures__/cohere.raw.json | 50 ++++ packages/collectors/src/gateways/cohere.ts | 111 ++++++++ 15 files changed, 1291 insertions(+) create mode 100644 .github/workflows/gateway-ai.yml create mode 100644 packages/collectors/manifest.json create mode 100644 packages/collectors/src/__tests__/generated-gateways.test.ts create mode 100644 packages/collectors/src/gateway-gen/__tests__/probe.test.ts create mode 100644 packages/collectors/src/gateway-gen/heal.ts create mode 100644 packages/collectors/src/gateway-gen/index.ts create mode 100644 packages/collectors/src/gateway-gen/llm.ts create mode 100644 packages/collectors/src/gateway-gen/manifest.ts create mode 100644 packages/collectors/src/gateway-gen/probe.ts create mode 100644 packages/collectors/src/gateway-gen/prompts.ts create mode 100644 packages/collectors/src/gateway-gen/write.ts create mode 100644 packages/collectors/src/gateways/__fixtures__/cohere.raw.json create mode 100644 packages/collectors/src/gateways/cohere.ts diff --git a/.github/workflows/gateway-ai.yml b/.github/workflows/gateway-ai.yml new file mode 100644 index 000000000..27cf4fbdd --- /dev/null +++ b/.github/workflows/gateway-ai.yml @@ -0,0 +1,142 @@ +name: AI Gateway Generation + +# Generates a custom gateway plugin with a free-tier LLM. +# 1. Register the gateway in packages/collectors/manifest.json (url + key + optional sample). +# 2. Add the gateway's API key as a repository secret (it must be listed in +# packages/collectors/src/core/gateway-secrets.ts before collection can use it). +# 3. Run this workflow with the gateway id, or open a PR that touches manifest.json +# to validate the manifest without generating anything. +on: + workflow_dispatch: + inputs: + gateway_id: + description: 'Gateway id listed in packages/collectors/manifest.json' + required: true + type: string + mode: + description: 'bootstrap (new plugin) or heal (fix existing plugin)' + required: false + default: bootstrap + type: choice + options: + - bootstrap + - heal + pull_request: + types: [opened, synchronize] + paths: + - 'packages/collectors/manifest.json' + +permissions: + contents: write + pull-requests: write + +jobs: + generate: + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install pnpm + uses: pnpm/action-setup@v3 + with: + version: 9 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build schema & registry + run: pnpm --filter @basemodel/schema --filter @basemodel/registry build + + - name: Generate gateway plugin via LLM + run: | + FLAGS="" + if [ "${{ github.event.inputs.mode }}" == "heal" ]; then FLAGS="--heal"; fi + pnpm --filter @basemodel/collectors run gen-gateway "${{ github.event.inputs.gateway_id }}" $FLAGS + env: + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GOOGLE_AI_API_KEY: ${{ secrets.GOOGLE_AI_API_KEY }} + GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} + TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} + DEEPINFRA_API_KEY: ${{ secrets.DEEPINFRA_API_KEY }} + FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} + CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} + HYPERBOLIC_API_KEY: ${{ secrets.HYPERBOLIC_API_KEY }} + REQUESTY_API_KEY: ${{ secrets.REQUESTY_API_KEY }} + PORTKEY_API_KEY: ${{ secrets.PORTKEY_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} + COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} + LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL }} + LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY }} + + - name: Validate generated plugin + run: | + pnpm --filter @basemodel/collectors run typecheck + pnpm --filter @basemodel/collectors run test + + - name: Commit generated plugin + run: | + BRANCH="ai/gateway/${{ github.event.inputs.gateway_id }}" + git config user.name "BaseModel Bot" + git config user.email "bot@basemodel" + git checkout -b "$BRANCH" + git add packages/collectors/src/gateways packages/collectors/manifest.json + git commit -m "feat(gateways): AI-generated plugin for ${{ github.event.inputs.gateway_id }}" || echo "No changes to commit" + git push -u origin "$BRANCH" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Open PR + run: | + BRANCH="ai/gateway/${{ github.event.inputs.gateway_id }}" + gh pr create \ + --base main \ + --head "$BRANCH" \ + --title "feat(gateways): AI-generated plugin for ${{ github.event.inputs.gateway_id }}" \ + --body "AI-generated gateway plugin for \`${{ github.event.inputs.gateway_id }}\` (mode: ${{ github.event.inputs.mode }}). + +Reviewed before merge, per docs/08_Gateway_Plugin_Security.md: +- Confirm the plugin only uses secrets registered in \`packages/collectors/src/core/gateway-secrets.ts\`. +- Add the gateway's API key as a repository secret so nightly collection can use it. +- Run \`pnpm --filter @basemodel/collectors run verify \` to smoke-test with a live key." \ + || echo "PR may already exist for this gateway." + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + validate-manifest: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install pnpm + uses: pnpm/action-setup@v3 + with: + version: 9 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build schema & registry + run: pnpm --filter @basemodel/schema --filter @basemodel/registry build + + - name: Validate manifest + run: | + pnpm --filter @basemodel/collectors exec tsx -e "(async () => { const m = await import('./src/gateway-gen/manifest.js'); await m.loadManifest(); console.log('manifest OK'); })()" diff --git a/packages/collectors/manifest.json b/packages/collectors/manifest.json new file mode 100644 index 000000000..c16f68ce0 --- /dev/null +++ b/packages/collectors/manifest.json @@ -0,0 +1,46 @@ +{ + "version": 1, + "gateways": [ + { + "id": "cohere", + "baseUrl": "https://api.cohere.com", + "endpoint": "/v1/models", + "auth": { + "type": "bearer", + "secret": "COHERE_API_KEY" + }, + "secrets": ["COHERE_API_KEY"], + "sample": { + "models": [ + { + "name": "command-r-plus", + "endpoints": ["chat", "embed", "generate", "summarize", "classify", "rerank"], + "context_length": 128000, + "tokenizer_url": "https://docs.cohere.com/docs/tokenizer" + }, + { + "name": "command-r", + "endpoints": ["chat", "embed", "generate", "summarize", "classify", "rerank"], + "context_length": 128000 + }, + { + "name": "command-light", + "endpoints": ["chat", "generate", "summarize"], + "context_length": 4096 + }, + { + "name": "embed-english-v3.0", + "endpoints": ["embed"], + "context_length": 512 + }, + { + "name": "rerank-english-v3.0", + "endpoints": ["rerank"], + "context_length": 4096 + } + ] + }, + "notes": "Example gateway for the AI generator: non-OpenAI-compatible response shape ({ models: [...] }). A static sample is provided so the generator can run without a live API key." + } + ] +} diff --git a/packages/collectors/package.json b/packages/collectors/package.json index d8e556468..346471429 100644 --- a/packages/collectors/package.json +++ b/packages/collectors/package.json @@ -16,6 +16,7 @@ "build": "tsup src/index.ts --format esm --dts --clean", "collect": "tsx src/run.ts", "enrich": "tsx src/enrich/run.ts", + "gen-gateway": "tsx src/gateway-gen/index.ts", "verify": "tsx src/core/verify.ts", "typecheck": "tsc --noEmit", "test": "vitest run --passWithNoTests", diff --git a/packages/collectors/src/__tests__/generated-gateways.test.ts b/packages/collectors/src/__tests__/generated-gateways.test.ts new file mode 100644 index 000000000..49711f19d --- /dev/null +++ b/packages/collectors/src/__tests__/generated-gateways.test.ts @@ -0,0 +1,69 @@ +import { existsSync } from 'node:fs'; +import { readFile } from 'node:fs/promises'; +import { ModelSchema } from '@basemodel/schema'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { getFixturePath, getGatewayPluginPath, loadManifest } from '../gateway-gen/manifest.js'; +import { fixtureExists } from '../gateway-gen/probe.js'; + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe('AI-generated gateway plugins', () => { + it('map their fixture sample into valid Models without a live API', async () => { + const manifest = await loadManifest(); + let exercised = 0; + + for (const gateway of manifest.gateways) { + const pluginPath = getGatewayPluginPath(gateway.id); + const fixturePath = getFixturePath(gateway.id); + if (!existsSync(pluginPath) || !fixtureExists(gateway.id)) continue; + + const fixture = JSON.parse(await readFile(fixturePath, 'utf-8')) as unknown; + const plugin = ((await import(pluginPath)) as { default?: unknown }).default as { + id?: string; + collect: (secrets: Record) => Promise<{ + provider_id: string; + models: Array>; + errors: string[]; + }>; + }; + expect(plugin.id).toBe(gateway.id); + + let fetchCalls = 0; + const fetchMock = vi.fn(async () => { + fetchCalls += 1; + if (fetchCalls > 2) { + throw new Error('mock: no more pages (pagination guard)'); + } + return new Response(JSON.stringify(fixture), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }); + vi.stubGlobal('fetch', fetchMock); + + const secrets: Record = { ...process.env }; + for (const name of [ + ...gateway.secrets, + ...(gateway.auth?.secret ? [gateway.auth.secret] : []), + ]) { + if (!secrets[name]) secrets[name] = 'test-key'; + } + + const result = await plugin.collect(secrets); + expect(fetchMock).toHaveBeenCalled(); + expect(result.provider_id).toBe(gateway.id); + for (const model of result.models) { + const parsed = ModelSchema.safeParse(model); + expect( + parsed.success, + `${model.model_id ?? '(no model_id)'} invalid: ${parsed.error?.message}`, + ).toBe(true); + } + exercised += 1; + } + + expect(exercised).toBeGreaterThan(0); + }); +}); diff --git a/packages/collectors/src/core/gateway-secrets.ts b/packages/collectors/src/core/gateway-secrets.ts index 6b831734e..ae075c5d2 100644 --- a/packages/collectors/src/core/gateway-secrets.ts +++ b/packages/collectors/src/core/gateway-secrets.ts @@ -6,6 +6,7 @@ export const GATEWAY_SECRET_KEYS = { anthropic: ['ANTHROPIC_API_KEY'], cerebras: ['CEREBRAS_API_KEY'], cloudflare: ['CLOUDFLARE_ACCOUNT_ID', 'CLOUDFLARE_API_TOKEN'], + cohere: ['COHERE_API_KEY'], deepinfra: ['DEEPINFRA_API_KEY'], fireworks: ['FIREWORKS_API_KEY'], google: ['GOOGLE_AI_API_KEY'], diff --git a/packages/collectors/src/gateway-gen/__tests__/probe.test.ts b/packages/collectors/src/gateway-gen/__tests__/probe.test.ts new file mode 100644 index 000000000..aa8a3244d --- /dev/null +++ b/packages/collectors/src/gateway-gen/__tests__/probe.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from 'vitest'; +import { loadManifest } from '../manifest.js'; +import { extractShape } from '../probe.js'; + +describe('extractShape', () => { + it('detects a model array with id/name keys', () => { + const raw = { + models: [ + { name: 'command-r-plus', endpoints: ['chat', 'embed'], context_length: 128000 }, + { name: 'embed-english-v3.0', endpoints: ['embed'] }, + ], + }; + const shape = extractShape(raw); + expect(shape.topLevel).toEqual({ models: 'array[2]' }); + expect(shape.modelArray).not.toBeNull(); + expect(shape.modelArray?.path).toBe('$.models'); + expect(shape.modelArray?.count).toBe(2); + expect(shape.modelArray?.keys).toEqual(['name', 'endpoints', 'context_length']); + }); + + it('returns null modelArray when no object with id/name exists', () => { + const shape = extractShape({ status: 'ok', total: 0, items: [1, 2, 3] }); + expect(shape.modelArray).toBeNull(); + expect(shape.topLevel).toEqual({ status: 'string', total: 'number', items: 'array[3]' }); + }); + + it('picks the largest array of model-like objects', () => { + const raw = { + small: [{ id: 'a' }], + big: [{ id: 'x' }, { id: 'y' }, { id: 'z' }], + }; + const shape = extractShape(raw); + expect(shape.modelArray?.path).toBe('$.big'); + expect(shape.modelArray?.count).toBe(3); + }); +}); + +describe('manifest', () => { + it('loads and contains the example cohere gateway', async () => { + const manifest = await loadManifest(); + expect(manifest.version).toBe(1); + const cohere = manifest.gateways.find((gateway) => gateway.id === 'cohere'); + expect(cohere).toBeDefined(); + expect(cohere?.baseUrl).toBe('https://api.cohere.com'); + expect(cohere?.auth?.secret).toBe('COHERE_API_KEY'); + }); +}); diff --git a/packages/collectors/src/gateway-gen/heal.ts b/packages/collectors/src/gateway-gen/heal.ts new file mode 100644 index 000000000..4d0dbf26c --- /dev/null +++ b/packages/collectors/src/gateway-gen/heal.ts @@ -0,0 +1,44 @@ +import { readFile, writeFile } from 'node:fs/promises'; +import { generateText } from './llm.js'; +import type { ManifestGateway } from './manifest.js'; +import { getGatewayPluginPath } from './manifest.js'; +import type { ShapeSummary } from './probe.js'; +import { buildHealPrompt } from './prompts.js'; +import { + ensureFinalNewline, + extractTsCode, + type GeneratedPlugin, + validatePluginModule, +} from './write.js'; + +export interface HealPluginOptions { + gateway: ManifestGateway; + shape: ShapeSummary; + raw: unknown; + errors?: string[]; + env?: NodeJS.ProcessEnv; + maxAttempts?: number; +} + +export async function healPlugin(options: HealPluginOptions): Promise { + const { gateway, shape, raw, errors = [], env = process.env, maxAttempts = 3 } = options; + const filePath = getGatewayPluginPath(gateway.id); + const currentCode = await readFile(filePath, 'utf-8'); + let prompt = buildHealPrompt(gateway, shape, raw, currentCode, errors); + const lastErrors: string[] = []; + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + const text = await generateText({ prompt }, env); + const code = ensureFinalNewline(extractTsCode(text)); + await writeFile(filePath, code, 'utf-8'); + const validation = await validatePluginModule(filePath, gateway.id); + if (validation.ok) return { filePath, code, attempts: attempt }; + lastErrors.push(...validation.errors); + console.warn(`[heal] attempt ${attempt} failed validation: ${validation.errors.join('; ')}`); + prompt += `\n\nThe previous attempt failed validation:\n${validation.errors.join('\n')}\n\nReturn the corrected file only.`; + } + + throw new Error( + `Failed to heal plugin for ${gateway.id} after ${maxAttempts} attempts:\n${lastErrors.join('\n')}`, + ); +} diff --git a/packages/collectors/src/gateway-gen/index.ts b/packages/collectors/src/gateway-gen/index.ts new file mode 100644 index 000000000..2199757a6 --- /dev/null +++ b/packages/collectors/src/gateway-gen/index.ts @@ -0,0 +1,93 @@ +import { readFile } from 'node:fs/promises'; +import { healPlugin } from './heal.js'; +import { findGateway, getGatewayPluginPath } from './manifest.js'; +import { extractShape, probeGateway, readFixture } from './probe.js'; +import { checkCollection, generatePlugin, validatePluginModule } from './write.js'; + +function hasAnySecret( + gateway: { secrets: string[]; auth?: { secret?: string } }, + env: NodeJS.ProcessEnv, +): boolean { + const names = [...gateway.secrets]; + if (gateway.auth?.secret) names.push(gateway.auth.secret); + return names.some((name) => Boolean(env[name])); +} + +async function runLiveCheck(gatewayId: string, env: NodeJS.ProcessEnv): Promise { + const filePath = getGatewayPluginPath(gatewayId); + const validation = await validatePluginModule(filePath, gatewayId); + if (!validation.ok || !validation.plugin) { + console.warn(` (skip live check: ${validation.errors.join('; ')})`); + return; + } + const check = await checkCollection(validation.plugin, env); + console.log( + ` live check : ${check.modelCount} models, ${check.validCount} valid`, + check.errors.length > 0 ? `| errors: ${check.errors.join(' | ')}` : '', + ); +} + +async function bootstrap(gatewayId: string, env: NodeJS.ProcessEnv): Promise { + const gateway = await findGateway(gatewayId); + console.log(`Probing ${gateway.id} (${gateway.baseUrl})...`); + const probe = await probeGateway(gateway, env); + console.log( + ` endpoint : ${probe.endpoint}${probe.fromSample ? ' (from manifest sample)' : ''}`, + ); + console.log(` fixture : ${probe.fixturePath}`); + console.log( + ` shape : ${probe.shape.modelArray ? `model array at "${probe.shape.modelArray.path}" (${probe.shape.modelArray.count} items)` : 'no model array detected'}`, + ); + + console.log(`Generating plugin via LLM...`); + const generated = await generatePlugin({ gateway, shape: probe.shape, raw: probe.raw, env }); + console.log(` wrote : ${generated.filePath} (attempt ${generated.attempts})`); + + if (hasAnySecret(gateway, env)) { + await runLiveCheck(gateway.id, env); + } else { + console.warn( + ` live check : skipped (no API key in env). The generated plugin was structurally validated; ` + + 'add a fixture-based unit test or set the gateway key to verify mapping.', + ); + } +} + +async function heal(gatewayId: string, env: NodeJS.ProcessEnv, errorsFile?: string): Promise { + const gateway = await findGateway(gatewayId); + const raw = await readFixture(gatewayId); + const shape = extractShape(raw); + let errors: string[] = []; + if (errorsFile) { + const parsed = JSON.parse(await readFile(errorsFile, 'utf-8')) as unknown; + errors = Array.isArray(parsed) ? parsed.map(String) : [String(parsed)]; + } + console.log(`Healing plugin for ${gateway.id}...`); + const healed = await healPlugin({ gateway, shape, raw, errors, env }); + console.log(` wrote : ${healed.filePath} (attempt ${healed.attempts})`); + if (hasAnySecret(gateway, env)) await runLiveCheck(gateway.id, env); +} + +async function main(): Promise { + const args = process.argv.slice(2); + const gatewayId = args.find((arg) => !arg.startsWith('--')); + if (!gatewayId) { + console.error('Usage: tsx src/gateway-gen/index.ts [--heal] [--errors ]'); + process.exit(1); + } + const env = process.env; + try { + if (args.includes('--heal')) { + const errorsIndex = args.indexOf('--errors'); + const errorsFile = errorsIndex >= 0 ? args[errorsIndex + 1] : undefined; + await heal(gatewayId, env, errorsFile); + } else { + await bootstrap(gatewayId, env); + } + } catch (error: unknown) { + console.error(`❌ ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); + } +} + +main(); diff --git a/packages/collectors/src/gateway-gen/llm.ts b/packages/collectors/src/gateway-gen/llm.ts new file mode 100644 index 000000000..8e72cf206 --- /dev/null +++ b/packages/collectors/src/gateway-gen/llm.ts @@ -0,0 +1,82 @@ +const GEMINI_ENDPOINT = 'https://generativelanguage.googleapis.com/v1beta/models'; +const OPENROUTER_ENDPOINT = 'https://openrouter.ai/api/v1/chat/completions'; + +export interface LlmConfig { + prompt: string; + temperature?: number; +} + +type Provider = 'gemini' | 'openrouter'; + +function pickProvider(env: NodeJS.ProcessEnv): Provider { + if (env.GEMINI_API_KEY) return 'gemini'; + if (env.OPENROUTER_API_KEY) return 'openrouter'; + throw new Error( + 'No LLM provider configured. Set GEMINI_API_KEY (Gemini free tier) or ' + + 'OPENROUTER_API_KEY (free models) to generate gateway plugins.', + ); +} + +async function callGemini(prompt: string, env: NodeJS.ProcessEnv): Promise { + const apiKey = env.GEMINI_API_KEY; + const model = env.GEMINI_MODEL ?? 'gemini-flash-lite-latest'; + const response = await fetch(`${GEMINI_ENDPOINT}/${model}:generateContent?key=${apiKey}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + contents: [{ role: 'user', parts: [{ text: prompt }] }], + generationConfig: { temperature: 0.2, maxOutputTokens: 8192 }, + }), + signal: AbortSignal.timeout(120_000), + }); + if (!response.ok) { + const body = await response.text().catch(() => ''); + throw new Error(`Gemini HTTP ${response.status}: ${body.slice(0, 300)}`); + } + const data = (await response.json()) as { + candidates?: Array<{ content?: { parts?: Array<{ text?: string }> } }>; + }; + const text = data.candidates?.[0]?.content?.parts?.map((part) => part.text ?? '').join(''); + if (!text) throw new Error('Gemini returned an empty response.'); + return text; +} + +async function callOpenRouter(prompt: string, env: NodeJS.ProcessEnv): Promise { + const apiKey = env.OPENROUTER_API_KEY; + const model = env.OPENROUTER_MODEL ?? 'deepseek/deepseek-chat-v3-0324:free'; + const response = await fetch(OPENROUTER_ENDPOINT, { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + 'HTTP-Referer': 'https://basemodel.ai', + 'X-Title': 'BaseModel gateway generator', + }, + body: JSON.stringify({ + model, + messages: [{ role: 'user', content: prompt }], + temperature: 0.2, + max_tokens: 8192, + }), + signal: AbortSignal.timeout(120_000), + }); + if (!response.ok) { + const body = await response.text().catch(() => ''); + throw new Error(`OpenRouter HTTP ${response.status}: ${body.slice(0, 300)}`); + } + const data = (await response.json()) as { + choices?: Array<{ message?: { content?: string } }>; + }; + const text = data.choices?.[0]?.message?.content; + if (!text) throw new Error('OpenRouter returned an empty response.'); + return text; +} + +export async function generateText( + config: LlmConfig, + env: NodeJS.ProcessEnv = process.env, +): Promise { + const provider = pickProvider(env); + if (provider === 'gemini') return callGemini(config.prompt, env); + return callOpenRouter(config.prompt, env); +} diff --git a/packages/collectors/src/gateway-gen/manifest.ts b/packages/collectors/src/gateway-gen/manifest.ts new file mode 100644 index 000000000..3c021f1b8 --- /dev/null +++ b/packages/collectors/src/gateway-gen/manifest.ts @@ -0,0 +1,78 @@ +import { existsSync } from 'node:fs'; +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { z } from 'zod'; + +export const GatewayAuthSchema = z + .object({ + type: z.enum(['bearer', 'header', 'query']), + secret: z.string().min(1), + headerName: z.string().optional(), + }) + .optional(); + +export const ManifestGatewaySchema = z.object({ + id: z.string().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/), + baseUrl: z.string().url(), + endpoint: z.string().optional(), + method: z.enum(['GET', 'POST']).default('GET'), + auth: GatewayAuthSchema, + extraHeaders: z.record(z.string(), z.string()).optional(), + secrets: z.array(z.string()).default([]), + sample: z.unknown().optional(), + notes: z.string().optional(), +}); + +export const ManifestSchema = z.object({ + version: z.number().int().positive().default(1), + gateways: z.array(ManifestGatewaySchema), +}); + +export type ManifestGateway = z.infer; +export type Manifest = z.infer; + +export const GATEWAYS_DIR = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '..', + 'gateways', +); + +export const FIXTURES_DIR = path.join(GATEWAYS_DIR, '__fixtures__'); + +export function getManifestPath(): string { + return path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..', 'manifest.json'); +} + +export async function loadManifest(): Promise { + const manifestPath = getManifestPath(); + if (!existsSync(manifestPath)) { + throw new Error(`Manifest not found: ${manifestPath}`); + } + const raw = await readFile(manifestPath, 'utf-8'); + const parsed = ManifestSchema.safeParse(JSON.parse(raw) as unknown); + if (!parsed.success) { + throw new Error(`Invalid gateway manifest: ${parsed.error.message}`); + } + return parsed.data; +} + +export async function findGateway(gatewayId: string): Promise { + const manifest = await loadManifest(); + const gateway = manifest.gateways.find((entry) => entry.id === gatewayId); + if (!gateway) { + throw new Error( + `Gateway "${gatewayId}" is not listed in ${getManifestPath()}. ` + + 'Add it first so the generator knows its endpoint and auth.', + ); + } + return gateway; +} + +export function getGatewayPluginPath(gatewayId: string): string { + return path.join(GATEWAYS_DIR, `${gatewayId}.ts`); +} + +export function getFixturePath(gatewayId: string): string { + return path.join(FIXTURES_DIR, `${gatewayId}.raw.json`); +} diff --git a/packages/collectors/src/gateway-gen/probe.ts b/packages/collectors/src/gateway-gen/probe.ts new file mode 100644 index 000000000..a46f1ca51 --- /dev/null +++ b/packages/collectors/src/gateway-gen/probe.ts @@ -0,0 +1,242 @@ +import { existsSync } from 'node:fs'; +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import type { ManifestGateway } from './manifest.js'; +import { FIXTURES_DIR, getFixturePath } from './manifest.js'; + +const DEFAULT_ENDPOINTS = [ + '/models', + '/v1/models', + '/api/models', + '/api/v1/models', + '/model', + '/models?limit=5', +]; + +const RETRYABLE_STATUSES = new Set([408, 429, 500, 502, 503, 504]); + +export interface ModelArrayCandidate { + path: string; + count: number; + keys: string[]; + example: unknown; +} + +export interface ShapeSummary { + topLevel: Record; + modelArray: ModelArrayCandidate | null; +} + +export interface ProbeResult { + gatewayId: string; + fixturePath: string; + endpoint: string; + fromSample: boolean; + raw: unknown; + shape: ShapeSummary; +} + +function typeName(value: unknown): string { + if (value === null) return 'null'; + if (Array.isArray(value)) return `array[${value.length}]`; + if (typeof value === 'object') return 'object'; + return typeof value; +} + +function truncateString(value: string, max = 120): string { + return value.length > max ? `${value.slice(0, max)}...` : value; +} + +function truncateExample(value: unknown, depth = 0): unknown { + if (depth > 3) return typeof value; + if (Array.isArray(value)) { + return value.slice(0, 5).map((item) => truncateExample(item, depth + 1)); + } + if (value !== null && typeof value === 'object') { + const out: Record = {}; + for (const [key, item] of Object.entries(value as Record)) { + out[key] = typeof item === 'string' ? truncateString(item) : truncateExample(item, depth + 1); + } + return out; + } + return value; +} + +function findModelArray(value: unknown): ModelArrayCandidate | null { + let best: ModelArrayCandidate | null = null; + const visit = (node: unknown, pathLabel: string): void => { + if (!node || typeof node !== 'object') return; + if (Array.isArray(node)) { + const objects = node.filter((item) => item !== null && typeof item === 'object'); + const hasIdOrName = objects.some( + (item) => + typeof (item as Record).id === 'string' || + typeof (item as Record).name === 'string', + ); + if (objects.length > 0 && hasIdOrName && (!best || objects.length > best.count)) { + const first = objects[0] as Record; + best = { + path: pathLabel, + count: objects.length, + keys: Object.keys(first), + example: truncateExample(first), + }; + } + for (const item of objects.slice(0, 5)) visit(item, `${pathLabel}[i]`); + return; + } + for (const [key, item] of Object.entries(node as Record)) { + visit(item, `${pathLabel}.${key}`); + } + }; + visit(value, '$'); + return best; +} + +export function extractShape(raw: unknown): ShapeSummary { + const topLevel: Record = {}; + if (raw !== null && typeof raw === 'object' && !Array.isArray(raw)) { + for (const [key, value] of Object.entries(raw as Record)) { + topLevel[key] = typeName(value); + } + } + return { topLevel, modelArray: findModelArray(raw) }; +} + +function redact(value: string, secrets: readonly string[]): string { + return secrets + .filter(Boolean) + .reduce((result, secret) => result.split(String(secret)).join('[REDACTED]'), value); +} + +function truncateArrays(value: unknown, depth = 0): unknown { + if (depth > 8) return typeof value; + if (Array.isArray(value)) { + return value.slice(0, 5).map((item) => truncateArrays(item, depth + 1)); + } + if (value !== null && typeof value === 'object') { + const out: Record = {}; + for (const [key, item] of Object.entries(value as Record)) { + out[key] = truncateArrays(item, depth + 1); + } + return out; + } + return value; +} + +function buildHeaders( + gateway: ManifestGateway, + env: NodeJS.ProcessEnv, +): { headers: Record; ok: boolean; missing: string[] } { + const headers: Record = { Accept: 'application/json' }; + if (gateway.extraHeaders) Object.assign(headers, gateway.extraHeaders); + const missing: string[] = []; + if (gateway.auth) { + const key = env[gateway.auth.secret]; + if (!key) { + missing.push(gateway.auth.secret); + } else if (gateway.auth.type === 'bearer') { + headers.Authorization = `Bearer ${key}`; + } else if (gateway.auth.type === 'header') { + headers[gateway.auth.headerName ?? 'x-api-key'] = key; + } + } + if (gateway.method === 'POST') headers['Content-Type'] = 'application/json'; + return { headers, ok: missing.length === 0, missing }; +} + +async function fetchJson( + url: string, + init: RequestInit, + attempts = 2, +): Promise<{ ok: boolean; status: number; data: unknown }> { + let last: Response | null = null; + for (let attempt = 1; attempt <= attempts; attempt++) { + const response = await fetch(url, init); + if (!RETRYABLE_STATUSES.has(response.status)) { + const data = await response.json().catch(() => null); + return { ok: response.ok, status: response.status, data }; + } + last = response; + if (attempt < attempts) await new Promise((resolve) => setTimeout(resolve, attempt * 500)); + } + return { ok: false, status: last?.status ?? 0, data: null }; +} + +export async function probeGateway( + gateway: ManifestGateway, + env: NodeJS.ProcessEnv = process.env, +): Promise { + if (gateway.sample !== undefined) { + const fixturePath = await saveFixture(gateway.id, gateway.sample, gateway.secrets); + return { + gatewayId: gateway.id, + fixturePath, + endpoint: gateway.endpoint ?? '(manifest sample)', + fromSample: true, + raw: gateway.sample, + shape: extractShape(gateway.sample), + }; + } + + const { headers, ok, missing } = buildHeaders(gateway, env); + if (!ok) { + throw new Error( + `Cannot probe ${gateway.id}: missing API key(s) ${missing.join(', ')}. ` + + 'Set them in the environment, or add a "sample" to the manifest to skip live probing.', + ); + } + + const endpoints = gateway.endpoint ? [gateway.endpoint] : DEFAULT_ENDPOINTS; + const failures: string[] = []; + for (const endpoint of endpoints) { + const url = new URL(endpoint, gateway.baseUrl).toString(); + try { + const result = await fetchJson(url, { + method: gateway.method, + headers, + signal: AbortSignal.timeout(15_000), + }); + if (result.ok && result.data !== null) { + const fixturePath = await saveFixture(gateway.id, result.data, gateway.secrets); + return { + gatewayId: gateway.id, + fixturePath, + endpoint, + fromSample: false, + raw: result.data, + shape: extractShape(result.data), + }; + } + failures.push(`${endpoint} -> HTTP ${result.status}`); + } catch (error: unknown) { + failures.push(`${endpoint} -> ${error instanceof Error ? error.message : String(error)}`); + } + } + + throw new Error( + `Probe for ${gateway.id} failed on all candidate endpoints: ${failures.join(' | ')}`, + ); +} + +export async function saveFixture( + gatewayId: string, + raw: unknown, + secrets: readonly string[], +): Promise { + await mkdir(FIXTURES_DIR, { recursive: true }); + const serialized = JSON.stringify(truncateArrays(raw), null, 2); + const safe = redact(serialized, secrets); + const fixturePath = getFixturePath(gatewayId); + await writeFile(fixturePath, `${safe}\n`, 'utf-8'); + return fixturePath; +} + +export async function readFixture(gatewayId: string): Promise { + const fixturePath = getFixturePath(gatewayId); + const raw = await readFile(fixturePath, 'utf-8'); + return JSON.parse(raw) as unknown; +} + +export function fixtureExists(gatewayId: string): boolean { + return existsSync(getFixturePath(gatewayId)); +} diff --git a/packages/collectors/src/gateway-gen/prompts.ts b/packages/collectors/src/gateway-gen/prompts.ts new file mode 100644 index 000000000..8f04ae749 --- /dev/null +++ b/packages/collectors/src/gateway-gen/prompts.ts @@ -0,0 +1,157 @@ +import type { ManifestGateway } from './manifest.js'; +import type { ShapeSummary } from './probe.js'; + +const MODEL_SCHEMA_DOC = ` +Canonical "Model" fields (TypeScript, from @basemodel/schema). Fields marked [required] must always be set: +- model_id: string [required] must match /^[a-z0-9-]+\\/[a-z0-9]+(?:[-.][a-z0-9]+)*$/ e.g. "cohere/command-r-plus" (provider slug / model slug) +- provider_id: string [required] e.g. "cohere" +- name: string [required] human-readable model name +- family: string (optional) e.g. "Command" +- version: string (optional) +- release_date: string (optional) YYYY-MM-DD +- description: string (optional) +- architecture: string (optional) +- parameter_size: string (optional) e.g. "70B" +- context_window: number (optional) in tokens +- modality: array of "text"|"image"|"audio"|"video"|"code"|"embedding" [required] +- open_weight: boolean [required] +- reasoning_support: boolean [required] +- function_calling: boolean [required] +- structured_output: boolean [required] +- vision_support: boolean [required] +- audio_support: boolean [required] +- image_generation: boolean [required] +- embedding_support: boolean [required] +- capability_ids: string[] (optional) +- license_id: string (optional) +- status: "active"|"preview"|"deprecated"|"discontinued" [required] +`; + +const PLUGIN_CONTRACT = ` +The plugin is a TypeScript module with a default export that satisfies CustomGateway: + +import { z } from 'zod'; +import type { CollectionResult, CustomGateway } from '../core/collector'; + +export default { + type: 'custom', + id: '', + async collect(secrets: Record): Promise { + const result: CollectionResult = { provider_id: '', models: [], errors: [] }; + // ... fetch the catalog, parse with zod, map entries to partial Models ... + return result; + }, +} satisfies CustomGateway; + +Guidelines: +- Build the URL from the manifest's baseUrl + endpoint. Use only the secret names granted by the manifest. +- Validate the raw JSON with a zod schema; on parse failure push the error message to result.errors and return. +- Map each catalog entry to a Model. model_id must be "/"; derive the slug from the provider's id/name using .toLowerCase() and replacing characters outside [a-z0-9.-] with '-', collapsing repeats. +- Set every required boolean field. Derive modality/capability flags from hints in the response (e.g. name/keywords like "embed", "image", "audio", "vision", "code") when available, otherwise default to text-only. +- Handle pagination if the response exposes it (next page URL / total counts); never loop forever (cap pages). +- Use AbortSignal.timeout(15_000) on fetch. Catch errors and push them to result.errors (do not throw). +- Never hardcode real API keys; only reference secrets by their env names. +- Do not add code comments. +`; + +function describeShape(shape: ShapeSummary, raw: unknown): string { + const lines: string[] = ['Raw response shape summary:']; + lines.push(` top-level keys: ${JSON.stringify(shape.topLevel)}`); + if (shape.modelArray) { + lines.push( + ` model array at "${shape.modelArray.path}" (${shape.modelArray.count} items), keys: ` + + JSON.stringify(shape.modelArray.keys), + ); + lines.push(` example record: ${JSON.stringify(shape.modelArray.example, null, 2)}`); + } else { + lines.push(' no obvious array of models detected; inspect the raw sample.'); + } + lines.push(''); + lines.push('Raw sample (truncated):'); + lines.push(JSON.stringify(raw, null, 2).slice(0, 6000)); + return lines.join('\n'); +} + +export function buildPluginPrompt( + gateway: ManifestGateway, + shape: ShapeSummary, + raw: unknown, +): string { + return [ + 'You are writing a BaseModel gateway plugin in TypeScript. BaseModel collects a catalog of ', + 'AI models from provider APIs. Write a complete, compilable custom gateway plugin for the ', + 'provider described below.', + '', + 'Gateway manifest entry:', + JSON.stringify( + { + id: gateway.id, + baseUrl: gateway.baseUrl, + endpoint: gateway.endpoint, + method: gateway.method, + auth: gateway.auth, + extraHeaders: gateway.extraHeaders, + secrets: gateway.secrets, + }, + null, + 2, + ), + '', + MODEL_SCHEMA_DOC, + '', + PLUGIN_CONTRACT, + '', + describeShape(shape, raw), + '', + 'Output ONLY the complete TypeScript file content. Do not wrap it in markdown fences, do not ', + 'add explanations or a leading title. The response must be valid TypeScript that can be ', + 'written verbatim to src/gateways/.ts.', + ].join('\n'); +} + +export function buildHealPrompt( + gateway: ManifestGateway, + shape: ShapeSummary, + raw: unknown, + currentCode: string, + errors: string[], +): string { + const errorBlock = + errors.length > 0 + ? ['Validation errors to fix:', ...errors.map((error) => `- ${error}`)].join('\n') + : 'No specific errors were provided; inspect the mapping carefully and fix anything that looks wrong.'; + + return [ + 'You previously generated a BaseModel gateway plugin for the provider below. It needs fixing.', + '', + 'Gateway manifest entry:', + JSON.stringify( + { + id: gateway.id, + baseUrl: gateway.baseUrl, + endpoint: gateway.endpoint, + auth: gateway.auth, + extraHeaders: gateway.extraHeaders, + secrets: gateway.secrets, + }, + null, + 2, + ), + '', + MODEL_SCHEMA_DOC, + '', + PLUGIN_CONTRACT, + '', + errorBlock, + '', + 'Current (broken) plugin:', + '```ts', + currentCode, + '```', + '', + describeShape(shape, raw), + '', + 'Output ONLY the complete corrected TypeScript file content. Do not wrap it in markdown ', + 'fences, do not add explanations.', + ].join('\n'); +} diff --git a/packages/collectors/src/gateway-gen/write.ts b/packages/collectors/src/gateway-gen/write.ts new file mode 100644 index 000000000..3be93885f --- /dev/null +++ b/packages/collectors/src/gateway-gen/write.ts @@ -0,0 +1,128 @@ +import { writeFile } from 'node:fs/promises'; +import { pathToFileURL } from 'node:url'; +import { ModelSchema } from '@basemodel/schema'; +import type { CollectionResult, GatewayPlugin } from '../core/collector.js'; +import { generateText } from './llm.js'; +import type { ManifestGateway } from './manifest.js'; +import { getGatewayPluginPath } from './manifest.js'; +import type { ShapeSummary } from './probe.js'; +import { buildPluginPrompt } from './prompts.js'; + +export interface GeneratePluginOptions { + gateway: ManifestGateway; + shape: ShapeSummary; + raw: unknown; + env?: NodeJS.ProcessEnv; + maxAttempts?: number; +} + +export interface GeneratedPlugin { + filePath: string; + code: string; + attempts: number; +} + +export function extractTsCode(text: string): string { + const fenced = text.match(/```(?:ts|typescript)?\s*([\s\S]*?)```/); + if (fenced) return (fenced[1] ?? text).trim(); + return text.trim(); +} + +export function checkForbiddenPatterns(code: string): string[] { + const problems: string[] = []; + if (/\b(?:catch\s*\([^)]*:\s*any\s*\)|as\s+any\b|:\s*any\s*[,;)\n])/.test(code)) { + problems.push('code uses the "any" type (catch (e: any), : any, as any)'); + } + if (/sk-[A-Za-z0-9]{8,}|AIza[A-Za-z0-9_-]{10,}|Bearer\s+[A-Za-z0-9_-]{20,}/.test(code)) { + problems.push('code appears to hardcode an API key'); + } + if (/<\s*%|process\.exit\s*\(|child_process|require\(['"]child_process/.test(code)) { + problems.push('code uses process.exit or child_process (not allowed)'); + } + return problems; +} + +export function ensureFinalNewline(code: string): string { + return code.endsWith('\n') ? code : `${code}\n`; +} + +export interface PluginValidation { + ok: boolean; + errors: string[]; + plugin?: GatewayPlugin; +} + +export async function validatePluginModule( + filePath: string, + expectedId: string, +): Promise { + try { + const moduleUrl = `${pathToFileURL(filePath).href}?v=${Date.now()}`; + const module = (await import(moduleUrl)) as { default?: unknown }; + const plugin = module.default as GatewayPlugin | undefined; + const errors: string[] = []; + if (!plugin || typeof plugin !== 'object') { + errors.push('module has no default export'); + } else { + if (plugin.type !== 'custom') errors.push(`expected type "custom", got "${plugin.type}"`); + if (plugin.id !== expectedId) errors.push(`expected id "${expectedId}", got "${plugin.id}"`); + if (typeof (plugin as { collect?: unknown }).collect !== 'function') { + errors.push('default export has no collect() function'); + } + } + if (errors.length > 0) return { ok: false, errors, plugin }; + return { ok: true, errors: [], plugin }; + } catch (error: unknown) { + return { ok: false, errors: [error instanceof Error ? error.message : String(error)] }; + } +} + +export interface CollectionCheck { + modelCount: number; + validCount: number; + errors: string[]; +} + +export async function checkCollection( + plugin: GatewayPlugin, + secrets: Record, +): Promise { + const result: CollectionResult = await ( + plugin as { + collect: (secrets: Record) => Promise; + } + ).collect(secrets); + let validCount = 0; + for (const model of result.models) { + const parsed = ModelSchema.safeParse(model); + if (parsed.success) validCount += 1; + } + if (result.models.length !== validCount) { + result.errors.push(`${result.models.length - validCount} models failed ModelSchema validation`); + } + return { modelCount: result.models.length, validCount, errors: result.errors }; +} + +export async function generatePlugin(options: GeneratePluginOptions): Promise { + const { gateway, shape, raw, env = process.env, maxAttempts = 3 } = options; + const filePath = getGatewayPluginPath(gateway.id); + let prompt = buildPluginPrompt(gateway, shape, raw); + const lastErrors: string[] = []; + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + const text = await generateText({ prompt }, env); + const code = ensureFinalNewline(extractTsCode(text)); + await writeFile(filePath, code, 'utf-8'); + const problems = checkForbiddenPatterns(code); + const validation = await validatePluginModule(filePath, gateway.id); + const allErrors = [...problems, ...validation.errors]; + if (allErrors.length === 0) return { filePath, code, attempts: attempt }; + lastErrors.push(...allErrors); + console.warn(`[gen] attempt ${attempt} failed validation: ${allErrors.join('; ')}`); + prompt += `\n\nThe previous attempt failed validation:\n${allErrors.join('\n')}\n\nReturn the corrected file only.`; + } + + throw new Error( + `Failed to generate a valid plugin for ${gateway.id} after ${maxAttempts} attempts:\n${lastErrors.join('\n')}`, + ); +} diff --git a/packages/collectors/src/gateways/__fixtures__/cohere.raw.json b/packages/collectors/src/gateways/__fixtures__/cohere.raw.json new file mode 100644 index 000000000..cc4379934 --- /dev/null +++ b/packages/collectors/src/gateways/__fixtures__/cohere.raw.json @@ -0,0 +1,50 @@ +{ + "models": [ + { + "name": "command-r-plus", + "endpoints": [ + "chat", + "embed", + "generate", + "summarize", + "classify" + ], + "context_length": 128000, + "tokenizer_url": "https://docs.cohere.com/docs/tokenizer" + }, + { + "name": "command-r", + "endpoints": [ + "chat", + "embed", + "generate", + "summarize", + "classify" + ], + "context_length": 128000 + }, + { + "name": "command-light", + "endpoints": [ + "chat", + "generate", + "summarize" + ], + "context_length": 4096 + }, + { + "name": "embed-english-v3.0", + "endpoints": [ + "embed" + ], + "context_length": 512 + }, + { + "name": "rerank-english-v3.0", + "endpoints": [ + "rerank" + ], + "context_length": 4096 + } + ] +} diff --git a/packages/collectors/src/gateways/cohere.ts b/packages/collectors/src/gateways/cohere.ts new file mode 100644 index 000000000..84f5416c6 --- /dev/null +++ b/packages/collectors/src/gateways/cohere.ts @@ -0,0 +1,111 @@ +import { z } from 'zod'; +import type { CollectionResult, CustomGateway } from '../core/collector'; + +const cohereModelSchema = z.object({ + name: z.string(), + endpoints: z.array(z.string()).optional(), + context_length: z.number().optional(), +}); + +const cohereResponseSchema = z.object({ + models: z.array(cohereModelSchema), + next_page_token: z.string().optional(), +}); + +export default { + type: 'custom', + id: 'cohere', + async collect(secrets: Record): Promise { + const result: CollectionResult = { provider_id: 'cohere', models: [], errors: [] }; + const apiKey = secrets.COHERE_API_KEY; + + if (!apiKey) { + result.errors.push('Missing secret COHERE_API_KEY'); + return result; + } + + let url: string | undefined = 'https://api.cohere.com/v1/models'; + let pagesFetched = 0; + const maxPages = 10; + + try { + while (url && pagesFetched < maxPages) { + pagesFetched++; + const response = await fetch(url, { + method: 'GET', + headers: { + Authorization: `Bearer ${apiKey}`, + Accept: 'application/json', + }, + signal: AbortSignal.timeout(15_000), + }); + + if (!response.ok) { + result.errors.push(`Failed to fetch models: ${response.status} ${response.statusText}`); + break; + } + + const json = await response.json(); + const parsed = cohereResponseSchema.safeParse(json); + + if (!parsed.success) { + result.errors.push(`Failed to parse response: ${parsed.error.message}`); + break; + } + + for (const item of parsed.data.models) { + const rawName = item.name; + const slug = rawName + .toLowerCase() + .replace(/[^a-z0-9.-]/g, '-') + .replace(/-+/g, '-') + .replace(/^-|-$/g, ''); + const model_id = `cohere/${slug}`; + + const endpoints = item.endpoints || []; + const isEmbed = endpoints.includes('embed') || rawName.includes('embed'); + const isChat = endpoints.includes('chat') || endpoints.includes('generate'); + + const modality: ('text' | 'image' | 'audio' | 'video' | 'code' | 'embedding')[] = []; + if (isChat) { + modality.push('text'); + } + if (isEmbed) { + modality.push('embedding'); + } + if (modality.length === 0) { + modality.push('text'); + } + + result.models.push({ + model_id, + provider_id: 'cohere', + name: rawName, + family: rawName.includes('command') ? 'Command' : undefined, + context_window: item.context_length, + modality, + open_weight: false, + reasoning_support: false, + function_calling: false, + structured_output: false, + vision_support: false, + audio_support: false, + image_generation: false, + embedding_support: isEmbed, + status: 'active', + }); + } + + url = undefined; + } + } catch (err: unknown) { + if (err instanceof Error) { + result.errors.push(err.message); + } else { + result.errors.push(String(err)); + } + } + + return result; + }, +} satisfies CustomGateway; From 0e2cbebc0ca791f15f266675ed9d9b2db6941fa2 Mon Sep 17 00:00:00 2001 From: ngoding sendiri Date: Sat, 1 Aug 2026 19:22:49 +0700 Subject: [PATCH 02/15] docs(gateways): document AI gateway generation flow --- README.md | 1 + docs/04_Pipeline.md | 21 +++++++++++++++++++++ docs/08_Gateway_Plugin_Security.md | 21 +++++++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/README.md b/README.md index 99ce80459..8b183d9d6 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ Useful package-level commands: - `pnpm --filter @basemodel/collectors run collect` - `pnpm --filter @basemodel/collectors run verify packages/collectors/src/gateways/openai.ts` +- `pnpm --filter @basemodel/collectors run gen-gateway ` - generate a custom gateway plugin for a gateway registered in `packages/collectors/manifest.json` ## License diff --git a/docs/04_Pipeline.md b/docs/04_Pipeline.md index d9d8213b8..e01c6a8c2 100644 --- a/docs/04_Pipeline.md +++ b/docs/04_Pipeline.md @@ -25,6 +25,26 @@ Collectors fetch data from official APIs, documentation, or other approved sourc Current collector support includes both OpenAI-compatible gateway plugins and custom gateway plugins. +#### AI Gateway Generation + +Providers whose APIs do not match the OpenAI-compatible shape can register a +gateway in `packages/collectors/manifest.json` (base URL, auth, and optionally a +static sample response). A free-tier LLM then generates a custom gateway plugin: + +1. `pnpm --filter @basemodel/collectors run gen-gateway ` probes the + endpoint (or uses the sample), captures a redacted fixture, and summarizes + the response shape. +2. The LLM writes `packages/collectors/src/gateways/.ts` implementing the + `CustomGateway` contract. +3. Output is structurally validated before use (importable module, correct id, + `collect()` present, no forbidden patterns such as `any` or hardcoded keys) + and retried up to 3 times. +4. `--heal` regenerates a plugin from its existing fixture when a sample changes + or a mapping breaks. + +Generated plugins are proposed through a pull request and must be reviewed +before merge. See `docs/08_Gateway_Plugin_Security.md`. + ### Validation Validation checks required fields, identifier formats, schema compliance, URL validity, and timestamp formats. @@ -98,3 +118,4 @@ The pipeline is automated through GitHub Actions: - `publish.yml` regenerates datasets on push to `main`. - `deploy-pages.yml` publishes the generated static files. - `verify-gateway.yml` checks gateway plugin changes. +- `gateway-ai.yml` generates gateway plugins with an LLM and opens a review PR; it also validates the manifest on PRs that touch it. diff --git a/docs/08_Gateway_Plugin_Security.md b/docs/08_Gateway_Plugin_Security.md index 0560040bf..2a9c1ae5f 100644 --- a/docs/08_Gateway_Plugin_Security.md +++ b/docs/08_Gateway_Plugin_Security.md @@ -26,6 +26,27 @@ the central secret registry. An unregistered gateway receives no secrets. A plugin file must not be able to escalate its own privileges by declaring a new secret name. +## AI-Generated Gateways + +`gateway-gen` writes custom gateway plugins with a free-tier LLM (Gemini +flash-lite by default, OpenRouter `:free` fallback) for providers registered in +`packages/collectors/manifest.json`. The generator stays deterministic about +data: it probes the endpoint (or reads a static sample) and feeds the LLM a +redacted fixture plus a shape summary, never live credentials. + +AI output is treated as untrusted code. Before a generated plugin is accepted: + +- It is structurally validated before writing (importable module, correct + gateway id, `collect()` present) and rejected when it uses forbidden patterns + (`any`, hardcoded API keys, `process.exit`) or refers to secrets not listed in + `gateway-secrets.ts`. +- The workflow opens a pull request; human review is required before merge. +- Reviewers re-check secret usage and add the provider's API key as a repository + secret; only then does nightly collection use the plugin. + +New custom gateways therefore pass through the same review gate as hand-written +plugins. + ## Remaining Risk Process isolation reduces exposure, but it is not an operating-system sandbox. From 24c947fb77512030dfeb4062134b1a425155bc26 Mon Sep 17 00:00:00 2001 From: ngoding sendiri Date: Sat, 1 Aug 2026 19:24:05 +0700 Subject: [PATCH 03/15] chore: reindex workflow metadata From 85427986d2de75ec800181ceac897e728a160c0f Mon Sep 17 00:00:00 2001 From: ngoding sendiri Date: Sat, 1 Aug 2026 19:24:42 +0700 Subject: [PATCH 04/15] test: add throwaway dispatch workflow --- .github/workflows/dispatch-test.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .github/workflows/dispatch-test.yml diff --git a/.github/workflows/dispatch-test.yml b/.github/workflows/dispatch-test.yml new file mode 100644 index 000000000..c6a0b5be8 --- /dev/null +++ b/.github/workflows/dispatch-test.yml @@ -0,0 +1,9 @@ +name: Dispatch Test +on: + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - run: echo hello From 8b3844143e665570116fd68cd1918a95e44232e4 Mon Sep 17 00:00:00 2001 From: ngoding sendiri Date: Sat, 1 Aug 2026 19:27:31 +0700 Subject: [PATCH 05/15] fix(workflow): indent multi-line PR body so YAML parses (block scalar terminated early) --- .github/workflows/dispatch-test.yml | 9 --------- .github/workflows/gateway-ai.yml | 8 ++++---- 2 files changed, 4 insertions(+), 13 deletions(-) delete mode 100644 .github/workflows/dispatch-test.yml diff --git a/.github/workflows/dispatch-test.yml b/.github/workflows/dispatch-test.yml deleted file mode 100644 index c6a0b5be8..000000000 --- a/.github/workflows/dispatch-test.yml +++ /dev/null @@ -1,9 +0,0 @@ -name: Dispatch Test -on: - workflow_dispatch: - -jobs: - test: - runs-on: ubuntu-latest - steps: - - run: echo hello diff --git a/.github/workflows/gateway-ai.yml b/.github/workflows/gateway-ai.yml index 27cf4fbdd..a4953f67b 100644 --- a/.github/workflows/gateway-ai.yml +++ b/.github/workflows/gateway-ai.yml @@ -105,10 +105,10 @@ jobs: --title "feat(gateways): AI-generated plugin for ${{ github.event.inputs.gateway_id }}" \ --body "AI-generated gateway plugin for \`${{ github.event.inputs.gateway_id }}\` (mode: ${{ github.event.inputs.mode }}). -Reviewed before merge, per docs/08_Gateway_Plugin_Security.md: -- Confirm the plugin only uses secrets registered in \`packages/collectors/src/core/gateway-secrets.ts\`. -- Add the gateway's API key as a repository secret so nightly collection can use it. -- Run \`pnpm --filter @basemodel/collectors run verify \` to smoke-test with a live key." \ + Reviewed before merge, per docs/08_Gateway_Plugin_Security.md: + - Confirm the plugin only uses secrets registered in \`packages/collectors/src/core/gateway-secrets.ts\`. + - Add the gateway's API key as a repository secret so nightly collection can use it. + - Run \`pnpm --filter @basemodel/collectors run verify \` to smoke-test with a live key." \ || echo "PR may already exist for this gateway." env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From c2220cc342f31bf6ad58c70683791bea6d311b60 Mon Sep 17 00:00:00 2001 From: ngoding sendiri Date: Sat, 1 Aug 2026 19:35:01 +0700 Subject: [PATCH 06/15] feat(gateway-gen): validate typecheck and live collection in the retry loop The generator now treats typecheck failures and live collection errors as validation failures, so the LLM is asked to fix them (up to 3 attempts) instead of shipping a plugin that compiles structurally but breaks typecheck or maps live API data to invalid Model records. - validateGeneratedPlugin: forbidden patterns + structural + tsc --noEmit + optional live check - runTypecheck: spawns node with the local tsc.js (no shell, cross-platform) - bootstrap/heal pass liveSecrets into the loop; duplicate post-hoc live check removed - cohere.ts: fix pagination field (next_page -> next_page_token) from review of AI output --- packages/collectors/src/gateway-gen/heal.ts | 15 +++- packages/collectors/src/gateway-gen/index.ts | 47 ++++++------ packages/collectors/src/gateway-gen/write.ts | 72 +++++++++++++++--- packages/collectors/src/gateways/cohere.ts | 78 +++++++++----------- 4 files changed, 132 insertions(+), 80 deletions(-) diff --git a/packages/collectors/src/gateway-gen/heal.ts b/packages/collectors/src/gateway-gen/heal.ts index 4d0dbf26c..90305240d 100644 --- a/packages/collectors/src/gateway-gen/heal.ts +++ b/packages/collectors/src/gateway-gen/heal.ts @@ -8,7 +8,7 @@ import { ensureFinalNewline, extractTsCode, type GeneratedPlugin, - validatePluginModule, + validateGeneratedPlugin, } from './write.js'; export interface HealPluginOptions { @@ -18,10 +18,19 @@ export interface HealPluginOptions { errors?: string[]; env?: NodeJS.ProcessEnv; maxAttempts?: number; + liveSecrets?: Record; } export async function healPlugin(options: HealPluginOptions): Promise { - const { gateway, shape, raw, errors = [], env = process.env, maxAttempts = 3 } = options; + const { + gateway, + shape, + raw, + errors = [], + env = process.env, + maxAttempts = 3, + liveSecrets, + } = options; const filePath = getGatewayPluginPath(gateway.id); const currentCode = await readFile(filePath, 'utf-8'); let prompt = buildHealPrompt(gateway, shape, raw, currentCode, errors); @@ -31,7 +40,7 @@ export async function healPlugin(options: HealPluginOptions): Promise Boolean(env[name])); } -async function runLiveCheck(gatewayId: string, env: NodeJS.ProcessEnv): Promise { - const filePath = getGatewayPluginPath(gatewayId); - const validation = await validatePluginModule(filePath, gatewayId); - if (!validation.ok || !validation.plugin) { - console.warn(` (skip live check: ${validation.errors.join('; ')})`); - return; - } - const check = await checkCollection(validation.plugin, env); - console.log( - ` live check : ${check.modelCount} models, ${check.validCount} valid`, - check.errors.length > 0 ? `| errors: ${check.errors.join(' | ')}` : '', - ); +function liveSecretsFor( + gateway: { secrets: string[]; auth?: { secret?: string } }, + env: NodeJS.ProcessEnv, +): Record | undefined { + return hasAnySecret(gateway, env) ? env : undefined; } async function bootstrap(gatewayId: string, env: NodeJS.ProcessEnv): Promise { @@ -39,15 +32,19 @@ async function bootstrap(gatewayId: string, env: NodeJS.ProcessEnv): Promise { diff --git a/packages/collectors/src/gateway-gen/write.ts b/packages/collectors/src/gateway-gen/write.ts index 3be93885f..fac1aa526 100644 --- a/packages/collectors/src/gateway-gen/write.ts +++ b/packages/collectors/src/gateway-gen/write.ts @@ -1,5 +1,7 @@ -import { writeFile } from 'node:fs/promises'; -import { pathToFileURL } from 'node:url'; +import { spawnSync } from 'node:child_process'; +import { readFile, writeFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; import { ModelSchema } from '@basemodel/schema'; import type { CollectionResult, GatewayPlugin } from '../core/collector.js'; import { generateText } from './llm.js'; @@ -8,12 +10,16 @@ import { getGatewayPluginPath } from './manifest.js'; import type { ShapeSummary } from './probe.js'; import { buildPluginPrompt } from './prompts.js'; +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..'); +const tscBin = resolve(packageRoot, 'node_modules', 'typescript', 'lib', 'tsc.js'); + export interface GeneratePluginOptions { gateway: ManifestGateway; shape: ShapeSummary; raw: unknown; env?: NodeJS.ProcessEnv; maxAttempts?: number; + liveSecrets?: Record; } export interface GeneratedPlugin { @@ -103,8 +109,56 @@ export async function checkCollection( return { modelCount: result.models.length, validCount, errors: result.errors }; } +export function runTypecheck(cwd: string = packageRoot): string[] { + const result = spawnSync(process.execPath, [tscBin, '--noEmit'], { + cwd, + encoding: 'utf-8', + timeout: 120_000, + }); + const output = `${result.stdout ?? ''}\n${result.stderr ?? ''}`; + if (result.status === 0) return []; + const errors = output + .split('\n') + .filter((line) => /error TS\d+/.test(line)) + .map((line) => line.trim()) + .filter(Boolean); + return errors.length > 0 ? errors : [result.error?.message ?? 'typecheck failed']; +} + +export interface GeneratedPluginValidation { + ok: boolean; + errors: string[]; + liveCheck?: CollectionCheck; +} + +export async function validateGeneratedPlugin( + filePath: string, + gatewayId: string, + liveSecrets?: Record, +): Promise { + const code = await readFile(filePath, 'utf-8'); + const errors = [...checkForbiddenPatterns(code)]; + const validation = await validatePluginModule(filePath, gatewayId); + errors.push(...validation.errors); + if (errors.length === 0) { + errors.push(...runTypecheck().map((e) => `typecheck: ${e}`)); + } + let liveCheck: CollectionCheck | undefined; + if (errors.length === 0 && liveSecrets && validation.plugin) { + liveCheck = await checkCollection(validation.plugin, liveSecrets); + console.log( + ` live check : ${liveCheck.modelCount} models, ${liveCheck.validCount} valid`, + liveCheck.errors.length > 0 ? `| errors: ${liveCheck.errors.join(' | ')}` : '', + ); + if (liveCheck.errors.length > 0) { + errors.push(...liveCheck.errors.map((e) => `live check: ${e}`)); + } + } + return { ok: errors.length === 0, errors, liveCheck }; +} + export async function generatePlugin(options: GeneratePluginOptions): Promise { - const { gateway, shape, raw, env = process.env, maxAttempts = 3 } = options; + const { gateway, shape, raw, env = process.env, maxAttempts = 3, liveSecrets } = options; const filePath = getGatewayPluginPath(gateway.id); let prompt = buildPluginPrompt(gateway, shape, raw); const lastErrors: string[] = []; @@ -113,13 +167,11 @@ export async function generatePlugin(options: GeneratePluginOptions): Promise = + isEmbed ? ['embedding'] : ['text']; + const embedding_support = isEmbed; + const function_calling = endpoints.includes('chat') && !isEmbed; + const structured_output = endpoints.includes('chat') && !isEmbed; result.models.push({ model_id, provider_id: 'cohere', name: rawName, - family: rawName.includes('command') ? 'Command' : undefined, - context_window: item.context_length, + family: rawName.toLowerCase().includes('command') ? 'Command' : undefined, + context_window: rawModel.context_length, modality, open_weight: false, reasoning_support: false, - function_calling: false, - structured_output: false, + function_calling, + structured_output, vision_support: false, audio_support: false, image_generation: false, - embedding_support: isEmbed, + embedding_support, status: 'active', }); } - url = undefined; - } - } catch (err: unknown) { - if (err instanceof Error) { - result.errors.push(err.message); - } else { - result.errors.push(String(err)); + url = parsed.data.next_page_token; } + } catch (error) { + result.errors.push( + `Network or parsing error: ${error instanceof Error ? error.message : String(error)}`, + ); } return result; From 9b545b56fa93a7c5ce3d6cd0b14201403bd9ae19 Mon Sep 17 00:00:00 2001 From: ngoding sendiri Date: Sat, 1 Aug 2026 19:39:05 +0700 Subject: [PATCH 07/15] feat(gateway-gen): prefer live probing when a key is available probeGateway now probes the live endpoint first when the gateway's key is available, falling back to the manifest sample when it is not (or when the live probe fails). This feeds the LLM the real response shape (field names, nulls, pagination) instead of a stale static sample, so generated plugins match the live API and the loop's live check converges. Also instruct the LLM to never use 'any' (use unknown + narrowing) so attempts are not wasted on forbidden patterns. --- packages/collectors/src/gateway-gen/probe.ts | 86 ++++++----- .../collectors/src/gateway-gen/prompts.ts | 1 + packages/collectors/src/gateways/cohere.ts | 138 +++++++++--------- 3 files changed, 118 insertions(+), 107 deletions(-) diff --git a/packages/collectors/src/gateway-gen/probe.ts b/packages/collectors/src/gateway-gen/probe.ts index a46f1ca51..0341bb1b2 100644 --- a/packages/collectors/src/gateway-gen/probe.ts +++ b/packages/collectors/src/gateway-gen/probe.ts @@ -166,7 +166,10 @@ export async function probeGateway( gateway: ManifestGateway, env: NodeJS.ProcessEnv = process.env, ): Promise { - if (gateway.sample !== undefined) { + const { headers, ok, missing } = buildHeaders(gateway, env); + const hasSample = gateway.sample !== undefined; + + const sampleResult = async (): Promise => { const fixturePath = await saveFixture(gateway.id, gateway.sample, gateway.secrets); return { gatewayId: gateway.id, @@ -176,46 +179,61 @@ export async function probeGateway( raw: gateway.sample, shape: extractShape(gateway.sample), }; - } + }; - const { headers, ok, missing } = buildHeaders(gateway, env); - if (!ok) { + const liveResult = async (): Promise => { + const endpoints = gateway.endpoint ? [gateway.endpoint] : DEFAULT_ENDPOINTS; + const failures: string[] = []; + for (const endpoint of endpoints) { + const url = new URL(endpoint, gateway.baseUrl).toString(); + try { + const result = await fetchJson(url, { + method: gateway.method, + headers, + signal: AbortSignal.timeout(15_000), + }); + if (result.ok && result.data !== null) { + const fixturePath = await saveFixture(gateway.id, result.data, gateway.secrets); + return { + gatewayId: gateway.id, + fixturePath, + endpoint, + fromSample: false, + raw: result.data, + shape: extractShape(result.data), + }; + } + failures.push(`${endpoint} -> HTTP ${result.status}`); + } catch (error: unknown) { + failures.push(`${endpoint} -> ${error instanceof Error ? error.message : String(error)}`); + } + } throw new Error( - `Cannot probe ${gateway.id}: missing API key(s) ${missing.join(', ')}. ` + - 'Set them in the environment, or add a "sample" to the manifest to skip live probing.', + `Probe for ${gateway.id} failed on all candidate endpoints: ${failures.join(' | ')}`, ); - } + }; - const endpoints = gateway.endpoint ? [gateway.endpoint] : DEFAULT_ENDPOINTS; - const failures: string[] = []; - for (const endpoint of endpoints) { - const url = new URL(endpoint, gateway.baseUrl).toString(); + if (hasSample && !ok) { + console.warn(` probe : no API key available, using manifest sample`); + return sampleResult(); + } + if (hasSample && ok) { try { - const result = await fetchJson(url, { - method: gateway.method, - headers, - signal: AbortSignal.timeout(15_000), - }); - if (result.ok && result.data !== null) { - const fixturePath = await saveFixture(gateway.id, result.data, gateway.secrets); - return { - gatewayId: gateway.id, - fixturePath, - endpoint, - fromSample: false, - raw: result.data, - shape: extractShape(result.data), - }; - } - failures.push(`${endpoint} -> HTTP ${result.status}`); - } catch (error: unknown) { - failures.push(`${endpoint} -> ${error instanceof Error ? error.message : String(error)}`); + return await liveResult(); + } catch (error) { + console.warn( + ` probe : live probe failed (${error instanceof Error ? error.message : String(error)}), using manifest sample`, + ); + return sampleResult(); } } - - throw new Error( - `Probe for ${gateway.id} failed on all candidate endpoints: ${failures.join(' | ')}`, - ); + if (!ok) { + throw new Error( + `Cannot probe ${gateway.id}: missing API key(s) ${missing.join(', ')}. ` + + 'Set them in the environment, or add a "sample" to the manifest to skip live probing.', + ); + } + return liveResult(); } export async function saveFixture( diff --git a/packages/collectors/src/gateway-gen/prompts.ts b/packages/collectors/src/gateway-gen/prompts.ts index 8f04ae749..6b22553ca 100644 --- a/packages/collectors/src/gateway-gen/prompts.ts +++ b/packages/collectors/src/gateway-gen/prompts.ts @@ -50,6 +50,7 @@ Guidelines: - Set every required boolean field. Derive modality/capability flags from hints in the response (e.g. name/keywords like "embed", "image", "audio", "vision", "code") when available, otherwise default to text-only. - Handle pagination if the response exposes it (next page URL / total counts); never loop forever (cap pages). - Use AbortSignal.timeout(15_000) on fetch. Catch errors and push them to result.errors (do not throw). +- Never use the "any" type anywhere. Type catch variables as "unknown" and narrow with instanceof/typeof. Do not use "as any". - Never hardcode real API keys; only reference secrets by their env names. - Do not add code comments. `; diff --git a/packages/collectors/src/gateways/cohere.ts b/packages/collectors/src/gateways/cohere.ts index dd4761ad6..8d4aa97d5 100644 --- a/packages/collectors/src/gateways/cohere.ts +++ b/packages/collectors/src/gateways/cohere.ts @@ -1,16 +1,15 @@ import { z } from 'zod'; import type { CollectionResult, CustomGateway } from '../core/collector'; -const cohereRawSchema = z.object({ - models: z.array( - z.object({ - name: z.string(), - endpoints: z.array(z.string()).optional(), - context_length: z.number().optional(), - tokenizer_url: z.string().optional(), - }), - ), - next_page_token: z.string().optional(), +const cohereModelSchema = z.object({ + name: z.string(), + endpoints: z.array(z.string()).optional(), + context_length: z.number().optional(), + tokenizer_url: z.string().optional(), +}); + +const cohereResponseSchema = z.object({ + models: z.array(cohereModelSchema), }); export default { @@ -18,80 +17,73 @@ export default { id: 'cohere', async collect(secrets: Record): Promise { const result: CollectionResult = { provider_id: 'cohere', models: [], errors: [] }; - const apiKey = secrets.COHERE_API_KEY; + const apiKey = secrets['COHERE_API_KEY']; - let url: string | undefined = 'https://api.cohere.com/v1/models'; - let pagesFetched = 0; - const maxPages = 10; + if (!apiKey) { + result.errors.push('Missing secret COHERE_API_KEY'); + return result; + } try { - while (url && pagesFetched < maxPages) { - pagesFetched++; - const response = await fetch(url, { - method: 'GET', - headers: { - ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}), - 'Content-Type': 'application/json', - }, - signal: AbortSignal.timeout(15_000), - }); - - if (!response.ok) { - result.errors.push(`Failed to fetch models: ${response.status} ${response.statusText}`); - return result; - } - - const json = await response.json(); - const parsed = cohereRawSchema.safeParse(json); + const response = await fetch('https://api.cohere.com/v1/models', { + method: 'GET', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + signal: AbortSignal.timeout(15_000), + }); - if (!parsed.success) { - result.errors.push(`Failed to parse response: ${parsed.error.message}`); - return result; - } + if (!response.ok) { + result.errors.push(`Failed to fetch models: ${response.status} ${response.statusText}`); + return result; + } - for (const rawModel of parsed.data.models) { - const rawName = rawModel.name; - const slug = rawName - .toLowerCase() - .replace(/[^a-z0-9.-]+/g, '-') - .replace(/-+/g, '-') - .replace(/^-|-$/g, ''); - const model_id = `cohere/${slug}`; + const rawJson: unknown = await response.json(); + const parseResult = cohereResponseSchema.safeParse(rawJson); - const endpoints = rawModel.endpoints ?? []; - const isEmbed = endpoints.includes('embed') || rawName.includes('embed'); + if (!parseResult.success) { + result.errors.push(`Validation error: ${parseResult.error.message}`); + return result; + } - const modality: Array<'text' | 'image' | 'audio' | 'video' | 'code' | 'embedding'> = - isEmbed ? ['embedding'] : ['text']; - const embedding_support = isEmbed; - const function_calling = endpoints.includes('chat') && !isEmbed; - const structured_output = endpoints.includes('chat') && !isEmbed; + for (const item of parseResult.data.models) { + const rawName = item.name; + const slug = rawName.toLowerCase().replace(/[^a-z0-9.-]/g, '-').replace(/-+/g, '-').replace(/^-|-$/g, ''); + const modelId = `cohere/${slug}`; - result.models.push({ - model_id, - provider_id: 'cohere', - name: rawName, - family: rawName.toLowerCase().includes('command') ? 'Command' : undefined, - context_window: rawModel.context_length, - modality, - open_weight: false, - reasoning_support: false, - function_calling, - structured_output, - vision_support: false, - audio_support: false, - image_generation: false, - embedding_support, - status: 'active', - }); + if (!/^[a-z0-9-]+\/[a-z0-9]+(?:[-.][a-z0-9]+)*$/.test(modelId)) { + continue; } - url = parsed.data.next_page_token; + const endpoints = item.endpoints ?? []; + const isEmbed = endpoints.includes('embed') || rawName.includes('embed'); + const modality: Array<'text' | 'image' | 'audio' | 'video' | 'code' | 'embedding'> = isEmbed ? ['embedding'] : ['text']; + + result.models.push({ + model_id: modelId, + provider_id: 'cohere', + name: rawName, + family: rawName.includes('command') ? 'Command' : undefined, + context_window: item.context_length, + modality, + open_weight: false, + reasoning_support: false, + function_calling: endpoints.includes('chat'), + structured_output: false, + vision_support: false, + audio_support: false, + image_generation: false, + embedding_support: isEmbed, + status: 'active', + }); + } + } catch (error: unknown) { + if (error instanceof Error) { + result.errors.push(error.message); + } else { + result.errors.push('An unknown error occurred during collection'); } - } catch (error) { - result.errors.push( - `Network or parsing error: ${error instanceof Error ? error.message : String(error)}`, - ); } return result; From eccb45c7988e07def08745dc62d16d2c6c9dfd66 Mon Sep 17 00:00:00 2001 From: ngoding sendiri Date: Sat, 1 Aug 2026 19:42:05 +0700 Subject: [PATCH 08/15] feat(gateway-gen): report per-record validation details to the LLM checkCollection now captures the first ModelSchema issue per invalid record (up to 5 unique examples) so the retry prompt tells the LLM exactly which field fails and why (e.g. 'context_window: expected number, received null') instead of a bare count, letting it converge within the attempt budget. --- packages/collectors/src/gateway-gen/write.ts | 34 ++++++++++++++++++-- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/packages/collectors/src/gateway-gen/write.ts b/packages/collectors/src/gateway-gen/write.ts index fac1aa526..229d12b27 100644 --- a/packages/collectors/src/gateway-gen/write.ts +++ b/packages/collectors/src/gateway-gen/write.ts @@ -89,6 +89,21 @@ export interface CollectionCheck { errors: string[]; } +function formatIssuePath(issue: { path?: ReadonlyArray }): string { + return (issue.path ?? []) + .map((segment) => (typeof segment === 'number' ? `[${segment}]` : `.${segment}`)) + .join('') + .replace(/^\./, ''); +} + +function formatModelIssue(issue: { message: string; expected?: unknown; received?: unknown }): string { + const extra = + issue.expected !== undefined + ? ` (expected ${String(issue.expected)}, received ${String(issue.received)})` + : ''; + return `${issue.message}${extra}`; +} + export async function checkCollection( plugin: GatewayPlugin, secrets: Record, @@ -99,12 +114,25 @@ export async function checkCollection( } ).collect(secrets); let validCount = 0; + const seen = new Set(); for (const model of result.models) { const parsed = ModelSchema.safeParse(model); - if (parsed.success) validCount += 1; + if (parsed.success) { + validCount += 1; + continue; + } + const issue = (parsed.error.issues ?? [])[0] as + | { path?: ReadonlyArray; message: string; expected?: unknown; received?: unknown } + | undefined; + if (issue) { + const label = `${formatIssuePath(issue)}: ${formatModelIssue(issue)}`; + if (seen.size < 5) seen.add(label); + } } - if (result.models.length !== validCount) { - result.errors.push(`${result.models.length - validCount} models failed ModelSchema validation`); + const invalid = result.models.length - validCount; + if (invalid > 0) { + result.errors.push(`${invalid} models failed ModelSchema validation`); + for (const label of seen) result.errors.push(` e.g. model ${label}`); } return { modelCount: result.models.length, validCount, errors: result.errors }; } From 530e8ecd3361db375d16a55b45c4461970eeb83a Mon Sep 17 00:00:00 2001 From: ngoding sendiri Date: Sat, 1 Aug 2026 19:55:10 +0700 Subject: [PATCH 09/15] feat(gateway-gen): prefer stronger free LLM (OpenRouter) with Gemini fallback OpenRouter (deepseek-chat-v3-0324:free by default) is now the primary LLM provider when OPENROUTER_API_KEY is set, falling back to Gemini on failure. LLM_PROVIDER=openrouter|gemini forces a single provider. This gives the retry loop a stronger code-generation model than flash-lite so plugins converge within the attempt budget. --- packages/collectors/src/gateway-gen/llm.ts | 39 ++++++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/packages/collectors/src/gateway-gen/llm.ts b/packages/collectors/src/gateway-gen/llm.ts index 8e72cf206..53ebda186 100644 --- a/packages/collectors/src/gateway-gen/llm.ts +++ b/packages/collectors/src/gateway-gen/llm.ts @@ -6,15 +6,22 @@ export interface LlmConfig { temperature?: number; } -type Provider = 'gemini' | 'openrouter'; +export type Provider = 'gemini' | 'openrouter'; -function pickProvider(env: NodeJS.ProcessEnv): Provider { - if (env.GEMINI_API_KEY) return 'gemini'; - if (env.OPENROUTER_API_KEY) return 'openrouter'; - throw new Error( - 'No LLM provider configured. Set GEMINI_API_KEY (Gemini free tier) or ' + - 'OPENROUTER_API_KEY (free models) to generate gateway plugins.', - ); +function resolveProviders(env: NodeJS.ProcessEnv): Provider[] { + const forced = env.LLM_PROVIDER; + if (forced === 'openrouter') return ['openrouter']; + if (forced === 'gemini') return ['gemini']; + const list: Provider[] = []; + if (env.OPENROUTER_API_KEY) list.push('openrouter'); + if (env.GEMINI_API_KEY) list.push('gemini'); + if (list.length === 0) { + throw new Error( + 'No LLM provider configured. Set OPENROUTER_API_KEY (free models) or ' + + 'GEMINI_API_KEY (Gemini free tier) to generate gateway plugins.', + ); + } + return list; } async function callGemini(prompt: string, env: NodeJS.ProcessEnv): Promise { @@ -76,7 +83,17 @@ export async function generateText( config: LlmConfig, env: NodeJS.ProcessEnv = process.env, ): Promise { - const provider = pickProvider(env); - if (provider === 'gemini') return callGemini(config.prompt, env); - return callOpenRouter(config.prompt, env); + const providers = resolveProviders(env); + const failures: string[] = []; + for (const provider of providers) { + try { + if (provider === 'gemini') return await callGemini(config.prompt, env); + return await callOpenRouter(config.prompt, env); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + failures.push(`${provider}: ${message}`); + console.warn(`[llm] ${provider} failed, ${providers.length - 1 > 0 ? 'trying next provider' : 'no fallback left'}: ${message}`); + } + } + throw new Error(`All LLM providers failed: ${failures.join(' | ')}`); } From 6e455bed7889d537344fbf88ef2fdcce606d4d8b Mon Sep 17 00:00:00 2001 From: ngoding sendiri Date: Sat, 1 Aug 2026 20:31:43 +0700 Subject: [PATCH 10/15] feat(gateway-gen): add Requesty LLM provider, richer sample, robustness - llm.ts: add Requesty provider (router.requesty.ai) with free model mistral/leanstral-1-5 by default; provider order requesty -> gemini -> openrouter; LLM_PROVIDER env override; OpenRouter free candidates rotate (deepseek :free no longer available). - gateway-ai.yml: pass REQUESTY_API_KEY to the generator env. - prompts.ts: forbid unused variables (noUnusedLocals) explicitly. - manifest sample: add next_page_token and null context_length/tokenizer_url entries so the LLM learns pagination and nullable fields. - generate/heal: raise default maxAttempts to 4. - cohere.ts: regenerated plugin handles pagination (next_page_token, maxPages) and nullable fields; passes forbidden/typecheck/unit validation. --- .github/workflows/gateway-ai.yml | 1 + packages/collectors/manifest.json | 9 +- packages/collectors/src/gateway-gen/heal.ts | 2 +- packages/collectors/src/gateway-gen/llm.ts | 121 ++++++++---- .../collectors/src/gateway-gen/prompts.ts | 1 + packages/collectors/src/gateway-gen/write.ts | 2 +- .../src/gateways/__fixtures__/cohere.raw.json | 3 +- packages/collectors/src/gateways/cohere.ts | 175 +++++++++++------- 8 files changed, 213 insertions(+), 101 deletions(-) diff --git a/.github/workflows/gateway-ai.yml b/.github/workflows/gateway-ai.yml index a4953f67b..5b72c7be9 100644 --- a/.github/workflows/gateway-ai.yml +++ b/.github/workflows/gateway-ai.yml @@ -76,6 +76,7 @@ jobs: PORTKEY_API_KEY: ${{ secrets.PORTKEY_API_KEY }} MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} + REQUESTY_API_KEY: ${{ secrets.REQUESTY_API_KEY }} LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL }} LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY }} diff --git a/packages/collectors/manifest.json b/packages/collectors/manifest.json index c16f68ce0..4e300d468 100644 --- a/packages/collectors/manifest.json +++ b/packages/collectors/manifest.json @@ -37,8 +37,15 @@ "name": "rerank-english-v3.0", "endpoints": ["rerank"], "context_length": 4096 + }, + { + "name": "embed-english-light-v3.0", + "endpoints": ["embed"], + "context_length": null, + "tokenizer_url": null } - ] + ], + "next_page_token": "cD0yCg==" }, "notes": "Example gateway for the AI generator: non-OpenAI-compatible response shape ({ models: [...] }). A static sample is provided so the generator can run without a live API key." } diff --git a/packages/collectors/src/gateway-gen/heal.ts b/packages/collectors/src/gateway-gen/heal.ts index 90305240d..8182bb17e 100644 --- a/packages/collectors/src/gateway-gen/heal.ts +++ b/packages/collectors/src/gateway-gen/heal.ts @@ -28,7 +28,7 @@ export async function healPlugin(options: HealPluginOptions): Promise { + const apiKey = env.REQUESTY_API_KEY; + const model = env.REQUESTY_MODEL ?? 'mistral/leanstral-1-5'; + const response = await fetch(REQUESTY_ENDPOINT, { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + 'HTTP-Referer': 'https://basemodel.ai', + 'X-Title': 'BaseModel gateway generator', + }, + body: JSON.stringify({ + model, + messages: [{ role: 'user', content: prompt }], + temperature: 0.2, + max_tokens: 8192, + }), + signal: AbortSignal.timeout(120_000), + }); + if (!response.ok) { + const body = await response.text().catch(() => ''); + throw new Error(`Requesty HTTP ${response.status}: ${body.slice(0, 300)}`); + } + const data = (await response.json()) as { + choices?: Array<{ message?: { content?: string } }>; + }; + const text = data.choices?.[0]?.message?.content; + if (!text) throw new Error('Requesty returned an empty response.'); + return text; +} + async function callGemini(prompt: string, env: NodeJS.ProcessEnv): Promise { const apiKey = env.GEMINI_API_KEY; const model = env.GEMINI_MODEL ?? 'gemini-flash-lite-latest'; @@ -48,35 +80,53 @@ async function callGemini(prompt: string, env: NodeJS.ProcessEnv): Promise { const apiKey = env.OPENROUTER_API_KEY; - const model = env.OPENROUTER_MODEL ?? 'deepseek/deepseek-chat-v3-0324:free'; - const response = await fetch(OPENROUTER_ENDPOINT, { - method: 'POST', - headers: { - Authorization: `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - 'HTTP-Referer': 'https://basemodel.ai', - 'X-Title': 'BaseModel gateway generator', - }, - body: JSON.stringify({ - model, - messages: [{ role: 'user', content: prompt }], - temperature: 0.2, - max_tokens: 8192, - }), - signal: AbortSignal.timeout(120_000), - }); - if (!response.ok) { - const body = await response.text().catch(() => ''); - throw new Error(`OpenRouter HTTP ${response.status}: ${body.slice(0, 300)}`); + const candidates = env.OPENROUTER_MODEL ? [env.OPENROUTER_MODEL] : OPENROUTER_FREE_CANDIDATES; + const failures: string[] = []; + for (const model of candidates) { + try { + const response = await fetch(OPENROUTER_ENDPOINT, { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + 'HTTP-Referer': 'https://basemodel.ai', + 'X-Title': 'BaseModel gateway generator', + }, + body: JSON.stringify({ + model, + messages: [{ role: 'user', content: prompt }], + temperature: 0.2, + max_tokens: 8192, + }), + signal: AbortSignal.timeout(120_000), + }); + if (!response.ok) { + const body = await response.text().catch(() => ''); + throw new Error(`HTTP ${response.status}: ${body.slice(0, 200)}`); + } + const data = (await response.json()) as { + choices?: Array<{ message?: { content?: string } }>; + }; + const text = data.choices?.[0]?.message?.content; + if (!text) throw new Error('empty response'); + return text; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + failures.push(`${model}: ${message}`); + console.warn(`[llm] openrouter ${model} failed: ${message}`); + } } - const data = (await response.json()) as { - choices?: Array<{ message?: { content?: string } }>; - }; - const text = data.choices?.[0]?.message?.content; - if (!text) throw new Error('OpenRouter returned an empty response.'); - return text; + throw new Error(`OpenRouter failed on all candidate models: ${failures.join(' | ')}`); } export async function generateText( @@ -88,11 +138,14 @@ export async function generateText( for (const provider of providers) { try { if (provider === 'gemini') return await callGemini(config.prompt, env); - return await callOpenRouter(config.prompt, env); + if (provider === 'openrouter') return await callOpenRouter(config.prompt, env); + return await callRequesty(config.prompt, env); } catch (error) { const message = error instanceof Error ? error.message : String(error); failures.push(`${provider}: ${message}`); - console.warn(`[llm] ${provider} failed, ${providers.length - 1 > 0 ? 'trying next provider' : 'no fallback left'}: ${message}`); + console.warn( + `[llm] ${provider} failed, ${providers.length > 1 ? 'trying next provider' : 'no fallback left'}: ${message}`, + ); } } throw new Error(`All LLM providers failed: ${failures.join(' | ')}`); diff --git a/packages/collectors/src/gateway-gen/prompts.ts b/packages/collectors/src/gateway-gen/prompts.ts index 6b22553ca..ac66768da 100644 --- a/packages/collectors/src/gateway-gen/prompts.ts +++ b/packages/collectors/src/gateway-gen/prompts.ts @@ -51,6 +51,7 @@ Guidelines: - Handle pagination if the response exposes it (next page URL / total counts); never loop forever (cap pages). - Use AbortSignal.timeout(15_000) on fetch. Catch errors and push them to result.errors (do not throw). - Never use the "any" type anywhere. Type catch variables as "unknown" and narrow with instanceof/typeof. Do not use "as any". +- Do not declare unused variables or helper functions; every declared name must be referenced, or the file fails typecheck (noUnusedLocals). - Never hardcode real API keys; only reference secrets by their env names. - Do not add code comments. `; diff --git a/packages/collectors/src/gateway-gen/write.ts b/packages/collectors/src/gateway-gen/write.ts index 229d12b27..510e559d7 100644 --- a/packages/collectors/src/gateway-gen/write.ts +++ b/packages/collectors/src/gateway-gen/write.ts @@ -186,7 +186,7 @@ export async function validateGeneratedPlugin( } export async function generatePlugin(options: GeneratePluginOptions): Promise { - const { gateway, shape, raw, env = process.env, maxAttempts = 3, liveSecrets } = options; + const { gateway, shape, raw, env = process.env, maxAttempts = 4, liveSecrets } = options; const filePath = getGatewayPluginPath(gateway.id); let prompt = buildPluginPrompt(gateway, shape, raw); const lastErrors: string[] = []; diff --git a/packages/collectors/src/gateways/__fixtures__/cohere.raw.json b/packages/collectors/src/gateways/__fixtures__/cohere.raw.json index cc4379934..264daa83f 100644 --- a/packages/collectors/src/gateways/__fixtures__/cohere.raw.json +++ b/packages/collectors/src/gateways/__fixtures__/cohere.raw.json @@ -46,5 +46,6 @@ ], "context_length": 4096 } - ] + ], + "next_page_token": "cD0yCg==" } diff --git a/packages/collectors/src/gateways/cohere.ts b/packages/collectors/src/gateways/cohere.ts index 8d4aa97d5..fe0023a4d 100644 --- a/packages/collectors/src/gateways/cohere.ts +++ b/packages/collectors/src/gateways/cohere.ts @@ -1,88 +1,137 @@ import { z } from 'zod'; import type { CollectionResult, CustomGateway } from '../core/collector'; -const cohereModelSchema = z.object({ +const ModelSchema = z.object({ name: z.string(), - endpoints: z.array(z.string()).optional(), - context_length: z.number().optional(), - tokenizer_url: z.string().optional(), + endpoints: z.array(z.string()), + context_length: z.number().nullable().optional(), + tokenizer_url: z.string().nullable().optional(), }); -const cohereResponseSchema = z.object({ - models: z.array(cohereModelSchema), +const ResponseSchema = z.object({ + models: z.array(ModelSchema), + next_page_token: z.string().optional(), }); +function slugify(name: string): string { + return name + .toLowerCase() + .replace(/[^a-z0-9.]+/g, '-') + .replace(/-+/g, '-') + .replace(/^-|-$/g, ''); +} + +function deriveModality(endpoints: string[]): ('text' | 'image' | 'audio' | 'video' | 'code' | 'embedding')[] { + const modalities: ('text' | 'image' | 'audio' | 'video' | 'code' | 'embedding')[] = ['text']; + if (endpoints.includes('embed')) modalities.push('embedding'); + if (endpoints.includes('classify') || endpoints.includes('rerank')) { + if (!modalities.includes('text')) modalities.push('text'); + } + return modalities; +} + +function deriveCapabilities(endpoints: string[]): string[] { + const caps: string[] = []; + if (endpoints.includes('embed')) caps.push('embedding'); + if (endpoints.includes('classify')) caps.push('classification'); + if (endpoints.includes('rerank')) caps.push('reranking'); + if (endpoints.includes('summarize')) caps.push('summarization'); + if (endpoints.includes('generate')) caps.push('generation'); + return caps; +} + +const gatewayId = 'cohere'; + export default { type: 'custom', - id: 'cohere', + id: gatewayId, async collect(secrets: Record): Promise { - const result: CollectionResult = { provider_id: 'cohere', models: [], errors: [] }; - const apiKey = secrets['COHERE_API_KEY']; + const result: CollectionResult = { provider_id: gatewayId, models: [], errors: [] }; + const apiKey = secrets['COHERE_API_KEY']; if (!apiKey) { - result.errors.push('Missing secret COHERE_API_KEY'); + result.errors.push('COHERE_API_KEY secret is missing'); return result; } - try { - const response = await fetch('https://api.cohere.com/v1/models', { - method: 'GET', - headers: { - Authorization: `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - }, - signal: AbortSignal.timeout(15_000), - }); - - if (!response.ok) { - result.errors.push(`Failed to fetch models: ${response.status} ${response.statusText}`); - return result; - } + const baseUrl = 'https://api.cohere.com'; + const endpoint = '/v1/models'; - const rawJson: unknown = await response.json(); - const parseResult = cohereResponseSchema.safeParse(rawJson); + let url = `${baseUrl}${endpoint}`; + let hasNextPage = true; + let pageCount = 0; + const maxPages = 5; - if (!parseResult.success) { - result.errors.push(`Validation error: ${parseResult.error.message}`); - return result; - } + while (hasNextPage && pageCount < maxPages) { + pageCount++; + try { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 15_000); + const response = await fetch(url, { + method: 'GET', + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + signal: controller.signal, + }); + clearTimeout(timeout); - for (const item of parseResult.data.models) { - const rawName = item.name; - const slug = rawName.toLowerCase().replace(/[^a-z0-9.-]/g, '-').replace(/-+/g, '-').replace(/^-|-$/g, ''); - const modelId = `cohere/${slug}`; + if (!response.ok) { + result.errors.push(`HTTP ${response.status}: ${response.statusText}`); + return result; + } - if (!/^[a-z0-9-]+\/[a-z0-9]+(?:[-.][a-z0-9]+)*$/.test(modelId)) { - continue; + const raw = await response.json() as unknown; + const parsed = ResponseSchema.safeParse(raw); + + if (!parsed.success) { + result.errors.push(`Validation error: ${parsed.error.message}`); + return result; } - const endpoints = item.endpoints ?? []; - const isEmbed = endpoints.includes('embed') || rawName.includes('embed'); - const modality: Array<'text' | 'image' | 'audio' | 'video' | 'code' | 'embedding'> = isEmbed ? ['embedding'] : ['text']; - - result.models.push({ - model_id: modelId, - provider_id: 'cohere', - name: rawName, - family: rawName.includes('command') ? 'Command' : undefined, - context_window: item.context_length, - modality, - open_weight: false, - reasoning_support: false, - function_calling: endpoints.includes('chat'), - structured_output: false, - vision_support: false, - audio_support: false, - image_generation: false, - embedding_support: isEmbed, - status: 'active', - }); - } - } catch (error: unknown) { - if (error instanceof Error) { - result.errors.push(error.message); - } else { - result.errors.push('An unknown error occurred during collection'); + const { models, next_page_token } = parsed.data; + + for (const model of models) { + const modelId = `${gatewayId}/${slugify(model.name)}`; + const modality = deriveModality(model.endpoints); + const capabilityIds = deriveCapabilities(model.endpoints); + const contextWindow = model.context_length ?? undefined; + + const modelEntry = { + model_id: modelId, + provider_id: gatewayId, + name: model.name, + family: model.name.split('-')[0] ?? undefined, + context_window: contextWindow, + modality, + open_weight: false, + reasoning_support: false, + function_calling: model.endpoints.includes('chat'), + structured_output: false, + vision_support: false, + audio_support: false, + image_generation: false, + embedding_support: model.endpoints.includes('embed'), + capability_ids: capabilityIds.length > 0 ? capabilityIds : undefined, + status: 'active' as const, + }; + + result.models.push(modelEntry); + } + + if (!next_page_token) { + hasNextPage = false; + } else { + url = `${baseUrl}${endpoint}?next_page_token=${encodeURIComponent(next_page_token)}`; + } + } catch (err: unknown) { + if (err instanceof Error) { + result.errors.push(`Fetch error: ${err.message}`); + } else { + result.errors.push('Unknown fetch error'); + } + return result; } } From 95e25312c6d8753cedd9a39c7794c760efc3325c Mon Sep 17 00:00:00 2001 From: ngoding sendiri Date: Sat, 1 Aug 2026 20:36:54 +0700 Subject: [PATCH 11/15] fix(workflow): remove duplicate REQUESTY_API_KEY env entry --- .github/workflows/gateway-ai.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/gateway-ai.yml b/.github/workflows/gateway-ai.yml index 5b72c7be9..a4953f67b 100644 --- a/.github/workflows/gateway-ai.yml +++ b/.github/workflows/gateway-ai.yml @@ -76,7 +76,6 @@ jobs: PORTKEY_API_KEY: ${{ secrets.PORTKEY_API_KEY }} MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} - REQUESTY_API_KEY: ${{ secrets.REQUESTY_API_KEY }} LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL }} LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY }} From a8274c59c5f64963edb8087467accb883776b575 Mon Sep 17 00:00:00 2001 From: ngoding sendiri Date: Sat, 1 Aug 2026 20:43:14 +0700 Subject: [PATCH 12/15] feat(gateway-gen): enrich cohere sample to mirror live shape, pin exact Model type in prompt --- packages/collectors/manifest.json | 30 ++- .../collectors/src/gateway-gen/prompts.ts | 56 ++-- .../src/gateways/__fixtures__/cohere.raw.json | 43 +++- packages/collectors/src/gateways/cohere.ts | 243 +++++++++++------- 4 files changed, 240 insertions(+), 132 deletions(-) diff --git a/packages/collectors/manifest.json b/packages/collectors/manifest.json index 4e300d468..e47bc7883 100644 --- a/packages/collectors/manifest.json +++ b/packages/collectors/manifest.json @@ -15,34 +15,52 @@ { "name": "command-r-plus", "endpoints": ["chat", "embed", "generate", "summarize", "classify", "rerank"], + "finetuned": false, "context_length": 128000, - "tokenizer_url": "https://docs.cohere.com/docs/tokenizer" + "tokenizer_url": "https://docs.cohere.com/docs/tokenizer", + "default_endpoints": ["chat"], + "features": ["supports_rag", "experimental_chat"] }, { "name": "command-r", "endpoints": ["chat", "embed", "generate", "summarize", "classify", "rerank"], - "context_length": 128000 + "finetuned": false, + "context_length": 128000, + "default_endpoints": ["chat"], + "features": ["supports_rag"] }, { "name": "command-light", "endpoints": ["chat", "generate", "summarize"], - "context_length": 4096 + "finetuned": false, + "context_length": 4096, + "default_endpoints": ["chat"], + "features": null }, { "name": "embed-english-v3.0", "endpoints": ["embed"], - "context_length": 512 + "finetuned": false, + "context_length": 512, + "default_endpoints": ["embed"], + "features": ["search", "classification"] }, { "name": "rerank-english-v3.0", "endpoints": ["rerank"], - "context_length": 4096 + "finetuned": false, + "context_length": 4096, + "default_endpoints": ["rerank"], + "features": null }, { "name": "embed-english-light-v3.0", "endpoints": ["embed"], + "finetuned": false, "context_length": null, - "tokenizer_url": null + "tokenizer_url": null, + "default_endpoints": ["embed"], + "features": null } ], "next_page_token": "cD0yCg==" diff --git a/packages/collectors/src/gateway-gen/prompts.ts b/packages/collectors/src/gateway-gen/prompts.ts index ac66768da..433c32e7c 100644 --- a/packages/collectors/src/gateway-gen/prompts.ts +++ b/packages/collectors/src/gateway-gen/prompts.ts @@ -2,29 +2,39 @@ import type { ManifestGateway } from './manifest.js'; import type { ShapeSummary } from './probe.js'; const MODEL_SCHEMA_DOC = ` -Canonical "Model" fields (TypeScript, from @basemodel/schema). Fields marked [required] must always be set: -- model_id: string [required] must match /^[a-z0-9-]+\\/[a-z0-9]+(?:[-.][a-z0-9]+)*$/ e.g. "cohere/command-r-plus" (provider slug / model slug) -- provider_id: string [required] e.g. "cohere" -- name: string [required] human-readable model name -- family: string (optional) e.g. "Command" -- version: string (optional) -- release_date: string (optional) YYYY-MM-DD -- description: string (optional) -- architecture: string (optional) -- parameter_size: string (optional) e.g. "70B" -- context_window: number (optional) in tokens -- modality: array of "text"|"image"|"audio"|"video"|"code"|"embedding" [required] -- open_weight: boolean [required] -- reasoning_support: boolean [required] -- function_calling: boolean [required] -- structured_output: boolean [required] -- vision_support: boolean [required] -- audio_support: boolean [required] -- image_generation: boolean [required] -- embedding_support: boolean [required] -- capability_ids: string[] (optional) -- license_id: string (optional) -- status: "active"|"preview"|"deprecated"|"discontinued" [required] +Canonical "Model" type (TypeScript, from @basemodel/schema). Each catalog entry must map to an object satisfying this shape: +type Model = { + model_id: string; // [required] matches /^[a-z0-9-]+\\/[a-z0-9]+(?:[-.][a-z0-9]+)*$/ e.g. "cohere/command-r-plus" + provider_id: string; // [required] e.g. "cohere" + name: string; // [required] human-readable name + family?: string; + version?: string; + release_date?: string; // ISO YYYY-MM-DD + description?: string; + architecture?: string; + parameter_size?: string; + context_window?: number; // positive integer (tokens); omit when unknown + modality: ('text'|'image'|'audio'|'video'|'code'|'embedding')[]; // [required] + open_weight: boolean; // [required] + reasoning_support: boolean; // [required] + function_calling: boolean; // [required] + structured_output: boolean; // [required] + vision_support: boolean; // [required] + audio_support: boolean; // [required] + image_generation: boolean; // [required] + embedding_support: boolean; // [required] + is_free?: boolean; + tier?: 'free'|'budget'|'balanced'|'premium'; + limits?: object; // omit unless you have exact pricing/limits data + capability_ids?: string[]; // defaults to [] + license_id?: string; + status: 'active'|'preview'|'deprecated'|'discontinued'; // [required] +}; +Rules: +- Set every required field; every key must be spelled exactly as above (this is a strict TS type). +- When a source value is null or missing, OMIT the optional field (or map to undefined) - never assign null. +- context_window must be a positive integer or absent - never null, never 0. +- Derive modality from source hints (e.g. "embed" -> 'embedding'); default to ['text']. `; const PLUGIN_CONTRACT = ` diff --git a/packages/collectors/src/gateways/__fixtures__/cohere.raw.json b/packages/collectors/src/gateways/__fixtures__/cohere.raw.json index 264daa83f..527811a9a 100644 --- a/packages/collectors/src/gateways/__fixtures__/cohere.raw.json +++ b/packages/collectors/src/gateways/__fixtures__/cohere.raw.json @@ -9,8 +9,16 @@ "summarize", "classify" ], + "finetuned": false, "context_length": 128000, - "tokenizer_url": "https://docs.cohere.com/docs/tokenizer" + "tokenizer_url": "https://docs.cohere.com/docs/tokenizer", + "default_endpoints": [ + "chat" + ], + "features": [ + "supports_rag", + "experimental_chat" + ] }, { "name": "command-r", @@ -21,7 +29,14 @@ "summarize", "classify" ], - "context_length": 128000 + "finetuned": false, + "context_length": 128000, + "default_endpoints": [ + "chat" + ], + "features": [ + "supports_rag" + ] }, { "name": "command-light", @@ -30,21 +45,39 @@ "generate", "summarize" ], - "context_length": 4096 + "finetuned": false, + "context_length": 4096, + "default_endpoints": [ + "chat" + ], + "features": null }, { "name": "embed-english-v3.0", "endpoints": [ "embed" ], - "context_length": 512 + "finetuned": false, + "context_length": 512, + "default_endpoints": [ + "embed" + ], + "features": [ + "search", + "classification" + ] }, { "name": "rerank-english-v3.0", "endpoints": [ "rerank" ], - "context_length": 4096 + "finetuned": false, + "context_length": 4096, + "default_endpoints": [ + "rerank" + ], + "features": null } ], "next_page_token": "cD0yCg==" diff --git a/packages/collectors/src/gateways/cohere.ts b/packages/collectors/src/gateways/cohere.ts index fe0023a4d..4c6dfa174 100644 --- a/packages/collectors/src/gateways/cohere.ts +++ b/packages/collectors/src/gateways/cohere.ts @@ -3,136 +3,183 @@ import type { CollectionResult, CustomGateway } from '../core/collector'; const ModelSchema = z.object({ name: z.string(), - endpoints: z.array(z.string()), + endpoints: z.array(z.string()).optional(), + finetuned: z.boolean().optional(), context_length: z.number().nullable().optional(), tokenizer_url: z.string().nullable().optional(), + default_endpoints: z.array(z.string()).optional(), + features: z.array(z.string()).nullable().optional(), }); -const ResponseSchema = z.object({ - models: z.array(ModelSchema), - next_page_token: z.string().optional(), -}); +type RawModel = z.infer; -function slugify(name: string): string { - return name +function slugify(text: string): string { + return text .toLowerCase() - .replace(/[^a-z0-9.]+/g, '-') + .replace(/[^a-z0-9.-]+/g, '-') .replace(/-+/g, '-') .replace(/^-|-$/g, ''); } -function deriveModality(endpoints: string[]): ('text' | 'image' | 'audio' | 'video' | 'code' | 'embedding')[] { +function deriveModality( + endpoints: string[] | undefined, + features: string[] | null | undefined, +): ('text' | 'image' | 'audio' | 'video' | 'code' | 'embedding')[] { const modalities: ('text' | 'image' | 'audio' | 'video' | 'code' | 'embedding')[] = ['text']; + if (!endpoints) return modalities; if (endpoints.includes('embed')) modalities.push('embedding'); - if (endpoints.includes('classify') || endpoints.includes('rerank')) { - if (!modalities.includes('text')) modalities.push('text'); - } + if (endpoints.includes('generate') && features?.includes('image_generation')) + modalities.push('image'); + if (endpoints.includes('audio')) modalities.push('audio'); + if (features?.includes('vision')) modalities.push('image'); + if (features?.includes('code')) modalities.push('code'); return modalities; } -function deriveCapabilities(endpoints: string[]): string[] { - const caps: string[] = []; - if (endpoints.includes('embed')) caps.push('embedding'); - if (endpoints.includes('classify')) caps.push('classification'); - if (endpoints.includes('rerank')) caps.push('reranking'); - if (endpoints.includes('summarize')) caps.push('summarization'); - if (endpoints.includes('generate')) caps.push('generation'); - return caps; +function deriveCapabilities( + endpoints: string[] | undefined, + features: string[] | null | undefined, +): { + reasoning_support: boolean; + function_calling: boolean; + structured_output: boolean; + vision_support: boolean; + audio_support: boolean; + image_generation: boolean; + embedding_support: boolean; +} { + const hasEndpoint = (ep: string) => endpoints?.includes(ep) ?? false; + const hasFeature = (f: string) => features?.includes(f) ?? false; + return { + reasoning_support: hasEndpoint('chat') && hasFeature('experimental_chat'), + function_calling: hasEndpoint('chat'), + structured_output: hasEndpoint('chat'), + vision_support: hasFeature('vision'), + audio_support: hasEndpoint('audio'), + image_generation: hasFeature('image_generation'), + embedding_support: hasEndpoint('embed'), + }; } -const gatewayId = 'cohere'; - export default { type: 'custom', - id: gatewayId, + id: 'cohere', async collect(secrets: Record): Promise { - const result: CollectionResult = { provider_id: gatewayId, models: [], errors: [] }; - + const result: CollectionResult = { provider_id: 'cohere', models: [], errors: [] }; const apiKey = secrets['COHERE_API_KEY']; if (!apiKey) { - result.errors.push('COHERE_API_KEY secret is missing'); + result.errors.push('COHERE_API_KEY secret is required'); return result; } const baseUrl = 'https://api.cohere.com'; const endpoint = '/v1/models'; + const url = `${baseUrl}${endpoint}`; + + const allModels: RawModel[] = []; + let nextPageToken: string | undefined; - let url = `${baseUrl}${endpoint}`; - let hasNextPage = true; - let pageCount = 0; - const maxPages = 5; + for (let page = 0; page < 10; page++) { + const params = new URLSearchParams({}); + if (nextPageToken) params.set('next_page_token', nextPageToken); + const fetchUrl = `${url}?${params.toString()}`; - while (hasNextPage && pageCount < maxPages) { - pageCount++; + let response: Response; try { - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), 15_000); - const response = await fetch(url, { - method: 'GET', - headers: { - 'Authorization': `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - }, - signal: controller.signal, + response = await fetch(fetchUrl, { + headers: { Authorization: `Bearer ${apiKey}` }, + signal: AbortSignal.timeout(15_000), }); - clearTimeout(timeout); - - if (!response.ok) { - result.errors.push(`HTTP ${response.status}: ${response.statusText}`); - return result; - } - - const raw = await response.json() as unknown; - const parsed = ResponseSchema.safeParse(raw); - - if (!parsed.success) { - result.errors.push(`Validation error: ${parsed.error.message}`); - return result; - } - - const { models, next_page_token } = parsed.data; - - for (const model of models) { - const modelId = `${gatewayId}/${slugify(model.name)}`; - const modality = deriveModality(model.endpoints); - const capabilityIds = deriveCapabilities(model.endpoints); - const contextWindow = model.context_length ?? undefined; - - const modelEntry = { - model_id: modelId, - provider_id: gatewayId, - name: model.name, - family: model.name.split('-')[0] ?? undefined, - context_window: contextWindow, - modality, - open_weight: false, - reasoning_support: false, - function_calling: model.endpoints.includes('chat'), - structured_output: false, - vision_support: false, - audio_support: false, - image_generation: false, - embedding_support: model.endpoints.includes('embed'), - capability_ids: capabilityIds.length > 0 ? capabilityIds : undefined, - status: 'active' as const, - }; - - result.models.push(modelEntry); - } - - if (!next_page_token) { - hasNextPage = false; - } else { - url = `${baseUrl}${endpoint}?next_page_token=${encodeURIComponent(next_page_token)}`; - } - } catch (err: unknown) { - if (err instanceof Error) { - result.errors.push(`Fetch error: ${err.message}`); - } else { - result.errors.push('Unknown fetch error'); - } - return result; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + result.errors.push(`Failed to fetch models: ${message}`); + break; } + + if (!response.ok) { + result.errors.push(`API returned ${response.status}: ${response.statusText}`); + break; + } + + let data: unknown; + try { + data = await response.json(); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + result.errors.push(`Failed to parse JSON: ${message}`); + break; + } + + const parsed = z + .object({ + models: z.array(ModelSchema), + next_page_token: z.string().optional(), + }) + .safeParse(data); + + if (!parsed.success) { + result.errors.push( + `Invalid response shape: ${parsed.error.issues.map((i) => i.message).join(', ')}`, + ); + break; + } + + allModels.push(...parsed.data.models); + nextPageToken = parsed.data.next_page_token; + + if (!nextPageToken) break; + } + + for (const raw of allModels) { + const modelId = `cohere/${slugify(raw.name)}`; + const modality = deriveModality(raw.endpoints, raw.features); + const caps = deriveCapabilities(raw.endpoints, raw.features); + + const model: { + model_id: string; + provider_id: string; + name: string; + family?: string; + version?: string; + release_date?: string; + description?: string; + architecture?: string; + parameter_size?: string; + context_window?: number; + modality: ('text' | 'image' | 'audio' | 'video' | 'code' | 'embedding')[]; + open_weight: boolean; + reasoning_support: boolean; + function_calling: boolean; + structured_output: boolean; + vision_support: boolean; + audio_support: boolean; + image_generation: boolean; + embedding_support: boolean; + is_free?: boolean; + tier?: 'free' | 'budget' | 'balanced' | 'premium'; + limits?: object; + capability_ids?: string[]; + license_id?: string; + status: 'active' | 'preview' | 'deprecated' | 'discontinued'; + } = { + model_id: modelId, + provider_id: 'cohere', + name: raw.name, + context_window: raw.context_length ?? undefined, + modality, + open_weight: false, + reasoning_support: caps.reasoning_support, + function_calling: caps.function_calling, + structured_output: caps.structured_output, + vision_support: caps.vision_support, + audio_support: caps.audio_support, + image_generation: caps.image_generation, + embedding_support: caps.embedding_support, + capability_ids: [], + status: 'active', + }; + + result.models.push(model); } return result; From 0bb03ffac99a3864b83c21c9f64bfb95b9519635 Mon Sep 17 00:00:00 2001 From: ngoding sendiri Date: Sat, 1 Aug 2026 20:50:56 +0700 Subject: [PATCH 13/15] feat(gateway-gen): separate raw response schema from Model constraints, truncate heal feedback, handle non-positive context_length --- packages/collectors/manifest.json | 8 + .../collectors/src/gateway-gen/prompts.ts | 5 +- packages/collectors/src/gateway-gen/write.ts | 5 +- packages/collectors/src/gateways/cohere.ts | 252 ++++++++---------- 4 files changed, 131 insertions(+), 139 deletions(-) diff --git a/packages/collectors/manifest.json b/packages/collectors/manifest.json index e47bc7883..e5f7c8d76 100644 --- a/packages/collectors/manifest.json +++ b/packages/collectors/manifest.json @@ -61,6 +61,14 @@ "tokenizer_url": null, "default_endpoints": ["embed"], "features": null + }, + { + "name": "command-nightly-finetuned", + "endpoints": [], + "finetuned": true, + "context_length": 0, + "default_endpoints": [], + "features": null } ], "next_page_token": "cD0yCg==" diff --git a/packages/collectors/src/gateway-gen/prompts.ts b/packages/collectors/src/gateway-gen/prompts.ts index 433c32e7c..2dbe24208 100644 --- a/packages/collectors/src/gateway-gen/prompts.ts +++ b/packages/collectors/src/gateway-gen/prompts.ts @@ -55,9 +55,12 @@ export default { Guidelines: - Build the URL from the manifest's baseUrl + endpoint. Use only the secret names granted by the manifest. -- Validate the raw JSON with a zod schema; on parse failure push the error message to result.errors and return. +- The raw response zod schema must describe ONLY what the API actually returns, using permissive types: numbers may be null or 0, arrays may be null, strings may be absent. Do NOT copy the BaseModel Model constraints above (model_id regex, positive context_window, status/modality enums) into the raw response schema - those apply ONLY when building the output Model objects. +- Validate the raw JSON with a zod schema; on parse failure push a short message to result.errors and return. - Map each catalog entry to a Model. model_id must be "/"; derive the slug from the provider's id/name using .toLowerCase() and replacing characters outside [a-z0-9.-] with '-', collapsing repeats. - Set every required boolean field. Derive modality/capability flags from hints in the response (e.g. name/keywords like "embed", "image", "audio", "vision", "code") when available, otherwise default to text-only. +- When building a Model, map raw context_length to context_window ONLY if it is a positive integer; otherwise omit context_window entirely. +- Raw nullable fields (e.g. features or endpoints that can be null) must be normalized before passing to helpers: write const features = raw.features ?? undefined and pass the normalized value, or type the helper parameter as 'string[] | null | undefined'. Never pass a string[] | null | undefined value to a parameter typed string[] | undefined. - Handle pagination if the response exposes it (next page URL / total counts); never loop forever (cap pages). - Use AbortSignal.timeout(15_000) on fetch. Catch errors and push them to result.errors (do not throw). - Never use the "any" type anywhere. Type catch variables as "unknown" and narrow with instanceof/typeof. Do not use "as any". diff --git a/packages/collectors/src/gateway-gen/write.ts b/packages/collectors/src/gateway-gen/write.ts index 510e559d7..e1750f4af 100644 --- a/packages/collectors/src/gateway-gen/write.ts +++ b/packages/collectors/src/gateway-gen/write.ts @@ -199,7 +199,10 @@ export async function generatePlugin(options: GeneratePluginOptions): Promise + e.length > 300 ? `${e.slice(0, 300)}... (truncated)` : e, + ); + prompt += `\n\nThe previous attempt failed validation:\n${truncated.join('\n')}\n\nReturn the corrected file only.`; } throw new Error( diff --git a/packages/collectors/src/gateways/cohere.ts b/packages/collectors/src/gateways/cohere.ts index 4c6dfa174..809e0506c 100644 --- a/packages/collectors/src/gateways/cohere.ts +++ b/packages/collectors/src/gateways/cohere.ts @@ -1,187 +1,165 @@ import { z } from 'zod'; import type { CollectionResult, CustomGateway } from '../core/collector'; -const ModelSchema = z.object({ +const rawModelSchema = z.object({ name: z.string(), - endpoints: z.array(z.string()).optional(), + endpoints: z.array(z.string()).nullable().optional(), finetuned: z.boolean().optional(), context_length: z.number().nullable().optional(), tokenizer_url: z.string().nullable().optional(), - default_endpoints: z.array(z.string()).optional(), + default_endpoints: z.array(z.string()).nullable().optional(), features: z.array(z.string()).nullable().optional(), }); -type RawModel = z.infer; +const rawResponseSchema = z.object({ + models: z.array(rawModelSchema), + next_page_token: z.string().nullable().optional(), +}); -function slugify(text: string): string { - return text +function slugify(name: string): string { + return name .toLowerCase() - .replace(/[^a-z0-9.-]+/g, '-') + .replace(/[^a-z0-9.]+/g, '-') .replace(/-+/g, '-') .replace(/^-|-$/g, ''); } function deriveModality( - endpoints: string[] | undefined, + endpoints: string[] | null | undefined, features: string[] | null | undefined, ): ('text' | 'image' | 'audio' | 'video' | 'code' | 'embedding')[] { const modalities: ('text' | 'image' | 'audio' | 'video' | 'code' | 'embedding')[] = ['text']; - if (!endpoints) return modalities; - if (endpoints.includes('embed')) modalities.push('embedding'); - if (endpoints.includes('generate') && features?.includes('image_generation')) - modalities.push('image'); - if (endpoints.includes('audio')) modalities.push('audio'); - if (features?.includes('vision')) modalities.push('image'); - if (features?.includes('code')) modalities.push('code'); + + if (endpoints?.includes('embed')) { + if (!modalities.includes('embedding')) modalities.push('embedding'); + } + if (endpoints?.includes('classify') || features?.includes('classification')) { + if (!modalities.includes('embedding')) modalities.push('embedding'); + } + if ( + endpoints?.includes('generate') || + features?.includes('image_generation') || + features?.includes('image') + ) { + if (!modalities.includes('image')) modalities.push('image'); + } + if (features?.includes('audio')) { + if (!modalities.includes('audio')) modalities.push('audio'); + } + if (features?.includes('code')) { + if (!modalities.includes('code')) modalities.push('code'); + } return modalities; } function deriveCapabilities( - endpoints: string[] | undefined, + endpoints: string[] | null | undefined, features: string[] | null | undefined, -): { - reasoning_support: boolean; - function_calling: boolean; - structured_output: boolean; - vision_support: boolean; - audio_support: boolean; - image_generation: boolean; - embedding_support: boolean; -} { - const hasEndpoint = (ep: string) => endpoints?.includes(ep) ?? false; - const hasFeature = (f: string) => features?.includes(f) ?? false; - return { - reasoning_support: hasEndpoint('chat') && hasFeature('experimental_chat'), - function_calling: hasEndpoint('chat'), - structured_output: hasEndpoint('chat'), - vision_support: hasFeature('vision'), - audio_support: hasEndpoint('audio'), - image_generation: hasFeature('image_generation'), - embedding_support: hasEndpoint('embed'), - }; +): string[] { + const caps: string[] = []; + if (endpoints?.includes('chat')) caps.push('chat'); + if (endpoints?.includes('embed') || features?.includes('classification')) caps.push('embedding'); + if (endpoints?.includes('classify')) caps.push('classification'); + if (endpoints?.includes('rerank')) caps.push('reranking'); + if (features?.includes('supports_rag')) caps.push('rag'); + if (features?.includes('search')) caps.push('search'); + if (endpoints?.includes('summarize')) caps.push('summarization'); + if (features?.includes('experimental_chat')) caps.push('experimental_chat'); + return caps; } export default { type: 'custom', id: 'cohere', async collect(secrets: Record): Promise { - const result: CollectionResult = { provider_id: 'cohere', models: [], errors: [] }; const apiKey = secrets['COHERE_API_KEY']; if (!apiKey) { - result.errors.push('COHERE_API_KEY secret is required'); - return result; + return { provider_id: 'cohere', models: [], errors: ['COHERE_API_KEY secret is missing'] }; } const baseUrl = 'https://api.cohere.com'; const endpoint = '/v1/models'; const url = `${baseUrl}${endpoint}`; - const allModels: RawModel[] = []; - let nextPageToken: string | undefined; - - for (let page = 0; page < 10; page++) { - const params = new URLSearchParams({}); - if (nextPageToken) params.set('next_page_token', nextPageToken); - const fetchUrl = `${url}?${params.toString()}`; - - let response: Response; - try { - response = await fetch(fetchUrl, { - headers: { Authorization: `Bearer ${apiKey}` }, - signal: AbortSignal.timeout(15_000), - }); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - result.errors.push(`Failed to fetch models: ${message}`); - break; - } - - if (!response.ok) { - result.errors.push(`API returned ${response.status}: ${response.statusText}`); - break; - } - - let data: unknown; - try { - data = await response.json(); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - result.errors.push(`Failed to parse JSON: ${message}`); - break; - } - - const parsed = z - .object({ - models: z.array(ModelSchema), - next_page_token: z.string().optional(), - }) - .safeParse(data); - - if (!parsed.success) { - result.errors.push( - `Invalid response shape: ${parsed.error.issues.map((i) => i.message).join(', ')}`, - ); - break; - } - - allModels.push(...parsed.data.models); - nextPageToken = parsed.data.next_page_token; + let response: Response; + try { + response = await fetch(url, { + method: 'GET', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + signal: AbortSignal.timeout(15_000), + }); + } catch (err) { + const message = err instanceof Error ? err.message : 'Unknown fetch error'; + return { provider_id: 'cohere', models: [], errors: [`Failed to fetch: ${message}`] }; + } - if (!nextPageToken) break; + let data: unknown; + try { + data = await response.json(); + } catch { + return { provider_id: 'cohere', models: [], errors: ['Invalid JSON response'] }; } - for (const raw of allModels) { - const modelId = `cohere/${slugify(raw.name)}`; - const modality = deriveModality(raw.endpoints, raw.features); - const caps = deriveCapabilities(raw.endpoints, raw.features); + const parsed = rawResponseSchema.safeParse(data); + if (!parsed.success) { + return { + provider_id: 'cohere', + models: [], + errors: [`Validation error: ${parsed.error.message}`], + }; + } - const model: { - model_id: string; - provider_id: string; - name: string; - family?: string; - version?: string; - release_date?: string; - description?: string; - architecture?: string; - parameter_size?: string; - context_window?: number; - modality: ('text' | 'image' | 'audio' | 'video' | 'code' | 'embedding')[]; - open_weight: boolean; - reasoning_support: boolean; - function_calling: boolean; - structured_output: boolean; - vision_support: boolean; - audio_support: boolean; - image_generation: boolean; - embedding_support: boolean; - is_free?: boolean; - tier?: 'free' | 'budget' | 'balanced' | 'premium'; - limits?: object; - capability_ids?: string[]; - license_id?: string; - status: 'active' | 'preview' | 'deprecated' | 'discontinued'; - } = { + const models: CollectionResult['models'] = []; + for (const raw of parsed.data.models) { + const name = raw.name; + const slug = slugify(name); + const modelId = `cohere/${slug}`; + const endpoints = raw.endpoints ?? undefined; + const features = raw.features ?? undefined; + const modality = deriveModality(endpoints, features); + const capabilities = deriveCapabilities(endpoints, features); + const contextLength = raw.context_length; + const contextWindow = + typeof contextLength === 'number' && contextLength > 0 ? contextLength : undefined; + + models.push({ model_id: modelId, provider_id: 'cohere', - name: raw.name, - context_window: raw.context_length ?? undefined, - modality, + name: name, + family: name.includes('command') + ? 'command' + : name.includes('embed') + ? 'embed' + : name.includes('rerank') + ? 'rerank' + : undefined, + version: name.includes('v') ? name.split('v')[1]?.split('.')[0] : undefined, + release_date: undefined, + description: undefined, + architecture: undefined, + parameter_size: undefined, + context_window: contextWindow, + modality: modality, open_weight: false, - reasoning_support: caps.reasoning_support, - function_calling: caps.function_calling, - structured_output: caps.structured_output, - vision_support: caps.vision_support, - audio_support: caps.audio_support, - image_generation: caps.image_generation, - embedding_support: caps.embedding_support, - capability_ids: [], - status: 'active', - }; - - result.models.push(model); + reasoning_support: false, + function_calling: endpoints?.includes('chat') ?? false, + structured_output: endpoints?.includes('chat') ?? false, + vision_support: endpoints?.includes('chat') ?? false, + audio_support: endpoints?.includes('chat') ?? false, + image_generation: false, + embedding_support: endpoints?.includes('embed') ?? false, + is_free: undefined, + tier: undefined, + limits: undefined, + capability_ids: capabilities.length > 0 ? capabilities : undefined, + license_id: undefined, + status: raw.finetuned ? 'preview' : 'active', + }); } - return result; + return { provider_id: 'cohere', models, errors: [] }; }, } satisfies CustomGateway; From 96dd960ce8b20e546b28027a38d2b56f124437f0 Mon Sep 17 00:00:00 2001 From: ngoding sendiri Date: Sat, 1 Aug 2026 20:54:06 +0700 Subject: [PATCH 14/15] feat(gateway-gen): pin slugify snippet, raise maxAttempts to 6 --- packages/collectors/src/gateway-gen/heal.ts | 2 +- packages/collectors/src/gateway-gen/prompts.ts | 4 ++++ packages/collectors/src/gateway-gen/write.ts | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/collectors/src/gateway-gen/heal.ts b/packages/collectors/src/gateway-gen/heal.ts index 8182bb17e..45f6497c4 100644 --- a/packages/collectors/src/gateway-gen/heal.ts +++ b/packages/collectors/src/gateway-gen/heal.ts @@ -28,7 +28,7 @@ export async function healPlugin(options: HealPluginOptions): Promise/"; derive the slug from the provider's id/name using .toLowerCase() and replacing characters outside [a-z0-9.-] with '-', collapsing repeats. +- Use this exact slugify helper (the two-argument form of .replace is required, otherwise typecheck fails with TS2554): + function slugify(text: string): string { + return text.toLowerCase().replace(/[^a-z0-9.-]+/g, '-').replace(/-+/g, '-').replace(/^-|-$/g, ''); + } - Set every required boolean field. Derive modality/capability flags from hints in the response (e.g. name/keywords like "embed", "image", "audio", "vision", "code") when available, otherwise default to text-only. - When building a Model, map raw context_length to context_window ONLY if it is a positive integer; otherwise omit context_window entirely. - Raw nullable fields (e.g. features or endpoints that can be null) must be normalized before passing to helpers: write const features = raw.features ?? undefined and pass the normalized value, or type the helper parameter as 'string[] | null | undefined'. Never pass a string[] | null | undefined value to a parameter typed string[] | undefined. diff --git a/packages/collectors/src/gateway-gen/write.ts b/packages/collectors/src/gateway-gen/write.ts index e1750f4af..9f0ed11d4 100644 --- a/packages/collectors/src/gateway-gen/write.ts +++ b/packages/collectors/src/gateway-gen/write.ts @@ -186,7 +186,7 @@ export async function validateGeneratedPlugin( } export async function generatePlugin(options: GeneratePluginOptions): Promise { - const { gateway, shape, raw, env = process.env, maxAttempts = 4, liveSecrets } = options; + const { gateway, shape, raw, env = process.env, maxAttempts = 6, liveSecrets } = options; const filePath = getGatewayPluginPath(gateway.id); let prompt = buildPluginPrompt(gateway, shape, raw); const lastErrors: string[] = []; From a7cead711c503173f9d1370e1b1872d0b880c55f Mon Sep 17 00:00:00 2001 From: BaseModel Bot Date: Sat, 1 Aug 2026 13:55:15 +0000 Subject: [PATCH 15/15] feat(gateways): AI-generated plugin for cohere --- .../src/gateways/__fixtures__/cohere.raw.json | 100 ++++----- packages/collectors/src/gateways/cohere.ts | 208 +++++++++--------- 2 files changed, 150 insertions(+), 158 deletions(-) diff --git a/packages/collectors/src/gateways/__fixtures__/cohere.raw.json b/packages/collectors/src/gateways/__fixtures__/cohere.raw.json index 527811a9a..4b7f685c1 100644 --- a/packages/collectors/src/gateways/__fixtures__/cohere.raw.json +++ b/packages/collectors/src/gateways/__fixtures__/cohere.raw.json @@ -1,84 +1,84 @@ { "models": [ { - "name": "command-r-plus", + "name": "c4ai-aya-expanse-32b", "endpoints": [ - "chat", - "embed", "generate", - "summarize", - "classify" + "chat" ], "finetuned": false, "context_length": 128000, - "tokenizer_url": "https://docs.cohere.com/docs/tokenizer", - "default_endpoints": [ - "chat" - ], - "features": [ - "supports_rag", - "experimental_chat" - ] + "tokenizer_url": "https://storage.googleapis.com/cohere-public/tokenizers/c4ai-aya-expanse-32b.json", + "features": null, + "default_endpoints": [] }, { - "name": "command-r", + "name": "c4ai-aya-vision-32b", "endpoints": [ - "chat", - "embed", - "generate", - "summarize", - "classify" - ], - "finetuned": false, - "context_length": 128000, - "default_endpoints": [ "chat" ], + "finetuned": false, + "context_length": 16384, + "tokenizer_url": "https://storage.googleapis.com/cohere-public/tokenizers/c4ai-aya-vision-32b.json", "features": [ - "supports_rag" - ] + "logprobs", + "vision", + "citations" + ], + "default_endpoints": [] }, { - "name": "command-light", + "name": "cohere-transcribe-03-2026", "endpoints": [ - "chat", - "generate", - "summarize" + "transcriptions" ], "finetuned": false, - "context_length": 4096, - "default_endpoints": [ - "chat" - ], - "features": null + "context_length": 32768, + "tokenizer_url": "https://storage.googleapis.com/cohere-public/tokenizers/cohere-transcribe-03-2026.json", + "features": null, + "default_endpoints": [] }, { - "name": "embed-english-v3.0", + "name": "command-a-03-2025", "endpoints": [ - "embed" + "chat" ], "finetuned": false, - "context_length": 512, - "default_endpoints": [ - "embed" - ], + "context_length": 288000, + "tokenizer_url": "https://storage.googleapis.com/cohere-public/tokenizers/command-a-03-2025.json", "features": [ - "search", - "classification" + "json_mode", + "json_schema", + "strict_tools", + "safety_modes", + "tools" + ], + "default_endpoints": [ + "chat", + "generate" ] }, { - "name": "rerank-english-v3.0", + "name": "command-a-plus-05-2026", "endpoints": [ - "rerank" + "generate", + "chat" ], "finetuned": false, - "context_length": 4096, - "default_endpoints": [ - "rerank" + "context_length": 436000, + "tokenizer_url": "https://storage.googleapis.com/cohere-public/tokenizers/command-a-plus-05-2026.json", + "features": [ + "logprobs", + "json_mode", + "json_schema", + "strict_tools", + "safety_modes" ], - "features": null + "default_endpoints": [], + "sampling_defaults": { + "temperature": 0.6, + "p": 0.95 + } } - ], - "next_page_token": "cD0yCg==" + ] } diff --git a/packages/collectors/src/gateways/cohere.ts b/packages/collectors/src/gateways/cohere.ts index 809e0506c..18a19ec9b 100644 --- a/packages/collectors/src/gateways/cohere.ts +++ b/packages/collectors/src/gateways/cohere.ts @@ -1,165 +1,157 @@ import { z } from 'zod'; import type { CollectionResult, CustomGateway } from '../core/collector'; +function slugify(text: string): string { + return text.toLowerCase().replace(/[^a-z0-9.-]+/g, '-').replace(/-+/g, '-').replace(/^-|-$/g, ''); +} + const rawModelSchema = z.object({ name: z.string(), - endpoints: z.array(z.string()).nullable().optional(), - finetuned: z.boolean().optional(), - context_length: z.number().nullable().optional(), - tokenizer_url: z.string().nullable().optional(), - default_endpoints: z.array(z.string()).nullable().optional(), - features: z.array(z.string()).nullable().optional(), + endpoints: z.array(z.string()).nullable(), + finetuned: z.boolean(), + context_length: z.number().nullable(), + tokenizer_url: z.string().nullable(), + features: z.array(z.string()).nullable(), + default_endpoints: z.array(z.string()).nullable(), }); const rawResponseSchema = z.object({ models: z.array(rawModelSchema), - next_page_token: z.string().nullable().optional(), }); -function slugify(name: string): string { - return name - .toLowerCase() - .replace(/[^a-z0-9.]+/g, '-') - .replace(/-+/g, '-') - .replace(/^-|-$/g, ''); -} - -function deriveModality( - endpoints: string[] | null | undefined, - features: string[] | null | undefined, -): ('text' | 'image' | 'audio' | 'video' | 'code' | 'embedding')[] { +function deriveModality(features: string[] | null | undefined): ('text' | 'image' | 'audio' | 'video' | 'code' | 'embedding')[] { + const f = features ?? []; const modalities: ('text' | 'image' | 'audio' | 'video' | 'code' | 'embedding')[] = ['text']; - - if (endpoints?.includes('embed')) { - if (!modalities.includes('embedding')) modalities.push('embedding'); - } - if (endpoints?.includes('classify') || features?.includes('classification')) { - if (!modalities.includes('embedding')) modalities.push('embedding'); - } - if ( - endpoints?.includes('generate') || - features?.includes('image_generation') || - features?.includes('image') - ) { - if (!modalities.includes('image')) modalities.push('image'); - } - if (features?.includes('audio')) { - if (!modalities.includes('audio')) modalities.push('audio'); - } - if (features?.includes('code')) { - if (!modalities.includes('code')) modalities.push('code'); - } + if (f.includes('vision')) modalities.push('image'); + if (f.includes('audio')) modalities.push('audio'); + if (f.includes('transcriptions')) modalities.push('audio'); + if (f.includes('embed')) modalities.push('embedding'); return modalities; } -function deriveCapabilities( - endpoints: string[] | null | undefined, - features: string[] | null | undefined, -): string[] { +function deriveCapabilities(features: string[] | null | undefined): string[] { + const f = features ?? []; const caps: string[] = []; - if (endpoints?.includes('chat')) caps.push('chat'); - if (endpoints?.includes('embed') || features?.includes('classification')) caps.push('embedding'); - if (endpoints?.includes('classify')) caps.push('classification'); - if (endpoints?.includes('rerank')) caps.push('reranking'); - if (features?.includes('supports_rag')) caps.push('rag'); - if (features?.includes('search')) caps.push('search'); - if (endpoints?.includes('summarize')) caps.push('summarization'); - if (features?.includes('experimental_chat')) caps.push('experimental_chat'); + if (f.includes('reasoning')) caps.push('reasoning'); + if (f.includes('tools')) caps.push('function_calling'); + if (f.includes('json_mode') || f.includes('json_schema')) caps.push('structured_output'); + if (f.includes('vision')) caps.push('vision_support'); + if (f.includes('logprobs')) caps.push('logprobs'); + if (f.includes('citations')) caps.push('citations'); + if (f.includes('safety_modes')) caps.push('safety_modes'); + if (f.includes('transcriptions')) caps.push('audio_support'); + if (f.includes('embed')) caps.push('embedding_support'); return caps; } +function deriveBooleanFlags(features: string[] | null | undefined): { + reasoning_support: boolean; + function_calling: boolean; + structured_output: boolean; + vision_support: boolean; + audio_support: boolean; + image_generation: boolean; + embedding_support: boolean; +} { + const f = features ?? []; + return { + reasoning_support: f.includes('reasoning'), + function_calling: f.includes('tools'), + structured_output: f.includes('json_mode') || f.includes('json_schema'), + vision_support: f.includes('vision'), + audio_support: f.includes('transcriptions'), + image_generation: false, + embedding_support: f.includes('embed'), + }; +} + export default { type: 'custom', id: 'cohere', async collect(secrets: Record): Promise { + const result: CollectionResult = { provider_id: 'cohere', models: [], errors: [] }; const apiKey = secrets['COHERE_API_KEY']; if (!apiKey) { - return { provider_id: 'cohere', models: [], errors: ['COHERE_API_KEY secret is missing'] }; + result.errors.push('COHERE_API_KEY secret is missing'); + return result; } - const baseUrl = 'https://api.cohere.com'; - const endpoint = '/v1/models'; - const url = `${baseUrl}${endpoint}`; - + const url = 'https://api.cohere.com/v1/models'; let response: Response; try { response = await fetch(url, { method: 'GET', - headers: { - Authorization: `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - }, + headers: { Authorization: `Bearer ${apiKey}` }, signal: AbortSignal.timeout(15_000), }); } catch (err) { const message = err instanceof Error ? err.message : 'Unknown fetch error'; - return { provider_id: 'cohere', models: [], errors: [`Failed to fetch: ${message}`] }; + result.errors.push(`Failed to fetch models: ${message}`); + return result; + } + + if (!response.ok) { + result.errors.push(`HTTP ${response.status}: ${response.statusText}`); + return result; } - let data: unknown; + let raw: unknown; try { - data = await response.json(); - } catch { - return { provider_id: 'cohere', models: [], errors: ['Invalid JSON response'] }; + raw = await response.json(); + } catch (err) { + const message = err instanceof Error ? err.message : 'Unknown parse error'; + result.errors.push(`Failed to parse JSON: ${message}`); + return result; } - const parsed = rawResponseSchema.safeParse(data); + const parsed = rawResponseSchema.safeParse(raw); if (!parsed.success) { - return { - provider_id: 'cohere', - models: [], - errors: [`Validation error: ${parsed.error.message}`], - }; + result.errors.push(`Invalid response shape: ${parsed.error.issues.map(i => i.message).join(', ')}`); + return result; } - const models: CollectionResult['models'] = []; - for (const raw of parsed.data.models) { - const name = raw.name; - const slug = slugify(name); + for (const rawModel of parsed.data.models) { + const slug = slugify(rawModel.name); const modelId = `cohere/${slug}`; - const endpoints = raw.endpoints ?? undefined; - const features = raw.features ?? undefined; - const modality = deriveModality(endpoints, features); - const capabilities = deriveCapabilities(endpoints, features); - const contextLength = raw.context_length; - const contextWindow = - typeof contextLength === 'number' && contextLength > 0 ? contextLength : undefined; - - models.push({ + const features = rawModel.features ?? undefined; + const modality = deriveModality(features); + const caps = deriveCapabilities(features); + const flags = deriveBooleanFlags(features); + + const contextWindow = typeof rawModel.context_length === 'number' && rawModel.context_length > 0 + ? rawModel.context_length + : undefined; + + const model = { model_id: modelId, provider_id: 'cohere', - name: name, - family: name.includes('command') - ? 'command' - : name.includes('embed') - ? 'embed' - : name.includes('rerank') - ? 'rerank' - : undefined, - version: name.includes('v') ? name.split('v')[1]?.split('.')[0] : undefined, - release_date: undefined, - description: undefined, - architecture: undefined, + name: rawModel.name, + family: rawModel.name.split('-').slice(0, -1).join('-') || undefined, + version: rawModel.name.split('-').pop() || undefined, + description: rawModel.name, + architecture: 'unknown', parameter_size: undefined, context_window: contextWindow, - modality: modality, + modality, open_weight: false, - reasoning_support: false, - function_calling: endpoints?.includes('chat') ?? false, - structured_output: endpoints?.includes('chat') ?? false, - vision_support: endpoints?.includes('chat') ?? false, - audio_support: endpoints?.includes('chat') ?? false, - image_generation: false, - embedding_support: endpoints?.includes('embed') ?? false, + reasoning_support: flags.reasoning_support, + function_calling: flags.function_calling, + structured_output: flags.structured_output, + vision_support: flags.vision_support, + audio_support: flags.audio_support, + image_generation: flags.image_generation, + embedding_support: flags.embedding_support, is_free: undefined, tier: undefined, limits: undefined, - capability_ids: capabilities.length > 0 ? capabilities : undefined, + capability_ids: caps, license_id: undefined, - status: raw.finetuned ? 'preview' : 'active', - }); + status: 'active' as const, + }; + + result.models.push(model); } - return { provider_id: 'cohere', models, errors: [] }; + return result; }, } satisfies CustomGateway;