diff --git a/.github/workflows/gateway-ai.yml b/.github/workflows/gateway-ai.yml new file mode 100644 index 000000000..a4953f67b --- /dev/null +++ b/.github/workflows/gateway-ai.yml @@ -0,0 +1,142 @@ +name: AI Gateway Generation + +# Generates a custom gateway plugin with a free-tier LLM. +# 1. Register the gateway in packages/collectors/manifest.json (url + key + optional sample). +# 2. Add the gateway's API key as a repository secret (it must be listed in +# packages/collectors/src/core/gateway-secrets.ts before collection can use it). +# 3. Run this workflow with the gateway id, or open a PR that touches manifest.json +# to validate the manifest without generating anything. +on: + workflow_dispatch: + inputs: + gateway_id: + description: 'Gateway id listed in packages/collectors/manifest.json' + required: true + type: string + mode: + description: 'bootstrap (new plugin) or heal (fix existing plugin)' + required: false + default: bootstrap + type: choice + options: + - bootstrap + - heal + pull_request: + types: [opened, synchronize] + paths: + - 'packages/collectors/manifest.json' + +permissions: + contents: write + pull-requests: write + +jobs: + generate: + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install pnpm + uses: pnpm/action-setup@v3 + with: + version: 9 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build schema & registry + run: pnpm --filter @basemodel/schema --filter @basemodel/registry build + + - name: Generate gateway plugin via LLM + run: | + FLAGS="" + if [ "${{ github.event.inputs.mode }}" == "heal" ]; then FLAGS="--heal"; fi + pnpm --filter @basemodel/collectors run gen-gateway "${{ github.event.inputs.gateway_id }}" $FLAGS + env: + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GOOGLE_AI_API_KEY: ${{ secrets.GOOGLE_AI_API_KEY }} + GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} + TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} + DEEPINFRA_API_KEY: ${{ secrets.DEEPINFRA_API_KEY }} + FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} + CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} + HYPERBOLIC_API_KEY: ${{ secrets.HYPERBOLIC_API_KEY }} + REQUESTY_API_KEY: ${{ secrets.REQUESTY_API_KEY }} + PORTKEY_API_KEY: ${{ secrets.PORTKEY_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} + COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} + LITELLM_BASE_URL: ${{ secrets.LITELLM_BASE_URL }} + LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY }} + + - name: Validate generated plugin + run: | + pnpm --filter @basemodel/collectors run typecheck + pnpm --filter @basemodel/collectors run test + + - name: Commit generated plugin + run: | + BRANCH="ai/gateway/${{ github.event.inputs.gateway_id }}" + git config user.name "BaseModel Bot" + git config user.email "bot@basemodel" + git checkout -b "$BRANCH" + git add packages/collectors/src/gateways packages/collectors/manifest.json + git commit -m "feat(gateways): AI-generated plugin for ${{ github.event.inputs.gateway_id }}" || echo "No changes to commit" + git push -u origin "$BRANCH" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Open PR + run: | + BRANCH="ai/gateway/${{ github.event.inputs.gateway_id }}" + gh pr create \ + --base main \ + --head "$BRANCH" \ + --title "feat(gateways): AI-generated plugin for ${{ github.event.inputs.gateway_id }}" \ + --body "AI-generated gateway plugin for \`${{ github.event.inputs.gateway_id }}\` (mode: ${{ github.event.inputs.mode }}). + + Reviewed before merge, per docs/08_Gateway_Plugin_Security.md: + - Confirm the plugin only uses secrets registered in \`packages/collectors/src/core/gateway-secrets.ts\`. + - Add the gateway's API key as a repository secret so nightly collection can use it. + - Run \`pnpm --filter @basemodel/collectors run verify \` to smoke-test with a live key." \ + || echo "PR may already exist for this gateway." + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + validate-manifest: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install pnpm + uses: pnpm/action-setup@v3 + with: + version: 9 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'pnpm' + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build schema & registry + run: pnpm --filter @basemodel/schema --filter @basemodel/registry build + + - name: Validate manifest + run: | + pnpm --filter @basemodel/collectors exec tsx -e "(async () => { const m = await import('./src/gateway-gen/manifest.js'); await m.loadManifest(); console.log('manifest OK'); })()" diff --git a/README.md b/README.md index 99ce80459..8b183d9d6 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ Useful package-level commands: - `pnpm --filter @basemodel/collectors run collect` - `pnpm --filter @basemodel/collectors run verify packages/collectors/src/gateways/openai.ts` +- `pnpm --filter @basemodel/collectors run gen-gateway ` - generate a custom gateway plugin for a gateway registered in `packages/collectors/manifest.json` ## License diff --git a/docs/04_Pipeline.md b/docs/04_Pipeline.md index d9d8213b8..e01c6a8c2 100644 --- a/docs/04_Pipeline.md +++ b/docs/04_Pipeline.md @@ -25,6 +25,26 @@ Collectors fetch data from official APIs, documentation, or other approved sourc Current collector support includes both OpenAI-compatible gateway plugins and custom gateway plugins. +#### AI Gateway Generation + +Providers whose APIs do not match the OpenAI-compatible shape can register a +gateway in `packages/collectors/manifest.json` (base URL, auth, and optionally a +static sample response). A free-tier LLM then generates a custom gateway plugin: + +1. `pnpm --filter @basemodel/collectors run gen-gateway ` probes the + endpoint (or uses the sample), captures a redacted fixture, and summarizes + the response shape. +2. The LLM writes `packages/collectors/src/gateways/.ts` implementing the + `CustomGateway` contract. +3. Output is structurally validated before use (importable module, correct id, + `collect()` present, no forbidden patterns such as `any` or hardcoded keys) + and retried up to 3 times. +4. `--heal` regenerates a plugin from its existing fixture when a sample changes + or a mapping breaks. + +Generated plugins are proposed through a pull request and must be reviewed +before merge. See `docs/08_Gateway_Plugin_Security.md`. + ### Validation Validation checks required fields, identifier formats, schema compliance, URL validity, and timestamp formats. @@ -98,3 +118,4 @@ The pipeline is automated through GitHub Actions: - `publish.yml` regenerates datasets on push to `main`. - `deploy-pages.yml` publishes the generated static files. - `verify-gateway.yml` checks gateway plugin changes. +- `gateway-ai.yml` generates gateway plugins with an LLM and opens a review PR; it also validates the manifest on PRs that touch it. diff --git a/docs/08_Gateway_Plugin_Security.md b/docs/08_Gateway_Plugin_Security.md index 0560040bf..2a9c1ae5f 100644 --- a/docs/08_Gateway_Plugin_Security.md +++ b/docs/08_Gateway_Plugin_Security.md @@ -26,6 +26,27 @@ the central secret registry. An unregistered gateway receives no secrets. A plugin file must not be able to escalate its own privileges by declaring a new secret name. +## AI-Generated Gateways + +`gateway-gen` writes custom gateway plugins with a free-tier LLM (Gemini +flash-lite by default, OpenRouter `:free` fallback) for providers registered in +`packages/collectors/manifest.json`. The generator stays deterministic about +data: it probes the endpoint (or reads a static sample) and feeds the LLM a +redacted fixture plus a shape summary, never live credentials. + +AI output is treated as untrusted code. Before a generated plugin is accepted: + +- It is structurally validated before writing (importable module, correct + gateway id, `collect()` present) and rejected when it uses forbidden patterns + (`any`, hardcoded API keys, `process.exit`) or refers to secrets not listed in + `gateway-secrets.ts`. +- The workflow opens a pull request; human review is required before merge. +- Reviewers re-check secret usage and add the provider's API key as a repository + secret; only then does nightly collection use the plugin. + +New custom gateways therefore pass through the same review gate as hand-written +plugins. + ## Remaining Risk Process isolation reduces exposure, but it is not an operating-system sandbox. diff --git a/packages/collectors/manifest.json b/packages/collectors/manifest.json new file mode 100644 index 000000000..e5f7c8d76 --- /dev/null +++ b/packages/collectors/manifest.json @@ -0,0 +1,79 @@ +{ + "version": 1, + "gateways": [ + { + "id": "cohere", + "baseUrl": "https://api.cohere.com", + "endpoint": "/v1/models", + "auth": { + "type": "bearer", + "secret": "COHERE_API_KEY" + }, + "secrets": ["COHERE_API_KEY"], + "sample": { + "models": [ + { + "name": "command-r-plus", + "endpoints": ["chat", "embed", "generate", "summarize", "classify", "rerank"], + "finetuned": false, + "context_length": 128000, + "tokenizer_url": "https://docs.cohere.com/docs/tokenizer", + "default_endpoints": ["chat"], + "features": ["supports_rag", "experimental_chat"] + }, + { + "name": "command-r", + "endpoints": ["chat", "embed", "generate", "summarize", "classify", "rerank"], + "finetuned": false, + "context_length": 128000, + "default_endpoints": ["chat"], + "features": ["supports_rag"] + }, + { + "name": "command-light", + "endpoints": ["chat", "generate", "summarize"], + "finetuned": false, + "context_length": 4096, + "default_endpoints": ["chat"], + "features": null + }, + { + "name": "embed-english-v3.0", + "endpoints": ["embed"], + "finetuned": false, + "context_length": 512, + "default_endpoints": ["embed"], + "features": ["search", "classification"] + }, + { + "name": "rerank-english-v3.0", + "endpoints": ["rerank"], + "finetuned": false, + "context_length": 4096, + "default_endpoints": ["rerank"], + "features": null + }, + { + "name": "embed-english-light-v3.0", + "endpoints": ["embed"], + "finetuned": false, + "context_length": null, + "tokenizer_url": null, + "default_endpoints": ["embed"], + "features": null + }, + { + "name": "command-nightly-finetuned", + "endpoints": [], + "finetuned": true, + "context_length": 0, + "default_endpoints": [], + "features": null + } + ], + "next_page_token": "cD0yCg==" + }, + "notes": "Example gateway for the AI generator: non-OpenAI-compatible response shape ({ models: [...] }). A static sample is provided so the generator can run without a live API key." + } + ] +} diff --git a/packages/collectors/package.json b/packages/collectors/package.json index d8e556468..346471429 100644 --- a/packages/collectors/package.json +++ b/packages/collectors/package.json @@ -16,6 +16,7 @@ "build": "tsup src/index.ts --format esm --dts --clean", "collect": "tsx src/run.ts", "enrich": "tsx src/enrich/run.ts", + "gen-gateway": "tsx src/gateway-gen/index.ts", "verify": "tsx src/core/verify.ts", "typecheck": "tsc --noEmit", "test": "vitest run --passWithNoTests", diff --git a/packages/collectors/src/__tests__/generated-gateways.test.ts b/packages/collectors/src/__tests__/generated-gateways.test.ts new file mode 100644 index 000000000..49711f19d --- /dev/null +++ b/packages/collectors/src/__tests__/generated-gateways.test.ts @@ -0,0 +1,69 @@ +import { existsSync } from 'node:fs'; +import { readFile } from 'node:fs/promises'; +import { ModelSchema } from '@basemodel/schema'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { getFixturePath, getGatewayPluginPath, loadManifest } from '../gateway-gen/manifest.js'; +import { fixtureExists } from '../gateway-gen/probe.js'; + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe('AI-generated gateway plugins', () => { + it('map their fixture sample into valid Models without a live API', async () => { + const manifest = await loadManifest(); + let exercised = 0; + + for (const gateway of manifest.gateways) { + const pluginPath = getGatewayPluginPath(gateway.id); + const fixturePath = getFixturePath(gateway.id); + if (!existsSync(pluginPath) || !fixtureExists(gateway.id)) continue; + + const fixture = JSON.parse(await readFile(fixturePath, 'utf-8')) as unknown; + const plugin = ((await import(pluginPath)) as { default?: unknown }).default as { + id?: string; + collect: (secrets: Record) => Promise<{ + provider_id: string; + models: Array>; + errors: string[]; + }>; + }; + expect(plugin.id).toBe(gateway.id); + + let fetchCalls = 0; + const fetchMock = vi.fn(async () => { + fetchCalls += 1; + if (fetchCalls > 2) { + throw new Error('mock: no more pages (pagination guard)'); + } + return new Response(JSON.stringify(fixture), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }); + vi.stubGlobal('fetch', fetchMock); + + const secrets: Record = { ...process.env }; + for (const name of [ + ...gateway.secrets, + ...(gateway.auth?.secret ? [gateway.auth.secret] : []), + ]) { + if (!secrets[name]) secrets[name] = 'test-key'; + } + + const result = await plugin.collect(secrets); + expect(fetchMock).toHaveBeenCalled(); + expect(result.provider_id).toBe(gateway.id); + for (const model of result.models) { + const parsed = ModelSchema.safeParse(model); + expect( + parsed.success, + `${model.model_id ?? '(no model_id)'} invalid: ${parsed.error?.message}`, + ).toBe(true); + } + exercised += 1; + } + + expect(exercised).toBeGreaterThan(0); + }); +}); diff --git a/packages/collectors/src/core/gateway-secrets.ts b/packages/collectors/src/core/gateway-secrets.ts index 6b831734e..ae075c5d2 100644 --- a/packages/collectors/src/core/gateway-secrets.ts +++ b/packages/collectors/src/core/gateway-secrets.ts @@ -6,6 +6,7 @@ export const GATEWAY_SECRET_KEYS = { anthropic: ['ANTHROPIC_API_KEY'], cerebras: ['CEREBRAS_API_KEY'], cloudflare: ['CLOUDFLARE_ACCOUNT_ID', 'CLOUDFLARE_API_TOKEN'], + cohere: ['COHERE_API_KEY'], deepinfra: ['DEEPINFRA_API_KEY'], fireworks: ['FIREWORKS_API_KEY'], google: ['GOOGLE_AI_API_KEY'], diff --git a/packages/collectors/src/gateway-gen/__tests__/probe.test.ts b/packages/collectors/src/gateway-gen/__tests__/probe.test.ts new file mode 100644 index 000000000..aa8a3244d --- /dev/null +++ b/packages/collectors/src/gateway-gen/__tests__/probe.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from 'vitest'; +import { loadManifest } from '../manifest.js'; +import { extractShape } from '../probe.js'; + +describe('extractShape', () => { + it('detects a model array with id/name keys', () => { + const raw = { + models: [ + { name: 'command-r-plus', endpoints: ['chat', 'embed'], context_length: 128000 }, + { name: 'embed-english-v3.0', endpoints: ['embed'] }, + ], + }; + const shape = extractShape(raw); + expect(shape.topLevel).toEqual({ models: 'array[2]' }); + expect(shape.modelArray).not.toBeNull(); + expect(shape.modelArray?.path).toBe('$.models'); + expect(shape.modelArray?.count).toBe(2); + expect(shape.modelArray?.keys).toEqual(['name', 'endpoints', 'context_length']); + }); + + it('returns null modelArray when no object with id/name exists', () => { + const shape = extractShape({ status: 'ok', total: 0, items: [1, 2, 3] }); + expect(shape.modelArray).toBeNull(); + expect(shape.topLevel).toEqual({ status: 'string', total: 'number', items: 'array[3]' }); + }); + + it('picks the largest array of model-like objects', () => { + const raw = { + small: [{ id: 'a' }], + big: [{ id: 'x' }, { id: 'y' }, { id: 'z' }], + }; + const shape = extractShape(raw); + expect(shape.modelArray?.path).toBe('$.big'); + expect(shape.modelArray?.count).toBe(3); + }); +}); + +describe('manifest', () => { + it('loads and contains the example cohere gateway', async () => { + const manifest = await loadManifest(); + expect(manifest.version).toBe(1); + const cohere = manifest.gateways.find((gateway) => gateway.id === 'cohere'); + expect(cohere).toBeDefined(); + expect(cohere?.baseUrl).toBe('https://api.cohere.com'); + expect(cohere?.auth?.secret).toBe('COHERE_API_KEY'); + }); +}); diff --git a/packages/collectors/src/gateway-gen/heal.ts b/packages/collectors/src/gateway-gen/heal.ts new file mode 100644 index 000000000..45f6497c4 --- /dev/null +++ b/packages/collectors/src/gateway-gen/heal.ts @@ -0,0 +1,53 @@ +import { readFile, writeFile } from 'node:fs/promises'; +import { generateText } from './llm.js'; +import type { ManifestGateway } from './manifest.js'; +import { getGatewayPluginPath } from './manifest.js'; +import type { ShapeSummary } from './probe.js'; +import { buildHealPrompt } from './prompts.js'; +import { + ensureFinalNewline, + extractTsCode, + type GeneratedPlugin, + validateGeneratedPlugin, +} from './write.js'; + +export interface HealPluginOptions { + gateway: ManifestGateway; + shape: ShapeSummary; + raw: unknown; + errors?: string[]; + env?: NodeJS.ProcessEnv; + maxAttempts?: number; + liveSecrets?: Record; +} + +export async function healPlugin(options: HealPluginOptions): Promise { + const { + gateway, + shape, + raw, + errors = [], + env = process.env, + maxAttempts = 6, + liveSecrets, + } = options; + const filePath = getGatewayPluginPath(gateway.id); + const currentCode = await readFile(filePath, 'utf-8'); + let prompt = buildHealPrompt(gateway, shape, raw, currentCode, errors); + const lastErrors: string[] = []; + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + const text = await generateText({ prompt }, env); + const code = ensureFinalNewline(extractTsCode(text)); + await writeFile(filePath, code, 'utf-8'); + const validation = await validateGeneratedPlugin(filePath, gateway.id, liveSecrets); + if (validation.ok) return { filePath, code, attempts: attempt }; + lastErrors.push(...validation.errors); + console.warn(`[heal] attempt ${attempt} failed validation: ${validation.errors.join('; ')}`); + prompt += `\n\nThe previous attempt failed validation:\n${validation.errors.join('\n')}\n\nReturn the corrected file only.`; + } + + throw new Error( + `Failed to heal plugin for ${gateway.id} after ${maxAttempts} attempts:\n${lastErrors.join('\n')}`, + ); +} diff --git a/packages/collectors/src/gateway-gen/index.ts b/packages/collectors/src/gateway-gen/index.ts new file mode 100644 index 000000000..aa79566f2 --- /dev/null +++ b/packages/collectors/src/gateway-gen/index.ts @@ -0,0 +1,96 @@ +import { readFile } from 'node:fs/promises'; +import { healPlugin } from './heal.js'; +import { findGateway } from './manifest.js'; +import { extractShape, probeGateway, readFixture } from './probe.js'; +import { generatePlugin } from './write.js'; + +function hasAnySecret( + gateway: { secrets: string[]; auth?: { secret?: string } }, + env: NodeJS.ProcessEnv, +): boolean { + const names = [...gateway.secrets]; + if (gateway.auth?.secret) names.push(gateway.auth.secret); + return names.some((name) => Boolean(env[name])); +} + +function liveSecretsFor( + gateway: { secrets: string[]; auth?: { secret?: string } }, + env: NodeJS.ProcessEnv, +): Record | undefined { + return hasAnySecret(gateway, env) ? env : undefined; +} + +async function bootstrap(gatewayId: string, env: NodeJS.ProcessEnv): Promise { + const gateway = await findGateway(gatewayId); + console.log(`Probing ${gateway.id} (${gateway.baseUrl})...`); + const probe = await probeGateway(gateway, env); + console.log( + ` endpoint : ${probe.endpoint}${probe.fromSample ? ' (from manifest sample)' : ''}`, + ); + console.log(` fixture : ${probe.fixturePath}`); + console.log( + ` shape : ${probe.shape.modelArray ? `model array at "${probe.shape.modelArray.path}" (${probe.shape.modelArray.count} items)` : 'no model array detected'}`, + ); + + const liveSecrets = liveSecretsFor(gateway, env); + console.log(`Generating plugin via LLM...`); + const generated = await generatePlugin({ + gateway, + shape: probe.shape, + raw: probe.raw, + env, + liveSecrets, + }); + console.log(` wrote : ${generated.filePath} (attempt ${generated.attempts})`); + if (!liveSecrets) { + console.warn( + ` live check : skipped (no API key in env). The generated plugin was validated for structure and types; ` + + 'add a fixture-based unit test or set the gateway key to verify mapping.', + ); + } +} + +async function heal(gatewayId: string, env: NodeJS.ProcessEnv, errorsFile?: string): Promise { + const gateway = await findGateway(gatewayId); + const raw = await readFixture(gatewayId); + const shape = extractShape(raw); + let errors: string[] = []; + if (errorsFile) { + const parsed = JSON.parse(await readFile(errorsFile, 'utf-8')) as unknown; + errors = Array.isArray(parsed) ? parsed.map(String) : [String(parsed)]; + } + console.log(`Healing plugin for ${gateway.id}...`); + const healed = await healPlugin({ + gateway, + shape, + raw, + errors, + env, + liveSecrets: liveSecretsFor(gateway, env), + }); + console.log(` wrote : ${healed.filePath} (attempt ${healed.attempts})`); +} + +async function main(): Promise { + const args = process.argv.slice(2); + const gatewayId = args.find((arg) => !arg.startsWith('--')); + if (!gatewayId) { + console.error('Usage: tsx src/gateway-gen/index.ts [--heal] [--errors ]'); + process.exit(1); + } + const env = process.env; + try { + if (args.includes('--heal')) { + const errorsIndex = args.indexOf('--errors'); + const errorsFile = errorsIndex >= 0 ? args[errorsIndex + 1] : undefined; + await heal(gatewayId, env, errorsFile); + } else { + await bootstrap(gatewayId, env); + } + } catch (error: unknown) { + console.error(`❌ ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); + } +} + +main(); diff --git a/packages/collectors/src/gateway-gen/llm.ts b/packages/collectors/src/gateway-gen/llm.ts new file mode 100644 index 000000000..69d95c3bc --- /dev/null +++ b/packages/collectors/src/gateway-gen/llm.ts @@ -0,0 +1,152 @@ +const GEMINI_ENDPOINT = 'https://generativelanguage.googleapis.com/v1beta/models'; +const OPENROUTER_ENDPOINT = 'https://openrouter.ai/api/v1/chat/completions'; +const REQUESTY_ENDPOINT = 'https://router.requesty.ai/v1/chat/completions'; + +export interface LlmConfig { + prompt: string; + temperature?: number; +} + +export type Provider = 'gemini' | 'openrouter' | 'requesty'; + +function resolveProviders(env: NodeJS.ProcessEnv): Provider[] { + const forced = env.LLM_PROVIDER; + if (forced === 'openrouter' || forced === 'gemini' || forced === 'requesty') return [forced]; + const list: Provider[] = []; + if (env.REQUESTY_API_KEY) list.push('requesty'); + if (env.GEMINI_API_KEY) list.push('gemini'); + if (env.OPENROUTER_API_KEY) list.push('openrouter'); + if (list.length === 0) { + throw new Error( + 'No LLM provider configured. Set REQUESTY_API_KEY, GEMINI_API_KEY, or ' + + 'OPENROUTER_API_KEY (all have free tiers) to generate gateway plugins.', + ); + } + return list; +} + +async function callRequesty(prompt: string, env: NodeJS.ProcessEnv): Promise { + const apiKey = env.REQUESTY_API_KEY; + const model = env.REQUESTY_MODEL ?? 'mistral/leanstral-1-5'; + const response = await fetch(REQUESTY_ENDPOINT, { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + 'HTTP-Referer': 'https://basemodel.ai', + 'X-Title': 'BaseModel gateway generator', + }, + body: JSON.stringify({ + model, + messages: [{ role: 'user', content: prompt }], + temperature: 0.2, + max_tokens: 8192, + }), + signal: AbortSignal.timeout(120_000), + }); + if (!response.ok) { + const body = await response.text().catch(() => ''); + throw new Error(`Requesty HTTP ${response.status}: ${body.slice(0, 300)}`); + } + const data = (await response.json()) as { + choices?: Array<{ message?: { content?: string } }>; + }; + const text = data.choices?.[0]?.message?.content; + if (!text) throw new Error('Requesty returned an empty response.'); + return text; +} + +async function callGemini(prompt: string, env: NodeJS.ProcessEnv): Promise { + const apiKey = env.GEMINI_API_KEY; + const model = env.GEMINI_MODEL ?? 'gemini-flash-lite-latest'; + const response = await fetch(`${GEMINI_ENDPOINT}/${model}:generateContent?key=${apiKey}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + contents: [{ role: 'user', parts: [{ text: prompt }] }], + generationConfig: { temperature: 0.2, maxOutputTokens: 8192 }, + }), + signal: AbortSignal.timeout(120_000), + }); + if (!response.ok) { + const body = await response.text().catch(() => ''); + throw new Error(`Gemini HTTP ${response.status}: ${body.slice(0, 300)}`); + } + const data = (await response.json()) as { + candidates?: Array<{ content?: { parts?: Array<{ text?: string }> } }>; + }; + const text = data.candidates?.[0]?.content?.parts?.map((part) => part.text ?? '').join(''); + if (!text) throw new Error('Gemini returned an empty response.'); + return text; +} + +const OPENROUTER_FREE_CANDIDATES = [ + 'cohere/north-mini-code:free', + 'google/gemma-4-31b-it:free', + 'openai/gpt-oss-20b:free', + 'google/gemma-4-26b-a4b-it:free', + 'poolside/laguna-s-2.1:free', +]; + +async function callOpenRouter(prompt: string, env: NodeJS.ProcessEnv): Promise { + const apiKey = env.OPENROUTER_API_KEY; + const candidates = env.OPENROUTER_MODEL ? [env.OPENROUTER_MODEL] : OPENROUTER_FREE_CANDIDATES; + const failures: string[] = []; + for (const model of candidates) { + try { + const response = await fetch(OPENROUTER_ENDPOINT, { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + 'HTTP-Referer': 'https://basemodel.ai', + 'X-Title': 'BaseModel gateway generator', + }, + body: JSON.stringify({ + model, + messages: [{ role: 'user', content: prompt }], + temperature: 0.2, + max_tokens: 8192, + }), + signal: AbortSignal.timeout(120_000), + }); + if (!response.ok) { + const body = await response.text().catch(() => ''); + throw new Error(`HTTP ${response.status}: ${body.slice(0, 200)}`); + } + const data = (await response.json()) as { + choices?: Array<{ message?: { content?: string } }>; + }; + const text = data.choices?.[0]?.message?.content; + if (!text) throw new Error('empty response'); + return text; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + failures.push(`${model}: ${message}`); + console.warn(`[llm] openrouter ${model} failed: ${message}`); + } + } + throw new Error(`OpenRouter failed on all candidate models: ${failures.join(' | ')}`); +} + +export async function generateText( + config: LlmConfig, + env: NodeJS.ProcessEnv = process.env, +): Promise { + const providers = resolveProviders(env); + const failures: string[] = []; + for (const provider of providers) { + try { + if (provider === 'gemini') return await callGemini(config.prompt, env); + if (provider === 'openrouter') return await callOpenRouter(config.prompt, env); + return await callRequesty(config.prompt, env); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + failures.push(`${provider}: ${message}`); + console.warn( + `[llm] ${provider} failed, ${providers.length > 1 ? 'trying next provider' : 'no fallback left'}: ${message}`, + ); + } + } + throw new Error(`All LLM providers failed: ${failures.join(' | ')}`); +} diff --git a/packages/collectors/src/gateway-gen/manifest.ts b/packages/collectors/src/gateway-gen/manifest.ts new file mode 100644 index 000000000..3c021f1b8 --- /dev/null +++ b/packages/collectors/src/gateway-gen/manifest.ts @@ -0,0 +1,78 @@ +import { existsSync } from 'node:fs'; +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { z } from 'zod'; + +export const GatewayAuthSchema = z + .object({ + type: z.enum(['bearer', 'header', 'query']), + secret: z.string().min(1), + headerName: z.string().optional(), + }) + .optional(); + +export const ManifestGatewaySchema = z.object({ + id: z.string().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/), + baseUrl: z.string().url(), + endpoint: z.string().optional(), + method: z.enum(['GET', 'POST']).default('GET'), + auth: GatewayAuthSchema, + extraHeaders: z.record(z.string(), z.string()).optional(), + secrets: z.array(z.string()).default([]), + sample: z.unknown().optional(), + notes: z.string().optional(), +}); + +export const ManifestSchema = z.object({ + version: z.number().int().positive().default(1), + gateways: z.array(ManifestGatewaySchema), +}); + +export type ManifestGateway = z.infer; +export type Manifest = z.infer; + +export const GATEWAYS_DIR = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '..', + 'gateways', +); + +export const FIXTURES_DIR = path.join(GATEWAYS_DIR, '__fixtures__'); + +export function getManifestPath(): string { + return path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..', 'manifest.json'); +} + +export async function loadManifest(): Promise { + const manifestPath = getManifestPath(); + if (!existsSync(manifestPath)) { + throw new Error(`Manifest not found: ${manifestPath}`); + } + const raw = await readFile(manifestPath, 'utf-8'); + const parsed = ManifestSchema.safeParse(JSON.parse(raw) as unknown); + if (!parsed.success) { + throw new Error(`Invalid gateway manifest: ${parsed.error.message}`); + } + return parsed.data; +} + +export async function findGateway(gatewayId: string): Promise { + const manifest = await loadManifest(); + const gateway = manifest.gateways.find((entry) => entry.id === gatewayId); + if (!gateway) { + throw new Error( + `Gateway "${gatewayId}" is not listed in ${getManifestPath()}. ` + + 'Add it first so the generator knows its endpoint and auth.', + ); + } + return gateway; +} + +export function getGatewayPluginPath(gatewayId: string): string { + return path.join(GATEWAYS_DIR, `${gatewayId}.ts`); +} + +export function getFixturePath(gatewayId: string): string { + return path.join(FIXTURES_DIR, `${gatewayId}.raw.json`); +} diff --git a/packages/collectors/src/gateway-gen/probe.ts b/packages/collectors/src/gateway-gen/probe.ts new file mode 100644 index 000000000..0341bb1b2 --- /dev/null +++ b/packages/collectors/src/gateway-gen/probe.ts @@ -0,0 +1,260 @@ +import { existsSync } from 'node:fs'; +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import type { ManifestGateway } from './manifest.js'; +import { FIXTURES_DIR, getFixturePath } from './manifest.js'; + +const DEFAULT_ENDPOINTS = [ + '/models', + '/v1/models', + '/api/models', + '/api/v1/models', + '/model', + '/models?limit=5', +]; + +const RETRYABLE_STATUSES = new Set([408, 429, 500, 502, 503, 504]); + +export interface ModelArrayCandidate { + path: string; + count: number; + keys: string[]; + example: unknown; +} + +export interface ShapeSummary { + topLevel: Record; + modelArray: ModelArrayCandidate | null; +} + +export interface ProbeResult { + gatewayId: string; + fixturePath: string; + endpoint: string; + fromSample: boolean; + raw: unknown; + shape: ShapeSummary; +} + +function typeName(value: unknown): string { + if (value === null) return 'null'; + if (Array.isArray(value)) return `array[${value.length}]`; + if (typeof value === 'object') return 'object'; + return typeof value; +} + +function truncateString(value: string, max = 120): string { + return value.length > max ? `${value.slice(0, max)}...` : value; +} + +function truncateExample(value: unknown, depth = 0): unknown { + if (depth > 3) return typeof value; + if (Array.isArray(value)) { + return value.slice(0, 5).map((item) => truncateExample(item, depth + 1)); + } + if (value !== null && typeof value === 'object') { + const out: Record = {}; + for (const [key, item] of Object.entries(value as Record)) { + out[key] = typeof item === 'string' ? truncateString(item) : truncateExample(item, depth + 1); + } + return out; + } + return value; +} + +function findModelArray(value: unknown): ModelArrayCandidate | null { + let best: ModelArrayCandidate | null = null; + const visit = (node: unknown, pathLabel: string): void => { + if (!node || typeof node !== 'object') return; + if (Array.isArray(node)) { + const objects = node.filter((item) => item !== null && typeof item === 'object'); + const hasIdOrName = objects.some( + (item) => + typeof (item as Record).id === 'string' || + typeof (item as Record).name === 'string', + ); + if (objects.length > 0 && hasIdOrName && (!best || objects.length > best.count)) { + const first = objects[0] as Record; + best = { + path: pathLabel, + count: objects.length, + keys: Object.keys(first), + example: truncateExample(first), + }; + } + for (const item of objects.slice(0, 5)) visit(item, `${pathLabel}[i]`); + return; + } + for (const [key, item] of Object.entries(node as Record)) { + visit(item, `${pathLabel}.${key}`); + } + }; + visit(value, '$'); + return best; +} + +export function extractShape(raw: unknown): ShapeSummary { + const topLevel: Record = {}; + if (raw !== null && typeof raw === 'object' && !Array.isArray(raw)) { + for (const [key, value] of Object.entries(raw as Record)) { + topLevel[key] = typeName(value); + } + } + return { topLevel, modelArray: findModelArray(raw) }; +} + +function redact(value: string, secrets: readonly string[]): string { + return secrets + .filter(Boolean) + .reduce((result, secret) => result.split(String(secret)).join('[REDACTED]'), value); +} + +function truncateArrays(value: unknown, depth = 0): unknown { + if (depth > 8) return typeof value; + if (Array.isArray(value)) { + return value.slice(0, 5).map((item) => truncateArrays(item, depth + 1)); + } + if (value !== null && typeof value === 'object') { + const out: Record = {}; + for (const [key, item] of Object.entries(value as Record)) { + out[key] = truncateArrays(item, depth + 1); + } + return out; + } + return value; +} + +function buildHeaders( + gateway: ManifestGateway, + env: NodeJS.ProcessEnv, +): { headers: Record; ok: boolean; missing: string[] } { + const headers: Record = { Accept: 'application/json' }; + if (gateway.extraHeaders) Object.assign(headers, gateway.extraHeaders); + const missing: string[] = []; + if (gateway.auth) { + const key = env[gateway.auth.secret]; + if (!key) { + missing.push(gateway.auth.secret); + } else if (gateway.auth.type === 'bearer') { + headers.Authorization = `Bearer ${key}`; + } else if (gateway.auth.type === 'header') { + headers[gateway.auth.headerName ?? 'x-api-key'] = key; + } + } + if (gateway.method === 'POST') headers['Content-Type'] = 'application/json'; + return { headers, ok: missing.length === 0, missing }; +} + +async function fetchJson( + url: string, + init: RequestInit, + attempts = 2, +): Promise<{ ok: boolean; status: number; data: unknown }> { + let last: Response | null = null; + for (let attempt = 1; attempt <= attempts; attempt++) { + const response = await fetch(url, init); + if (!RETRYABLE_STATUSES.has(response.status)) { + const data = await response.json().catch(() => null); + return { ok: response.ok, status: response.status, data }; + } + last = response; + if (attempt < attempts) await new Promise((resolve) => setTimeout(resolve, attempt * 500)); + } + return { ok: false, status: last?.status ?? 0, data: null }; +} + +export async function probeGateway( + gateway: ManifestGateway, + env: NodeJS.ProcessEnv = process.env, +): Promise { + const { headers, ok, missing } = buildHeaders(gateway, env); + const hasSample = gateway.sample !== undefined; + + const sampleResult = async (): Promise => { + const fixturePath = await saveFixture(gateway.id, gateway.sample, gateway.secrets); + return { + gatewayId: gateway.id, + fixturePath, + endpoint: gateway.endpoint ?? '(manifest sample)', + fromSample: true, + raw: gateway.sample, + shape: extractShape(gateway.sample), + }; + }; + + const liveResult = async (): Promise => { + const endpoints = gateway.endpoint ? [gateway.endpoint] : DEFAULT_ENDPOINTS; + const failures: string[] = []; + for (const endpoint of endpoints) { + const url = new URL(endpoint, gateway.baseUrl).toString(); + try { + const result = await fetchJson(url, { + method: gateway.method, + headers, + signal: AbortSignal.timeout(15_000), + }); + if (result.ok && result.data !== null) { + const fixturePath = await saveFixture(gateway.id, result.data, gateway.secrets); + return { + gatewayId: gateway.id, + fixturePath, + endpoint, + fromSample: false, + raw: result.data, + shape: extractShape(result.data), + }; + } + failures.push(`${endpoint} -> HTTP ${result.status}`); + } catch (error: unknown) { + failures.push(`${endpoint} -> ${error instanceof Error ? error.message : String(error)}`); + } + } + throw new Error( + `Probe for ${gateway.id} failed on all candidate endpoints: ${failures.join(' | ')}`, + ); + }; + + if (hasSample && !ok) { + console.warn(` probe : no API key available, using manifest sample`); + return sampleResult(); + } + if (hasSample && ok) { + try { + return await liveResult(); + } catch (error) { + console.warn( + ` probe : live probe failed (${error instanceof Error ? error.message : String(error)}), using manifest sample`, + ); + return sampleResult(); + } + } + if (!ok) { + throw new Error( + `Cannot probe ${gateway.id}: missing API key(s) ${missing.join(', ')}. ` + + 'Set them in the environment, or add a "sample" to the manifest to skip live probing.', + ); + } + return liveResult(); +} + +export async function saveFixture( + gatewayId: string, + raw: unknown, + secrets: readonly string[], +): Promise { + await mkdir(FIXTURES_DIR, { recursive: true }); + const serialized = JSON.stringify(truncateArrays(raw), null, 2); + const safe = redact(serialized, secrets); + const fixturePath = getFixturePath(gatewayId); + await writeFile(fixturePath, `${safe}\n`, 'utf-8'); + return fixturePath; +} + +export async function readFixture(gatewayId: string): Promise { + const fixturePath = getFixturePath(gatewayId); + const raw = await readFile(fixturePath, 'utf-8'); + return JSON.parse(raw) as unknown; +} + +export function fixtureExists(gatewayId: string): boolean { + return existsSync(getFixturePath(gatewayId)); +} diff --git a/packages/collectors/src/gateway-gen/prompts.ts b/packages/collectors/src/gateway-gen/prompts.ts new file mode 100644 index 000000000..9a580b7ee --- /dev/null +++ b/packages/collectors/src/gateway-gen/prompts.ts @@ -0,0 +1,176 @@ +import type { ManifestGateway } from './manifest.js'; +import type { ShapeSummary } from './probe.js'; + +const MODEL_SCHEMA_DOC = ` +Canonical "Model" type (TypeScript, from @basemodel/schema). Each catalog entry must map to an object satisfying this shape: +type Model = { + model_id: string; // [required] matches /^[a-z0-9-]+\\/[a-z0-9]+(?:[-.][a-z0-9]+)*$/ e.g. "cohere/command-r-plus" + provider_id: string; // [required] e.g. "cohere" + name: string; // [required] human-readable name + family?: string; + version?: string; + release_date?: string; // ISO YYYY-MM-DD + description?: string; + architecture?: string; + parameter_size?: string; + context_window?: number; // positive integer (tokens); omit when unknown + modality: ('text'|'image'|'audio'|'video'|'code'|'embedding')[]; // [required] + open_weight: boolean; // [required] + reasoning_support: boolean; // [required] + function_calling: boolean; // [required] + structured_output: boolean; // [required] + vision_support: boolean; // [required] + audio_support: boolean; // [required] + image_generation: boolean; // [required] + embedding_support: boolean; // [required] + is_free?: boolean; + tier?: 'free'|'budget'|'balanced'|'premium'; + limits?: object; // omit unless you have exact pricing/limits data + capability_ids?: string[]; // defaults to [] + license_id?: string; + status: 'active'|'preview'|'deprecated'|'discontinued'; // [required] +}; +Rules: +- Set every required field; every key must be spelled exactly as above (this is a strict TS type). +- When a source value is null or missing, OMIT the optional field (or map to undefined) - never assign null. +- context_window must be a positive integer or absent - never null, never 0. +- Derive modality from source hints (e.g. "embed" -> 'embedding'); default to ['text']. +`; + +const PLUGIN_CONTRACT = ` +The plugin is a TypeScript module with a default export that satisfies CustomGateway: + +import { z } from 'zod'; +import type { CollectionResult, CustomGateway } from '../core/collector'; + +export default { + type: 'custom', + id: '', + async collect(secrets: Record): Promise { + const result: CollectionResult = { provider_id: '', models: [], errors: [] }; + // ... fetch the catalog, parse with zod, map entries to partial Models ... + return result; + }, +} satisfies CustomGateway; + +Guidelines: +- Build the URL from the manifest's baseUrl + endpoint. Use only the secret names granted by the manifest. +- The raw response zod schema must describe ONLY what the API actually returns, using permissive types: numbers may be null or 0, arrays may be null, strings may be absent. Do NOT copy the BaseModel Model constraints above (model_id regex, positive context_window, status/modality enums) into the raw response schema - those apply ONLY when building the output Model objects. +- Validate the raw JSON with a zod schema; on parse failure push a short message to result.errors and return. +- Map each catalog entry to a Model. model_id must be "/"; derive the slug from the provider's id/name using .toLowerCase() and replacing characters outside [a-z0-9.-] with '-', collapsing repeats. +- Use this exact slugify helper (the two-argument form of .replace is required, otherwise typecheck fails with TS2554): + function slugify(text: string): string { + return text.toLowerCase().replace(/[^a-z0-9.-]+/g, '-').replace(/-+/g, '-').replace(/^-|-$/g, ''); + } +- Set every required boolean field. Derive modality/capability flags from hints in the response (e.g. name/keywords like "embed", "image", "audio", "vision", "code") when available, otherwise default to text-only. +- When building a Model, map raw context_length to context_window ONLY if it is a positive integer; otherwise omit context_window entirely. +- Raw nullable fields (e.g. features or endpoints that can be null) must be normalized before passing to helpers: write const features = raw.features ?? undefined and pass the normalized value, or type the helper parameter as 'string[] | null | undefined'. Never pass a string[] | null | undefined value to a parameter typed string[] | undefined. +- Handle pagination if the response exposes it (next page URL / total counts); never loop forever (cap pages). +- Use AbortSignal.timeout(15_000) on fetch. Catch errors and push them to result.errors (do not throw). +- Never use the "any" type anywhere. Type catch variables as "unknown" and narrow with instanceof/typeof. Do not use "as any". +- Do not declare unused variables or helper functions; every declared name must be referenced, or the file fails typecheck (noUnusedLocals). +- Never hardcode real API keys; only reference secrets by their env names. +- Do not add code comments. +`; + +function describeShape(shape: ShapeSummary, raw: unknown): string { + const lines: string[] = ['Raw response shape summary:']; + lines.push(` top-level keys: ${JSON.stringify(shape.topLevel)}`); + if (shape.modelArray) { + lines.push( + ` model array at "${shape.modelArray.path}" (${shape.modelArray.count} items), keys: ` + + JSON.stringify(shape.modelArray.keys), + ); + lines.push(` example record: ${JSON.stringify(shape.modelArray.example, null, 2)}`); + } else { + lines.push(' no obvious array of models detected; inspect the raw sample.'); + } + lines.push(''); + lines.push('Raw sample (truncated):'); + lines.push(JSON.stringify(raw, null, 2).slice(0, 6000)); + return lines.join('\n'); +} + +export function buildPluginPrompt( + gateway: ManifestGateway, + shape: ShapeSummary, + raw: unknown, +): string { + return [ + 'You are writing a BaseModel gateway plugin in TypeScript. BaseModel collects a catalog of ', + 'AI models from provider APIs. Write a complete, compilable custom gateway plugin for the ', + 'provider described below.', + '', + 'Gateway manifest entry:', + JSON.stringify( + { + id: gateway.id, + baseUrl: gateway.baseUrl, + endpoint: gateway.endpoint, + method: gateway.method, + auth: gateway.auth, + extraHeaders: gateway.extraHeaders, + secrets: gateway.secrets, + }, + null, + 2, + ), + '', + MODEL_SCHEMA_DOC, + '', + PLUGIN_CONTRACT, + '', + describeShape(shape, raw), + '', + 'Output ONLY the complete TypeScript file content. Do not wrap it in markdown fences, do not ', + 'add explanations or a leading title. The response must be valid TypeScript that can be ', + 'written verbatim to src/gateways/.ts.', + ].join('\n'); +} + +export function buildHealPrompt( + gateway: ManifestGateway, + shape: ShapeSummary, + raw: unknown, + currentCode: string, + errors: string[], +): string { + const errorBlock = + errors.length > 0 + ? ['Validation errors to fix:', ...errors.map((error) => `- ${error}`)].join('\n') + : 'No specific errors were provided; inspect the mapping carefully and fix anything that looks wrong.'; + + return [ + 'You previously generated a BaseModel gateway plugin for the provider below. It needs fixing.', + '', + 'Gateway manifest entry:', + JSON.stringify( + { + id: gateway.id, + baseUrl: gateway.baseUrl, + endpoint: gateway.endpoint, + auth: gateway.auth, + extraHeaders: gateway.extraHeaders, + secrets: gateway.secrets, + }, + null, + 2, + ), + '', + MODEL_SCHEMA_DOC, + '', + PLUGIN_CONTRACT, + '', + errorBlock, + '', + 'Current (broken) plugin:', + '```ts', + currentCode, + '```', + '', + describeShape(shape, raw), + '', + 'Output ONLY the complete corrected TypeScript file content. Do not wrap it in markdown ', + 'fences, do not add explanations.', + ].join('\n'); +} diff --git a/packages/collectors/src/gateway-gen/write.ts b/packages/collectors/src/gateway-gen/write.ts new file mode 100644 index 000000000..9f0ed11d4 --- /dev/null +++ b/packages/collectors/src/gateway-gen/write.ts @@ -0,0 +1,211 @@ +import { spawnSync } from 'node:child_process'; +import { readFile, writeFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { ModelSchema } from '@basemodel/schema'; +import type { CollectionResult, GatewayPlugin } from '../core/collector.js'; +import { generateText } from './llm.js'; +import type { ManifestGateway } from './manifest.js'; +import { getGatewayPluginPath } from './manifest.js'; +import type { ShapeSummary } from './probe.js'; +import { buildPluginPrompt } from './prompts.js'; + +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..'); +const tscBin = resolve(packageRoot, 'node_modules', 'typescript', 'lib', 'tsc.js'); + +export interface GeneratePluginOptions { + gateway: ManifestGateway; + shape: ShapeSummary; + raw: unknown; + env?: NodeJS.ProcessEnv; + maxAttempts?: number; + liveSecrets?: Record; +} + +export interface GeneratedPlugin { + filePath: string; + code: string; + attempts: number; +} + +export function extractTsCode(text: string): string { + const fenced = text.match(/```(?:ts|typescript)?\s*([\s\S]*?)```/); + if (fenced) return (fenced[1] ?? text).trim(); + return text.trim(); +} + +export function checkForbiddenPatterns(code: string): string[] { + const problems: string[] = []; + if (/\b(?:catch\s*\([^)]*:\s*any\s*\)|as\s+any\b|:\s*any\s*[,;)\n])/.test(code)) { + problems.push('code uses the "any" type (catch (e: any), : any, as any)'); + } + if (/sk-[A-Za-z0-9]{8,}|AIza[A-Za-z0-9_-]{10,}|Bearer\s+[A-Za-z0-9_-]{20,}/.test(code)) { + problems.push('code appears to hardcode an API key'); + } + if (/<\s*%|process\.exit\s*\(|child_process|require\(['"]child_process/.test(code)) { + problems.push('code uses process.exit or child_process (not allowed)'); + } + return problems; +} + +export function ensureFinalNewline(code: string): string { + return code.endsWith('\n') ? code : `${code}\n`; +} + +export interface PluginValidation { + ok: boolean; + errors: string[]; + plugin?: GatewayPlugin; +} + +export async function validatePluginModule( + filePath: string, + expectedId: string, +): Promise { + try { + const moduleUrl = `${pathToFileURL(filePath).href}?v=${Date.now()}`; + const module = (await import(moduleUrl)) as { default?: unknown }; + const plugin = module.default as GatewayPlugin | undefined; + const errors: string[] = []; + if (!plugin || typeof plugin !== 'object') { + errors.push('module has no default export'); + } else { + if (plugin.type !== 'custom') errors.push(`expected type "custom", got "${plugin.type}"`); + if (plugin.id !== expectedId) errors.push(`expected id "${expectedId}", got "${plugin.id}"`); + if (typeof (plugin as { collect?: unknown }).collect !== 'function') { + errors.push('default export has no collect() function'); + } + } + if (errors.length > 0) return { ok: false, errors, plugin }; + return { ok: true, errors: [], plugin }; + } catch (error: unknown) { + return { ok: false, errors: [error instanceof Error ? error.message : String(error)] }; + } +} + +export interface CollectionCheck { + modelCount: number; + validCount: number; + errors: string[]; +} + +function formatIssuePath(issue: { path?: ReadonlyArray }): string { + return (issue.path ?? []) + .map((segment) => (typeof segment === 'number' ? `[${segment}]` : `.${segment}`)) + .join('') + .replace(/^\./, ''); +} + +function formatModelIssue(issue: { message: string; expected?: unknown; received?: unknown }): string { + const extra = + issue.expected !== undefined + ? ` (expected ${String(issue.expected)}, received ${String(issue.received)})` + : ''; + return `${issue.message}${extra}`; +} + +export async function checkCollection( + plugin: GatewayPlugin, + secrets: Record, +): Promise { + const result: CollectionResult = await ( + plugin as { + collect: (secrets: Record) => Promise; + } + ).collect(secrets); + let validCount = 0; + const seen = new Set(); + for (const model of result.models) { + const parsed = ModelSchema.safeParse(model); + if (parsed.success) { + validCount += 1; + continue; + } + const issue = (parsed.error.issues ?? [])[0] as + | { path?: ReadonlyArray; message: string; expected?: unknown; received?: unknown } + | undefined; + if (issue) { + const label = `${formatIssuePath(issue)}: ${formatModelIssue(issue)}`; + if (seen.size < 5) seen.add(label); + } + } + const invalid = result.models.length - validCount; + if (invalid > 0) { + result.errors.push(`${invalid} models failed ModelSchema validation`); + for (const label of seen) result.errors.push(` e.g. model ${label}`); + } + return { modelCount: result.models.length, validCount, errors: result.errors }; +} + +export function runTypecheck(cwd: string = packageRoot): string[] { + const result = spawnSync(process.execPath, [tscBin, '--noEmit'], { + cwd, + encoding: 'utf-8', + timeout: 120_000, + }); + const output = `${result.stdout ?? ''}\n${result.stderr ?? ''}`; + if (result.status === 0) return []; + const errors = output + .split('\n') + .filter((line) => /error TS\d+/.test(line)) + .map((line) => line.trim()) + .filter(Boolean); + return errors.length > 0 ? errors : [result.error?.message ?? 'typecheck failed']; +} + +export interface GeneratedPluginValidation { + ok: boolean; + errors: string[]; + liveCheck?: CollectionCheck; +} + +export async function validateGeneratedPlugin( + filePath: string, + gatewayId: string, + liveSecrets?: Record, +): Promise { + const code = await readFile(filePath, 'utf-8'); + const errors = [...checkForbiddenPatterns(code)]; + const validation = await validatePluginModule(filePath, gatewayId); + errors.push(...validation.errors); + if (errors.length === 0) { + errors.push(...runTypecheck().map((e) => `typecheck: ${e}`)); + } + let liveCheck: CollectionCheck | undefined; + if (errors.length === 0 && liveSecrets && validation.plugin) { + liveCheck = await checkCollection(validation.plugin, liveSecrets); + console.log( + ` live check : ${liveCheck.modelCount} models, ${liveCheck.validCount} valid`, + liveCheck.errors.length > 0 ? `| errors: ${liveCheck.errors.join(' | ')}` : '', + ); + if (liveCheck.errors.length > 0) { + errors.push(...liveCheck.errors.map((e) => `live check: ${e}`)); + } + } + return { ok: errors.length === 0, errors, liveCheck }; +} + +export async function generatePlugin(options: GeneratePluginOptions): Promise { + const { gateway, shape, raw, env = process.env, maxAttempts = 6, liveSecrets } = options; + const filePath = getGatewayPluginPath(gateway.id); + let prompt = buildPluginPrompt(gateway, shape, raw); + const lastErrors: string[] = []; + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + const text = await generateText({ prompt }, env); + const code = ensureFinalNewline(extractTsCode(text)); + await writeFile(filePath, code, 'utf-8'); + const validation = await validateGeneratedPlugin(filePath, gateway.id, liveSecrets); + if (validation.ok) return { filePath, code, attempts: attempt }; + lastErrors.push(...validation.errors); + console.warn(`[gen] attempt ${attempt} failed validation: ${validation.errors.join('; ')}`); + const truncated = validation.errors.map((e) => + e.length > 300 ? `${e.slice(0, 300)}... (truncated)` : e, + ); + prompt += `\n\nThe previous attempt failed validation:\n${truncated.join('\n')}\n\nReturn the corrected file only.`; + } + + throw new Error( + `Failed to generate a valid plugin for ${gateway.id} after ${maxAttempts} attempts:\n${lastErrors.join('\n')}`, + ); +} diff --git a/packages/collectors/src/gateways/__fixtures__/cohere.raw.json b/packages/collectors/src/gateways/__fixtures__/cohere.raw.json new file mode 100644 index 000000000..4b7f685c1 --- /dev/null +++ b/packages/collectors/src/gateways/__fixtures__/cohere.raw.json @@ -0,0 +1,84 @@ +{ + "models": [ + { + "name": "c4ai-aya-expanse-32b", + "endpoints": [ + "generate", + "chat" + ], + "finetuned": false, + "context_length": 128000, + "tokenizer_url": "https://storage.googleapis.com/cohere-public/tokenizers/c4ai-aya-expanse-32b.json", + "features": null, + "default_endpoints": [] + }, + { + "name": "c4ai-aya-vision-32b", + "endpoints": [ + "chat" + ], + "finetuned": false, + "context_length": 16384, + "tokenizer_url": "https://storage.googleapis.com/cohere-public/tokenizers/c4ai-aya-vision-32b.json", + "features": [ + "logprobs", + "vision", + "citations" + ], + "default_endpoints": [] + }, + { + "name": "cohere-transcribe-03-2026", + "endpoints": [ + "transcriptions" + ], + "finetuned": false, + "context_length": 32768, + "tokenizer_url": "https://storage.googleapis.com/cohere-public/tokenizers/cohere-transcribe-03-2026.json", + "features": null, + "default_endpoints": [] + }, + { + "name": "command-a-03-2025", + "endpoints": [ + "chat" + ], + "finetuned": false, + "context_length": 288000, + "tokenizer_url": "https://storage.googleapis.com/cohere-public/tokenizers/command-a-03-2025.json", + "features": [ + "json_mode", + "json_schema", + "strict_tools", + "safety_modes", + "tools" + ], + "default_endpoints": [ + "chat", + "generate" + ] + }, + { + "name": "command-a-plus-05-2026", + "endpoints": [ + "generate", + "chat" + ], + "finetuned": false, + "context_length": 436000, + "tokenizer_url": "https://storage.googleapis.com/cohere-public/tokenizers/command-a-plus-05-2026.json", + "features": [ + "logprobs", + "json_mode", + "json_schema", + "strict_tools", + "safety_modes" + ], + "default_endpoints": [], + "sampling_defaults": { + "temperature": 0.6, + "p": 0.95 + } + } + ] +} diff --git a/packages/collectors/src/gateways/cohere.ts b/packages/collectors/src/gateways/cohere.ts new file mode 100644 index 000000000..18a19ec9b --- /dev/null +++ b/packages/collectors/src/gateways/cohere.ts @@ -0,0 +1,157 @@ +import { z } from 'zod'; +import type { CollectionResult, CustomGateway } from '../core/collector'; + +function slugify(text: string): string { + return text.toLowerCase().replace(/[^a-z0-9.-]+/g, '-').replace(/-+/g, '-').replace(/^-|-$/g, ''); +} + +const rawModelSchema = z.object({ + name: z.string(), + endpoints: z.array(z.string()).nullable(), + finetuned: z.boolean(), + context_length: z.number().nullable(), + tokenizer_url: z.string().nullable(), + features: z.array(z.string()).nullable(), + default_endpoints: z.array(z.string()).nullable(), +}); + +const rawResponseSchema = z.object({ + models: z.array(rawModelSchema), +}); + +function deriveModality(features: string[] | null | undefined): ('text' | 'image' | 'audio' | 'video' | 'code' | 'embedding')[] { + const f = features ?? []; + const modalities: ('text' | 'image' | 'audio' | 'video' | 'code' | 'embedding')[] = ['text']; + if (f.includes('vision')) modalities.push('image'); + if (f.includes('audio')) modalities.push('audio'); + if (f.includes('transcriptions')) modalities.push('audio'); + if (f.includes('embed')) modalities.push('embedding'); + return modalities; +} + +function deriveCapabilities(features: string[] | null | undefined): string[] { + const f = features ?? []; + const caps: string[] = []; + if (f.includes('reasoning')) caps.push('reasoning'); + if (f.includes('tools')) caps.push('function_calling'); + if (f.includes('json_mode') || f.includes('json_schema')) caps.push('structured_output'); + if (f.includes('vision')) caps.push('vision_support'); + if (f.includes('logprobs')) caps.push('logprobs'); + if (f.includes('citations')) caps.push('citations'); + if (f.includes('safety_modes')) caps.push('safety_modes'); + if (f.includes('transcriptions')) caps.push('audio_support'); + if (f.includes('embed')) caps.push('embedding_support'); + return caps; +} + +function deriveBooleanFlags(features: string[] | null | undefined): { + reasoning_support: boolean; + function_calling: boolean; + structured_output: boolean; + vision_support: boolean; + audio_support: boolean; + image_generation: boolean; + embedding_support: boolean; +} { + const f = features ?? []; + return { + reasoning_support: f.includes('reasoning'), + function_calling: f.includes('tools'), + structured_output: f.includes('json_mode') || f.includes('json_schema'), + vision_support: f.includes('vision'), + audio_support: f.includes('transcriptions'), + image_generation: false, + embedding_support: f.includes('embed'), + }; +} + +export default { + type: 'custom', + id: 'cohere', + async collect(secrets: Record): Promise { + const result: CollectionResult = { provider_id: 'cohere', models: [], errors: [] }; + const apiKey = secrets['COHERE_API_KEY']; + if (!apiKey) { + result.errors.push('COHERE_API_KEY secret is missing'); + return result; + } + + const url = 'https://api.cohere.com/v1/models'; + let response: Response; + try { + response = await fetch(url, { + method: 'GET', + headers: { Authorization: `Bearer ${apiKey}` }, + signal: AbortSignal.timeout(15_000), + }); + } catch (err) { + const message = err instanceof Error ? err.message : 'Unknown fetch error'; + result.errors.push(`Failed to fetch models: ${message}`); + return result; + } + + if (!response.ok) { + result.errors.push(`HTTP ${response.status}: ${response.statusText}`); + return result; + } + + let raw: unknown; + try { + raw = await response.json(); + } catch (err) { + const message = err instanceof Error ? err.message : 'Unknown parse error'; + result.errors.push(`Failed to parse JSON: ${message}`); + return result; + } + + const parsed = rawResponseSchema.safeParse(raw); + if (!parsed.success) { + result.errors.push(`Invalid response shape: ${parsed.error.issues.map(i => i.message).join(', ')}`); + return result; + } + + for (const rawModel of parsed.data.models) { + const slug = slugify(rawModel.name); + const modelId = `cohere/${slug}`; + const features = rawModel.features ?? undefined; + const modality = deriveModality(features); + const caps = deriveCapabilities(features); + const flags = deriveBooleanFlags(features); + + const contextWindow = typeof rawModel.context_length === 'number' && rawModel.context_length > 0 + ? rawModel.context_length + : undefined; + + const model = { + model_id: modelId, + provider_id: 'cohere', + name: rawModel.name, + family: rawModel.name.split('-').slice(0, -1).join('-') || undefined, + version: rawModel.name.split('-').pop() || undefined, + description: rawModel.name, + architecture: 'unknown', + parameter_size: undefined, + context_window: contextWindow, + modality, + open_weight: false, + reasoning_support: flags.reasoning_support, + function_calling: flags.function_calling, + structured_output: flags.structured_output, + vision_support: flags.vision_support, + audio_support: flags.audio_support, + image_generation: flags.image_generation, + embedding_support: flags.embedding_support, + is_free: undefined, + tier: undefined, + limits: undefined, + capability_ids: caps, + license_id: undefined, + status: 'active' as const, + }; + + result.models.push(model); + } + + return result; + }, +} satisfies CustomGateway;