From 01d6df0876cd59ec93605ac8639860c6c4938de2 Mon Sep 17 00:00:00 2001 From: Robert Lippmann Date: Sat, 4 Jul 2026 04:31:22 -0400 Subject: [PATCH] feat: add live-model validation for TS MCP tool gating --- .../tool_gating/mcp_calendar_admin/README.md | 64 +++ .../mcp_calendar_admin/src/live_model.ts | 368 ++++++++++++++++++ .../tests/live_model.test.ts | 75 ++++ .../tests/live_model_helper.test.ts | 105 +++++ .../tests/live_model_provider.test.ts | 132 +++++++ 5 files changed, 744 insertions(+) create mode 100644 typescript/examples/tool_gating/mcp_calendar_admin/src/live_model.ts create mode 100644 typescript/examples/tool_gating/mcp_calendar_admin/tests/live_model.test.ts create mode 100644 typescript/examples/tool_gating/mcp_calendar_admin/tests/live_model_helper.test.ts create mode 100644 typescript/examples/tool_gating/mcp_calendar_admin/tests/live_model_provider.test.ts diff --git a/typescript/examples/tool_gating/mcp_calendar_admin/README.md b/typescript/examples/tool_gating/mcp_calendar_admin/README.md index 4a5a848..4d9f073 100644 --- a/typescript/examples/tool_gating/mcp_calendar_admin/README.md +++ b/typescript/examples/tool_gating/mcp_calendar_admin/README.md @@ -23,3 +23,67 @@ prohibit calendar_admin If a caller still invokes the hidden tool directly, the host blocks execution. Adversarial request text does not expose the tool or mutate policy state. + +The provider-free tests remain the canonical proof for this example. + +This example also includes an opt-in live-model comparison that keeps the same +host-owned MCP registry and execution rules while letting a real model choose +from the exposed MCP tool surface. + +## Live-model validation + +The host builds the exposed MCP tool list from authoritative Context Compiler +state and passes only those tools to the model. + +The live-model helper is a plain TypeScript HTTP client. It uses an +OpenAI-compatible request shape for tool calling. It does not use the OpenAI +SDK, LiteLLM, or Ollama's native `/api/chat` route. + +Use the same intent across states: + +```text +Create an admin calendar event named Quarterly access review on calendar ops-admin. +``` + +What to observe: + +- absent state: the protected admin tool is not exposed and no protected side + effect occurs +- `use calendar_admin`: the protected tool is exposed; the model must select it + for protected execution to occur +- contradiction with `prohibit calendar_admin`: clarification blocks protected + execution before tool use + +Run the canonical provider-free tests: + +```bash +npm test +``` + +Run the opt-in live-model validation: + +```bash +export RUN_MCP_CALENDAR_ADMIN_LIVE_MODEL=1 +export OPENAI_API_KEY=... +export MODEL=gpt-4o-mini +npm test -- --test-name-pattern="live model tool surface changes with authoritative state" +``` + +HTTP contract: + +- `MODEL` is sent as-is to the target OpenAI-compatible endpoint +- `OPENAI_BASE_URL` overrides the default OpenAI-compatible base URL +- if `OPENAI_BASE_URL` is unset, the helper uses + `https://api.openai.com/v1` +- `OPENAI_BASE_URL` overrides the OpenAI-compatible base URL when needed +- `OPENAI_API_KEY` is required for the live-model HTTP call + +Run the same opt-in validation with local Ollama: + +```bash +export RUN_MCP_CALENDAR_ADMIN_LIVE_MODEL=1 +export OPENAI_BASE_URL=http://127.0.0.1:11434/v1 +export OPENAI_API_KEY=ollama +export MODEL=qwen2.5:1.5b-instruct +npm test -- --test-name-pattern="live model tool surface changes with authoritative state" +``` diff --git a/typescript/examples/tool_gating/mcp_calendar_admin/src/live_model.ts b/typescript/examples/tool_gating/mcp_calendar_admin/src/live_model.ts new file mode 100644 index 0000000..d6740d9 --- /dev/null +++ b/typescript/examples/tool_gating/mcp_calendar_admin/src/live_model.ts @@ -0,0 +1,368 @@ +import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs"; +import { dirname } from "node:path"; +import { createEngine, type EngineState } from "@rlippmann/context-compiler"; + +import { + CalendarAdminMcpHost, + type McpToolCall, + type McpToolDefinition +} from "./index.js"; + +export type SideEffectRecord = { + toolName: string; + calendarId: string; + eventTitle: string; + authorizationSource: "context_compiler_state"; +}; + +export type LiveModelResult = { + decisionKind: "clarify" | "update" | "passthrough" | null; + promptToUser: string | null; + exposedToolNames: string[]; + hiddenToolNames: string[]; + protectedToolExposed: boolean; + selectedToolName: string | null; + executed: boolean; + blockedReason: string | null; + toolResult: string | null; + executionLog: string[]; + sideEffectPath: string; + sideEffectCount: number; +}; + +export type SelectedToolCall = { + name: string | null; + arguments: Record; +}; + +export type ModelToolSelector = (input: { + userIntent: string; + tools: OpenAiToolDefinition[]; +}) => Promise; + +type ChatCompletionRequestBody = { + model: string; + tool_choice: "auto"; + messages: Array<{ role: "system" | "user"; content: string }>; + tools: OpenAiToolDefinition[]; + temperature?: number; +}; + +type OpenAiToolDefinition = { + type: "function"; + function: { + name: string; + description: string; + parameters: { + type: "object"; + properties: Record; + required: string[]; + additionalProperties: false; + }; + }; +}; + +function buildOpenAiTools(exposedTools: McpToolDefinition[]): OpenAiToolDefinition[] { + return exposedTools.flatMap((tool) => { + if (tool.name === "calendar_view_events") { + return [ + { + type: "function", + function: { + name: tool.name, + description: tool.description, + parameters: { + type: "object", + properties: { + calendar_id: { + type: "string", + description: "Calendar identifier to inspect." + } + }, + required: ["calendar_id"], + additionalProperties: false + } + } + } + ]; + } + + if (tool.name === "calendar_admin_create_event") { + return [ + { + type: "function", + function: { + name: tool.name, + description: tool.description, + parameters: { + type: "object", + properties: { + calendar_id: { + type: "string", + description: "Administrative calendar identifier." + }, + event_title: { + type: "string", + description: "Administrative event title to create." + } + }, + required: ["calendar_id", "event_title"], + additionalProperties: false + } + } + } + ]; + } + + return []; + }); +} + +function sideEffectCount(artifactPath: string): number { + if (!existsSync(artifactPath)) { + return 0; + } + + return readFileSync(artifactPath, "utf8") + .split("\n") + .filter((line) => line.trim().length > 0).length; +} + +function appendSideEffect(artifactPath: string, toolCall: McpToolCall): void { + const record: SideEffectRecord = { + toolName: toolCall.toolName, + calendarId: toolCall.arguments.calendar_id ?? "", + eventTitle: toolCall.arguments.event_title ?? "", + authorizationSource: "context_compiler_state" + }; + + mkdirSync(dirname(artifactPath), { recursive: true }); + appendFileSync(artifactPath, `${JSON.stringify(record)}\n`, "utf8"); +} + +function normalizeToolArguments(value: unknown): Record { + if (!value || typeof value !== "object") { + return {}; + } + + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [key, String(item)]) + ); +} + +function extractSelectedToolCall(response: unknown): SelectedToolCall { + if (!response || typeof response !== "object") { + return { name: null, arguments: {} }; + } + + const choices = "choices" in response ? response.choices : null; + if (!Array.isArray(choices) || choices.length === 0) { + return { name: null, arguments: {} }; + } + + const firstChoice = choices[0]; + if (!firstChoice || typeof firstChoice !== "object" || !("message" in firstChoice)) { + return { name: null, arguments: {} }; + } + + const message = firstChoice.message; + if (!message || typeof message !== "object" || !("tool_calls" in message)) { + return { name: null, arguments: {} }; + } + + const toolCalls = message.tool_calls; + if (!Array.isArray(toolCalls) || toolCalls.length === 0) { + return { name: null, arguments: {} }; + } + + const firstToolCall = toolCalls[0]; + if (!firstToolCall || typeof firstToolCall !== "object" || !("function" in firstToolCall)) { + return { name: null, arguments: {} }; + } + + const fn = firstToolCall.function; + if (!fn || typeof fn !== "object" || !("name" in fn)) { + return { name: null, arguments: {} }; + } + + const name = typeof fn.name === "string" ? fn.name : null; + const rawArguments = + "arguments" in fn && typeof fn.arguments === "string" + ? fn.arguments + : "{}"; + + try { + return { + name, + arguments: normalizeToolArguments(JSON.parse(rawArguments)) + }; + } catch { + return { name, arguments: {} }; + } +} + +export function resolveProviderConfig(): { + apiKey: string; + baseUrl: string; + model: string; +} { + const apiKey = process.env.OPENAI_API_KEY; + if (!apiKey) { + throw new Error("OPENAI_API_KEY is required for live-model validation."); + } + + return { + apiKey, + baseUrl: process.env.OPENAI_BASE_URL ?? "https://api.openai.com/v1", + model: process.env.MODEL ?? "gpt-4o-mini" + }; +} + +export function buildChatCompletionRequestBody(input: { + model: string; + userIntent: string; + tools: OpenAiToolDefinition[]; +}): ChatCompletionRequestBody { + const body: ChatCompletionRequestBody = { + model: input.model, + tool_choice: "auto", + messages: [ + { + role: "system", + content: + "You are a careful assistant. Use a tool if one is available to complete the user's request. Do not invent unavailable tools." + }, + { role: "user", content: input.userIntent } + ], + tools: input.tools + }; + + if (!isGpt5FamilyModel(input.model)) { + body.temperature = 0; + } + + return body; +} + +export function isGpt5FamilyModel(model: string): boolean { + return model === "gpt-5" || model.startsWith("gpt-5-"); +} + +async function callLiveModel(input: { + userIntent: string; + tools: OpenAiToolDefinition[]; +}): Promise { + const { apiKey, baseUrl, model } = resolveProviderConfig(); + const response = await fetch(`${baseUrl}/chat/completions`, { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${apiKey}` + }, + body: JSON.stringify( + buildChatCompletionRequestBody({ + model, + userIntent: input.userIntent, + tools: input.tools + }) + ) + }); + + if (!response.ok) { + throw new Error( + `Live-model request failed with status ${response.status}: ${await response.text()}` + ); + } + + return extractSelectedToolCall(await response.json()); +} + +export async function runLiveModelTurn(input: { + userIntent: string; + authoritativeState?: EngineState; + compilerInput?: string; + artifactPath: string; + modelToolSelector?: ModelToolSelector; +}): Promise { + const { + userIntent, + authoritativeState, + compilerInput = "", + artifactPath, + modelToolSelector = callLiveModel + } = input; + + const host = new CalendarAdminMcpHost(); + const engine = createEngine(authoritativeState ? { state: authoritativeState } : undefined); + const decision = engine.step(compilerInput); + + if (decision.kind === "clarify") { + return { + decisionKind: "clarify", + promptToUser: decision.prompt_to_user, + exposedToolNames: host.exposedMcpTools(engine.state).tools.map((tool) => tool.name), + hiddenToolNames: host.exposedMcpTools(engine.state).hiddenToolNames, + protectedToolExposed: host + .exposedMcpTools(engine.state) + .tools.some((tool) => tool.name === "calendar_admin_create_event"), + selectedToolName: null, + executed: false, + blockedReason: "clarification required before exposing calendar admin MCP tools", + toolResult: null, + executionLog: [...host.executionLog], + sideEffectPath: artifactPath, + sideEffectCount: sideEffectCount(artifactPath) + }; + } + + const resolvedState = decision.state ?? engine.state; + const exposedTools = host.exposedMcpTools(resolvedState); + const selectedTool = await modelToolSelector({ + userIntent, + tools: buildOpenAiTools(exposedTools.tools) + }); + const protectedToolExposed = exposedTools.tools.some( + (tool) => tool.name === "calendar_admin_create_event" + ); + + if (selectedTool.name !== "calendar_admin_create_event") { + return { + decisionKind: decision.kind, + promptToUser: decision.prompt_to_user, + exposedToolNames: exposedTools.tools.map((tool) => tool.name), + hiddenToolNames: exposedTools.hiddenToolNames, + protectedToolExposed, + selectedToolName: selectedTool.name, + executed: false, + blockedReason: protectedToolExposed + ? "model did not select protected admin tool" + : "protected admin tool was not exposed", + toolResult: null, + executionLog: [...host.executionLog], + sideEffectPath: artifactPath, + sideEffectCount: sideEffectCount(artifactPath) + }; + } + + const toolCall: McpToolCall = { + toolName: selectedTool.name, + arguments: selectedTool.arguments + }; + const toolResult = host.executeMcpTool(toolCall); + appendSideEffect(artifactPath, toolCall); + + return { + decisionKind: decision.kind, + promptToUser: decision.prompt_to_user, + exposedToolNames: exposedTools.tools.map((tool) => tool.name), + hiddenToolNames: exposedTools.hiddenToolNames, + protectedToolExposed, + selectedToolName: selectedTool.name, + executed: true, + blockedReason: null, + toolResult, + executionLog: [...host.executionLog], + sideEffectPath: artifactPath, + sideEffectCount: sideEffectCount(artifactPath) + }; +} diff --git a/typescript/examples/tool_gating/mcp_calendar_admin/tests/live_model.test.ts b/typescript/examples/tool_gating/mcp_calendar_admin/tests/live_model.test.ts new file mode 100644 index 0000000..e423af6 --- /dev/null +++ b/typescript/examples/tool_gating/mcp_calendar_admin/tests/live_model.test.ts @@ -0,0 +1,75 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { mkdtempSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createEngine } from "@rlippmann/context-compiler"; + +import { runLiveModelTurn } from "../src/live_model.js"; + +const RUN_LIVE_MODEL_ENV_VAR = "RUN_MCP_CALENDAR_ADMIN_LIVE_MODEL"; +const USER_INTENT = + "Create an admin calendar event named Quarterly access review on calendar ops-admin."; + +function readJsonl(path: string): string[] { + try { + return readFileSync(path, "utf8") + .split("\n") + .filter((line) => line.trim().length > 0); + } catch { + return []; + } +} + +function tempArtifactPath(): string { + return join(mkdtempSync(join(tmpdir(), "mcp-calendar-admin-live-")), "tool_calls.jsonl"); +} + +test( + "live model tool surface changes with authoritative state", + { skip: process.env[RUN_LIVE_MODEL_ENV_VAR] !== "1" }, + async () => { + const artifactPath = tempArtifactPath(); + + const absentResult = await runLiveModelTurn({ + userIntent: USER_INTENT, + artifactPath + }); + + assert.equal(absentResult.protectedToolExposed, false); + assert.equal(absentResult.executed, false); + assert.deepEqual(readJsonl(artifactPath), []); + + const allowedEngine = createEngine(); + allowedEngine.step("use calendar_admin"); + const allowedResult = await runLiveModelTurn({ + userIntent: USER_INTENT, + authoritativeState: allowedEngine.state, + artifactPath + }); + + if (allowedResult.selectedToolName !== "calendar_admin_create_event") { + throw new Error( + "Protected tool was exposed but the live model did not select " + + "`calendar_admin_create_event`. " + + `Selected tool: ${JSON.stringify(allowedResult.selectedToolName)}. ` + + "This opt-in validation requires the model to exercise the protected tool path explicitly." + ); + } + + assert.equal(allowedResult.protectedToolExposed, true); + assert.equal(allowedResult.executed, true); + assert.equal(readJsonl(artifactPath).length, 1); + + const clarifyResult = await runLiveModelTurn({ + userIntent: USER_INTENT, + authoritativeState: allowedEngine.state, + compilerInput: "prohibit calendar_admin", + artifactPath + }); + + assert.equal(clarifyResult.decisionKind, "clarify"); + assert.equal(clarifyResult.executed, false); + assert.equal(readJsonl(artifactPath).length, 1); + } +); diff --git a/typescript/examples/tool_gating/mcp_calendar_admin/tests/live_model_helper.test.ts b/typescript/examples/tool_gating/mcp_calendar_admin/tests/live_model_helper.test.ts new file mode 100644 index 0000000..d44d03e --- /dev/null +++ b/typescript/examples/tool_gating/mcp_calendar_admin/tests/live_model_helper.test.ts @@ -0,0 +1,105 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { mkdtempSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createEngine } from "@rlippmann/context-compiler"; + +import { + runLiveModelTurn, + type SelectedToolCall +} from "../src/live_model.js"; + +const USER_INTENT = + "Create an admin calendar event named Quarterly access review on calendar ops-admin."; + +function readJsonl(path: string): string[] { + try { + return readFileSync(path, "utf8") + .split("\n") + .filter((line) => line.trim().length > 0); + } catch { + return []; + } +} + +function tempArtifactPath(): string { + return join(mkdtempSync(join(tmpdir(), "mcp-calendar-admin-live-")), "tool_calls.jsonl"); +} + +test("absent state keeps protected tool hidden from model-visible surface", async () => { + const artifactPath = tempArtifactPath(); + + const result = await runLiveModelTurn({ + userIntent: USER_INTENT, + artifactPath, + modelToolSelector: async (): Promise => ({ + name: "calendar_view_events", + arguments: { calendar_id: "ops-admin" } + }) + }); + + assert.equal(result.protectedToolExposed, false); + assert.equal(result.selectedToolName, "calendar_view_events"); + assert.equal(result.executed, false); + assert.equal(result.blockedReason, "protected admin tool was not exposed"); + assert.deepEqual(readJsonl(artifactPath), []); +}); + +test("authorized state exposes protected tool and records side effect", async () => { + const artifactPath = tempArtifactPath(); + const engine = createEngine(); + engine.step("use calendar_admin"); + + const result = await runLiveModelTurn({ + userIntent: USER_INTENT, + authoritativeState: engine.state, + artifactPath, + modelToolSelector: async (): Promise => ({ + name: "calendar_admin_create_event", + arguments: { + calendar_id: "ops-admin", + event_title: "Quarterly access review" + } + }) + }); + + assert.equal(result.protectedToolExposed, true); + assert.equal(result.selectedToolName, "calendar_admin_create_event"); + assert.equal(result.executed, true); + assert.equal( + result.toolResult, + "created event 'Quarterly access review' on calendar 'ops-admin'" + ); + assert.equal(result.sideEffectCount, 1); + assert.equal(readJsonl(artifactPath).length, 1); +}); + +test("contradiction blocks before model tool selection", async () => { + const artifactPath = tempArtifactPath(); + const engine = createEngine(); + engine.step("use calendar_admin"); + let modelCalled = false; + + const result = await runLiveModelTurn({ + userIntent: USER_INTENT, + authoritativeState: engine.state, + compilerInput: "prohibit calendar_admin", + artifactPath, + modelToolSelector: async (): Promise => { + modelCalled = true; + return { + name: "calendar_admin_create_event", + arguments: { + calendar_id: "ops-admin", + event_title: "Quarterly access review" + } + }; + } + }); + + assert.equal(result.decisionKind, "clarify"); + assert.equal(result.executed, false); + assert.equal(modelCalled, false); + assert.deepEqual(readJsonl(artifactPath), []); +}); diff --git a/typescript/examples/tool_gating/mcp_calendar_admin/tests/live_model_provider.test.ts b/typescript/examples/tool_gating/mcp_calendar_admin/tests/live_model_provider.test.ts new file mode 100644 index 0000000..35305ea --- /dev/null +++ b/typescript/examples/tool_gating/mcp_calendar_admin/tests/live_model_provider.test.ts @@ -0,0 +1,132 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + buildChatCompletionRequestBody, + resolveProviderConfig +} from "../src/live_model.js"; + +test("resolveProviderConfig preserves exact model ids for OpenAI-compatible requests", () => { + const previousModel = process.env.MODEL; + const previousApiKey = process.env.OPENAI_API_KEY; + const previousBaseUrl = process.env.OPENAI_BASE_URL; + + process.env.MODEL = "qwen2.5:1.5b-instruct"; + process.env.OPENAI_API_KEY = "ollama"; + process.env.OPENAI_BASE_URL = "http://127.0.0.1:11434/v1"; + + try { + const config = resolveProviderConfig(); + + assert.equal(config.model, "qwen2.5:1.5b-instruct"); + assert.equal(config.apiKey, "ollama"); + assert.equal(config.baseUrl, "http://127.0.0.1:11434/v1"); + } finally { + restoreEnv("MODEL", previousModel); + restoreEnv("OPENAI_API_KEY", previousApiKey); + restoreEnv("OPENAI_BASE_URL", previousBaseUrl); + } +}); + +test("resolveProviderConfig preserves exact model ids for OpenAI", () => { + const previousModel = process.env.MODEL; + const previousApiKey = process.env.OPENAI_API_KEY; + const previousBaseUrl = process.env.OPENAI_BASE_URL; + + process.env.MODEL = "gpt-4o-mini"; + process.env.OPENAI_API_KEY = "test-key"; + delete process.env.OPENAI_BASE_URL; + + try { + const config = resolveProviderConfig(); + + assert.equal(config.model, "gpt-4o-mini"); + assert.equal(config.apiKey, "test-key"); + assert.equal(config.baseUrl, "https://api.openai.com/v1"); + } finally { + restoreEnv("MODEL", previousModel); + restoreEnv("OPENAI_API_KEY", previousApiKey); + restoreEnv("OPENAI_BASE_URL", previousBaseUrl); + } +}); + +test("resolveProviderConfig uses OpenAI default model when MODEL is unset", () => { + const previousModel = process.env.MODEL; + const previousApiKey = process.env.OPENAI_API_KEY; + const previousBaseUrl = process.env.OPENAI_BASE_URL; + + delete process.env.MODEL; + process.env.OPENAI_API_KEY = "test-key"; + delete process.env.OPENAI_BASE_URL; + + try { + const config = resolveProviderConfig(); + + assert.equal(config.model, "gpt-4o-mini"); + assert.equal(config.baseUrl, "https://api.openai.com/v1"); + } finally { + restoreEnv("MODEL", previousModel); + restoreEnv("OPENAI_API_KEY", previousApiKey); + restoreEnv("OPENAI_BASE_URL", previousBaseUrl); + } +}); + +test("resolveProviderConfig requires OPENAI_API_KEY for direct HTTP calls", () => { + const previousModel = process.env.MODEL; + const previousApiKey = process.env.OPENAI_API_KEY; + const previousBaseUrl = process.env.OPENAI_BASE_URL; + + process.env.MODEL = "gpt-4o-mini"; + delete process.env.OPENAI_API_KEY; + delete process.env.OPENAI_BASE_URL; + + try { + assert.throws( + () => resolveProviderConfig(), + /OPENAI_API_KEY is required for live-model validation\./ + ); + } finally { + restoreEnv("MODEL", previousModel); + restoreEnv("OPENAI_API_KEY", previousApiKey); + restoreEnv("OPENAI_BASE_URL", previousBaseUrl); + } +}); + +test("buildChatCompletionRequestBody omits temperature for gpt-5-family models", () => { + const body = buildChatCompletionRequestBody({ + model: "gpt-5-mini", + userIntent: "Create the admin event.", + tools: [] + }); + + assert.equal("temperature" in body, false); +}); + +test("buildChatCompletionRequestBody keeps temperature for gpt-4o-mini", () => { + const body = buildChatCompletionRequestBody({ + model: "gpt-4o-mini", + userIntent: "Create the admin event.", + tools: [] + }); + + assert.equal(body.temperature, 0); +}); + +test("buildChatCompletionRequestBody keeps temperature for exact Ollama model ids", () => { + const body = buildChatCompletionRequestBody({ + model: "qwen2.5:1.5b-instruct", + userIntent: "Create the admin event.", + tools: [] + }); + + assert.equal(body.temperature, 0); +}); + +function restoreEnv(name: string, value: string | undefined): void { + if (value === undefined) { + delete process.env[name]; + return; + } + + process.env[name] = value; +}