From 6b1c06f9335816eb73e533bcf88d5e43d093d498 Mon Sep 17 00:00:00 2001 From: Austin Cherry Date: Wed, 22 Apr 2026 17:48:28 -0500 Subject: [PATCH] feat: add dry-run mode for execute (Phase 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a dry_run flag to the execute meta-tool that validates arguments and checks integration health without actually performing the operation. This makes LLMs more willing to call mutation tools (create, update, delete) since they can preview what would happen first. Tiered approach: - Native: integrations implementing DryRunIntegration can return a custom preview (e.g. AWS Lambda DryRun invocation type) - Simulated fallback: validates args, checks circuit breaker and health, returns a preview with validated_args and status - The tool is never actually executed in either path New DryRunIntegration optional interface in mcp.go — integrations can adopt it incrementally. The execute tool schema now includes a dry_run boolean parameter. 8 new tests: simulated fallback, validation failure, native integration, native-declines-falls-back, unhealthy integration, tool not found, does-not-execute verification, no-pinning verification. 🐘 Generated with Crush Co-Authored-By: Claude Opus 4 --- mcp.go | 9 ++ server/dryrun.go | 65 ++++++++++++ server/dryrun_test.go | 234 ++++++++++++++++++++++++++++++++++++++++++ server/server.go | 9 ++ 4 files changed, 317 insertions(+) create mode 100644 server/dryrun.go create mode 100644 server/dryrun_test.go diff --git a/mcp.go b/mcp.go index f1d9a88c..0e97f18e 100644 --- a/mcp.go +++ b/mcp.go @@ -267,6 +267,15 @@ func (tn *ToolName) UnmarshalJSON(b []byte) error { // Alias lets adapter authors use mcp.Markdown without importing the markdown subpackage. type Markdown = markdown.Markdown +// DryRunIntegration is an optional interface that integrations can implement +// to provide native dry-run previews for mutation tools. When dry_run is set +// on an execute call, the server checks this interface first. If the integration +// returns (result, true), that result is used directly. Otherwise the server +// falls back to a simulated dry-run (arg validation + preview). +type DryRunIntegration interface { + DryRun(ctx context.Context, toolName ToolName, args map[string]any) (*ToolResult, bool) +} + // MarkdownIntegration is an optional interface that integrations can implement // to render tool responses as Markdown instead of JSON. The server calls // RenderMarkdown in processResult before compaction — if it returns rendered diff --git a/server/dryrun.go b/server/dryrun.go new file mode 100644 index 00000000..5cd62479 --- /dev/null +++ b/server/dryrun.go @@ -0,0 +1,65 @@ +package server + +import ( + "context" + "fmt" + + mcp "github.com/daltoniam/switchboard" + mcpsdk "github.com/modelcontextprotocol/go-sdk/mcp" +) + +func (s *Server) handleDryRun(ctx context.Context, toolName mcp.ToolName, args map[string]any) (*mcpsdk.CallToolResult, error) { + integration, toolDef, err := s.findTool(toolName) + if err != nil { + return errorResult(err.Error()), nil + } + + if err := validateArgs(toolDef, args); err != nil { + return errorResult(fmt.Sprintf("dry-run validation failed: %s", err)), nil + } + + cb := s.getBreaker(integration.Name()) + if !cb.allow() { + cb.recordSuccess() + return errorResult(fmt.Sprintf( + "dry-run: integration %q temporarily unavailable (circuit breaker open)", + integration.Name(), + )), nil + } + cb.recordSuccess() + + if !integration.Healthy(ctx) { + return errorResult(fmt.Sprintf( + "dry-run: integration %q is unhealthy — call would likely fail", + integration.Name(), + )), nil + } + + if dri, ok := integration.(mcp.DryRunIntegration); ok { + if result, handled := dri.DryRun(ctx, toolName, args); handled { + return &mcpsdk.CallToolResult{ + Content: []mcpsdk.Content{ + &mcpsdk.TextContent{Text: result.Data}, + }, + IsError: result.IsError, + }, nil + } + } + + result, err := mcp.JSONResult(map[string]any{ + "dry_run": true, + "tool": toolName, + "integration": integration.Name(), + "validated_args": args, + "status": "ok", + "note": "Simulated dry-run: arguments are valid, integration is healthy. This tool does not support native dry-run preview.", + }) + if err != nil { + return errorResult(err.Error()), nil + } + return &mcpsdk.CallToolResult{ + Content: []mcpsdk.Content{ + &mcpsdk.TextContent{Text: result.Data}, + }, + }, nil +} diff --git a/server/dryrun_test.go b/server/dryrun_test.go new file mode 100644 index 00000000..025b6ac8 --- /dev/null +++ b/server/dryrun_test.go @@ -0,0 +1,234 @@ +package server + +import ( + "context" + "encoding/json" + "testing" + + mcp "github.com/daltoniam/switchboard" + mcpsdk "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type mockDryRunIntegration struct { + mockIntegration + dryRunFn func(ctx context.Context, toolName mcp.ToolName, args map[string]any) (*mcp.ToolResult, bool) +} + +func (m *mockDryRunIntegration) DryRun(ctx context.Context, toolName mcp.ToolName, args map[string]any) (*mcp.ToolResult, bool) { + if m.dryRunFn != nil { + return m.dryRunFn(ctx, toolName, args) + } + return nil, false +} + +func dryRunRequest(toolName string, args map[string]any) *mcpsdk.CallToolRequest { + data, _ := json.Marshal(map[string]any{ + "tool_name": toolName, + "arguments": args, + "dry_run": true, + }) + return &mcpsdk.CallToolRequest{ + Params: &mcpsdk.CallToolParamsRaw{ + Name: "execute", + Arguments: json.RawMessage(data), + }, + } +} + +func TestDryRun_SimulatedFallback(t *testing.T) { + mi := &mockIntegration{ + name: "github", + healthy: true, + tools: []mcp.ToolDefinition{ + { + Name: "github_create_issue", + Description: "Create issue", + Parameters: map[string]string{"owner": "Owner", "repo": "Repo", "title": "Title"}, + Required: []string{"owner", "repo", "title"}, + }, + }, + } + s := setupTestServer(mi) + ctx := context.Background() + + result, err := s.handleExecute(ctx, dryRunRequest("github_create_issue", map[string]any{ + "owner": "daltoniam", + "repo": "switchboard", + "title": "Test issue", + })) + require.NoError(t, err) + require.False(t, result.IsError) + + var resp map[string]any + require.NoError(t, json.Unmarshal([]byte(result.Content[0].(*mcpsdk.TextContent).Text), &resp)) + assert.Equal(t, true, resp["dry_run"]) + assert.Equal(t, "github_create_issue", resp["tool"]) + assert.Equal(t, "github", resp["integration"]) + assert.Equal(t, "ok", resp["status"]) + args := resp["validated_args"].(map[string]any) + assert.Equal(t, "daltoniam", args["owner"]) +} + +func TestDryRun_ValidationFails(t *testing.T) { + mi := &mockIntegration{ + name: "github", + healthy: true, + tools: []mcp.ToolDefinition{ + { + Name: "github_create_issue", + Parameters: map[string]string{"owner": "Owner", "repo": "Repo", "title": "Title"}, + Required: []string{"owner", "repo", "title"}, + }, + }, + } + s := setupTestServer(mi) + ctx := context.Background() + + result, err := s.handleExecute(ctx, dryRunRequest("github_create_issue", map[string]any{ + "owner": "daltoniam", + })) + require.NoError(t, err) + assert.True(t, result.IsError) + assert.Contains(t, result.Content[0].(*mcpsdk.TextContent).Text, "dry-run validation failed") + assert.Contains(t, result.Content[0].(*mcpsdk.TextContent).Text, "repo") +} + +func TestDryRun_NativeIntegration(t *testing.T) { + mi := &mockDryRunIntegration{ + mockIntegration: mockIntegration{ + name: "aws", + healthy: true, + tools: []mcp.ToolDefinition{ + { + Name: "aws_lambda_invoke", + Parameters: map[string]string{"function_name": "Function"}, + Required: []string{"function_name"}, + }, + }, + }, + dryRunFn: func(_ context.Context, _ mcp.ToolName, args map[string]any) (*mcp.ToolResult, bool) { + return &mcp.ToolResult{ + Data: `{"dry_run":true,"native":true,"would_invoke":"` + args["function_name"].(string) + `"}`, + }, true + }, + } + s := setupTestServerWithIntegration(mi) + ctx := context.Background() + + result, err := s.handleExecute(ctx, dryRunRequest("aws_lambda_invoke", map[string]any{ + "function_name": "my-func", + })) + require.NoError(t, err) + require.False(t, result.IsError) + + var resp map[string]any + require.NoError(t, json.Unmarshal([]byte(result.Content[0].(*mcpsdk.TextContent).Text), &resp)) + assert.Equal(t, true, resp["native"]) + assert.Equal(t, "my-func", resp["would_invoke"]) +} + +func TestDryRun_NativeDeclines_FallsBackToSimulated(t *testing.T) { + mi := &mockDryRunIntegration{ + mockIntegration: mockIntegration{ + name: "aws", + healthy: true, + tools: []mcp.ToolDefinition{ + { + Name: "aws_s3_list", + Parameters: map[string]string{"bucket": "Bucket"}, + Required: []string{"bucket"}, + }, + }, + }, + dryRunFn: func(_ context.Context, _ mcp.ToolName, _ map[string]any) (*mcp.ToolResult, bool) { + return nil, false + }, + } + s := setupTestServerWithIntegration(mi) + ctx := context.Background() + + result, err := s.handleExecute(ctx, dryRunRequest("aws_s3_list", map[string]any{ + "bucket": "my-bucket", + })) + require.NoError(t, err) + require.False(t, result.IsError) + + var resp map[string]any + require.NoError(t, json.Unmarshal([]byte(result.Content[0].(*mcpsdk.TextContent).Text), &resp)) + assert.Equal(t, true, resp["dry_run"]) + assert.Equal(t, "ok", resp["status"]) +} + +func TestDryRun_UnhealthyIntegration(t *testing.T) { + mi := &mockIntegration{ + name: "github", + healthy: false, + tools: []mcp.ToolDefinition{ + {Name: "github_create_issue", Parameters: map[string]string{"title": "Title"}, Required: []string{"title"}}, + }, + } + s := setupTestServer(mi) + ctx := context.Background() + + result, err := s.handleExecute(ctx, dryRunRequest("github_create_issue", map[string]any{ + "title": "test", + })) + require.NoError(t, err) + assert.True(t, result.IsError) + assert.Contains(t, result.Content[0].(*mcpsdk.TextContent).Text, "unhealthy") +} + +func TestDryRun_ToolNotFound(t *testing.T) { + s := setupTestServer(&mockIntegration{name: "test", healthy: true}) + ctx := context.Background() + + result, err := s.handleExecute(ctx, dryRunRequest("nonexistent_tool", nil)) + require.NoError(t, err) + assert.True(t, result.IsError) + assert.Contains(t, result.Content[0].(*mcpsdk.TextContent).Text, "not found") +} + +func TestDryRun_DoesNotExecute(t *testing.T) { + executed := false + mi := &mockIntegration{ + name: "github", + healthy: true, + tools: []mcp.ToolDefinition{ + {Name: "github_create_issue", Parameters: map[string]string{"title": "Title"}, Required: []string{"title"}}, + }, + execFn: func(_ context.Context, _ mcp.ToolName, _ map[string]any) (*mcp.ToolResult, error) { + executed = true + return &mcp.ToolResult{Data: `{"id":1}`}, nil + }, + } + s := setupTestServer(mi) + ctx := context.Background() + + _, err := s.handleExecute(ctx, dryRunRequest("github_create_issue", map[string]any{ + "title": "test", + })) + require.NoError(t, err) + assert.False(t, executed, "dry-run should not call Execute") +} + +func TestDryRun_NoPinning(t *testing.T) { + mi := &mockIntegration{ + name: "github", + healthy: true, + tools: []mcp.ToolDefinition{ + {Name: "github_create_issue", Parameters: map[string]string{"title": "Title"}, Required: []string{"title"}}, + }, + } + s := setupTestServer(mi) + ctx := context.Background() + + _, err := s.handleExecute(ctx, dryRunRequest("github_create_issue", map[string]any{ + "title": "test", + })) + require.NoError(t, err) + + sess := s.sessionStore.GetOrCreate("default") + assert.Equal(t, 0, sess.PinnedCount(), "dry-run should not pin results") +} diff --git a/server/server.go b/server/server.go index 0b6c274b..277243fa 100644 --- a/server/server.go +++ b/server/server.go @@ -233,6 +233,10 @@ List issues with server-side projection (only id, title, labels — no manual .m "type": "string", "description": "ES5 JavaScript code to execute server-side. Use var (not let/const), function() (not =>), string + concatenation (not template literals). Use api.call(toolName, args, {fields: [...]}) to invoke tools with optional field projection. Return the final result. (mutually exclusive with tool_name)", }, + "dry_run": map[string]any{ + "type": "boolean", + "description": "If true, validate arguments and show what would happen without executing. Works with tool_name only (not scripts).", + }, }, nil), } @@ -643,6 +647,7 @@ func (s *Server) handleExecute(ctx context.Context, req *mcpsdk.CallToolRequest) ToolName mcp.ToolName `json:"tool_name"` Arguments map[string]any `json:"arguments"` Script string `json:"script"` + DryRun bool `json:"dry_run"` } if err := json.Unmarshal(req.Params.Arguments, &args); err != nil { return errorResult("invalid arguments: " + err.Error()), nil @@ -664,6 +669,10 @@ func (s *Server) handleExecute(ctx context.Context, req *mcpsdk.CallToolRequest) args.Arguments = map[string]any{} } + if args.DryRun { + return s.handleDryRun(ctx, args.ToolName, args.Arguments) + } + sess := sessionFromCtx(ctx) if sess == nil { sess = s.sessionStore.GetOrCreate(sessionIDFromReq(req.Session))