From 673a2c158b6f54ec09528676668cae093a46320b Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Mon, 10 Aug 2026 12:58:35 -0400 Subject: [PATCH 01/11] feat(harness): add aforge do parity across SDKs --- docs/design/harness-v2-design.md | 2 +- docs/harness-providers.md | 33 ++ sdk/go/agent/harness.go | 2 +- sdk/go/harness/aforge.go | 264 +++++++++++++++ sdk/go/harness/aforge_integration_test.go | 77 +++++ sdk/go/harness/aforge_test.go | 315 ++++++++++++++++++ sdk/go/harness/cli.go | 14 + sdk/go/harness/factory.go | 8 +- sdk/go/harness/provider.go | 4 +- sdk/go/harness/runner.go | 16 +- sdk/go/harness/runner_invariant_test.go | 8 + sdk/go/harness/runner_test.go | 5 +- sdk/go/harness/testmain_test.go | 5 + .../agentfield/harness/providers/aforge.py | 106 +++--- .../tests/test_harness_provider_aforge.py | 80 +++-- sdk/typescript/package.json | 2 +- sdk/typescript/src/harness/cli.ts | 15 +- .../src/harness/providers/aforge.ts | 229 +++++++++++++ .../src/harness/providers/factory.ts | 6 +- sdk/typescript/src/harness/providers/index.ts | 1 + sdk/typescript/src/harness/runner.ts | 21 +- sdk/typescript/src/harness/types.ts | 8 +- sdk/typescript/tests/harness_cli.test.ts | 20 +- .../tests/harness_functional.test.ts | 54 ++- .../tests/harness_provider_aforge.test.ts | 171 ++++++++++ sdk/typescript/tests/harness_runner.test.ts | 51 ++- sdk/typescript/vitest.functional.config.ts | 9 + 27 files changed, 1424 insertions(+), 102 deletions(-) create mode 100644 sdk/go/harness/aforge.go create mode 100644 sdk/go/harness/aforge_integration_test.go create mode 100644 sdk/go/harness/aforge_test.go create mode 100644 sdk/typescript/src/harness/providers/aforge.ts create mode 100644 sdk/typescript/tests/harness_provider_aforge.test.ts create mode 100644 sdk/typescript/vitest.functional.config.ts diff --git a/docs/design/harness-v2-design.md b/docs/design/harness-v2-design.md index 9ea47165a..f3e8fbd90 100644 --- a/docs/design/harness-v2-design.md +++ b/docs/design/harness-v2-design.md @@ -142,7 +142,7 @@ async def fix_issue(issue: dict) -> dict: ``` Agent ├── .ai() → AIConfig → LiteLLM → LLM APIs (100+ providers) -└── .harness() → HarnessConfig → HarnessRunner → Provider → {Claude Code, Codex, Gemini, OpenCode} +└── .harness() → HarnessConfig → HarnessRunner → Provider → {Aforge, Claude Code, Codex, Gemini, OpenCode} ``` ### 3.2 Component Stack diff --git a/docs/harness-providers.md b/docs/harness-providers.md index 62995a11d..ef4d0b633 100644 --- a/docs/harness-providers.md +++ b/docs/harness-providers.md @@ -37,6 +37,39 @@ The extras install Python wrappers. They do not replace the runtime preflight: Aforge and Gemini are CLI-only, and Codex or OpenCode may still require a separately available executable depending on the wrapper and platform. +### Aforge drop-in usage + +Aforge is registered as `aforge` in the Python, Go, and TypeScript SDKs. The +adapters all invoke the same non-interactive contract: `aforge do --json +--yes-spend`, send the task over stdin, and map Aforge's usage ledger into +AgentField turns, token counts, and cost metrics. Install a current aforge-v2 +binary on `PATH`, set `OPENROUTER_API_KEY`, then select the provider: + +```python +result = await app.harness(task, provider="aforge", model="openrouter/z-ai/glm-5.2") +``` + +```go +result, err := harness.NewRunner(harness.Options{Provider: harness.ProviderAforge}).Run( + ctx, task, nil, nil, harness.Options{Cwd: repoRoot}, +) +``` + +```ts +const result = await new HarnessRunner().run(task, { + provider: 'aforge', + model: 'openrouter/z-ai/glm-5.2', + cwd: repoRoot, +}); +``` + +Set `AFORGE_MAX_CONCURRENT` to cap simultaneous Aforge subprocesses. The +default is 8. `AGENTFIELD_HARNESS_TIMEOUT_SECONDS` is the outer watchdog; each +adapter gives Aforge a five-second landing window to emit its exit-2 timeout +envelope. Schema runs use a unique output directory per invocation so parallel +benchmark jobs can safely share a checkout. Set `AFORGE_BIN` to an absolute +path when the unreleased binary is not installed on `PATH`. + ## Model selection and reasoning-effort variants Every provider accepts a `model` option on `.harness()` calls. The model string diff --git a/sdk/go/agent/harness.go b/sdk/go/agent/harness.go index 29462934c..2783ef09c 100644 --- a/sdk/go/agent/harness.go +++ b/sdk/go/agent/harness.go @@ -10,7 +10,7 @@ import ( // providing lazy initialization and a convenience Harness() method. // HarnessConfig configures the default harness runner for the agent. type HarnessConfig struct { - // Provider is the default provider: "claude-code", "codex", "gemini", or "opencode". + // Provider is the default provider: "aforge", "claude-code", "codex", "gemini", or "opencode". Provider string // Model is the default model identifier. It may carry a diff --git a/sdk/go/harness/aforge.go b/sdk/go/harness/aforge.go new file mode 100644 index 000000000..f6d5482d5 --- /dev/null +++ b/sdk/go/harness/aforge.go @@ -0,0 +1,264 @@ +package harness + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + "sync" + "time" +) + +const ( + defaultAforgeMaxConcurrent = 8 + defaultAforgeTimeout = 1800 + aforgeLandingWindow = 5 +) + +var ( + aforgeSemaphore chan struct{} + aforgeSemOnce sync.Once +) + +// AforgeProvider invokes aforge's one-shot machine-readable harness mode. +type AforgeProvider struct { + BinPath string + runCLI func(ctx context.Context, cmd []string, env map[string]string, cwd string, timeout, idleSeconds int, stdin []byte) (*CLIResult, error) +} + +// NewAforgeProvider creates an Aforge provider. If binPath is empty, it +// defaults to "aforge". +func NewAforgeProvider(binPath string) *AforgeProvider { + if binPath == "" { + binPath = strings.TrimSpace(os.Getenv("AFORGE_BIN")) + if binPath == "" { + binPath = "aforge" + } + } + return &AforgeProvider{BinPath: binPath, runCLI: runCLIWithStdinIdle} +} + +func getAforgeSemaphore() chan struct{} { + aforgeSemOnce.Do(func() { + maxConcurrent := defaultAforgeMaxConcurrent + if raw := strings.TrimSpace(os.Getenv("AFORGE_MAX_CONCURRENT")); raw != "" { + if parsed, err := strconv.Atoi(raw); err == nil && parsed > 0 { + maxConcurrent = parsed + } + } + aforgeSemaphore = make(chan struct{}, maxConcurrent) + }) + return aforgeSemaphore +} + +func aforgeTimeout(options Options) int { + if options.Timeout > 0 { + return options.Timeout + } + if raw := strings.TrimSpace(os.Getenv("AGENTFIELD_HARNESS_TIMEOUT_SECONDS")); raw != "" { + if parsed, err := strconv.Atoi(raw); err == nil && parsed > 0 { + return parsed + } + } + return defaultAforgeTimeout +} + +func aforgeInnerTimeout(outer int) int { + if outer > aforgeLandingWindow { + return outer - aforgeLandingWindow + } + return 1 +} + +func stripOpenRouterPrefix(model string) string { + return strings.TrimPrefix(model, "openrouter/") +} + +func supportedAforgeVariant(variant string) (string, bool) { + normalized := strings.ToLower(strings.TrimSpace(variant)) + switch normalized { + case "off", "low", "medium", "high": + return normalized, true + default: + return "", false + } +} + +func aforgeTaskInput(prompt, systemPrompt string) string { + if systemPrompt = strings.TrimSpace(systemPrompt); systemPrompt != "" { + return systemPrompt + "\n\nTask:\n" + prompt + } + return prompt +} + +func parseAforgeEnvelope(stdout string) map[string]any { + lines := strings.Split(stdout, "\n") + for i := len(lines) - 1; i >= 0; i-- { + line := strings.TrimSpace(lines[i]) + if line == "" { + continue + } + var envelope map[string]any + if err := json.Unmarshal([]byte(line), &envelope); err != nil { + continue + } + if _, ok := envelope["deliverable"]; ok { + return envelope + } + } + return nil +} + +func aforgeNumber(value any) (float64, bool) { + switch number := value.(type) { + case float64: + return number, true + case float32: + return float64(number), true + case int: + return float64(number), true + case int64: + return float64(number), true + case json.Number: + parsed, err := number.Float64() + return parsed, err == nil + default: + return 0, false + } +} + +func aforgeCrashMessage(returnCode int, blockedOn, deliverable, stderr string) string { + cleanStderr := StripANSI(strings.TrimSpace(stderr)) + exitContext := fmt.Sprintf("aforge exit code %d", returnCode) + message := exitContext + if returnCode < 0 { + message = fmt.Sprintf("Process killed by signal %d. %s", -returnCode, exitContext) + } + switch { + case cleanStderr != "": + message += ". stderr: " + truncate(cleanStderr, 1000) + case blockedOn != "": + message += ". blocked_on: " + truncate(blockedOn, 1000) + case deliverable != "": + message += ". partial: " + truncate(deliverable, 1000) + } + return message +} + +func (p *AforgeProvider) Execute(ctx context.Context, prompt string, options Options) (*RawResult, error) { + sem := getAforgeSemaphore() + select { + case sem <- struct{}{}: + defer func() { <-sem }() + case <-ctx.Done(): + return nil, ctx.Err() + } + + root := options.ProjectDir + if root == "" { + root = options.Cwd + } + if root == "" { + root = "." + } + outerTimeout := aforgeTimeout(options) + cmd := []string{ + p.BinPath, "do", "--json", "--yes-spend", "-w", root, + "--timeout", strconv.Itoa(aforgeInnerTimeout(outerTimeout)), + } + + model, variant := options.resolveModelAndVariant() + env := make(map[string]string) + if model != "" { + env["AFORGE_MODEL"] = stripOpenRouterPrefix(model) + } + if normalized, ok := supportedAforgeVariant(variant); ok { + env["AFORGE_EXEC_REASONING"] = normalized + } + // Caller-supplied environment wins over values derived from model/variant. + for key, value := range options.Env { + env[key] = value + } + + started := time.Now() + cliResult, err := p.runCLI(ctx, cmd, env, "", outerTimeout, 0, []byte(aforgeTaskInput(prompt, options.SystemPrompt))) + apiMS := int(time.Since(started).Milliseconds()) + if err != nil { + if isExecNotFound(err) { + return &RawResult{ + IsError: true, + ErrorMessage: fmt.Sprintf("Aforge binary not found at '%s'. Build it from https://github.com/Agent-Field/aforge-v2", p.BinPath), + FailureType: FailureCrash, + Metrics: Metrics{DurationAPIMS: apiMS}, + }, nil + } + if strings.Contains(strings.ToLower(err.Error()), "timed out") || strings.Contains(strings.ToLower(err.Error()), "deadline exceeded") { + return &RawResult{ + IsError: true, + ErrorMessage: err.Error(), + FailureType: FailureTimeout, + Metrics: Metrics{DurationAPIMS: apiMS}, + }, nil + } + return nil, err + } + + envelope := parseAforgeEnvelope(cliResult.Stdout) + resultText := "" + blockedOn := "" + usage := map[string]any{} + if envelope != nil { + if text, ok := envelope["deliverable"].(string); ok { + resultText = strings.TrimSpace(text) + } + if value, ok := envelope["blocked_on"].(string); ok { + blockedOn = strings.TrimSpace(value) + } + if value, ok := envelope["usage"].(map[string]any); ok { + usage = value + } + } + + isError := cliResult.ReturnCode != 0 || resultText == "" || blockedOn != "" + metrics := Metrics{DurationAPIMS: apiMS} + if value, ok := aforgeNumber(usage["calls"]); ok { + metrics.NumTurns = int(value) + } + if value, ok := aforgeNumber(usage["prompt_tokens"]); ok { + metrics.InputTokens = int(value) + } + if value, ok := aforgeNumber(usage["completion_tokens"]); ok { + metrics.OutputTokens = int(value) + } + if value, ok := aforgeNumber(usage["cached_tokens"]); ok { + metrics.CacheReadTokens = int(value) + } + if value, ok := aforgeNumber(usage["cost"]); ok && value > 0 { + cost := value + metrics.CostUSD = &cost + } + + messages := []map[string]any(nil) + if envelope != nil { + messages = []map[string]any{envelope} + } + raw := &RawResult{ + Result: resultText, + Messages: messages, + Metrics: metrics, + IsError: isError, + FailureType: FailureNone, + ReturnCode: cliResult.ReturnCode, + } + if isError { + if cliResult.ReturnCode == 2 { + raw.FailureType = FailureTimeout + } else { + raw.FailureType = FailureCrash + } + raw.ErrorMessage = aforgeCrashMessage(cliResult.ReturnCode, blockedOn, resultText, cliResult.Stderr) + } + return raw, nil +} diff --git a/sdk/go/harness/aforge_integration_test.go b/sdk/go/harness/aforge_integration_test.go new file mode 100644 index 000000000..59e371a2a --- /dev/null +++ b/sdk/go/harness/aforge_integration_test.go @@ -0,0 +1,77 @@ +//go:build integration + +package harness + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestAforgeProvider_Integration drives the real aforge binary and its `do` +// JSON envelope. Set AFORGE_INTEGRATION=1 so TestMain does not shadow it. +func TestAforgeProvider_Integration(t *testing.T) { + binPath, err := exec.LookPath("aforge") + if err != nil { + t.Skip("aforge binary not installed") + } + if os.Getenv("OPENROUTER_API_KEY") == "" && os.Getenv("OPENAI_API_KEY") == "" { + t.Skip("OPENROUTER_API_KEY or OPENAI_API_KEY is required") + } + + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second) + defer cancel() + raw, err := NewAforgeProvider(binPath).Execute(ctx, "Reply with exactly: HELLO_AGENTFIELD", Options{Timeout: 300}) + require.NoError(t, err) + require.NotNil(t, raw) + assert.False(t, raw.IsError, raw.ErrorMessage) + assert.Contains(t, raw.Result, "HELLO_AGENTFIELD") + assert.NotEmpty(t, raw.Messages) + assert.Greater(t, raw.Metrics.NumTurns, 0) + assert.Greater(t, raw.Metrics.InputTokens, 0) + assert.Greater(t, raw.Metrics.OutputTokens, 0) + require.NotNil(t, raw.Metrics.CostUSD) + assert.Greater(t, *raw.Metrics.CostUSD, 0.0) +} + +func TestAforgeRunner_Integration_Schema(t *testing.T) { + if _, err := exec.LookPath("aforge"); err != nil { + t.Skip("aforge binary not installed") + } + if os.Getenv("OPENROUTER_API_KEY") == "" && os.Getenv("OPENAI_API_KEY") == "" { + t.Skip("OPENROUTER_API_KEY or OPENAI_API_KEY is required") + } + + workDir := t.TempDir() + schema := map[string]any{ + "type": "object", + "properties": map[string]any{ + "greeting": map[string]any{"type": "string"}, + "number": map[string]any{"type": "integer"}, + }, + "required": []string{"greeting", "number"}, + } + + var parsed map[string]any + result, err := NewRunner(Options{Provider: ProviderAforge}).Run( + context.Background(), + `Return greeting="Hello from Aforge" and number=42. Follow the OUTPUT REQUIREMENTS precisely.`, + schema, + &parsed, + Options{Cwd: workDir, MaxRetries: 1, Timeout: 300}, + ) + require.NoError(t, err) + require.NotNil(t, result) + assert.False(t, result.IsError, result.ErrorMessage) + assert.Equal(t, "Hello from Aforge", parsed["greeting"]) + assert.EqualValues(t, 42, parsed["number"]) + matches, err := filepath.Glob(filepath.Join(workDir, ".agentfield-out-*")) + require.NoError(t, err) + assert.Empty(t, matches) +} diff --git a/sdk/go/harness/aforge_test.go b/sdk/go/harness/aforge_test.go new file mode 100644 index 000000000..df8125576 --- /dev/null +++ b/sdk/go/harness/aforge_test.go @@ -0,0 +1,315 @@ +package harness + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func aforgeEnvelope(deliverable string, settled bool, blockedOn, usage string) string { + if usage == "" { + usage = `{"calls":1,"prompt_tokens":0,"completion_tokens":0,"cached_tokens":0,"cost":0}` + } + return fmt.Sprintf(`{"settled":%t,"deliverable":%q,"blocked_on":%q,"spend_usd":0.0123,"elapsed_ms":12,"usage":%s}`, + settled, deliverable, blockedOn, usage) +} + +func TestAforgeProviderMapsDoCommandEnvelopeAndMetrics(t *testing.T) { + var capturedCmd []string + var capturedEnv map[string]string + var capturedCwd string + var capturedTimeout, capturedIdle int + var capturedStdin []byte + + p := NewAforgeProvider("/opt/aforge") + p.runCLI = func(_ context.Context, cmd []string, env map[string]string, cwd string, timeout, idleSeconds int, stdin []byte) (*CLIResult, error) { + capturedCmd = append([]string(nil), cmd...) + capturedEnv = env + capturedCwd = cwd + capturedTimeout = timeout + capturedIdle = idleSeconds + capturedStdin = append([]byte(nil), stdin...) + return &CLIResult{ + Stdout: aforgeEnvelope(" final answer ", true, "", + `{"calls":3,"prompt_tokens":100,"completion_tokens":50,"cached_tokens":20,"cost":0.0123}`), + ReturnCode: 0, + }, nil + } + + raw, err := p.Execute(context.Background(), "prompt that stays off argv", Options{ + ProjectDir: "/project", + Cwd: "/project/nested", + SystemPrompt: " be precise ", + Model: "openrouter/z-ai/glm-5.2#high", + }) + require.NoError(t, err) + require.NotNil(t, raw) + assert.Equal(t, []string{ + "/opt/aforge", "do", "--json", "--yes-spend", "-w", "/project", + "--timeout", "1795", + }, capturedCmd) + assert.Equal(t, "z-ai/glm-5.2", capturedEnv["AFORGE_MODEL"]) + assert.Equal(t, "high", capturedEnv["AFORGE_EXEC_REASONING"]) + assert.Empty(t, capturedCwd) + assert.Equal(t, defaultAforgeTimeout, capturedTimeout) + assert.Zero(t, capturedIdle) + assert.Equal(t, "be precise\n\nTask:\nprompt that stays off argv", string(capturedStdin)) + for _, arg := range capturedCmd { + assert.NotContains(t, arg, "prompt that stays off argv") + } + + assert.Equal(t, "final answer", raw.Result) + assert.False(t, raw.IsError) + assert.Equal(t, FailureNone, raw.FailureType) + assert.Equal(t, 0, raw.ReturnCode) + assert.Equal(t, 3, raw.Metrics.NumTurns) + assert.Equal(t, 100, raw.Metrics.InputTokens) + assert.Equal(t, 50, raw.Metrics.OutputTokens) + assert.Equal(t, 20, raw.Metrics.CacheReadTokens) + require.NotNil(t, raw.Metrics.CostUSD) + assert.InDelta(t, 0.0123, *raw.Metrics.CostUSD, 1e-9) + require.Len(t, raw.Messages, 1) + assert.Equal(t, " final answer ", raw.Messages[0]["deliverable"]) +} + +func TestAforgeProviderBinaryEnvironmentOverride(t *testing.T) { + t.Setenv("AFORGE_BIN", "/opt/aforge-env") + assert.Equal(t, "/opt/aforge-env", NewAforgeProvider("").BinPath) + assert.Equal(t, "/explicit/aforge", NewAforgeProvider("/explicit/aforge").BinPath) +} + +func TestAforgeProviderModelVariantAndEnvironmentPrecedence(t *testing.T) { + var captured []map[string]string + p := NewAforgeProvider("aforge") + p.runCLI = func(_ context.Context, _ []string, env map[string]string, _ string, _, _ int, _ []byte) (*CLIResult, error) { + copyEnv := make(map[string]string, len(env)) + for key, value := range env { + copyEnv[key] = value + } + captured = append(captured, copyEnv) + return &CLIResult{Stdout: aforgeEnvelope("done", true, "", ""), ReturnCode: 0}, nil + } + + _, err := p.Execute(context.Background(), "hello", Options{Model: "openrouter/x/y#turbo"}) + require.NoError(t, err) + _, err = p.Execute(context.Background(), "hello", Options{ + Model: "openrouter/x/y#low", + Variant: " HIGH ", + Env: map[string]string{ + "AFORGE_MODEL": "override/model", + "AFORGE_EXEC_REASONING": "off", + "EXTRA": "1", + }, + }) + require.NoError(t, err) + + assert.Equal(t, "x/y", captured[0]["AFORGE_MODEL"]) + assert.NotContains(t, captured[0], "AFORGE_EXEC_REASONING") + assert.Equal(t, "override/model", captured[1]["AFORGE_MODEL"]) + assert.Equal(t, "off", captured[1]["AFORGE_EXEC_REASONING"]) + assert.Equal(t, "1", captured[1]["EXTRA"]) +} + +func TestAforgeProviderRootAndTimeoutResolution(t *testing.T) { + t.Setenv("AGENTFIELD_HARNESS_TIMEOUT_SECONDS", "2400") + var commands [][]string + var timeouts []int + p := NewAforgeProvider("aforge") + p.runCLI = func(_ context.Context, cmd []string, _ map[string]string, _ string, timeout, idleSeconds int, _ []byte) (*CLIResult, error) { + commands = append(commands, append([]string(nil), cmd...)) + timeouts = append(timeouts, timeout) + assert.Zero(t, idleSeconds) + return &CLIResult{Stdout: aforgeEnvelope("done", true, "", ""), ReturnCode: 0}, nil + } + + _, err := p.Execute(context.Background(), "hello", Options{Cwd: "/cwd-only"}) + require.NoError(t, err) + _, err = p.Execute(context.Background(), "hello", Options{Timeout: 7}) + require.NoError(t, err) + + assert.Equal(t, []string{"aforge", "do", "--json", "--yes-spend", "-w", "/cwd-only", "--timeout", "2395"}, commands[0]) + assert.Equal(t, []string{"aforge", "do", "--json", "--yes-spend", "-w", ".", "--timeout", "2"}, commands[1]) + assert.Equal(t, []int{2400, 7}, timeouts) +} + +func TestAforgeProviderExitSemantics(t *testing.T) { + tests := []struct { + name string + code int + deliverable string + blockedOn string + stderr string + wantFailure FailureType + wantMessage string + }{ + {name: "success", code: 0, deliverable: "done", wantFailure: FailureNone}, + {name: "timeout with partial", code: 2, deliverable: "usable", wantFailure: FailureTimeout, wantMessage: "partial: usable"}, + {name: "blocked", code: 1, blockedOn: "Which repository?", wantFailure: FailureCrash, wantMessage: "blocked_on: Which repository?"}, + {name: "error", code: 1, stderr: "\x1b[31mauthentication exploded\x1b[0m", wantFailure: FailureCrash, wantMessage: "authentication exploded"}, + {name: "zero without deliverable", code: 0, wantFailure: FailureCrash, wantMessage: "aforge exit code 0"}, + {name: "signal", code: -9, wantFailure: FailureCrash, wantMessage: "Process killed by signal 9"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + p := NewAforgeProvider("aforge") + p.runCLI = func(context.Context, []string, map[string]string, string, int, int, []byte) (*CLIResult, error) { + return &CLIResult{ + Stdout: aforgeEnvelope(test.deliverable, test.code == 0, test.blockedOn, ""), + Stderr: test.stderr, + ReturnCode: test.code, + }, nil + } + + raw, err := p.Execute(context.Background(), "hello", Options{}) + require.NoError(t, err) + assert.Equal(t, test.wantFailure != FailureNone, raw.IsError) + assert.Equal(t, test.wantFailure, raw.FailureType) + if test.wantMessage != "" { + assert.Contains(t, raw.ErrorMessage, test.wantMessage) + } + }) + } +} + +func TestAforgeProviderParsesLastEnvelopeAndLeavesZeroCostUnknown(t *testing.T) { + p := NewAforgeProvider("aforge") + p.runCLI = func(context.Context, []string, map[string]string, string, int, int, []byte) (*CLIResult, error) { + return &CLIResult{Stdout: "stray diagnostic\n{\"type\":\"event\"}\n" + + aforgeEnvelope("real result", true, "", `{"calls":1,"cost":0}`), ReturnCode: 0}, nil + } + + raw, err := p.Execute(context.Background(), "hello", Options{}) + require.NoError(t, err) + assert.Equal(t, "real result", raw.Result) + assert.Nil(t, raw.Metrics.CostUSD, "zero provider cost remains unknown") +} + +func TestAforgeProviderMissingBinaryAndTimeout(t *testing.T) { + t.Run("missing binary", func(t *testing.T) { + p := NewAforgeProvider("aforge-missing") + p.runCLI = func(context.Context, []string, map[string]string, string, int, int, []byte) (*CLIResult, error) { + return nil, fmt.Errorf("exec: executable file not found in $PATH") + } + raw, err := p.Execute(context.Background(), "hello", Options{}) + require.NoError(t, err) + assert.True(t, raw.IsError) + assert.Equal(t, FailureCrash, raw.FailureType) + assert.Contains(t, raw.ErrorMessage, "aforge-missing") + }) + + t.Run("timeout", func(t *testing.T) { + p := NewAforgeProvider("aforge") + p.runCLI = func(context.Context, []string, map[string]string, string, int, int, []byte) (*CLIResult, error) { + return nil, fmt.Errorf("CLI command timed out after 1s: aforge do") + } + raw, err := p.Execute(context.Background(), "hello", Options{}) + require.NoError(t, err) + assert.True(t, raw.IsError) + assert.Equal(t, FailureTimeout, raw.FailureType) + }) +} + +func TestAforgeProviderConcurrencyLimit(t *testing.T) { + t.Setenv("AFORGE_MAX_CONCURRENT", "2") + aforgeSemaphore = nil + aforgeSemOnce = sync.Once{} + t.Cleanup(func() { + aforgeSemaphore = nil + aforgeSemOnce = sync.Once{} + }) + + var current int64 + var maxSeen int64 + p := NewAforgeProvider("aforge") + p.runCLI = func(context.Context, []string, map[string]string, string, int, int, []byte) (*CLIResult, error) { + active := atomic.AddInt64(¤t, 1) + for { + previous := atomic.LoadInt64(&maxSeen) + if active <= previous || atomic.CompareAndSwapInt64(&maxSeen, previous, active) { + break + } + } + time.Sleep(50 * time.Millisecond) + atomic.AddInt64(¤t, -1) + return &CLIResult{Stdout: aforgeEnvelope("done", true, "", ""), ReturnCode: 0}, nil + } + + var wg sync.WaitGroup + for i := 0; i < 6; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, _ = p.Execute(context.Background(), "hello", Options{}) + }() + } + wg.Wait() + assert.LessOrEqual(t, maxSeen, int64(2)) +} + +func TestAforgeRunnerConcurrentSameCwdUsesIsolatedSchemaFiles(t *testing.T) { + cwd := t.TempDir() + script := writeTestScript(t, cwd, "aforge-test", `#!/bin/sh +prompt=$(cat) +output_path=$(printf '%s' "$prompt" | tr '\n' ' ' | sed -n 's/.*create this file: \([^ ]*\.agentfield_output\.json\).*/\1/p') +case "$prompt" in + *first*) payload='{"name":"first","count":1}' ;; + *) payload='{"name":"second","count":2}' ;; +esac +mkdir -p "$(dirname "$output_path")" +printf '%s' "$payload" > "$output_path" +printf '%s\n' '{"settled":true,"deliverable":"done","blocked_on":"","spend_usd":0,"elapsed_ms":1,"usage":{"calls":1,"prompt_tokens":0,"completion_tokens":0,"cached_tokens":0,"cost":0}}' +`) + + type output struct { + Name string `json:"name"` + Count int `json:"count"` + } + schema := map[string]any{ + "type": "object", + "properties": map[string]any{ + "name": map[string]any{"type": "string"}, + "count": map[string]any{"type": "integer"}, + }, + "required": []any{"name", "count"}, + } + runner := NewRunner(Options{Provider: ProviderAforge, BinPath: script}) + + type runResult struct { + result *Result + dest output + err error + } + results := make(chan runResult, 2) + for _, prompt := range []string{"first", "second"} { + prompt := prompt + go func() { + var dest output + result, err := runner.Run(context.Background(), prompt, schema, &dest, Options{Cwd: cwd}) + results <- runResult{result: result, dest: dest, err: err} + }() + } + + seen := map[string]int{} + for i := 0; i < 2; i++ { + got := <-results + require.NoError(t, got.err) + require.NotNil(t, got.result) + assert.False(t, got.result.IsError, got.result.ErrorMessage) + seen[got.dest.Name] = got.dest.Count + } + assert.Equal(t, map[string]int{"first": 1, "second": 2}, seen) + matches, err := filepath.Glob(filepath.Join(cwd, ".agentfield-out-*")) + require.NoError(t, err) + assert.Empty(t, matches) + _, err = os.Stat(filepath.Join(cwd, outputFilename)) + assert.True(t, os.IsNotExist(err)) +} diff --git a/sdk/go/harness/cli.go b/sdk/go/harness/cli.go index 24d090a1a..c2ce6936f 100644 --- a/sdk/go/harness/cli.go +++ b/sdk/go/harness/cli.go @@ -74,6 +74,17 @@ func RunCLI(ctx context.Context, cmd []string, env map[string]string, cwd string // string value ("") causes that variable to be removed from the environment // rather than set to empty — use this to unset inherited variables. func RunCLIWithStdin(ctx context.Context, cmd []string, env map[string]string, cwd string, timeout int, stdin []byte) (*CLIResult, error) { + return runCLIWithStdin(ctx, cmd, env, cwd, timeout, stdin, nil) +} + +// runCLIWithStdinIdle is the provider-internal variant with an explicit idle +// watchdog. A value <= 0 disables the watchdog. Aforge uses this because its +// machine-readable mode is intentionally silent until the final JSON envelope. +func runCLIWithStdinIdle(ctx context.Context, cmd []string, env map[string]string, cwd string, timeout, idleSeconds int, stdin []byte) (*CLIResult, error) { + return runCLIWithStdin(ctx, cmd, env, cwd, timeout, stdin, &idleSeconds) +} + +func runCLIWithStdin(ctx context.Context, cmd []string, env map[string]string, cwd string, timeout int, stdin []byte, idleOverride *int) (*CLIResult, error) { if timeout > 0 { var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, time.Duration(timeout)*time.Second) @@ -191,6 +202,9 @@ func RunCLIWithStdin(ctx context.Context, cmd []string, env map[string]string, c }() idleSeconds := resolveIdleSeconds() + if idleOverride != nil { + idleSeconds = *idleOverride + } idleTimedOut := false var waitErr error diff --git a/sdk/go/harness/factory.go b/sdk/go/harness/factory.go index c58420207..9d07474b6 100644 --- a/sdk/go/harness/factory.go +++ b/sdk/go/harness/factory.go @@ -3,9 +3,11 @@ package harness import "fmt" // BuildProvider creates a Provider instance for the given provider name. -// Supported providers: "claude-code", "codex", "gemini", "opencode". +// Supported providers: "aforge", "claude-code", "codex", "gemini", "opencode". func BuildProvider(name string, binPath string) (Provider, error) { switch name { + case ProviderAforge: + return NewAforgeProvider(binPath), nil case ProviderClaudeCode: return NewClaudeCodeProvider(binPath), nil case ProviderCodex: @@ -16,8 +18,8 @@ func BuildProvider(name string, binPath string) (Provider, error) { return NewOpenCodeProvider(binPath, ""), nil default: return nil, fmt.Errorf( - "unknown harness provider: %q (supported: %s, %s, %s, %s)", - name, ProviderClaudeCode, ProviderCodex, ProviderGemini, ProviderOpenCode, + "unknown harness provider: %q (supported: %s, %s, %s, %s, %s)", + name, ProviderAforge, ProviderClaudeCode, ProviderCodex, ProviderGemini, ProviderOpenCode, ) } } diff --git a/sdk/go/harness/provider.go b/sdk/go/harness/provider.go index 2f6ab37fd..2e7b499f5 100644 --- a/sdk/go/harness/provider.go +++ b/sdk/go/harness/provider.go @@ -3,6 +3,8 @@ package harness import "context" const ( + // ProviderAforge is the provider name for the Aforge CLI. + ProviderAforge = "aforge" // ProviderOpenCode is the provider name for OpenCode CLI. ProviderOpenCode = "opencode" // ProviderClaudeCode is the provider name for Claude Code CLI. @@ -23,7 +25,7 @@ type Provider interface { // Options control a single harness invocation. Fields are optional; // zero values mean "use default". type Options struct { - // Provider name: "opencode", "claude-code". + // Provider name: "aforge", "opencode", "claude-code", "codex", or "gemini". Provider string // Model identifier passed to the coding agent. It may carry a diff --git a/sdk/go/harness/runner.go b/sdk/go/harness/runner.go index 45f811d0d..cb4f8b7ba 100644 --- a/sdk/go/harness/runner.go +++ b/sdk/go/harness/runner.go @@ -64,14 +64,22 @@ func (r *Runner) Run(ctx context.Context, prompt string, schema map[string]any, return nil, err } - // Determine output directory for schema files. - outputDir := opts.Cwd + // Always isolate schema output in a per-run directory under the effective + // project root. Concurrent calls sharing one checkout must not overwrite or + // clean up each other's fixed .agentfield_output.json file. + outputDir := opts.ProjectDir + if outputDir == "" { + outputDir = opts.Cwd + } if outputDir == "" { outputDir = "." } var tempOutputDir string - if opts.ProjectDir != "" { - tempOutputDir, err = os.MkdirTemp(opts.ProjectDir, ".agentfield-out-") + if schema != nil { + if err = os.MkdirAll(outputDir, 0o700); err != nil { + return nil, fmt.Errorf("creating schema output root: %w", err) + } + tempOutputDir, err = os.MkdirTemp(outputDir, ".agentfield-out-") if err != nil { return nil, fmt.Errorf("creating temp output dir: %w", err) } diff --git a/sdk/go/harness/runner_invariant_test.go b/sdk/go/harness/runner_invariant_test.go index 20157e06b..11251f4f6 100644 --- a/sdk/go/harness/runner_invariant_test.go +++ b/sdk/go/harness/runner_invariant_test.go @@ -178,6 +178,7 @@ func TestInvariant_Runner_RepairJSON_ProjectionExtended(t *testing.T) { // registered provider name returns a non-nil provider from BuildProvider. func TestInvariant_Runner_ProviderFactoryExhaustiveness(t *testing.T) { knownProviders := []string{ + ProviderAforge, ProviderClaudeCode, ProviderCodex, ProviderGemini, @@ -310,6 +311,13 @@ func TestInvariant_Runner_BuildProviderWithBinPath(t *testing.T) { provider string checkPath func(t *testing.T, prov Provider) }{ + { + provider: ProviderAforge, + checkPath: func(t *testing.T, prov Provider) { + p := prov.(*AforgeProvider) + assert.Equal(t, customBinPath, p.BinPath) + }, + }, { provider: ProviderClaudeCode, checkPath: func(t *testing.T, prov Provider) { diff --git a/sdk/go/harness/runner_test.go b/sdk/go/harness/runner_test.go index 4d744ec3d..bf95c5b39 100644 --- a/sdk/go/harness/runner_test.go +++ b/sdk/go/harness/runner_test.go @@ -567,6 +567,7 @@ func TestBuildProvider(t *testing.T) { wantType string wantErr bool }{ + {"aforge", "aforge", "*harness.AforgeProvider", false}, {"claude-code", "claude-code", "*harness.ClaudeCodeProvider", false}, {"codex", "codex", "*harness.CodexProvider", false}, {"gemini", "gemini", "*harness.GeminiProvider", false}, @@ -589,8 +590,8 @@ func TestBuildProvider(t *testing.T) { } func TestRunner_BuildProvider_UsesFactory(t *testing.T) { - // Verify the runner can now build all 4 providers - for _, name := range []string{"claude-code", "codex", "gemini", "opencode"} { + // Verify the runner can build every registered provider. + for _, name := range []string{"aforge", "claude-code", "codex", "gemini", "opencode"} { t.Run(name, func(t *testing.T) { runner := NewRunner(Options{Provider: name}) _, err := runner.Run(context.Background(), "test", nil, nil, Options{}) diff --git a/sdk/go/harness/testmain_test.go b/sdk/go/harness/testmain_test.go index f574d80d5..de877dd55 100644 --- a/sdk/go/harness/testmain_test.go +++ b/sdk/go/harness/testmain_test.go @@ -27,6 +27,11 @@ printf '%s\n' 'stub gemini result' printf '%s\n' 'stub opencode result' `, } + if os.Getenv("AFORGE_INTEGRATION") != "1" { + stubs["aforge"] = `#!/bin/sh +printf '%s\n' '{"settled":true,"deliverable":"stub aforge result","blocked_on":"","spend_usd":0,"elapsed_ms":1,"usage":{"calls":1,"prompt_tokens":0,"completion_tokens":0,"cached_tokens":0,"cost":0}}' +` + } for name, content := range stubs { if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o755); err != nil { diff --git a/sdk/python/agentfield/harness/providers/aforge.py b/sdk/python/agentfield/harness/providers/aforge.py index 1b36d4539..a77b05415 100644 --- a/sdk/python/agentfield/harness/providers/aforge.py +++ b/sdk/python/agentfield/harness/providers/aforge.py @@ -7,7 +7,7 @@ import logging import os import time -from typing import ClassVar, Dict, Optional +from typing import ClassVar from agentfield.harness._availability import ensure_cli_available, provider_unavailable from agentfield.harness._cli import ( @@ -25,12 +25,11 @@ def _strip_openrouter_prefix(model: str) -> str: """Strip one leading ``openrouter/`` prefix from a model slug.""" - prefix = "openrouter/" - return model[len(prefix) :] if model.startswith(prefix) else model + return model.removeprefix("openrouter/") def _parse_envelope(stdout: str) -> dict[str, object] | None: - """Return the last JSON object containing an aforge ``text`` field.""" + """Return the last JSON object containing an aforge deliverable.""" for line in reversed( [line.strip() for line in stdout.splitlines() if line.strip()] ): @@ -38,7 +37,7 @@ def _parse_envelope(stdout: str) -> dict[str, object] | None: value = json.loads(line) except ValueError: continue - if isinstance(value, dict) and "text" in value: + if isinstance(value, dict) and "deliverable" in value: return value return None @@ -50,27 +49,44 @@ def _numeric(value: object) -> int | float | None: return None -def _crash_message(returncode: int, stop: str, stderr: str) -> str: +def _task_input(prompt: str, system_prompt: object) -> str: + if isinstance(system_prompt, str) and system_prompt.strip(): + return f"{system_prompt.strip()}\n\nTask:\n{prompt}" + return prompt + + +def _crash_message( + returncode: int, + blocked_on: str, + deliverable: str | None, + stderr: str, +) -> str: """Build a consistent, bounded aforge crash message.""" clean_stderr = strip_ansi(stderr.strip()) - exit_context = f"aforge exit code {returncode}, stop={stop!s}" + exit_context = f"aforge exit code {returncode}" if returncode < 0: message = f"Process killed by signal {-returncode}. {exit_context}" else: message = exit_context if clean_stderr: message += f". stderr: {clean_stderr[:1000]}" + elif blocked_on: + message += f". blocked_on: {blocked_on[:1000]}" + elif deliverable: + message += f". partial: {deliverable[:1000]}" return message class AforgeProvider: - """Aforge CLI provider. Invokes ``aforge exec --json`` subprocess.""" + """Aforge CLI provider. Invokes canonical ``aforge do --json`` mode.""" _MAX_CONCURRENT: ClassVar[int] = int(os.environ.get("AFORGE_MAX_CONCURRENT", "8")) - _concurrency_sem: ClassVar[Optional[asyncio.Semaphore]] = None + _concurrency_sem: ClassVar[asyncio.Semaphore | None] = None def __init__(self, bin_path: str = "aforge"): - self._bin = bin_path + self._bin = ( + os.environ.get("AFORGE_BIN", bin_path) if bin_path == "aforge" else bin_path + ) @classmethod def _get_semaphore(cls) -> asyncio.Semaphore: @@ -93,14 +109,26 @@ async def _execute_impl(self, prompt: str, options: dict[str, object]) -> RawRes # project_dir is the canonical agent root; a nested task cwd must not # restrict access to sibling paths under the shared project root. root = str(options.get("project_dir") or options.get("cwd") or ".") - cmd = [self._bin, "exec", "--json", "-w", root] - - system_prompt = options.get("system_prompt") - if isinstance(system_prompt, str) and system_prompt.strip(): - cmd.extend(["--system", system_prompt.strip()]) + timeout_seconds = int( + os.environ.get("AGENTFIELD_HARNESS_TIMEOUT_SECONDS", "1800") + ) + # Leave a small landing window so aforge can emit its honest timeout + # envelope before the outer subprocess watchdog has to kill it. + aforge_timeout = max(1, timeout_seconds - 5) + cmd = [ + self._bin, + "do", + "--json", + "--yes-spend", + "-w", + root, + "--timeout", + str(aforge_timeout), + ] + input_text = _task_input(prompt, options.get("system_prompt")) model_value, variant_value = resolve_model_and_variant(options) - env: Dict[str, str] = {} + env: dict[str, str] = {} if model_value: env["AFORGE_MODEL"] = _strip_openrouter_prefix(model_value) @@ -121,9 +149,6 @@ async def _execute_impl(self, prompt: str, options: dict[str, object]) -> RawRes } ) - timeout_seconds = int( - os.environ.get("AGENTFIELD_HARNESS_TIMEOUT_SECONDS", "1800") - ) start_api = time.monotonic() try: @@ -135,7 +160,7 @@ async def _execute_impl(self, prompt: str, options: dict[str, object]) -> RawRes # Aforge is stdout-silent until its final envelope; disable the # no-progress watchdog so legitimate long runs are not killed. idle_seconds=0, - input_text=prompt, + input_text=input_text, ) except FileNotFoundError as exc: raise provider_unavailable("aforge", self._bin) from exc @@ -151,22 +176,22 @@ async def _execute_impl(self, prompt: str, options: dict[str, object]) -> RawRes envelope = _parse_envelope(stdout) result_text: str | None = None - stop = "" usage: dict[object, object] = {} - turns = 0 + calls = 0 + blocked_on = "" if envelope is not None: - text_value = envelope.get("text") + text_value = envelope.get("deliverable") if isinstance(text_value, str) and text_value.strip(): result_text = text_value.strip() - stop_value = envelope.get("stop") - if isinstance(stop_value, str): - stop = stop_value + blocked_value = envelope.get("blocked_on") + if isinstance(blocked_value, str): + blocked_on = blocked_value.strip() usage_value = envelope.get("usage") if isinstance(usage_value, dict): usage = usage_value - turns_value = _numeric(envelope.get("turns")) - if turns_value is not None: - turns = int(turns_value) + calls_value = _numeric(usage.get("calls")) + if calls_value is not None: + calls = int(calls_value) clean_stderr = strip_ansi(stderr.strip()) if stderr else "" logger.info( @@ -178,17 +203,18 @@ async def _execute_impl(self, prompt: str, options: dict[str, object]) -> RawRes if not result_text and clean_stderr: logger.warning("aforge no text. stderr: %s", clean_stderr[:800]) - if returncode < 0: - is_error = True - elif returncode in (2, 3) and result_text: - is_error = False - elif returncode != 0: - is_error = True + is_error = returncode != 0 or result_text is None or bool(blocked_on) + if not is_error: + failure_type = FailureType.NONE + elif returncode == 2: + failure_type = FailureType.TIMEOUT else: - is_error = result_text is None - - failure_type = FailureType.CRASH if is_error else FailureType.NONE - error_message = _crash_message(returncode, stop, stderr) if is_error else None + failure_type = FailureType.CRASH + error_message = ( + _crash_message(returncode, blocked_on, result_text, stderr) + if is_error + else None + ) input_tokens_value = _numeric(usage.get("prompt_tokens")) output_tokens_value = _numeric(usage.get("completion_tokens")) @@ -208,7 +234,7 @@ async def _execute_impl(self, prompt: str, options: dict[str, object]) -> RawRes messages=[envelope] if envelope is not None else [], metrics=Metrics( duration_api_ms=api_ms, - num_turns=turns, + num_turns=calls, total_cost_usd=total_cost, session_id="", input_tokens=int(input_tokens_value or 0), diff --git a/sdk/python/tests/test_harness_provider_aforge.py b/sdk/python/tests/test_harness_provider_aforge.py index 1ae720b0f..a5f3d14be 100644 --- a/sdk/python/tests/test_harness_provider_aforge.py +++ b/sdk/python/tests/test_harness_provider_aforge.py @@ -21,20 +21,22 @@ def mock_aforge_available(monkeypatch: pytest.MonkeyPatch) -> None: def _envelope( text: str = "done", *, - stop: str = "done", usage: dict[str, object] | None = None, - turns: int = 1, + settled: bool = True, + blocked_on: str | None = None, ) -> str: - return json.dumps( - { - "text": text, - "stop": stop, - "usage": usage or {}, - "artifacts": [], - "turns": turns, - "elapsed_ms": 12, - } - ) + envelope = { + "deliverable": text, + "usage": usage or {}, + "artifacts": [], + "nodes": 2, + "seconds": 0.012, + "settled": settled, + "spend": 0.0, + } + if blocked_on is not None: + envelope["blocked_on"] = blocked_on + return json.dumps(envelope) @pytest.mark.asyncio @@ -55,7 +57,6 @@ async def fake_run_cli( "cached_tokens": 20, "cost": 0.0123, }, - turns=4, ), "", 0, @@ -72,12 +73,18 @@ async def fake_run_cli( assert raw.metrics.output_tokens == 50 assert raw.metrics.cache_read_tokens == 20 assert raw.metrics.cache_creation_tokens == 0 - assert raw.metrics.num_turns == 4 + assert raw.metrics.num_turns == 3 assert raw.metrics.total_cost_usd == 0.0123 assert raw.metrics.model == "openrouter/z-ai/glm-5.2" assert raw.metrics.duration_api_ms >= 0 assert raw.returncode == 0 - assert raw.messages[0]["text"] == " final answer " + assert raw.messages[0]["deliverable"] == " final answer " + + +def test_aforge_binary_environment_override(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("AFORGE_BIN", "/opt/aforge-env") + assert AforgeProvider()._bin == "/opt/aforge-env" + assert AforgeProvider("/explicit/aforge")._bin == "/explicit/aforge" @pytest.mark.asyncio @@ -200,7 +207,7 @@ async def fake_run_cli( @pytest.mark.asyncio -async def test_aforge_passes_stripped_system_prompt_flag( +async def test_aforge_prepends_stripped_system_prompt_to_stdin( monkeypatch: pytest.MonkeyPatch, ): captured: dict[str, Any] = {} @@ -208,15 +215,17 @@ async def test_aforge_passes_stripped_system_prompt_flag( async def fake_run_cli( cmd, *, env=None, cwd=None, timeout=None, idle_seconds=None, input_text=None ): - _ = env, cwd, timeout, idle_seconds, input_text + _ = env, cwd, timeout, idle_seconds captured["cmd"] = cmd + captured["input_text"] = input_text return _envelope(), "", 0 monkeypatch.setattr("agentfield.harness.providers.aforge.run_cli", fake_run_cli) await AforgeProvider().execute("hello", {"system_prompt": " be precise "}) - assert captured["cmd"][-2:] == ["--system", "be precise"] + assert "--system" not in captured["cmd"] + assert captured["input_text"] == "be precise\n\nTask:\nhello" @pytest.mark.asyncio @@ -240,19 +249,25 @@ async def fake_run_cli( ) await provider.execute("hello", {"cwd": "/cwd-only"}) - assert captured_cmds[0][:6] == [ + assert captured_cmds[0] == [ "aforge", - "exec", + "do", "--json", + "--yes-spend", "-w", "/project", + "--timeout", + "1795", ] - assert captured_cmds[1][:6] == [ + assert captured_cmds[1] == [ "aforge", - "exec", + "do", "--json", + "--yes-spend", "-w", "/cwd-only", + "--timeout", + "1795", ] @@ -281,7 +296,7 @@ async def fake_run_cli( cmd, *, env=None, cwd=None, timeout=None, idle_seconds=None, input_text=None ): _ = cmd, env, cwd, timeout, idle_seconds, input_text - return _envelope("", stop="error"), "authentication exploded", 5 + return _envelope(""), "authentication exploded", 1 monkeypatch.setattr("agentfield.harness.providers.aforge.run_cli", fake_run_cli) @@ -289,38 +304,38 @@ async def fake_run_cli( assert raw.is_error is True assert raw.failure_type is FailureType.CRASH - assert "aforge exit code 5" in (raw.error_message or "") + assert "aforge exit code 1" in (raw.error_message or "") assert "authentication exploded" in (raw.error_message or "") @pytest.mark.asyncio -async def test_aforge_budget_exit_with_text_is_success( +async def test_aforge_timeout_exit_with_partial_is_timeout( monkeypatch: pytest.MonkeyPatch, ): async def fake_run_cli( cmd, *, env=None, cwd=None, timeout=None, idle_seconds=None, input_text=None ): _ = cmd, env, cwd, timeout, idle_seconds, input_text - return _envelope("usable landing", stop="budget"), "", 2 + return _envelope("usable partial", settled=False), "", 2 monkeypatch.setattr("agentfield.harness.providers.aforge.run_cli", fake_run_cli) raw = await AforgeProvider().execute("hello", {}) - assert raw.result == "usable landing" - assert raw.is_error is False - assert raw.failure_type is FailureType.NONE + assert raw.result == "usable partial" + assert raw.is_error is True + assert raw.failure_type is FailureType.TIMEOUT @pytest.mark.asyncio -async def test_aforge_turn_cap_without_text_is_crash( +async def test_aforge_blocked_question_is_crash( monkeypatch: pytest.MonkeyPatch, ): async def fake_run_cli( cmd, *, env=None, cwd=None, timeout=None, idle_seconds=None, input_text=None ): _ = cmd, env, cwd, timeout, idle_seconds, input_text - return _envelope("", stop="turn-cap"), "turn cap reached", 3 + return _envelope("", blocked_on="Which repository?"), "", 1 monkeypatch.setattr("agentfield.harness.providers.aforge.run_cli", fake_run_cli) @@ -328,7 +343,8 @@ async def fake_run_cli( assert raw.is_error is True assert raw.failure_type is FailureType.CRASH - assert "aforge exit code 3" in (raw.error_message or "") + assert "aforge exit code 1" in (raw.error_message or "") + assert "Which repository?" in (raw.error_message or "") @pytest.mark.asyncio diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json index 49696665b..1c597ad8f 100644 --- a/sdk/typescript/package.json +++ b/sdk/typescript/package.json @@ -14,7 +14,7 @@ "dev": "tsup --watch", "lint": "tsc --noEmit", "test": "vitest run --config vitest.config.ts", - "test:functional": "vitest run tests/harness_functional.test.ts --timeout=300000", + "test:functional": "vitest run --config vitest.functional.config.ts", "test:core": "vitest run --config vitest.config.ts", "test:coverage": "vitest run --config vitest.config.ts --coverage", "test:coverage:core": "vitest run --config vitest.config.ts --coverage", diff --git a/sdk/typescript/src/harness/cli.ts b/sdk/typescript/src/harness/cli.ts index f279460fc..2ed9e82f8 100644 --- a/sdk/typescript/src/harness/cli.ts +++ b/sdk/typescript/src/harness/cli.ts @@ -33,20 +33,27 @@ export function runCli( cwd?: string; timeout?: number; idleSeconds?: number; + inputText?: string; } ): Promise { return new Promise((resolve, reject) => { const [bin, ...args] = cmd; const env = { ...process.env, ...options?.env }; applyOpenRouterAttributionEnv(env); + const hasInput = options?.inputText !== undefined; // 'ignore' on stdin gives the child an immediate EOF instead of an open - // pipe that never closes (a hang risk if the child probes stdin). + // pipe that never closes (a hang risk if the child probes stdin). Providers + // with large or sensitive prompts can explicitly pipe text over stdin. const proc = spawn(bin, args, { env, cwd: options?.cwd, - stdio: ['ignore', 'pipe', 'pipe'], + stdio: [hasInput ? 'pipe' : 'ignore', 'pipe', 'pipe'], }); + if (hasInput) { + proc.stdin?.end(options.inputText); + } + let stdout = ''; let stderr = ''; let settled = false; @@ -54,11 +61,11 @@ export function runCli( // Both stdout and stderr are drained concurrently via their own 'data' // listeners, so a full stderr pipe cannot deadlock the read of stdout. - proc.stdout.on('data', (data: Uint8Array | string) => { + proc.stdout!.on('data', (data: Uint8Array | string) => { stdout += data.toString(); lastActivity = Date.now(); }); - proc.stderr.on('data', (data: Uint8Array | string) => { + proc.stderr!.on('data', (data: Uint8Array | string) => { stderr += data.toString(); lastActivity = Date.now(); }); diff --git a/sdk/typescript/src/harness/providers/aforge.ts b/sdk/typescript/src/harness/providers/aforge.ts new file mode 100644 index 000000000..c2d32575d --- /dev/null +++ b/sdk/typescript/src/harness/providers/aforge.ts @@ -0,0 +1,229 @@ +import type { HarnessProvider } from './base.js'; +import type { RawResult } from '../types.js'; +import { createMetrics, createRawResult } from '../types.js'; +import { runCli } from '../cli.js'; +import { resolveModelAndVariant } from '../modelVariant.js'; + +const REASONING_VARIANTS = new Set(['off', 'low', 'medium', 'high']); +const DEFAULT_TIMEOUT_SECONDS = 1800; +const LANDING_WINDOW_SECONDS = 5; +const DEFAULT_MAX_CONCURRENT = 8; +const ANSI_PATTERN = /\x1B\[[0-?]*[ -/]*[@-~]/g; + +class Semaphore { + private active = 0; + private readonly waiters: Array<() => void> = []; + + public constructor(private readonly limit: number) {} + + public async use(operation: () => Promise): Promise { + await this.acquire(); + try { + return await operation(); + } finally { + this.release(); + } + } + + private acquire(): Promise { + if (this.active < this.limit) { + this.active += 1; + return Promise.resolve(); + } + return new Promise((resolve) => { + this.waiters.push(() => { + this.active += 1; + resolve(); + }); + }); + } + + private release(): void { + this.active -= 1; + this.waiters.shift()?.(); + } +} + +function resolveMaxConcurrent(): number { + const parsed = Number.parseInt(process.env.AFORGE_MAX_CONCURRENT ?? '', 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_MAX_CONCURRENT; +} + +const aforgeSemaphore = new Semaphore(resolveMaxConcurrent()); + +function stripOpenRouterPrefix(model: string): string { + return model.startsWith('openrouter/') ? model.slice('openrouter/'.length) : model; +} + +function parseEnvelope(stdout: string): Record | undefined { + const lines = stdout.split('\n').map((line) => line.trim()).filter(Boolean); + for (let index = lines.length - 1; index >= 0; index -= 1) { + try { + const value: unknown = JSON.parse(lines[index]); + if (typeof value === 'object' && value !== null && !Array.isArray(value) && 'deliverable' in value) { + return value as Record; + } + } catch { + // Tolerate stray stdout from a wrapper and keep looking for the envelope. + } + } + return undefined; +} + +function numeric(value: unknown): number | undefined { + return typeof value === 'number' && Number.isFinite(value) ? value : undefined; +} + +function timeoutSeconds(): number { + const parsed = Number.parseInt(process.env.AGENTFIELD_HARNESS_TIMEOUT_SECONDS ?? '', 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_TIMEOUT_SECONDS; +} + +function innerTimeout(outer: number): number { + return outer > LANDING_WINDOW_SECONDS ? outer - LANDING_WINDOW_SECONDS : 1; +} + +function taskInput(prompt: string, systemPrompt: unknown): string { + return typeof systemPrompt === 'string' && systemPrompt.trim() + ? `${systemPrompt.trim()}\n\nTask:\n${prompt}` + : prompt; +} + +function crashMessage(exitCode: number, blockedOn: string, deliverable: string | undefined, stderr: string): string { + const cleanStderr = stderr.trim().replace(ANSI_PATTERN, ''); + const exitContext = `aforge exit code ${exitCode}`; + let message = exitCode < 0 ? `Process killed by signal ${-exitCode}. ${exitContext}` : exitContext; + if (cleanStderr) { + message += `. stderr: ${cleanStderr.slice(0, 1000)}`; + } else if (blockedOn) { + message += `. blocked_on: ${blockedOn.slice(0, 1000)}`; + } else if (deliverable) { + message += `. partial: ${deliverable.slice(0, 1000)}`; + } + return message; +} + +function stringOptions(value: unknown): Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return {}; + } + const result: Record = {}; + for (const [key, item] of Object.entries(value)) { + if (typeof item === 'string') { + result[key] = item; + } + } + return result; +} + +/** Aforge CLI provider using canonical `aforge do --json` one-shot mode. */ +export class AforgeProvider implements HarnessProvider { + private readonly bin: string; + + public constructor(bin = 'aforge') { + this.bin = bin === 'aforge' ? (process.env.AFORGE_BIN?.trim() || bin) : bin; + } + + public async execute(prompt: string, options: Record): Promise { + return aforgeSemaphore.use(() => this.executeImpl(prompt, options)); + } + + private async executeImpl(prompt: string, options: Record): Promise { + const projectDir = typeof options.projectDir === 'string' + ? options.projectDir + : typeof options.project_dir === 'string' + ? options.project_dir + : undefined; + const cwd = typeof options.cwd === 'string' ? options.cwd : undefined; + const root = projectDir ?? cwd ?? '.'; + const outerTimeout = timeoutSeconds(); + const cmd = [ + this.bin, + 'do', + '--json', + '--yes-spend', + '-w', + root, + '--timeout', + String(innerTimeout(outerTimeout)), + ]; + + const { model, variant } = resolveModelAndVariant(options); + const env: Record = {}; + if (model) { + env.AFORGE_MODEL = stripOpenRouterPrefix(model); + } + if (variant) { + const normalized = variant.trim().toLowerCase(); + if (REASONING_VARIANTS.has(normalized)) { + env.AFORGE_EXEC_REASONING = normalized; + } + } + Object.assign(env, stringOptions(options.env)); + + const startApi = Date.now(); + try { + const { stdout, stderr, exitCode } = await runCli(cmd, { + env, + cwd: undefined, + timeout: outerTimeout * 1000, + idleSeconds: 0, + inputText: taskInput(prompt, options.systemPrompt ?? options.system_prompt), + }); + const envelope = parseEnvelope(stdout); + const resultText = typeof envelope?.deliverable === 'string' && envelope.deliverable.trim() + ? envelope.deliverable.trim() + : undefined; + const blockedOn = typeof envelope?.blocked_on === 'string' ? envelope.blocked_on.trim() : ''; + const usage = typeof envelope?.usage === 'object' && envelope.usage !== null && !Array.isArray(envelope.usage) + ? envelope.usage as Record + : {}; + + const isError = exitCode !== 0 || resultText === undefined || blockedOn !== ''; + const inputTokens = Math.trunc(numeric(usage.prompt_tokens) ?? 0); + const outputTokens = Math.trunc(numeric(usage.completion_tokens) ?? 0); + const cacheReadTokens = Math.trunc(numeric(usage.cached_tokens) ?? 0); + const calls = Math.trunc(numeric(usage.calls) ?? 0); + const providerCost = numeric(usage.cost); + + return createRawResult({ + result: resultText, + messages: envelope ? [envelope] : [], + metrics: createMetrics({ + durationApiMs: Date.now() - startApi, + numTurns: calls, + totalCostUsd: providerCost !== undefined && providerCost > 0 ? providerCost : undefined, + usage, + sessionId: '', + inputTokens, + outputTokens, + cacheReadTokens, + cacheCreationTokens: 0, + totalTokens: inputTokens + outputTokens, + model, + }), + isError, + errorMessage: isError ? crashMessage(exitCode, blockedOn, resultText, stderr) : undefined, + failureType: isError ? (exitCode === 2 ? 'timeout' : 'crash') : 'none', + returnCode: exitCode, + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (message.includes('ENOENT')) { + return createRawResult({ + isError: true, + errorMessage: `Aforge binary not found at '${this.bin}'. Build it from https://github.com/Agent-Field/aforge-v2`, + failureType: 'crash', + metrics: createMetrics({ durationApiMs: Date.now() - startApi }), + }); + } + const timedOut = /timed out|deadline exceeded|no progress/i.test(message); + return createRawResult({ + isError: true, + errorMessage: message, + failureType: timedOut ? 'timeout' : 'crash', + metrics: createMetrics({ durationApiMs: Date.now() - startApi }), + }); + } + } +} diff --git a/sdk/typescript/src/harness/providers/factory.ts b/sdk/typescript/src/harness/providers/factory.ts index 432d904a7..53a2a9d74 100644 --- a/sdk/typescript/src/harness/providers/factory.ts +++ b/sdk/typescript/src/harness/providers/factory.ts @@ -1,7 +1,7 @@ import type { HarnessProvider } from './base.js'; import type { HarnessConfig } from '../types.js'; -export const SUPPORTED_PROVIDERS = new Set(['claude-code', 'codex', 'gemini', 'opencode']); +export const SUPPORTED_PROVIDERS = new Set(['aforge', 'claude-code', 'codex', 'gemini', 'opencode']); export async function buildProvider(config: HarnessConfig): Promise { if (!SUPPORTED_PROVIDERS.has(config.provider)) { @@ -9,6 +9,10 @@ export async function buildProvider(config: HarnessConfig): Promise & { initialDelay?: number; maxDelay?: number; backoffFactor?: number; + projectDir?: string; + aforgeBin?: string; codexBin?: string; geminiBin?: string; opencodeBin?: string; @@ -57,16 +61,22 @@ export class HarnessRunner { throw new Error("No harness provider specified. Set 'provider' in HarnessConfig or pass it to .harness() call."); } - const cwd = resolved.cwd ?? '.'; const provider = await this.buildProvider(resolved.provider, resolved); - const effectivePrompt = schema === undefined ? prompt : `${prompt}${buildPromptSuffix(schema, cwd)}`; + const cwd = resolved.cwd ?? '.'; + const outputRoot = resolved.projectDir ?? cwd; + let outputDir: string | undefined; + if (schema !== undefined) { + fs.mkdirSync(outputRoot, { recursive: true }); + outputDir = fs.mkdtempSync(path.join(outputRoot, '.agentfield-out-')); + } + const effectivePrompt = schema === undefined ? prompt : `${prompt}${buildPromptSuffix(schema, outputDir!)}`; const startTime = Date.now(); try { const raw = await this.executeWithRetry(provider, effectivePrompt, resolved); if (schema !== undefined) { - return this.handleSchemaOutput(raw, schema, cwd, startTime); + return this.handleSchemaOutput(raw, schema, outputDir!, startTime); } return createHarnessResult({ @@ -82,7 +92,8 @@ export class HarnessRunner { }); } finally { if (schema !== undefined) { - cleanupTempFiles(cwd); + cleanupTempFiles(outputDir!); + fs.rmSync(outputDir!, { recursive: true, force: true }); } } } @@ -105,6 +116,8 @@ export class HarnessRunner { 'systemPrompt', 'env', 'cwd', + 'projectDir', + 'aforgeBin', 'codexBin', 'geminiBin', 'opencodeBin', diff --git a/sdk/typescript/src/harness/types.ts b/sdk/typescript/src/harness/types.ts index 397fc320f..a824fd29e 100644 --- a/sdk/typescript/src/harness/types.ts +++ b/sdk/typescript/src/harness/types.ts @@ -1,5 +1,5 @@ export interface HarnessConfig { - provider: 'claude-code' | 'codex' | 'gemini' | 'opencode'; + provider: 'aforge' | 'claude-code' | 'codex' | 'gemini' | 'opencode'; model?: string; /** * Provider-specific reasoning-effort variant (e.g. `high`, `minimal`). @@ -17,6 +17,8 @@ export interface HarnessConfig { systemPrompt?: string; env?: Record; cwd?: string; + projectDir?: string; + aforgeBin?: string; codexBin?: string; geminiBin?: string; opencodeBin?: string; @@ -41,6 +43,8 @@ export interface HarnessOptions { systemPrompt?: string; env?: Record; cwd?: string; + projectDir?: string; + aforgeBin?: string; codexBin?: string; geminiBin?: string; opencodeBin?: string; @@ -70,6 +74,8 @@ export interface RawResult { metrics: Metrics; isError: boolean; errorMessage?: string; + failureType?: 'none' | 'crash' | 'timeout' | 'api_error' | 'no_output' | 'schema'; + returnCode?: number; } export interface HarnessResult { diff --git a/sdk/typescript/tests/harness_cli.test.ts b/sdk/typescript/tests/harness_cli.test.ts index 367308806..896c9a01b 100644 --- a/sdk/typescript/tests/harness_cli.test.ts +++ b/sdk/typescript/tests/harness_cli.test.ts @@ -20,10 +20,11 @@ class MockStream extends EventEmitter { } type MockChild = EventEmitter & - Pick; + Pick; const createProcess = (): MockChild => { const proc = new EventEmitter() as MockChild; + proc.stdin = { end: vi.fn() } as unknown as ChildProcessWithoutNullStreams['stdin']; proc.stdout = new MockStream() as ChildProcessWithoutNullStreams['stdout']; proc.stderr = new MockStream() as ChildProcessWithoutNullStreams['stderr']; proc.kill = vi.fn(); @@ -90,6 +91,23 @@ describe('harness cli utilities', () => { await expect(pending).resolves.toMatchObject({ exitCode: 0 }); }); + it('pipes an explicit prompt over stdin and closes the stream', async () => { + const proc = createProcess(); + spawnMock.mockReturnValueOnce(proc as unknown as ReturnType); + + const pending = runCli(['aforge', 'do', '--json'], { inputText: 'keep me off argv' }); + + expect(spawnMock).toHaveBeenCalledWith('aforge', ['do', '--json'], { + env: expect.any(Object), + cwd: undefined, + stdio: ['pipe', 'pipe', 'pipe'] + }); + expect(proc.stdin.end).toHaveBeenCalledWith('keep me off argv'); + + proc.emit('close', 0); + await expect(pending).resolves.toMatchObject({ exitCode: 0 }); + }); + it('rejects on child process errors and on timeouts', async () => { const errorProc = createProcess(); spawnMock.mockReturnValueOnce(errorProc as unknown as ReturnType); diff --git a/sdk/typescript/tests/harness_functional.test.ts b/sdk/typescript/tests/harness_functional.test.ts index 3f0993562..b77711dfb 100644 --- a/sdk/typescript/tests/harness_functional.test.ts +++ b/sdk/typescript/tests/harness_functional.test.ts @@ -4,11 +4,12 @@ * These tests invoke REAL coding agents and make real API calls. * They are NOT included in the default `vitest run` — run explicitly: * - * npx vitest run tests/harness_functional.test.ts --timeout=300000 + * npx vitest run --config vitest.functional.config.ts * * Run a single provider: - * npx vitest run tests/harness_functional.test.ts -t "Codex" --timeout=300000 - * npx vitest run tests/harness_functional.test.ts -t "OpenCode" --timeout=300000 + * npx vitest run --config vitest.functional.config.ts -t "Aforge" + * npx vitest run --config vitest.functional.config.ts -t "Codex" + * npx vitest run --config vitest.functional.config.ts -t "OpenCode" */ import { describe, it, expect, beforeEach, afterEach } from 'vitest'; @@ -33,6 +34,8 @@ function hasBinary(name: string): boolean { const HAS_CODEX = hasBinary('codex'); const HAS_OPENCODE = hasBinary('opencode'); +const HAS_AFORGE = hasBinary('aforge') + && Boolean(process.env.OPENROUTER_API_KEY || process.env.OPENAI_API_KEY); /** Plain JSON Schema — no Zod dependency needed. */ const simpleSchema = { @@ -77,6 +80,51 @@ function cleanupWorkDir(dir: string): void { } } +// ════════════════════════════════════════════════════════════════════════ +// AFORGE +// ════════════════════════════════════════════════════════════════════════ + +describe.skipIf(!HAS_AFORGE)('Aforge Functional', () => { + beforeEach(() => { + workDir = createWorkDir(); + }); + afterEach(() => { + cleanupWorkDir(workDir); + }); + + it('basic prompt returns the do JSON deliverable and benchmark metrics', async () => { + const { AforgeProvider } = await import('../src/harness/providers/aforge.js'); + const result = await new AforgeProvider().execute( + 'Reply with exactly: HELLO_AGENTFIELD', + { cwd: workDir }, + ); + + expect(result.isError).toBe(false); + expect(result.result).toContain('HELLO_AGENTFIELD'); + expect(result.messages).toHaveLength(1); + expect(result.metrics.numTurns).toBeGreaterThan(0); + expect(result.metrics.inputTokens).toBeGreaterThan(0); + expect(result.metrics.outputTokens).toBeGreaterThan(0); + expect(result.metrics.totalCostUsd).toBeGreaterThan(0); + }, 300_000); + + it('schema pipeline writes, parses, and cleans isolated output', async () => { + const result = await new HarnessRunner().run( + 'Return exactly: greeting="Hello from Aforge" and number=42. Follow the OUTPUT REQUIREMENTS below precisely.', + { + provider: 'aforge', + schema: simpleSchema, + cwd: workDir, + maxRetries: 1, + }, + ); + + expect(result.isError).toBe(false); + expect(result.parsed).toMatchObject({ greeting: 'Hello from Aforge', number: 42 }); + expect(fs.readdirSync(workDir).filter((name) => name.startsWith('.agentfield-out-'))).toEqual([]); + }, 300_000); +}); + // ════════════════════════════════════════════════════════════════════════ // CODEX // ════════════════════════════════════════════════════════════════════════ diff --git a/sdk/typescript/tests/harness_provider_aforge.test.ts b/sdk/typescript/tests/harness_provider_aforge.test.ts new file mode 100644 index 000000000..78161eccb --- /dev/null +++ b/sdk/typescript/tests/harness_provider_aforge.test.ts @@ -0,0 +1,171 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { AforgeProvider } from '../src/harness/providers/aforge.js'; +import { buildProvider, SUPPORTED_PROVIDERS } from '../src/harness/providers/factory.js'; +import * as cli from '../src/harness/cli.js'; + +function envelope( + deliverable = 'done', + options: { settled?: boolean; blockedOn?: string; usage?: Record } = {} +): string { + return JSON.stringify({ + settled: options.settled ?? true, + deliverable, + blocked_on: options.blockedOn ?? '', + spend_usd: 0.0123, + elapsed_ms: 12, + usage: options.usage ?? {}, + }); +} + +afterEach(() => { + vi.restoreAllMocks(); + delete process.env.AGENTFIELD_HARNESS_TIMEOUT_SECONDS; + delete process.env.AFORGE_BIN; +}); + +describe('aforge provider', () => { + it('honors AFORGE_BIN unless a binary is explicit', async () => { + process.env.AFORGE_BIN = '/opt/aforge-env'; + vi.spyOn(cli, 'runCli').mockResolvedValue({ stdout: envelope(), stderr: '', exitCode: 0 }); + + await new AforgeProvider().execute('hello', {}); + await new AforgeProvider('/explicit/aforge').execute('hello', {}); + + expect(vi.mocked(cli.runCli).mock.calls[0][0][0]).toBe('/opt/aforge-env'); + expect(vi.mocked(cli.runCli).mock.calls[1][0][0]).toBe('/explicit/aforge'); + }); + + it('maps the do command, stdin prompt, JSON envelope, and metrics', async () => { + vi.spyOn(cli, 'runCli').mockResolvedValue({ + stdout: envelope(' final answer ', { + usage: { + calls: 3, + prompt_tokens: 100, + completion_tokens: 50, + cached_tokens: 20, + cost: 0.0123, + }, + }), + stderr: '', + exitCode: 0, + }); + + const result = await new AforgeProvider('/opt/aforge').execute('prompt that stays off argv', { + projectDir: '/project', + cwd: '/project/nested', + systemPrompt: ' be precise ', + model: 'openrouter/z-ai/glm-5.2#high', + }); + + expect(cli.runCli).toHaveBeenCalledWith( + ['/opt/aforge', 'do', '--json', '--yes-spend', '-w', '/project', '--timeout', '1795'], + { + env: { AFORGE_MODEL: 'z-ai/glm-5.2', AFORGE_EXEC_REASONING: 'high' }, + cwd: undefined, + timeout: 1_800_000, + idleSeconds: 0, + inputText: 'be precise\n\nTask:\nprompt that stays off argv', + } + ); + expect(result.result).toBe('final answer'); + expect(result.isError).toBe(false); + expect(result.failureType).toBe('none'); + expect(result.returnCode).toBe(0); + expect(result.metrics).toMatchObject({ + numTurns: 3, + totalCostUsd: 0.0123, + inputTokens: 100, + outputTokens: 50, + cacheReadTokens: 20, + cacheCreationTokens: 0, + totalTokens: 150, + model: 'openrouter/z-ai/glm-5.2', + }); + expect(result.messages[0].deliverable).toBe(' final answer '); + }); + + it('uses cwd as the root and gives aforge a timeout landing window', async () => { + process.env.AGENTFIELD_HARNESS_TIMEOUT_SECONDS = '2400'; + vi.spyOn(cli, 'runCli').mockResolvedValue({ stdout: envelope(), stderr: '', exitCode: 0 }); + + await new AforgeProvider().execute('hello', { cwd: '/cwd-only' }); + + expect(cli.runCli).toHaveBeenCalledWith( + ['aforge', 'do', '--json', '--yes-spend', '-w', '/cwd-only', '--timeout', '2395'], + expect.objectContaining({ timeout: 2_400_000, idleSeconds: 0, inputText: 'hello' }) + ); + }); + + it('ignores unknown variants and lets caller env override derived env', async () => { + vi.spyOn(cli, 'runCli').mockResolvedValue({ stdout: envelope(), stderr: '', exitCode: 0 }); + const provider = new AforgeProvider(); + + await provider.execute('hello', { model: 'openrouter/x/y#turbo' }); + await provider.execute('hello', { + model: 'openrouter/x/y#low', + variant: 'HIGH', + env: { AFORGE_MODEL: 'override/model', AFORGE_EXEC_REASONING: 'off', EXTRA: '1' }, + }); + + expect(vi.mocked(cli.runCli).mock.calls[0][1]?.env).toEqual({ AFORGE_MODEL: 'x/y' }); + expect(vi.mocked(cli.runCli).mock.calls[1][1]?.env).toEqual({ + AFORGE_MODEL: 'override/model', + AFORGE_EXEC_REASONING: 'off', + EXTRA: '1', + }); + }); + + it.each([ + ['success', 0, 'done', '', 'none', false], + ['timeout with partial', 2, 'usable', '', 'timeout', true], + ['blocked', 1, '', 'Which repository?', 'crash', true], + ['zero without deliverable', 0, '', '', 'crash', true], + ['signal', -9, '', '', 'crash', true], + ])('applies %s exit semantics', async (_name, exitCode, deliverable, blockedOn, failureType, wantError) => { + vi.spyOn(cli, 'runCli').mockResolvedValue({ + stdout: envelope(deliverable as string, { settled: exitCode === 0, blockedOn: blockedOn as string }), + stderr: '', + exitCode: exitCode as number, + }); + + const result = await new AforgeProvider().execute('hello', {}); + + expect(result.isError).toBe(wantError); + expect(result.failureType).toBe(failureType); + }); + + it('parses the last envelope and leaves zero cost unknown', async () => { + vi.spyOn(cli, 'runCli').mockResolvedValue({ + stdout: `stray diagnostic\n{"type":"event"}\n${envelope('real result', { usage: { calls: 1, cost: 0 } })}`, + stderr: '', + exitCode: 0, + }); + + const result = await new AforgeProvider().execute('hello', {}); + + expect(result.result).toBe('real result'); + expect(result.metrics.totalCostUsd).toBeUndefined(); + }); + + it('classifies missing binaries and timeouts', async () => { + vi.spyOn(cli, 'runCli').mockRejectedValueOnce(new Error('spawn aforge ENOENT')); + const missing = await new AforgeProvider('aforge-missing').execute('hello', {}); + expect(missing.isError).toBe(true); + expect(missing.failureType).toBe('crash'); + expect(missing.errorMessage).toContain('aforge-missing'); + + vi.spyOn(cli, 'runCli').mockRejectedValueOnce(new Error('CLI timed out after 1ms')); + const timeout = await new AforgeProvider().execute('hello', {}); + expect(timeout.isError).toBe(true); + expect(timeout.failureType).toBe('timeout'); + }); +}); + +describe('aforge provider factory', () => { + it('registers aforge and threads aforgeBin', async () => { + expect(SUPPORTED_PROVIDERS.has('aforge')).toBe(true); + const provider = await buildProvider({ provider: 'aforge', aforgeBin: '/opt/aforge' }); + expect(provider).toBeInstanceOf(AforgeProvider); + }); +}); diff --git a/sdk/typescript/tests/harness_runner.test.ts b/sdk/typescript/tests/harness_runner.test.ts index 1fe393bc3..4631f5981 100644 --- a/sdk/typescript/tests/harness_runner.test.ts +++ b/sdk/typescript/tests/harness_runner.test.ts @@ -45,13 +45,19 @@ class MockProvider implements HarnessProvider { } class FileWritingProvider extends MockProvider { + public readonly outputPaths: string[] = []; + public constructor(private readonly payload: string, result?: RawResult) { super([result ?? createRawResult({ result: 'ok' })]); } public override async execute(prompt: string, options: Record): Promise { - const cwd = typeof options.cwd === 'string' ? options.cwd : '.'; - fs.writeFileSync(getOutputPath(cwd), this.payload, 'utf8'); + const match = prompt.match(/(\S*\.agentfield_output\.json)/); + if (!match) { + throw new Error('schema prompt did not contain an output path'); + } + this.outputPaths.push(match[1]); + fs.writeFileSync(match[1], this.payload, 'utf8'); return super.execute(prompt, options); } } @@ -68,6 +74,8 @@ describe('harness runner', () => { systemPrompt: 'base', env: { A: '1' }, cwd: '/tmp/base', + projectDir: '/tmp/project', + aforgeBin: 'aforge', codexBin: 'codex', geminiBin: 'gemini', opencodeBin: 'opencode', @@ -88,6 +96,8 @@ describe('harness runner', () => { expect(options.maxBudgetUsd).toBe(2); expect(options.env).toEqual({ B: '2' }); expect(options.cwd).toBe('/tmp/override'); + expect(options.projectDir).toBe('/tmp/project'); + expect(options.aforgeBin).toBe('aforge'); }); it('isTransient matches transient errors and rejects non-transient', () => { @@ -154,9 +164,44 @@ describe('harness runner', () => { const result = await runner.run('produce json', { provider: 'codex', schema, cwd }); expect(provider.lastPrompt).toContain('OUTPUT REQUIREMENTS'); - expect(provider.lastPrompt).toContain(getOutputPath(cwd)); + expect(provider.outputPaths).toHaveLength(1); + expect(provider.lastPrompt).toContain(provider.outputPaths[0]); + expect(provider.outputPaths[0]).toMatch(/\.agentfield-out-[^/]+\/\.agentfield_output\.json$/); expect(result.isError).toBe(false); expect(result.parsed).toEqual({ name: 'ok', count: 1 }); + expect(fs.existsSync(provider.outputPaths[0])).toBe(false); + }); + + it('isolates concurrent schema runs that share one cwd', async () => { + const cwd = makeTempDir(); + const schema = z.object({ name: z.string(), count: z.number() }); + const outputPaths: string[] = []; + const provider: HarnessProvider = { + async execute(prompt: string): Promise { + const match = prompt.match(/(\S*\.agentfield_output\.json)/); + if (!match) { + throw new Error('schema prompt did not contain an output path'); + } + outputPaths.push(match[1]); + const first = prompt.startsWith('first'); + await new Promise((resolve) => setTimeout(resolve, 10)); + fs.writeFileSync(match[1], JSON.stringify({ name: first ? 'first' : 'second', count: first ? 1 : 2 })); + return createRawResult({ result: 'done' }); + }, + }; + vi.spyOn(factory, 'buildProvider').mockResolvedValue(provider); + + const runner = new HarnessRunner(); + const [first, second] = await Promise.all([ + runner.run('first', { provider: 'aforge', schema, cwd }), + runner.run('second', { provider: 'aforge', schema, cwd }), + ]); + + expect(first.parsed).toEqual({ name: 'first', count: 1 }); + expect(second.parsed).toEqual({ name: 'second', count: 2 }); + expect(new Set(outputPaths).size).toBe(2); + expect(fs.readdirSync(cwd).filter((name) => name.startsWith('.agentfield-out-'))).toEqual([]); + expect(fs.existsSync(getOutputPath(cwd))).toBe(false); }); it('run throws when no provider is configured', async () => { diff --git a/sdk/typescript/vitest.functional.config.ts b/sdk/typescript/vitest.functional.config.ts new file mode 100644 index 000000000..32814c370 --- /dev/null +++ b/sdk/typescript/vitest.functional.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + include: ['tests/harness_functional.test.ts'], + testTimeout: 300_000, + hookTimeout: 30_000, + }, +}); From d8a99da3580b00806837407ee3967a1ef2a0c8fb Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Mon, 10 Aug 2026 13:03:04 -0400 Subject: [PATCH 02/11] feat(harness): preserve TypeScript failure metadata --- sdk/typescript/src/harness/runner.ts | 6 ++++++ sdk/typescript/src/harness/types.ts | 6 +++++- sdk/typescript/tests/harness_runner.test.ts | 4 ++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/sdk/typescript/src/harness/runner.ts b/sdk/typescript/src/harness/runner.ts index 42058a108..edcb3fd08 100644 --- a/sdk/typescript/src/harness/runner.ts +++ b/sdk/typescript/src/harness/runner.ts @@ -83,6 +83,8 @@ export class HarnessRunner { result: raw.result, isError: raw.isError, errorMessage: raw.errorMessage, + failureType: raw.failureType, + returnCode: raw.returnCode, costUsd: raw.metrics.totalCostUsd, numTurns: raw.metrics.numTurns, durationMs: Date.now() - startTime, @@ -192,6 +194,8 @@ export class HarnessRunner { result: raw.result, parsed, isError: false, + failureType: raw.failureType, + returnCode: raw.returnCode, costUsd: raw.metrics.totalCostUsd, numTurns: raw.metrics.numTurns, durationMs: Date.now() - startTime, @@ -205,6 +209,8 @@ export class HarnessRunner { result: raw.result, isError: true, errorMessage: 'Schema validation failed after parse and cosmetic repair attempts.', + failureType: 'schema', + returnCode: raw.returnCode, costUsd: raw.metrics.totalCostUsd, numTurns: raw.metrics.numTurns, durationMs: Date.now() - startTime, diff --git a/sdk/typescript/src/harness/types.ts b/sdk/typescript/src/harness/types.ts index a824fd29e..a04f98f35 100644 --- a/sdk/typescript/src/harness/types.ts +++ b/sdk/typescript/src/harness/types.ts @@ -68,13 +68,15 @@ export interface Metrics { model?: string; } +export type FailureType = 'none' | 'crash' | 'timeout' | 'api_error' | 'no_output' | 'schema'; + export interface RawResult { result?: string; messages: Array>; metrics: Metrics; isError: boolean; errorMessage?: string; - failureType?: 'none' | 'crash' | 'timeout' | 'api_error' | 'no_output' | 'schema'; + failureType?: FailureType; returnCode?: number; } @@ -83,6 +85,8 @@ export interface HarnessResult { parsed?: unknown; isError: boolean; errorMessage?: string; + failureType?: FailureType; + returnCode?: number; costUsd?: number; numTurns: number; durationMs: number; diff --git a/sdk/typescript/tests/harness_runner.test.ts b/sdk/typescript/tests/harness_runner.test.ts index 4631f5981..cf3a903d3 100644 --- a/sdk/typescript/tests/harness_runner.test.ts +++ b/sdk/typescript/tests/harness_runner.test.ts @@ -138,6 +138,8 @@ describe('harness runner', () => { const provider = new MockProvider([ createRawResult({ result: 'done', + failureType: 'none', + returnCode: 0, metrics: createMetrics({ numTurns: 2, totalCostUsd: 0.42, sessionId: 'sess-1' }), }), ]); @@ -152,6 +154,8 @@ describe('harness runner', () => { expect(result.costUsd).toBe(0.42); expect(result.numTurns).toBe(2); expect(result.sessionId).toBe('sess-1'); + expect(result.failureType).toBe('none'); + expect(result.returnCode).toBe(0); }); it('run with schema injects suffix and parses output', async () => { From 2235a19497ddab210aeb4762410179058fae6c1d Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Mon, 10 Aug 2026 13:20:27 -0400 Subject: [PATCH 03/11] fix(harness): parse live aforge JSON envelopes --- sdk/go/harness/aforge.go | 12 ++- sdk/go/harness/aforge_integration_test.go | 11 +++ sdk/go/harness/aforge_test.go | 17 ++++ .../agentfield/harness/providers/aforge.py | 11 +++ sdk/python/tests/test_harness_functional.py | 81 ++++++++++++++++++- .../tests/test_harness_provider_aforge.py | 18 +++++ .../src/harness/providers/aforge.ts | 12 +++ .../tests/harness_functional.test.ts | 9 +++ .../tests/harness_provider_aforge.test.ts | 10 +++ 9 files changed, 176 insertions(+), 5 deletions(-) diff --git a/sdk/go/harness/aforge.go b/sdk/go/harness/aforge.go index f6d5482d5..0f5d4eabb 100644 --- a/sdk/go/harness/aforge.go +++ b/sdk/go/harness/aforge.go @@ -94,13 +94,23 @@ func aforgeTaskInput(prompt, systemPrompt string) string { } func parseAforgeEnvelope(stdout string) map[string]any { + // Canonical `aforge do --json` output is one object, currently formatted + // across multiple lines. Parse that shape before falling back to the + // line-oriented form tolerated for wrappers that prepend diagnostics. + var envelope map[string]any + if err := json.Unmarshal([]byte(strings.TrimSpace(stdout)), &envelope); err == nil { + if _, ok := envelope["deliverable"]; ok { + return envelope + } + } + lines := strings.Split(stdout, "\n") for i := len(lines) - 1; i >= 0; i-- { line := strings.TrimSpace(lines[i]) if line == "" { continue } - var envelope map[string]any + envelope = nil if err := json.Unmarshal([]byte(line), &envelope); err != nil { continue } diff --git a/sdk/go/harness/aforge_integration_test.go b/sdk/go/harness/aforge_integration_test.go index 59e371a2a..b32d04afa 100644 --- a/sdk/go/harness/aforge_integration_test.go +++ b/sdk/go/harness/aforge_integration_test.go @@ -38,6 +38,11 @@ func TestAforgeProvider_Integration(t *testing.T) { assert.Greater(t, raw.Metrics.OutputTokens, 0) require.NotNil(t, raw.Metrics.CostUSD) assert.Greater(t, *raw.Metrics.CostUSD, 0.0) + t.Logf( + "aforge metrics: duration_ms=%d calls=%d input_tokens=%d output_tokens=%d cache_read_tokens=%d cost_usd=%.10f", + raw.Metrics.DurationAPIMS, raw.Metrics.NumTurns, raw.Metrics.InputTokens, + raw.Metrics.OutputTokens, raw.Metrics.CacheReadTokens, *raw.Metrics.CostUSD, + ) } func TestAforgeRunner_Integration_Schema(t *testing.T) { @@ -71,6 +76,12 @@ func TestAforgeRunner_Integration_Schema(t *testing.T) { assert.False(t, result.IsError, result.ErrorMessage) assert.Equal(t, "Hello from Aforge", parsed["greeting"]) assert.EqualValues(t, 42, parsed["number"]) + require.NotNil(t, result.CostUSD) + t.Logf( + "aforge schema metrics: duration_ms=%d calls=%d input_tokens=%d output_tokens=%d cache_read_tokens=%d cost_usd=%.10f", + result.DurationMS, result.NumTurns, result.InputTokens, result.OutputTokens, + result.CacheReadTokens, *result.CostUSD, + ) matches, err := filepath.Glob(filepath.Join(workDir, ".agentfield-out-*")) require.NoError(t, err) assert.Empty(t, matches) diff --git a/sdk/go/harness/aforge_test.go b/sdk/go/harness/aforge_test.go index df8125576..37c773d8e 100644 --- a/sdk/go/harness/aforge_test.go +++ b/sdk/go/harness/aforge_test.go @@ -2,6 +2,7 @@ package harness import ( "context" + "encoding/json" "fmt" "os" "path/filepath" @@ -193,6 +194,22 @@ func TestAforgeProviderParsesLastEnvelopeAndLeavesZeroCostUnknown(t *testing.T) assert.Nil(t, raw.Metrics.CostUSD, "zero provider cost remains unknown") } +func TestAforgeProviderParsesPrettyPrintedEnvelope(t *testing.T) { + p := NewAforgeProvider("aforge") + p.runCLI = func(context.Context, []string, map[string]string, string, int, int, []byte) (*CLIResult, error) { + var value map[string]any + require.NoError(t, json.Unmarshal([]byte(aforgeEnvelope("pretty result", true, "", "")), &value)) + pretty, err := json.MarshalIndent(value, "", " ") + require.NoError(t, err) + return &CLIResult{Stdout: string(pretty), ReturnCode: 0}, nil + } + + raw, err := p.Execute(context.Background(), "hello", Options{}) + require.NoError(t, err) + assert.Equal(t, "pretty result", raw.Result) + assert.False(t, raw.IsError) +} + func TestAforgeProviderMissingBinaryAndTimeout(t *testing.T) { t.Run("missing binary", func(t *testing.T) { p := NewAforgeProvider("aforge-missing") diff --git a/sdk/python/agentfield/harness/providers/aforge.py b/sdk/python/agentfield/harness/providers/aforge.py index a77b05415..46b2bc8f6 100644 --- a/sdk/python/agentfield/harness/providers/aforge.py +++ b/sdk/python/agentfield/harness/providers/aforge.py @@ -30,6 +30,17 @@ def _strip_openrouter_prefix(model: str) -> str: def _parse_envelope(stdout: str) -> dict[str, object] | None: """Return the last JSON object containing an aforge deliverable.""" + # Canonical `aforge do --json` output is one object, currently formatted + # across multiple lines. Parse that shape before falling back to the + # line-oriented form tolerated for wrappers that prepend diagnostics. + try: + value = json.loads(stdout.strip()) + except ValueError: + pass + else: + if isinstance(value, dict) and "deliverable" in value: + return value + for line in reversed( [line.strip() for line in stdout.splitlines() if line.strip()] ): diff --git a/sdk/python/tests/test_harness_functional.py b/sdk/python/tests/test_harness_functional.py index 000ada1c8..43c1c994b 100644 --- a/sdk/python/tests/test_harness_functional.py +++ b/sdk/python/tests/test_harness_functional.py @@ -5,7 +5,7 @@ Prerequisites ~~~~~~~~~~~~~ -- Coding agent CLIs installed (claude, codex, opencode) +- Coding agent CLIs installed (aforge, claude, codex, opencode) - Valid API keys / auth configured for each provider - Internet access for API calls @@ -18,6 +18,7 @@ pytest tests/test_harness_functional.py -m harness_live -v -k codex --timeout=300 pytest tests/test_harness_functional.py -m harness_live -v -k claude --timeout=300 pytest tests/test_harness_functional.py -m harness_live -v -k opencode --timeout=300 + pytest tests/test_harness_functional.py -m harness_live -v -k aforge --timeout=300 """ from __future__ import annotations @@ -26,13 +27,12 @@ import shutil import subprocess import tempfile -from typing import List import pytest from pydantic import BaseModel -from agentfield.harness._runner import HarnessRunner from agentfield.harness._result import HarnessResult # noqa: F401 +from agentfield.harness._runner import HarnessRunner from agentfield.types import HarnessConfig # ──────────────────────────────────────────────────────────────────────── @@ -52,7 +52,7 @@ class CodeReviewResponse(BaseModel): summary: str score: int - suggestions: List[str] + suggestions: list[str] # ──────────────────────────────────────────────────────────────────────── @@ -61,6 +61,7 @@ class CodeReviewResponse(BaseModel): HAS_CODEX = shutil.which("codex") is not None HAS_OPENCODE = shutil.which("opencode") is not None +HAS_AFORGE = shutil.which("aforge") is not None try: import claude_agent_sdk # noqa: F401 @@ -106,6 +107,78 @@ def work_dir(): pytestmark = [pytest.mark.harness_live, pytest.mark.asyncio] +# ════════════════════════════════════════════════════════════════════════ +# AFORGE +# ════════════════════════════════════════════════════════════════════════ + + +@pytest.mark.skipif(not HAS_AFORGE, reason="aforge CLI not installed") +class TestAforgeLive: + """Live tests against Aforge's canonical ``aforge do --json`` mode.""" + + async def test_basic_prompt(self, work_dir: str) -> None: + """Provider returns text plus measured benchmark usage.""" + from agentfield.harness.providers.aforge import AforgeProvider + + result = await AforgeProvider().execute( + "Reply with exactly: HELLO_AGENTFIELD", + {"cwd": work_dir}, + ) + + assert not result.is_error, f"Aforge returned error: {result.error_message}" + assert result.result is not None + assert "HELLO_AGENTFIELD" in result.result + assert result.metrics.num_turns > 0 + assert result.metrics.input_tokens > 0 + assert result.metrics.output_tokens > 0 + assert result.metrics.total_cost_usd is not None + assert result.metrics.total_cost_usd > 0 + print( + "aforge metrics", + { + "duration_ms": result.metrics.duration_api_ms, + "calls": result.metrics.num_turns, + "input_tokens": result.metrics.input_tokens, + "output_tokens": result.metrics.output_tokens, + "cache_read_tokens": result.metrics.cache_read_tokens, + "cost_usd": result.metrics.total_cost_usd, + }, + ) + + async def test_schema_pipeline(self, work_dir: str) -> None: + """Full schema pipeline writes, parses, and cleans isolated output.""" + result = await HarnessRunner().run( + 'Return exactly: greeting="Hello from Aforge" and number=42. ' + "Follow the OUTPUT REQUIREMENTS below precisely.", + provider="aforge", + schema=SimpleResponse, + cwd=work_dir, + max_retries=1, + ) + + assert not result.is_error, ( + f"Schema pipeline failed: {result.error_message}\n" + f"Raw result: {result.result!r}" + ) + assert isinstance(result.parsed, SimpleResponse) + assert result.parsed.greeting == "Hello from Aforge" + assert result.parsed.number == 42 + print( + "aforge schema metrics", + { + "duration_ms": result.duration_ms, + "calls": result.num_turns, + "input_tokens": result.input_tokens, + "output_tokens": result.output_tokens, + "cache_read_tokens": result.cache_read_tokens, + "cost_usd": result.cost_usd, + }, + ) + assert not any( + name.startswith(".agentfield-out-") for name in os.listdir(work_dir) + ) + + # ════════════════════════════════════════════════════════════════════════ # CODEX # ════════════════════════════════════════════════════════════════════════ diff --git a/sdk/python/tests/test_harness_provider_aforge.py b/sdk/python/tests/test_harness_provider_aforge.py index a5f3d14be..ba5cade16 100644 --- a/sdk/python/tests/test_harness_provider_aforge.py +++ b/sdk/python/tests/test_harness_provider_aforge.py @@ -405,6 +405,24 @@ async def fake_run_cli( assert raw.is_error is False +@pytest.mark.asyncio +async def test_aforge_parses_pretty_printed_envelope( + monkeypatch: pytest.MonkeyPatch, +): + async def fake_run_cli( + cmd, *, env=None, cwd=None, timeout=None, idle_seconds=None, input_text=None + ): + _ = cmd, env, cwd, timeout, idle_seconds, input_text + return json.dumps(json.loads(_envelope("pretty result")), indent=2), "", 0 + + monkeypatch.setattr("agentfield.harness.providers.aforge.run_cli", fake_run_cli) + + raw = await AforgeProvider().execute("hello", {}) + + assert raw.result == "pretty result" + assert raw.is_error is False + + @pytest.mark.asyncio async def test_aforge_missing_or_zero_cost_falls_back_to_estimate( monkeypatch: pytest.MonkeyPatch, diff --git a/sdk/typescript/src/harness/providers/aforge.ts b/sdk/typescript/src/harness/providers/aforge.ts index c2d32575d..6cb1f4b35 100644 --- a/sdk/typescript/src/harness/providers/aforge.ts +++ b/sdk/typescript/src/harness/providers/aforge.ts @@ -56,6 +56,18 @@ function stripOpenRouterPrefix(model: string): string { } function parseEnvelope(stdout: string): Record | undefined { + // Canonical `aforge do --json` output is one object, currently formatted + // across multiple lines. Parse that shape before falling back to the + // line-oriented form tolerated for wrappers that prepend diagnostics. + try { + const value: unknown = JSON.parse(stdout.trim()); + if (typeof value === 'object' && value !== null && !Array.isArray(value) && 'deliverable' in value) { + return value as Record; + } + } catch { + // Fall through to the wrapper-compatible line scan. + } + const lines = stdout.split('\n').map((line) => line.trim()).filter(Boolean); for (let index = lines.length - 1; index >= 0; index -= 1) { try { diff --git a/sdk/typescript/tests/harness_functional.test.ts b/sdk/typescript/tests/harness_functional.test.ts index b77711dfb..706830c15 100644 --- a/sdk/typescript/tests/harness_functional.test.ts +++ b/sdk/typescript/tests/harness_functional.test.ts @@ -106,6 +106,7 @@ describe.skipIf(!HAS_AFORGE)('Aforge Functional', () => { expect(result.metrics.inputTokens).toBeGreaterThan(0); expect(result.metrics.outputTokens).toBeGreaterThan(0); expect(result.metrics.totalCostUsd).toBeGreaterThan(0); + console.info('aforge metrics', JSON.stringify(result.metrics)); }, 300_000); it('schema pipeline writes, parses, and cleans isolated output', async () => { @@ -121,6 +122,14 @@ describe.skipIf(!HAS_AFORGE)('Aforge Functional', () => { expect(result.isError).toBe(false); expect(result.parsed).toMatchObject({ greeting: 'Hello from Aforge', number: 42 }); + console.info('aforge schema metrics', JSON.stringify({ + durationMs: result.durationMs, + calls: result.numTurns, + inputTokens: result.inputTokens, + outputTokens: result.outputTokens, + cacheReadTokens: result.cacheReadTokens, + costUsd: result.costUsd, + })); expect(fs.readdirSync(workDir).filter((name) => name.startsWith('.agentfield-out-'))).toEqual([]); }, 300_000); }); diff --git a/sdk/typescript/tests/harness_provider_aforge.test.ts b/sdk/typescript/tests/harness_provider_aforge.test.ts index 78161eccb..091f131f6 100644 --- a/sdk/typescript/tests/harness_provider_aforge.test.ts +++ b/sdk/typescript/tests/harness_provider_aforge.test.ts @@ -148,6 +148,16 @@ describe('aforge provider', () => { expect(result.metrics.totalCostUsd).toBeUndefined(); }); + it('parses the canonical pretty-printed envelope', async () => { + const pretty = JSON.stringify(JSON.parse(envelope('pretty result')), null, 2); + vi.spyOn(cli, 'runCli').mockResolvedValue({ stdout: pretty, stderr: '', exitCode: 0 }); + + const result = await new AforgeProvider().execute('hello', {}); + + expect(result.result).toBe('pretty result'); + expect(result.isError).toBe(false); + }); + it('classifies missing binaries and timeouts', async () => { vi.spyOn(cli, 'runCli').mockRejectedValueOnce(new Error('spawn aforge ENOENT')); const missing = await new AforgeProvider('aforge-missing').execute('hello', {}); From b8a04393d160e34d7a270b5b804065bb4bcfc6ca Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Fri, 14 Aug 2026 20:32:06 -0400 Subject: [PATCH 04/11] feat(harness): support aforge exec across SDKs --- sdk/go/harness/aforge.go | 96 ++++++++++++--- sdk/go/harness/aforge_test.go | 60 ++++++++++ .../agentfield/harness/providers/aforge.py | 112 +++++++++++++----- .../tests/test_harness_provider_aforge.py | 104 ++++++++++++++++ .../src/harness/providers/aforge.ts | 97 +++++++++++---- .../tests/harness_provider_aforge.test.ts | 80 +++++++++++++ 6 files changed, 481 insertions(+), 68 deletions(-) diff --git a/sdk/go/harness/aforge.go b/sdk/go/harness/aforge.go index 0f5d4eabb..786dbe9e6 100644 --- a/sdk/go/harness/aforge.go +++ b/sdk/go/harness/aforge.go @@ -93,13 +93,25 @@ func aforgeTaskInput(prompt, systemPrompt string) string { return prompt } +func aforgeCommand() (string, error) { + command := strings.ToLower(strings.TrimSpace(os.Getenv("AGENTFIELD_AFORGE_COMMAND"))) + if command == "" { + return "do", nil + } + if command != "do" && command != "exec" { + return "", fmt.Errorf("AGENTFIELD_AFORGE_COMMAND must be do or exec, got %q", command) + } + return command, nil +} + func parseAforgeEnvelope(stdout string) map[string]any { - // Canonical `aforge do --json` output is one object, currently formatted - // across multiple lines. Parse that shape before falling back to the - // line-oriented form tolerated for wrappers that prepend diagnostics. + // Both canonical `do` and `exec` print one JSON object. Parse that shape + // before the wrapper-compatible line scan. var envelope map[string]any if err := json.Unmarshal([]byte(strings.TrimSpace(stdout)), &envelope); err == nil { - if _, ok := envelope["deliverable"]; ok { + _, hasDeliverable := envelope["deliverable"] + _, hasText := envelope["text"] + if hasDeliverable || hasText { return envelope } } @@ -114,7 +126,9 @@ func parseAforgeEnvelope(stdout string) map[string]any { if err := json.Unmarshal([]byte(line), &envelope); err != nil { continue } - if _, ok := envelope["deliverable"]; ok { + _, hasDeliverable := envelope["deliverable"] + _, hasText := envelope["text"] + if hasDeliverable || hasText { return envelope } } @@ -174,15 +188,43 @@ func (p *AforgeProvider) Execute(ctx context.Context, prompt string, options Opt root = "." } outerTimeout := aforgeTimeout(options) - cmd := []string{ - p.BinPath, "do", "--json", "--yes-spend", "-w", root, - "--timeout", strconv.Itoa(aforgeInnerTimeout(outerTimeout)), + command, commandErr := aforgeCommand() + if commandErr != nil { + return &RawResult{ + IsError: true, ErrorMessage: commandErr.Error(), FailureType: FailureCrash, + Metrics: Metrics{}, + }, nil + } + var cmd []string + input := []byte(prompt) + if command == "exec" { + cmd = []string{ + p.BinPath, "exec", "--json", "-w", root, + "--timeout", strconv.Itoa(aforgeInnerTimeout(outerTimeout)), + "--context-fill", "60", "--completion-reserve", "65536", + } + if systemPrompt := strings.TrimSpace(options.SystemPrompt); systemPrompt != "" { + cmd = append(cmd, "--system", systemPrompt) + } + } else { + cmd = []string{ + p.BinPath, "do", "--json", "--yes-spend", "-w", root, + "--timeout", strconv.Itoa(aforgeInnerTimeout(outerTimeout)), + } + input = []byte(aforgeTaskInput(prompt, options.SystemPrompt)) } model, variant := options.resolveModelAndVariant() env := make(map[string]string) + if command == "exec" { + env["AFORGE_MODELS"] = "" + } if model != "" { - env["AFORGE_MODEL"] = stripOpenRouterPrefix(model) + modelSlug := stripOpenRouterPrefix(model) + env["AFORGE_MODEL"] = modelSlug + if command == "exec" { + cmd = append(cmd, "--model", modelSlug, "--plan-model", modelSlug) + } } if normalized, ok := supportedAforgeVariant(variant); ok { env["AFORGE_EXEC_REASONING"] = normalized @@ -193,7 +235,7 @@ func (p *AforgeProvider) Execute(ctx context.Context, prompt string, options Opt } started := time.Now() - cliResult, err := p.runCLI(ctx, cmd, env, "", outerTimeout, 0, []byte(aforgeTaskInput(prompt, options.SystemPrompt))) + cliResult, err := p.runCLI(ctx, cmd, env, "", outerTimeout, 0, input) apiMS := int(time.Since(started).Milliseconds()) if err != nil { if isExecNotFound(err) { @@ -218,9 +260,14 @@ func (p *AforgeProvider) Execute(ctx context.Context, prompt string, options Opt envelope := parseAforgeEnvelope(cliResult.Stdout) resultText := "" blockedOn := "" + stop := "" usage := map[string]any{} if envelope != nil { - if text, ok := envelope["deliverable"].(string); ok { + textKey := "deliverable" + if command == "exec" { + textKey = "text" + } + if text, ok := envelope[textKey].(string); ok { resultText = strings.TrimSpace(text) } if value, ok := envelope["blocked_on"].(string); ok { @@ -229,13 +276,27 @@ func (p *AforgeProvider) Execute(ctx context.Context, prompt string, options Opt if value, ok := envelope["usage"].(map[string]any); ok { usage = value } + if command == "exec" { + if value, ok := envelope["stop"].(string); ok { + stop = strings.TrimSpace(value) + } + } } isError := cliResult.ReturnCode != 0 || resultText == "" || blockedOn != "" + if command == "exec" { + isError = cliResult.ReturnCode < 0 || resultText == "" || + (cliResult.ReturnCode != 0 && cliResult.ReturnCode != 2 && cliResult.ReturnCode != 3) + } metrics := Metrics{DurationAPIMS: apiMS} if value, ok := aforgeNumber(usage["calls"]); ok { metrics.NumTurns = int(value) } + if command == "exec" { + if value, ok := aforgeNumber(envelope["turns"]); ok { + metrics.NumTurns = int(value) + } + } if value, ok := aforgeNumber(usage["prompt_tokens"]); ok { metrics.InputTokens = int(value) } @@ -245,8 +306,13 @@ func (p *AforgeProvider) Execute(ctx context.Context, prompt string, options Opt if value, ok := aforgeNumber(usage["cached_tokens"]); ok { metrics.CacheReadTokens = int(value) } - if value, ok := aforgeNumber(usage["cost"]); ok && value > 0 { - cost := value + spend, hasSpend := aforgeNumber(envelope["spend"]) + legacyCost, hasLegacyCost := aforgeNumber(usage["cost"]) + if hasSpend && spend > 0 { + cost := spend + metrics.CostUSD = &cost + } else if hasLegacyCost && legacyCost > 0 { + cost := legacyCost metrics.CostUSD = &cost } @@ -263,12 +329,12 @@ func (p *AforgeProvider) Execute(ctx context.Context, prompt string, options Opt ReturnCode: cliResult.ReturnCode, } if isError { - if cliResult.ReturnCode == 2 { + if (command == "do" && cliResult.ReturnCode == 2) || (command == "exec" && cliResult.ReturnCode == 4) { raw.FailureType = FailureTimeout } else { raw.FailureType = FailureCrash } - raw.ErrorMessage = aforgeCrashMessage(cliResult.ReturnCode, blockedOn, resultText, cliResult.Stderr) + raw.ErrorMessage = aforgeCrashMessage(cliResult.ReturnCode, firstNonEmpty(blockedOn, stop), resultText, cliResult.Stderr) } return raw, nil } diff --git a/sdk/go/harness/aforge_test.go b/sdk/go/harness/aforge_test.go index 37c773d8e..d09b9f367 100644 --- a/sdk/go/harness/aforge_test.go +++ b/sdk/go/harness/aforge_test.go @@ -23,6 +23,14 @@ func aforgeEnvelope(deliverable string, settled bool, blockedOn, usage string) s settled, deliverable, blockedOn, usage) } +func aforgeExecEnvelope(text, stop, usage string, turns int) string { + if usage == "" { + usage = `{"calls":1,"prompt_tokens":0,"completion_tokens":0,"cached_tokens":0,"cost":0}` + } + return fmt.Sprintf(`{"text":%q,"stop":%q,"usage":%s,"artifacts":[],"turns":%d,"elapsed_ms":12}`, + text, stop, usage, turns) +} + func TestAforgeProviderMapsDoCommandEnvelopeAndMetrics(t *testing.T) { var capturedCmd []string var capturedEnv map[string]string @@ -81,6 +89,58 @@ func TestAforgeProviderMapsDoCommandEnvelopeAndMetrics(t *testing.T) { assert.Equal(t, " final answer ", raw.Messages[0]["deliverable"]) } +func TestAforgeProviderMapsExecCommandEnvelopeAndMetrics(t *testing.T) { + t.Setenv("AGENTFIELD_AFORGE_COMMAND", "exec") + var capturedCmd []string + var capturedEnv map[string]string + var capturedStdin []byte + p := NewAforgeProvider("/opt/aforge") + p.runCLI = func(_ context.Context, cmd []string, env map[string]string, _ string, _, _ int, stdin []byte) (*CLIResult, error) { + capturedCmd = append([]string(nil), cmd...) + capturedEnv = env + capturedStdin = append([]byte(nil), stdin...) + return &CLIResult{Stdout: aforgeExecEnvelope(" linear answer ", "done", + `{"calls":3,"prompt_tokens":100,"completion_tokens":50,"cached_tokens":20,"cost":0.0123}`, 4)}, nil + } + + raw, err := p.Execute(context.Background(), "prompt that stays off argv", Options{ + ProjectDir: "/project", + SystemPrompt: " be precise ", + Model: "openrouter/deepseek/deepseek-v4-flash-0731", + }) + require.NoError(t, err) + assert.Equal(t, []string{ + "/opt/aforge", "exec", "--json", "-w", "/project", + "--timeout", "1795", "--context-fill", "60", "--completion-reserve", "65536", + "--system", "be precise", + "--model", "deepseek/deepseek-v4-flash-0731", + "--plan-model", "deepseek/deepseek-v4-flash-0731", + }, capturedCmd) + assert.Equal(t, "", capturedEnv["AFORGE_MODELS"]) + assert.Equal(t, "deepseek/deepseek-v4-flash-0731", capturedEnv["AFORGE_MODEL"]) + assert.Equal(t, "prompt that stays off argv", string(capturedStdin)) + assert.Equal(t, "linear answer", raw.Result) + assert.False(t, raw.IsError) + assert.Equal(t, 4, raw.Metrics.NumTurns) + assert.Equal(t, 100, raw.Metrics.InputTokens) + require.NotNil(t, raw.Metrics.CostUSD) + assert.InDelta(t, 0.0123, *raw.Metrics.CostUSD, 1e-9) +} + +func TestAforgeProviderExecBudgetPartialIsUsable(t *testing.T) { + t.Setenv("AGENTFIELD_AFORGE_COMMAND", "exec") + p := NewAforgeProvider("aforge") + p.runCLI = func(context.Context, []string, map[string]string, string, int, int, []byte) (*CLIResult, error) { + return &CLIResult{Stdout: aforgeExecEnvelope("usable", "budget", "", 2), ReturnCode: 2}, nil + } + + raw, err := p.Execute(context.Background(), "hello", Options{}) + require.NoError(t, err) + assert.False(t, raw.IsError) + assert.Equal(t, FailureNone, raw.FailureType) + assert.Equal(t, "usable", raw.Result) +} + func TestAforgeProviderBinaryEnvironmentOverride(t *testing.T) { t.Setenv("AFORGE_BIN", "/opt/aforge-env") assert.Equal(t, "/opt/aforge-env", NewAforgeProvider("").BinPath) diff --git a/sdk/python/agentfield/harness/providers/aforge.py b/sdk/python/agentfield/harness/providers/aforge.py index 46b2bc8f6..be969eec0 100644 --- a/sdk/python/agentfield/harness/providers/aforge.py +++ b/sdk/python/agentfield/harness/providers/aforge.py @@ -29,16 +29,15 @@ def _strip_openrouter_prefix(model: str) -> str: def _parse_envelope(stdout: str) -> dict[str, object] | None: - """Return the last JSON object containing an aforge deliverable.""" - # Canonical `aforge do --json` output is one object, currently formatted - # across multiple lines. Parse that shape before falling back to the - # line-oriented form tolerated for wrappers that prepend diagnostics. + """Return the last canonical ``do`` or ``exec`` envelope.""" + # Both surfaces print one JSON object. Parse that shape before falling back + # to the line-oriented form tolerated for wrappers that prepend diagnostics. try: value = json.loads(stdout.strip()) except ValueError: pass else: - if isinstance(value, dict) and "deliverable" in value: + if isinstance(value, dict) and ("deliverable" in value or "text" in value): return value for line in reversed( @@ -48,7 +47,7 @@ def _parse_envelope(stdout: str) -> dict[str, object] | None: value = json.loads(line) except ValueError: continue - if isinstance(value, dict) and "deliverable" in value: + if isinstance(value, dict) and ("deliverable" in value or "text" in value): return value return None @@ -89,7 +88,11 @@ def _crash_message( class AforgeProvider: - """Aforge CLI provider. Invokes canonical ``aforge do --json`` mode.""" + """Aforge CLI provider. + + ``do`` remains the default. Set ``AGENTFIELD_AFORGE_COMMAND=exec`` to use + Aforge's direct one-shot harness entry point. + """ _MAX_CONCURRENT: ClassVar[int] = int(os.environ.get("AFORGE_MAX_CONCURRENT", "8")) _concurrency_sem: ClassVar[asyncio.Semaphore | None] = None @@ -123,25 +126,58 @@ async def _execute_impl(self, prompt: str, options: dict[str, object]) -> RawRes timeout_seconds = int( os.environ.get("AGENTFIELD_HARNESS_TIMEOUT_SECONDS", "1800") ) + model_value, variant_value = resolve_model_and_variant(options) # Leave a small landing window so aforge can emit its honest timeout # envelope before the outer subprocess watchdog has to kill it. aforge_timeout = max(1, timeout_seconds - 5) - cmd = [ - self._bin, - "do", - "--json", - "--yes-spend", - "-w", - root, - "--timeout", - str(aforge_timeout), - ] - input_text = _task_input(prompt, options.get("system_prompt")) - - model_value, variant_value = resolve_model_and_variant(options) - env: dict[str, str] = {} + command = os.environ.get("AGENTFIELD_AFORGE_COMMAND", "do").strip().lower() + if command not in {"do", "exec"}: + return RawResult( + is_error=True, + error_message=( + "AGENTFIELD_AFORGE_COMMAND must be 'do' or 'exec', " + f"got {command!r}" + ), + failure_type=FailureType.CRASH, + metrics=Metrics(), + ) + if command == "exec": + cmd = [ + self._bin, + "exec", + "--json", + "-w", + root, + "--timeout", + str(aforge_timeout), + "--context-fill", + "60", + "--completion-reserve", + "65536", + ] + system_prompt = options.get("system_prompt") + if isinstance(system_prompt, str) and system_prompt.strip(): + cmd.extend(["--system", system_prompt.strip()]) + input_text = prompt + else: + cmd = [ + self._bin, + "do", + "--json", + "--yes-spend", + "-w", + root, + "--timeout", + str(aforge_timeout), + ] + input_text = _task_input(prompt, options.get("system_prompt")) + + env: dict[str, str] = {"AFORGE_MODELS": ""} if command == "exec" else {} if model_value: - env["AFORGE_MODEL"] = _strip_openrouter_prefix(model_value) + model_slug = _strip_openrouter_prefix(model_value) + env["AFORGE_MODEL"] = model_slug + if command == "exec": + cmd.extend(["--model", model_slug, "--plan-model", model_slug]) if variant_value: normalized_variant = variant_value.strip().lower() @@ -190,8 +226,9 @@ async def _execute_impl(self, prompt: str, options: dict[str, object]) -> RawRes usage: dict[object, object] = {} calls = 0 blocked_on = "" + stop = "" if envelope is not None: - text_value = envelope.get("deliverable") + text_value = envelope.get("text" if command == "exec" else "deliverable") if isinstance(text_value, str) and text_value.strip(): result_text = text_value.strip() blocked_value = envelope.get("blocked_on") @@ -203,6 +240,13 @@ async def _execute_impl(self, prompt: str, options: dict[str, object]) -> RawRes calls_value = _numeric(usage.get("calls")) if calls_value is not None: calls = int(calls_value) + if command == "exec": + stop_value = envelope.get("stop") + if isinstance(stop_value, str): + stop = stop_value.strip() + turns_value = _numeric(envelope.get("turns")) + if turns_value is not None: + calls = int(turns_value) clean_stderr = strip_ansi(stderr.strip()) if stderr else "" logger.info( @@ -214,15 +258,26 @@ async def _execute_impl(self, prompt: str, options: dict[str, object]) -> RawRes if not result_text and clean_stderr: logger.warning("aforge no text. stderr: %s", clean_stderr[:800]) - is_error = returncode != 0 or result_text is None or bool(blocked_on) + if command == "exec": + # Budget and turn-cap exits with a usable landing are partial + # successes under the original exec adapter contract. + is_error = ( + returncode < 0 + or result_text is None + or returncode not in {0, 2, 3} + ) + else: + is_error = returncode != 0 or result_text is None or bool(blocked_on) if not is_error: failure_type = FailureType.NONE - elif returncode == 2: + elif (command == "do" and returncode == 2) or ( + command == "exec" and returncode == 4 + ): failure_type = FailureType.TIMEOUT else: failure_type = FailureType.CRASH error_message = ( - _crash_message(returncode, blocked_on, result_text, stderr) + _crash_message(returncode, blocked_on or stop, result_text, stderr) if is_error else None ) @@ -230,8 +285,11 @@ async def _execute_impl(self, prompt: str, options: dict[str, object]) -> RawRes input_tokens_value = _numeric(usage.get("prompt_tokens")) output_tokens_value = _numeric(usage.get("completion_tokens")) cached_tokens_value = _numeric(usage.get("cached_tokens")) + spend_value = _numeric(envelope.get("spend")) if envelope else None cost_value = _numeric(usage.get("cost")) - if cost_value is not None and cost_value > 0: + if spend_value is not None and spend_value > 0: + total_cost = float(spend_value) + elif cost_value is not None and cost_value > 0: total_cost = float(cost_value) else: total_cost = estimate_cli_cost( diff --git a/sdk/python/tests/test_harness_provider_aforge.py b/sdk/python/tests/test_harness_provider_aforge.py index ba5cade16..ab10ec7b9 100644 --- a/sdk/python/tests/test_harness_provider_aforge.py +++ b/sdk/python/tests/test_harness_provider_aforge.py @@ -39,6 +39,25 @@ def _envelope( return json.dumps(envelope) +def _exec_envelope( + text: str = "done", + *, + stop: str = "done", + usage: dict[str, object] | None = None, + turns: int = 1, +) -> str: + return json.dumps( + { + "text": text, + "stop": stop, + "usage": usage or {}, + "artifacts": [], + "turns": turns, + "elapsed_ms": 12, + } + ) + + @pytest.mark.asyncio async def test_aforge_success_maps_envelope_and_metrics( monkeypatch: pytest.MonkeyPatch, @@ -81,6 +100,91 @@ async def fake_run_cli( assert raw.messages[0]["deliverable"] == " final answer " +@pytest.mark.asyncio +async def test_aforge_exec_mode_maps_original_contract_and_pins_model( + monkeypatch: pytest.MonkeyPatch, +): + captured: dict[str, object] = {} + + async def fake_run_cli( + cmd, *, env=None, cwd=None, timeout=None, idle_seconds=None, input_text=None + ): + captured.update(cmd=cmd, env=env, input_text=input_text) + return ( + _exec_envelope( + " linear answer ", + usage={ + "calls": 3, + "prompt_tokens": 100, + "completion_tokens": 50, + "cached_tokens": 20, + "cost": 0.0123, + }, + turns=4, + ), + "", + 0, + ) + + monkeypatch.setenv("AGENTFIELD_AFORGE_COMMAND", "exec") + monkeypatch.setattr("agentfield.harness.providers.aforge.run_cli", fake_run_cli) + raw = await AforgeProvider("/opt/aforge").execute( + "prompt that stays off argv", + { + "project_dir": "/project", + "system_prompt": " be precise ", + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + }, + ) + + assert captured["cmd"] == [ + "/opt/aforge", + "exec", + "--json", + "-w", + "/project", + "--timeout", + "1795", + "--context-fill", + "60", + "--completion-reserve", + "65536", + "--system", + "be precise", + "--model", + "deepseek/deepseek-v4-flash-0731", + "--plan-model", + "deepseek/deepseek-v4-flash-0731", + ] + assert captured["env"] == { + "AFORGE_MODELS": "", + "AFORGE_MODEL": "deepseek/deepseek-v4-flash-0731", + } + assert captured["input_text"] == "prompt that stays off argv" + assert raw.result == "linear answer" + assert raw.is_error is False + assert raw.metrics.num_turns == 4 + assert raw.metrics.input_tokens == 100 + assert raw.metrics.total_cost_usd == 0.0123 + + +@pytest.mark.asyncio +async def test_aforge_exec_mode_accepts_budget_partial( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setenv("AGENTFIELD_AFORGE_COMMAND", "exec") + monkeypatch.setattr( + "agentfield.harness.providers.aforge.run_cli", + AsyncMock(return_value=(_exec_envelope("usable", stop="budget"), "", 2)), + ) + + raw = await AforgeProvider().execute("hello", {}) + + assert raw.result == "usable" + assert raw.is_error is False + assert raw.failure_type is FailureType.NONE + + def test_aforge_binary_environment_override(monkeypatch: pytest.MonkeyPatch): monkeypatch.setenv("AFORGE_BIN", "/opt/aforge-env") assert AforgeProvider()._bin == "/opt/aforge-env" diff --git a/sdk/typescript/src/harness/providers/aforge.ts b/sdk/typescript/src/harness/providers/aforge.ts index 6cb1f4b35..b22f8ed4e 100644 --- a/sdk/typescript/src/harness/providers/aforge.ts +++ b/sdk/typescript/src/harness/providers/aforge.ts @@ -56,12 +56,12 @@ function stripOpenRouterPrefix(model: string): string { } function parseEnvelope(stdout: string): Record | undefined { - // Canonical `aforge do --json` output is one object, currently formatted - // across multiple lines. Parse that shape before falling back to the - // line-oriented form tolerated for wrappers that prepend diagnostics. + // Both canonical `do` and `exec` print one JSON object. Parse that shape + // before the wrapper-compatible line scan. try { const value: unknown = JSON.parse(stdout.trim()); - if (typeof value === 'object' && value !== null && !Array.isArray(value) && 'deliverable' in value) { + if (typeof value === 'object' && value !== null && !Array.isArray(value) + && ('deliverable' in value || 'text' in value)) { return value as Record; } } catch { @@ -72,7 +72,8 @@ function parseEnvelope(stdout: string): Record | undefined { for (let index = lines.length - 1; index >= 0; index -= 1) { try { const value: unknown = JSON.parse(lines[index]); - if (typeof value === 'object' && value !== null && !Array.isArray(value) && 'deliverable' in value) { + if (typeof value === 'object' && value !== null && !Array.isArray(value) + && ('deliverable' in value || 'text' in value)) { return value as Record; } } catch { @@ -128,7 +129,10 @@ function stringOptions(value: unknown): Record { return result; } -/** Aforge CLI provider using canonical `aforge do --json` one-shot mode. */ +/** + * Aforge CLI provider. `do` is the default; set + * `AGENTFIELD_AFORGE_COMMAND=exec` for the direct one-shot entry point. + */ export class AforgeProvider implements HarnessProvider { private readonly bin: string; @@ -149,21 +153,52 @@ export class AforgeProvider implements HarnessProvider { const cwd = typeof options.cwd === 'string' ? options.cwd : undefined; const root = projectDir ?? cwd ?? '.'; const outerTimeout = timeoutSeconds(); - const cmd = [ - this.bin, - 'do', - '--json', - '--yes-spend', - '-w', - root, - '--timeout', - String(innerTimeout(outerTimeout)), - ]; + const command = (process.env.AGENTFIELD_AFORGE_COMMAND ?? 'do').trim().toLowerCase(); + if (command !== 'do' && command !== 'exec') { + return createRawResult({ + isError: true, + errorMessage: `AGENTFIELD_AFORGE_COMMAND must be 'do' or 'exec', got ${JSON.stringify(command)}`, + failureType: 'crash', + metrics: createMetrics(), + }); + } + const systemPrompt = options.systemPrompt ?? options.system_prompt; + const cmd = command === 'exec' + ? [ + this.bin, + 'exec', + '--json', + '-w', + root, + '--timeout', + String(innerTimeout(outerTimeout)), + '--context-fill', + '60', + '--completion-reserve', + '65536', + ] + : [ + this.bin, + 'do', + '--json', + '--yes-spend', + '-w', + root, + '--timeout', + String(innerTimeout(outerTimeout)), + ]; + if (command === 'exec' && typeof systemPrompt === 'string' && systemPrompt.trim()) { + cmd.push('--system', systemPrompt.trim()); + } const { model, variant } = resolveModelAndVariant(options); - const env: Record = {}; + const env: Record = command === 'exec' ? { AFORGE_MODELS: '' } : {}; if (model) { - env.AFORGE_MODEL = stripOpenRouterPrefix(model); + const slug = stripOpenRouterPrefix(model); + env.AFORGE_MODEL = slug; + if (command === 'exec') { + cmd.push('--model', slug, '--plan-model', slug); + } } if (variant) { const normalized = variant.trim().toLowerCase(); @@ -180,23 +215,29 @@ export class AforgeProvider implements HarnessProvider { cwd: undefined, timeout: outerTimeout * 1000, idleSeconds: 0, - inputText: taskInput(prompt, options.systemPrompt ?? options.system_prompt), + inputText: command === 'exec' ? prompt : taskInput(prompt, systemPrompt), }); const envelope = parseEnvelope(stdout); - const resultText = typeof envelope?.deliverable === 'string' && envelope.deliverable.trim() - ? envelope.deliverable.trim() + const outputValue = command === 'exec' ? envelope?.text : envelope?.deliverable; + const resultText = typeof outputValue === 'string' && outputValue.trim() + ? outputValue.trim() : undefined; const blockedOn = typeof envelope?.blocked_on === 'string' ? envelope.blocked_on.trim() : ''; + const stop = typeof envelope?.stop === 'string' ? envelope.stop.trim() : ''; const usage = typeof envelope?.usage === 'object' && envelope.usage !== null && !Array.isArray(envelope.usage) ? envelope.usage as Record : {}; - const isError = exitCode !== 0 || resultText === undefined || blockedOn !== ''; + const isError = command === 'exec' + ? exitCode < 0 || resultText === undefined || ![0, 2, 3].includes(exitCode) + : exitCode !== 0 || resultText === undefined || blockedOn !== ''; const inputTokens = Math.trunc(numeric(usage.prompt_tokens) ?? 0); const outputTokens = Math.trunc(numeric(usage.completion_tokens) ?? 0); const cacheReadTokens = Math.trunc(numeric(usage.cached_tokens) ?? 0); - const calls = Math.trunc(numeric(usage.calls) ?? 0); - const providerCost = numeric(usage.cost); + const calls = Math.trunc(numeric(command === 'exec' ? envelope?.turns : usage.calls) ?? 0); + const nativeSpend = numeric(envelope?.spend); + const legacyCost = numeric(usage.cost); + const providerCost = nativeSpend !== undefined && nativeSpend > 0 ? nativeSpend : legacyCost; return createRawResult({ result: resultText, @@ -215,8 +256,12 @@ export class AforgeProvider implements HarnessProvider { model, }), isError, - errorMessage: isError ? crashMessage(exitCode, blockedOn, resultText, stderr) : undefined, - failureType: isError ? (exitCode === 2 ? 'timeout' : 'crash') : 'none', + errorMessage: isError ? crashMessage(exitCode, blockedOn || stop, resultText, stderr) : undefined, + failureType: isError + ? ((command === 'do' && exitCode === 2) || (command === 'exec' && exitCode === 4) + ? 'timeout' + : 'crash') + : 'none', returnCode: exitCode, }); } catch (error) { diff --git a/sdk/typescript/tests/harness_provider_aforge.test.ts b/sdk/typescript/tests/harness_provider_aforge.test.ts index 091f131f6..08aea3fa2 100644 --- a/sdk/typescript/tests/harness_provider_aforge.test.ts +++ b/sdk/typescript/tests/harness_provider_aforge.test.ts @@ -18,10 +18,25 @@ function envelope( }); } +function execEnvelope( + text = 'done', + options: { stop?: string; usage?: Record; turns?: number } = {} +): string { + return JSON.stringify({ + text, + stop: options.stop ?? 'done', + usage: options.usage ?? {}, + artifacts: [], + turns: options.turns ?? 1, + elapsed_ms: 12, + }); +} + afterEach(() => { vi.restoreAllMocks(); delete process.env.AGENTFIELD_HARNESS_TIMEOUT_SECONDS; delete process.env.AFORGE_BIN; + delete process.env.AGENTFIELD_AFORGE_COMMAND; }); describe('aforge provider', () => { @@ -85,6 +100,71 @@ describe('aforge provider', () => { expect(result.messages[0].deliverable).toBe(' final answer '); }); + it('maps the opt-in exec command and original envelope contract', async () => { + process.env.AGENTFIELD_AFORGE_COMMAND = 'exec'; + vi.spyOn(cli, 'runCli').mockResolvedValue({ + stdout: execEnvelope(' linear answer ', { + turns: 4, + usage: { + calls: 3, + prompt_tokens: 100, + completion_tokens: 50, + cached_tokens: 20, + cost: 0.0123, + }, + }), + stderr: '', + exitCode: 0, + }); + + const result = await new AforgeProvider('/opt/aforge').execute('prompt that stays off argv', { + projectDir: '/project', + systemPrompt: ' be precise ', + model: 'openrouter/deepseek/deepseek-v4-flash-0731#high', + }); + + expect(cli.runCli).toHaveBeenCalledWith( + [ + '/opt/aforge', 'exec', '--json', '-w', '/project', '--timeout', '1795', + '--context-fill', '60', '--completion-reserve', '65536', + '--system', 'be precise', + '--model', 'deepseek/deepseek-v4-flash-0731', + '--plan-model', 'deepseek/deepseek-v4-flash-0731', + ], + { + env: { + AFORGE_MODELS: '', + AFORGE_MODEL: 'deepseek/deepseek-v4-flash-0731', + AFORGE_EXEC_REASONING: 'high', + }, + cwd: undefined, + timeout: 1_800_000, + idleSeconds: 0, + inputText: 'prompt that stays off argv', + } + ); + expect(result.result).toBe('linear answer'); + expect(result.isError).toBe(false); + expect(result.metrics.numTurns).toBe(4); + expect(result.metrics.inputTokens).toBe(100); + expect(result.metrics.totalCostUsd).toBe(0.0123); + }); + + it('accepts an exec budget partial with usable text', async () => { + process.env.AGENTFIELD_AFORGE_COMMAND = 'exec'; + vi.spyOn(cli, 'runCli').mockResolvedValue({ + stdout: execEnvelope('usable', { stop: 'budget', turns: 2 }), + stderr: '', + exitCode: 2, + }); + + const result = await new AforgeProvider().execute('hello', {}); + + expect(result.result).toBe('usable'); + expect(result.isError).toBe(false); + expect(result.failureType).toBe('none'); + }); + it('uses cwd as the root and gives aforge a timeout landing window', async () => { process.env.AGENTFIELD_HARNESS_TIMEOUT_SECONDS = '2400'; vi.spyOn(cli, 'runCli').mockResolvedValue({ stdout: envelope(), stderr: '', exitCode: 0 }); From 901af4b56e7e0ee11954923983843004f59f0d2b Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Fri, 14 Aug 2026 23:15:30 -0400 Subject: [PATCH 05/11] feat(harness): default aforge to exec --- docs/harness-providers.md | 13 +++++++------ sdk/go/harness/aforge.go | 2 +- sdk/go/harness/aforge_integration_test.go | 4 ++-- sdk/go/harness/aforge_test.go | 16 +++++++++++++++- .../agentfield/harness/providers/aforge.py | 6 +++--- sdk/python/tests/test_harness_provider_aforge.py | 3 ++- sdk/typescript/src/harness/providers/aforge.ts | 6 +++--- sdk/typescript/tests/harness_functional.test.ts | 2 +- .../tests/harness_provider_aforge.test.ts | 8 ++++++-- 9 files changed, 40 insertions(+), 20 deletions(-) diff --git a/docs/harness-providers.md b/docs/harness-providers.md index ef4d0b633..a39a89f90 100644 --- a/docs/harness-providers.md +++ b/docs/harness-providers.md @@ -40,10 +40,11 @@ separately available executable depending on the wrapper and platform. ### Aforge drop-in usage Aforge is registered as `aforge` in the Python, Go, and TypeScript SDKs. The -adapters all invoke the same non-interactive contract: `aforge do --json ---yes-spend`, send the task over stdin, and map Aforge's usage ledger into -AgentField turns, token counts, and cost metrics. Install a current aforge-v2 -binary on `PATH`, set `OPENROUTER_API_KEY`, then select the provider: +adapters default to the direct non-interactive contract, `aforge exec --json`, +send the task over stdin, and map Aforge's usage ledger into AgentField turns, +token counts, and cost metrics. Set `AGENTFIELD_AFORGE_COMMAND=do` to opt into +the routed `aforge do --json --yes-spend` workflow instead. Install a current +aforge-v2 binary on `PATH`, set `OPENROUTER_API_KEY`, then select the provider: ```python result = await app.harness(task, provider="aforge", model="openrouter/z-ai/glm-5.2") @@ -67,7 +68,7 @@ Set `AFORGE_MAX_CONCURRENT` to cap simultaneous Aforge subprocesses. The default is 8. `AGENTFIELD_HARNESS_TIMEOUT_SECONDS` is the outer watchdog; each adapter gives Aforge a five-second landing window to emit its exit-2 timeout envelope. Schema runs use a unique output directory per invocation so parallel -benchmark jobs can safely share a checkout. Set `AFORGE_BIN` to an absolute +jobs can safely share a checkout. Set `AFORGE_BIN` to an absolute path when the unreleased binary is not installed on `PATH`. ## Model selection and reasoning-effort variants @@ -87,7 +88,7 @@ An explicit `variant="high"` keyword wins over the suffix. Per provider: | Provider | Model flag | Variant handling | | --- | --- | --- | -| `aforge` | `AFORGE_MODEL` env var with a bare OpenRouter slug (a leading `openrouter/` is stripped) | `AFORGE_EXEC_REASONING` (`off`, `low`, `medium`, or `high`) | +| `aforge` | `exec`: `--model` and `--plan-model`; `do`: `AFORGE_MODEL` (a leading `openrouter/` is stripped) | `AFORGE_EXEC_REASONING` (`off`, `low`, `medium`, or `high`) | | OpenCode | `-m ` | `--variant ` (provider-specific effort, e.g. `high`, `max`, `minimal`) | | Codex | `-m ` | `-c model_reasoning_effort=` | | Claude Code | SDK `model` option | No effort control — variant is dropped with a debug log | diff --git a/sdk/go/harness/aforge.go b/sdk/go/harness/aforge.go index 786dbe9e6..8a85e1359 100644 --- a/sdk/go/harness/aforge.go +++ b/sdk/go/harness/aforge.go @@ -96,7 +96,7 @@ func aforgeTaskInput(prompt, systemPrompt string) string { func aforgeCommand() (string, error) { command := strings.ToLower(strings.TrimSpace(os.Getenv("AGENTFIELD_AFORGE_COMMAND"))) if command == "" { - return "do", nil + return "exec", nil } if command != "do" && command != "exec" { return "", fmt.Errorf("AGENTFIELD_AFORGE_COMMAND must be do or exec, got %q", command) diff --git a/sdk/go/harness/aforge_integration_test.go b/sdk/go/harness/aforge_integration_test.go index b32d04afa..d86138a09 100644 --- a/sdk/go/harness/aforge_integration_test.go +++ b/sdk/go/harness/aforge_integration_test.go @@ -14,8 +14,8 @@ import ( "github.com/stretchr/testify/require" ) -// TestAforgeProvider_Integration drives the real aforge binary and its `do` -// JSON envelope. Set AFORGE_INTEGRATION=1 so TestMain does not shadow it. +// TestAforgeProvider_Integration drives the real aforge binary and its default +// `exec` JSON envelope. Set AFORGE_INTEGRATION=1 so TestMain does not shadow it. func TestAforgeProvider_Integration(t *testing.T) { binPath, err := exec.LookPath("aforge") if err != nil { diff --git a/sdk/go/harness/aforge_test.go b/sdk/go/harness/aforge_test.go index d09b9f367..434dbfa0c 100644 --- a/sdk/go/harness/aforge_test.go +++ b/sdk/go/harness/aforge_test.go @@ -31,7 +31,13 @@ func aforgeExecEnvelope(text, stop, usage string, turns int) string { text, stop, usage, turns) } +func useAforgeDo(t *testing.T) { + t.Helper() + t.Setenv("AGENTFIELD_AFORGE_COMMAND", "do") +} + func TestAforgeProviderMapsDoCommandEnvelopeAndMetrics(t *testing.T) { + useAforgeDo(t) var capturedCmd []string var capturedEnv map[string]string var capturedCwd string @@ -90,7 +96,7 @@ func TestAforgeProviderMapsDoCommandEnvelopeAndMetrics(t *testing.T) { } func TestAforgeProviderMapsExecCommandEnvelopeAndMetrics(t *testing.T) { - t.Setenv("AGENTFIELD_AFORGE_COMMAND", "exec") + t.Setenv("AGENTFIELD_AFORGE_COMMAND", "") var capturedCmd []string var capturedEnv map[string]string var capturedStdin []byte @@ -148,6 +154,7 @@ func TestAforgeProviderBinaryEnvironmentOverride(t *testing.T) { } func TestAforgeProviderModelVariantAndEnvironmentPrecedence(t *testing.T) { + useAforgeDo(t) var captured []map[string]string p := NewAforgeProvider("aforge") p.runCLI = func(_ context.Context, _ []string, env map[string]string, _ string, _, _ int, _ []byte) (*CLIResult, error) { @@ -180,6 +187,7 @@ func TestAforgeProviderModelVariantAndEnvironmentPrecedence(t *testing.T) { } func TestAforgeProviderRootAndTimeoutResolution(t *testing.T) { + useAforgeDo(t) t.Setenv("AGENTFIELD_HARNESS_TIMEOUT_SECONDS", "2400") var commands [][]string var timeouts []int @@ -202,6 +210,7 @@ func TestAforgeProviderRootAndTimeoutResolution(t *testing.T) { } func TestAforgeProviderExitSemantics(t *testing.T) { + useAforgeDo(t) tests := []struct { name string code int @@ -242,6 +251,7 @@ func TestAforgeProviderExitSemantics(t *testing.T) { } func TestAforgeProviderParsesLastEnvelopeAndLeavesZeroCostUnknown(t *testing.T) { + useAforgeDo(t) p := NewAforgeProvider("aforge") p.runCLI = func(context.Context, []string, map[string]string, string, int, int, []byte) (*CLIResult, error) { return &CLIResult{Stdout: "stray diagnostic\n{\"type\":\"event\"}\n" + @@ -255,6 +265,7 @@ func TestAforgeProviderParsesLastEnvelopeAndLeavesZeroCostUnknown(t *testing.T) } func TestAforgeProviderParsesPrettyPrintedEnvelope(t *testing.T) { + useAforgeDo(t) p := NewAforgeProvider("aforge") p.runCLI = func(context.Context, []string, map[string]string, string, int, int, []byte) (*CLIResult, error) { var value map[string]any @@ -271,6 +282,7 @@ func TestAforgeProviderParsesPrettyPrintedEnvelope(t *testing.T) { } func TestAforgeProviderMissingBinaryAndTimeout(t *testing.T) { + useAforgeDo(t) t.Run("missing binary", func(t *testing.T) { p := NewAforgeProvider("aforge-missing") p.runCLI = func(context.Context, []string, map[string]string, string, int, int, []byte) (*CLIResult, error) { @@ -296,6 +308,7 @@ func TestAforgeProviderMissingBinaryAndTimeout(t *testing.T) { } func TestAforgeProviderConcurrencyLimit(t *testing.T) { + useAforgeDo(t) t.Setenv("AFORGE_MAX_CONCURRENT", "2") aforgeSemaphore = nil aforgeSemOnce = sync.Once{} @@ -333,6 +346,7 @@ func TestAforgeProviderConcurrencyLimit(t *testing.T) { } func TestAforgeRunnerConcurrentSameCwdUsesIsolatedSchemaFiles(t *testing.T) { + useAforgeDo(t) cwd := t.TempDir() script := writeTestScript(t, cwd, "aforge-test", `#!/bin/sh prompt=$(cat) diff --git a/sdk/python/agentfield/harness/providers/aforge.py b/sdk/python/agentfield/harness/providers/aforge.py index be969eec0..f6e32f7e0 100644 --- a/sdk/python/agentfield/harness/providers/aforge.py +++ b/sdk/python/agentfield/harness/providers/aforge.py @@ -90,8 +90,8 @@ def _crash_message( class AforgeProvider: """Aforge CLI provider. - ``do`` remains the default. Set ``AGENTFIELD_AFORGE_COMMAND=exec`` to use - Aforge's direct one-shot harness entry point. + ``exec`` is the default direct one-shot entry point. Set + ``AGENTFIELD_AFORGE_COMMAND=do`` to opt into Aforge's routed workflow. """ _MAX_CONCURRENT: ClassVar[int] = int(os.environ.get("AFORGE_MAX_CONCURRENT", "8")) @@ -130,7 +130,7 @@ async def _execute_impl(self, prompt: str, options: dict[str, object]) -> RawRes # Leave a small landing window so aforge can emit its honest timeout # envelope before the outer subprocess watchdog has to kill it. aforge_timeout = max(1, timeout_seconds - 5) - command = os.environ.get("AGENTFIELD_AFORGE_COMMAND", "do").strip().lower() + command = os.environ.get("AGENTFIELD_AFORGE_COMMAND", "exec").strip().lower() if command not in {"do", "exec"}: return RawResult( is_error=True, diff --git a/sdk/python/tests/test_harness_provider_aforge.py b/sdk/python/tests/test_harness_provider_aforge.py index ab10ec7b9..6f38498ea 100644 --- a/sdk/python/tests/test_harness_provider_aforge.py +++ b/sdk/python/tests/test_harness_provider_aforge.py @@ -16,6 +16,7 @@ def mock_aforge_available(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( "agentfield.harness._availability.shutil.which", lambda path: path ) + monkeypatch.setenv("AGENTFIELD_AFORGE_COMMAND", "do") def _envelope( @@ -126,7 +127,7 @@ async def fake_run_cli( 0, ) - monkeypatch.setenv("AGENTFIELD_AFORGE_COMMAND", "exec") + monkeypatch.delenv("AGENTFIELD_AFORGE_COMMAND") monkeypatch.setattr("agentfield.harness.providers.aforge.run_cli", fake_run_cli) raw = await AforgeProvider("/opt/aforge").execute( "prompt that stays off argv", diff --git a/sdk/typescript/src/harness/providers/aforge.ts b/sdk/typescript/src/harness/providers/aforge.ts index b22f8ed4e..ef8e0ef48 100644 --- a/sdk/typescript/src/harness/providers/aforge.ts +++ b/sdk/typescript/src/harness/providers/aforge.ts @@ -130,8 +130,8 @@ function stringOptions(value: unknown): Record { } /** - * Aforge CLI provider. `do` is the default; set - * `AGENTFIELD_AFORGE_COMMAND=exec` for the direct one-shot entry point. + * Aforge CLI provider. `exec` is the default direct one-shot entry point; set + * `AGENTFIELD_AFORGE_COMMAND=do` to opt into Aforge's routed workflow. */ export class AforgeProvider implements HarnessProvider { private readonly bin: string; @@ -153,7 +153,7 @@ export class AforgeProvider implements HarnessProvider { const cwd = typeof options.cwd === 'string' ? options.cwd : undefined; const root = projectDir ?? cwd ?? '.'; const outerTimeout = timeoutSeconds(); - const command = (process.env.AGENTFIELD_AFORGE_COMMAND ?? 'do').trim().toLowerCase(); + const command = (process.env.AGENTFIELD_AFORGE_COMMAND ?? 'exec').trim().toLowerCase(); if (command !== 'do' && command !== 'exec') { return createRawResult({ isError: true, diff --git a/sdk/typescript/tests/harness_functional.test.ts b/sdk/typescript/tests/harness_functional.test.ts index 706830c15..a0214696f 100644 --- a/sdk/typescript/tests/harness_functional.test.ts +++ b/sdk/typescript/tests/harness_functional.test.ts @@ -92,7 +92,7 @@ describe.skipIf(!HAS_AFORGE)('Aforge Functional', () => { cleanupWorkDir(workDir); }); - it('basic prompt returns the do JSON deliverable and benchmark metrics', async () => { + it('basic prompt returns the exec JSON deliverable and provider metrics', async () => { const { AforgeProvider } = await import('../src/harness/providers/aforge.js'); const result = await new AforgeProvider().execute( 'Reply with exactly: HELLO_AGENTFIELD', diff --git a/sdk/typescript/tests/harness_provider_aforge.test.ts b/sdk/typescript/tests/harness_provider_aforge.test.ts index 08aea3fa2..267d51ca9 100644 --- a/sdk/typescript/tests/harness_provider_aforge.test.ts +++ b/sdk/typescript/tests/harness_provider_aforge.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { AforgeProvider } from '../src/harness/providers/aforge.js'; import { buildProvider, SUPPORTED_PROVIDERS } from '../src/harness/providers/factory.js'; @@ -39,6 +39,10 @@ afterEach(() => { delete process.env.AGENTFIELD_AFORGE_COMMAND; }); +beforeEach(() => { + process.env.AGENTFIELD_AFORGE_COMMAND = 'do'; +}); + describe('aforge provider', () => { it('honors AFORGE_BIN unless a binary is explicit', async () => { process.env.AFORGE_BIN = '/opt/aforge-env'; @@ -101,7 +105,7 @@ describe('aforge provider', () => { }); it('maps the opt-in exec command and original envelope contract', async () => { - process.env.AGENTFIELD_AFORGE_COMMAND = 'exec'; + delete process.env.AGENTFIELD_AFORGE_COMMAND; vi.spyOn(cli, 'runCli').mockResolvedValue({ stdout: execEnvelope(' linear answer ', { turns: 4, From fa70333ab06437af7d048960200d42f0193fb018 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Mon, 17 Aug 2026 11:21:28 -0400 Subject: [PATCH 06/11] feat(harness): make aforge the default provider in the Python SDK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `app.harness("...")` with nothing configured now runs AForge, AgentField's native coding harness, instead of raising "No harness provider specified". Provider precedence is explicit value > AGENTFIELD_HARNESS_PROVIDER > "aforge", implemented once in harness/_defaults.py and applied both where HarnessConfig materialises its default and where the runner resolves options (so a runner built without any config follows the same chain). HarnessConfig.model stops defaulting to "sonnet" — that was Claude-specific and wrong for every other provider. The default is now empty, meaning "use the provider's own default", and the claude-code provider carries "sonnet" internally so explicit claude-code users see no change. Also drops the private-repo build instruction from the aforge install hint; the binary ships alongside `af` and `af aforge ensure` (re)installs it. Co-Authored-By: Claude Fable 5 --- sdk/python/agentfield/agent.py | 5 +- sdk/python/agentfield/harness/_defaults.py | 24 ++++++ sdk/python/agentfield/harness/_runner.py | 9 +-- .../agentfield/harness/providers/claude.py | 8 +- sdk/python/agentfield/types.py | 22 +++++- sdk/python/tests/test_harness_defaults.py | 78 +++++++++++++++++++ .../tests/test_harness_provider_claude.py | 24 ++++++ sdk/python/tests/test_harness_runner.py | 17 +++- sdk/python/tests/test_harness_types.py | 10 +-- sdk/python/tests/test_types.py | 8 +- 10 files changed, 178 insertions(+), 27 deletions(-) create mode 100644 sdk/python/agentfield/harness/_defaults.py create mode 100644 sdk/python/tests/test_harness_defaults.py diff --git a/sdk/python/agentfield/agent.py b/sdk/python/agentfield/agent.py index 004164e57..cf92f9d3a 100644 --- a/sdk/python/agentfield/agent.py +++ b/sdk/python/agentfield/agent.py @@ -3735,8 +3735,9 @@ async def harness( prompt: Task description for the coding agent. schema: Pydantic BaseModel class for structured output validation. provider: Override provider ("aforge", "claude-code", "codex", "gemini", - "opencode"). - model: Override model identifier. + "opencode", "grok"). Omit to use ``AGENTFIELD_HARNESS_PROVIDER`` + when set, otherwise ``aforge``. + model: Override model identifier. Empty uses the provider's own default. max_turns: Maximum agent iterations. max_budget_usd: Cost cap in USD. tools: Allowed tools list. diff --git a/sdk/python/agentfield/harness/_defaults.py b/sdk/python/agentfield/harness/_defaults.py new file mode 100644 index 000000000..b9b731718 --- /dev/null +++ b/sdk/python/agentfield/harness/_defaults.py @@ -0,0 +1,24 @@ +"""Default harness provider selection. + +`aforge` is AgentField's native harness. Provider precedence is +explicit value > AGENTFIELD_HARNESS_PROVIDER env var > DEFAULT_HARNESS_PROVIDER. +""" + +from __future__ import annotations + +import os +from typing import Optional + +DEFAULT_HARNESS_PROVIDER = "aforge" +HARNESS_PROVIDER_ENV_VAR = "AGENTFIELD_HARNESS_PROVIDER" + + +def resolve_harness_provider(explicit: Optional[str] = None) -> str: + if isinstance(explicit, str) and explicit.strip(): + return explicit.strip() + + env_provider = os.environ.get(HARNESS_PROVIDER_ENV_VAR) + if isinstance(env_provider, str) and env_provider.strip(): + return env_provider.strip() + + return DEFAULT_HARNESS_PROVIDER diff --git a/sdk/python/agentfield/harness/_runner.py b/sdk/python/agentfield/harness/_runner.py index 46fe61ecc..216a58b22 100644 --- a/sdk/python/agentfield/harness/_runner.py +++ b/sdk/python/agentfield/harness/_runner.py @@ -10,6 +10,7 @@ import time from typing import Any, Dict, List, Optional +from agentfield.harness._defaults import resolve_harness_provider from agentfield.harness._result import FailureType, HarnessResult, RawResult from agentfield.harness._schema import ( build_followup_prompt, @@ -275,12 +276,8 @@ async def run( } options = _resolve_options(self._config, overrides) - resolved_provider = options.get("provider") - if not resolved_provider: - raise ValueError( - "No harness provider specified. Set 'provider' in HarnessConfig " - "or pass it to .harness() call." - ) + resolved_provider = resolve_harness_provider(options.get("provider")) + options["provider"] = resolved_provider resolved_cwd = str(options.get("cwd") or ".") provider_instance = self._build_provider(str(resolved_provider), options) diff --git a/sdk/python/agentfield/harness/providers/claude.py b/sdk/python/agentfield/harness/providers/claude.py index 39bdb8059..5c5cbeb1d 100644 --- a/sdk/python/agentfield/harness/providers/claude.py +++ b/sdk/python/agentfield/harness/providers/claude.py @@ -16,6 +16,11 @@ logger = logging.getLogger("agentfield.harness.claude") +# Claude Code's historical default in AgentField. HarnessConfig.model used to +# default to "sonnet"; now the default is empty and each provider owns its own +# default, so claude-code keeps sonnet here. +DEFAULT_CLAUDE_MODEL = "sonnet" + def _get_claude_sdk() -> Any: """Lazy import of claude_agent_sdk.""" @@ -75,8 +80,7 @@ async def execute(self, prompt: str, options: dict[str, object]) -> RawResult: agent_options: dict[str, object] = {} model_value, variant_value = resolve_model_and_variant(options) - if model_value is not None: - agent_options["model"] = model_value + agent_options["model"] = model_value or DEFAULT_CLAUDE_MODEL if variant_value: # claude_agent_sdk has no reasoning-effort knob; drop the variant # rather than passing an invalid "model#variant" model id. diff --git a/sdk/python/agentfield/types.py b/sdk/python/agentfield/types.py index f9091d3ad..abe3860c7 100644 --- a/sdk/python/agentfield/types.py +++ b/sdk/python/agentfield/types.py @@ -273,15 +273,29 @@ class DiscoveryResult: xml: Optional[str] = None +def _default_harness_provider() -> str: + # Imported lazily: agentfield.harness imports agentfield.types. + from agentfield.harness._defaults import resolve_harness_provider + + return resolve_harness_provider() + + class HarnessConfig(BaseModel): provider: str = Field( - ..., + default_factory=_default_harness_provider, + description=( + 'Coding agent provider: "aforge" (default) | "claude-code" | "codex" | ' + '"gemini" | "opencode" | "grok". Unset resolves to the ' + 'AGENTFIELD_HARNESS_PROVIDER env var when present, else "aforge".' + ), + ) + model: Optional[str] = Field( + default=None, description=( - 'Coding agent provider: "aforge" | "claude-code" | "codex" | ' - '"gemini" | "opencode" | "grok"' + "Model identifier. None/empty means the provider's own default " + "(aforge picks its own; claude-code uses sonnet)." ), ) - model: str = Field(default="sonnet", description="Default model identifier.") max_turns: int = Field(default=30, description="Maximum agent iterations.") max_budget_usd: Optional[float] = Field( default=None, description="Cost cap in USD." diff --git a/sdk/python/tests/test_harness_defaults.py b/sdk/python/tests/test_harness_defaults.py new file mode 100644 index 000000000..58ffcf39a --- /dev/null +++ b/sdk/python/tests/test_harness_defaults.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +from typing import Any +from unittest.mock import patch + +import pytest + +from agentfield.harness._result import RawResult +from agentfield.harness._runner import HarnessRunner +from agentfield.types import HarnessConfig + + +class _Provider: + async def execute(self, prompt: str, options: dict[str, Any]) -> RawResult: + return RawResult(result="ok") + + +def test_default_provider_is_aforge(monkeypatch): + monkeypatch.delenv("AGENTFIELD_HARNESS_PROVIDER", raising=False) + assert HarnessConfig().provider == "aforge" + + +def test_env_provider_is_honoured(monkeypatch): + monkeypatch.setenv("AGENTFIELD_HARNESS_PROVIDER", "codex") + assert HarnessConfig().provider == "codex" + + +def test_explicit_provider_beats_env(monkeypatch): + monkeypatch.setenv("AGENTFIELD_HARNESS_PROVIDER", "codex") + assert HarnessConfig(provider="gemini").provider == "gemini" + + +def test_blank_env_is_ignored(monkeypatch): + monkeypatch.setenv("AGENTFIELD_HARNESS_PROVIDER", " ") + assert HarnessConfig().provider == "aforge" + + +async def _run_and_capture_provider( + tmp_path, config: HarnessConfig | None = None, **overrides: Any +) -> str: + captured: dict[str, str] = {} + + def fake_build_provider(factory_config): + captured["provider"] = factory_config.provider + return _Provider() + + with patch("agentfield.harness._runner.build_provider", fake_build_provider): + await HarnessRunner(config=config).run( + "hello", cwd=str(tmp_path), **overrides + ) + return captured["provider"] + + +@pytest.mark.asyncio +async def test_zero_setup_runner_selects_aforge(tmp_path, monkeypatch): + monkeypatch.delenv("AGENTFIELD_HARNESS_PROVIDER", raising=False) + assert await _run_and_capture_provider(tmp_path) == "aforge" + + +@pytest.mark.asyncio +async def test_runner_honours_env_provider(tmp_path, monkeypatch): + monkeypatch.setenv("AGENTFIELD_HARNESS_PROVIDER", "opencode") + assert await _run_and_capture_provider(tmp_path) == "opencode" + + +@pytest.mark.asyncio +async def test_runner_explicit_provider_beats_env_and_config(tmp_path, monkeypatch): + monkeypatch.setenv("AGENTFIELD_HARNESS_PROVIDER", "gemini") + config = HarnessConfig(provider="codex") + assert ( + await _run_and_capture_provider(tmp_path, config, provider="opencode") + == "opencode" + ) + + +def test_model_default_is_empty(monkeypatch): + monkeypatch.delenv("AGENTFIELD_HARNESS_PROVIDER", raising=False) + assert HarnessConfig().model is None diff --git a/sdk/python/tests/test_harness_provider_claude.py b/sdk/python/tests/test_harness_provider_claude.py index 1fa3b0b86..0997b2116 100644 --- a/sdk/python/tests/test_harness_provider_claude.py +++ b/sdk/python/tests/test_harness_provider_claude.py @@ -87,6 +87,30 @@ def fake_query(*, prompt: str, options: FakeClaudeAgentOptions): assert len(raw.messages) == 2 +@pytest.mark.asyncio +async def test_execute_defaults_model_to_sonnet(monkeypatch): + from agentfield.harness.providers.claude import ClaudeCodeProvider + + captured: dict[str, Any] = {} + + class FakeClaudeAgentOptions: + def __init__(self, **kwargs: Any) -> None: + self.kwargs = kwargs + + def fake_query(*, prompt: str, options: FakeClaudeAgentOptions): + captured["options"] = options + return _AsyncStream([{"type": "result", "result": "ok"}]) + + fake_sdk = ModuleType("claude_agent_sdk") + setattr(fake_sdk, "ClaudeAgentOptions", FakeClaudeAgentOptions) + setattr(fake_sdk, "query", fake_query) + monkeypatch.setitem(__import__("sys").modules, "claude_agent_sdk", fake_sdk) + + await ClaudeCodeProvider().execute("hello", {}) + + assert captured["options"].kwargs["model"] == "sonnet" + + @pytest.mark.asyncio async def test_execute_extracts_result_from_subtype_success(monkeypatch): """Claude Agent SDK sends subtype='success' instead of type='result'.""" diff --git a/sdk/python/tests/test_harness_runner.py b/sdk/python/tests/test_harness_runner.py index cbb689ff0..033f428e6 100644 --- a/sdk/python/tests/test_harness_runner.py +++ b/sdk/python/tests/test_harness_runner.py @@ -265,10 +265,21 @@ async def test_run_with_schema_injects_prompt_suffix_and_parses_output(tmp_path) @pytest.mark.asyncio -async def test_run_raises_when_no_provider_set(tmp_path): +async def test_run_defaults_to_aforge_when_no_provider_set(tmp_path, monkeypatch): + monkeypatch.delenv("AGENTFIELD_HARNESS_PROVIDER", raising=False) runner = HarnessRunner() - with pytest.raises(ValueError, match="No harness provider specified"): - await runner.run("hello", cwd=str(tmp_path)) + provider = MockProvider([RawResult(result="ok")]) + captured: dict[str, str] = {} + + def fake_build_provider(config): + captured["provider"] = config.provider + return provider + + with patch("agentfield.harness._runner.build_provider", fake_build_provider): + result = await runner.run("hello", cwd=str(tmp_path)) + + assert captured["provider"] == "aforge" + assert result.result == "ok" @pytest.mark.asyncio diff --git a/sdk/python/tests/test_harness_types.py b/sdk/python/tests/test_harness_types.py index 26f206c45..1a95db42a 100644 --- a/sdk/python/tests/test_harness_types.py +++ b/sdk/python/tests/test_harness_types.py @@ -1,23 +1,21 @@ # pyright: reportMissingImports=false import pytest -from pydantic import ValidationError - from agentfield.harness._result import HarnessResult, Metrics, RawResult from agentfield.harness.providers._factory import build_provider from agentfield.types import HarnessConfig -def test_harness_config_provider_required(): - with pytest.raises(ValidationError): - HarnessConfig() +def test_harness_config_provider_defaults_to_aforge(monkeypatch): + monkeypatch.delenv("AGENTFIELD_HARNESS_PROVIDER", raising=False) + assert HarnessConfig().provider == "aforge" def test_harness_config_defaults(): cfg = HarnessConfig(provider="codex") assert cfg.provider == "codex" - assert cfg.model == "sonnet" + assert cfg.model is None assert cfg.max_turns == 30 assert cfg.max_budget_usd is None assert cfg.max_retries == 3 diff --git a/sdk/python/tests/test_types.py b/sdk/python/tests/test_types.py index 9ee9805ef..03bfeb696 100644 --- a/sdk/python/tests/test_types.py +++ b/sdk/python/tests/test_types.py @@ -470,7 +470,7 @@ class TestHarnessConfig: def test_defaults(self): hc = HarnessConfig(provider="claude-code") assert hc.provider == "claude-code" - assert hc.model == "sonnet" + assert hc.model is None assert hc.max_turns == 30 assert hc.max_budget_usd is None assert hc.max_retries == 3 @@ -493,9 +493,9 @@ def test_custom_values(self): assert hc.tools == ["Bash"] assert hc.permission_mode == "auto" - def test_provider_required(self): - with pytest.raises(Exception): - HarnessConfig() # type: ignore[call-arg] + def test_provider_defaults_to_aforge(self, monkeypatch): + monkeypatch.delenv("AGENTFIELD_HARNESS_PROVIDER", raising=False) + assert HarnessConfig().provider == "aforge" def test_json_roundtrip(self): hc = HarnessConfig(provider="gemini", model="gemini-2.5-flash") From 40c31279e850593fef312e9d8fc9f0e1cfd32448 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Mon, 17 Aug 2026 11:21:34 -0400 Subject: [PATCH 07/11] feat(harness): make aforge the default provider in the Go SDK Runner.Run no longer errors on an empty Options.Provider. BuildProvider and Run both route the name through ResolveProviderName, which applies the same precedence as the Python SDK: explicit value > AGENTFIELD_HARNESS_PROVIDER > DefaultProvider ("aforge"). The resolved name is written back onto the options so error messages and provider construction see the real provider. Model stays empty by default and means "the provider's own default"; the Harness doc example no longer suggests a Claude-specific "sonnet". Co-Authored-By: Claude Fable 5 --- sdk/go/agent/harness.go | 12 ++++--- sdk/go/agent/harness_test.go | 22 +++++++----- sdk/go/harness/factory.go | 20 ++++++++++- sdk/go/harness/factory_test.go | 48 +++++++++++++++++++++++++ sdk/go/harness/provider.go | 15 +++++++- sdk/go/harness/runner.go | 7 +--- sdk/go/harness/runner_invariant_test.go | 1 - sdk/go/harness/runner_test.go | 14 +++++--- 8 files changed, 111 insertions(+), 28 deletions(-) create mode 100644 sdk/go/harness/factory_test.go diff --git a/sdk/go/agent/harness.go b/sdk/go/agent/harness.go index 2783ef09c..f39d06970 100644 --- a/sdk/go/agent/harness.go +++ b/sdk/go/agent/harness.go @@ -10,10 +10,14 @@ import ( // providing lazy initialization and a convenience Harness() method. // HarnessConfig configures the default harness runner for the agent. type HarnessConfig struct { - // Provider is the default provider: "aforge", "claude-code", "codex", "gemini", or "opencode". + // Provider is the default provider: "aforge", "claude-code", "codex", + // "gemini", or "opencode". When empty, AGENTFIELD_HARNESS_PROVIDER + // overrides the default, "aforge" (AgentField's native harness). An + // explicit value always wins. Provider string - // Model is the default model identifier. It may carry a + // Model is the default model identifier. Empty means the provider's own + // default. It may carry a // reasoning-effort variant after a "#" separator (e.g. // "openrouter/z-ai/glm-5.2#high"). Model string @@ -86,9 +90,7 @@ func (a *Agent) HarnessRunner() *harness.Runner { // } // var result ReviewResult // schema, _ := harness.StructToJSONSchema(result) -// hr, err := agent.Harness(ctx, "Review this code...", schema, &result, harness.Options{ -// Model: "sonnet", -// }) +// hr, err := agent.Harness(ctx, "Review this code...", schema, &result, harness.Options{}) func (a *Agent) Harness(ctx context.Context, prompt string, schema map[string]any, dest any, opts harness.Options) (*harness.Result, error) { result, err := a.HarnessRunner().Run(ctx, prompt, schema, dest, opts) if err == nil { diff --git a/sdk/go/agent/harness_test.go b/sdk/go/agent/harness_test.go index 78b51fc76..98bdb269d 100644 --- a/sdk/go/agent/harness_test.go +++ b/sdk/go/agent/harness_test.go @@ -4,6 +4,7 @@ import ( "context" "io" "log" + "path/filepath" "testing" "github.com/Agent-Field/agentfield/sdk/go/harness" @@ -112,28 +113,31 @@ func TestHarnessRunner_ConcurrentAccess(t *testing.T) { } func TestHarness_ErrorWithoutProvider(t *testing.T) { - // Harness() should fail when no provider is configured. - // The runner will return an error about a missing provider. + // With no provider configured, Harness() defaults to aforge and reaches + // provider execution. A missing binary may fail, but provider resolution does not. + t.Setenv(harness.ProviderEnvVar, "") a := newTestAgentForHarness(t) - _, err := a.Harness(context.Background(), "do something", nil, nil, harness.Options{}) - assert.Error(t, err) - assert.Contains(t, err.Error(), "provider") + missingAforge := filepath.Join(t.TempDir(), "missing-aforge") + result, err := a.Harness(context.Background(), "do something", nil, nil, harness.Options{BinPath: missingAforge}) + require.NoError(t, err) + require.NotNil(t, result) + assert.True(t, result.IsError) + assert.Contains(t, result.ErrorMessage, "missing-aforge") + assert.NotContains(t, result.ErrorMessage, "no harness provider specified") } func TestHarness_PassesOptsToRunner(t *testing.T) { // Verify that per-call Options are forwarded to the runner. // Using a non-existent provider triggers a provider-build error, - // which confirms the Options reached Run() (otherwise we'd get - // the "no harness provider specified" error instead). + // which confirms the Options reached Run(). a := newTestAgentForHarness(t) _, err := a.Harness(context.Background(), "test", nil, nil, harness.Options{ Provider: "nonexistent-provider", }) assert.Error(t, err) - // Should be a provider-build error, NOT the "no harness provider specified" error - assert.NotContains(t, err.Error(), "no harness provider specified") + assert.Contains(t, err.Error(), "unknown harness provider") } func TestHarnessConfig_PartialOverride(t *testing.T) { diff --git a/sdk/go/harness/factory.go b/sdk/go/harness/factory.go index 9d07474b6..9b8a0d3d0 100644 --- a/sdk/go/harness/factory.go +++ b/sdk/go/harness/factory.go @@ -1,10 +1,28 @@ package harness -import "fmt" +import ( + "fmt" + "os" + "strings" +) + +// ResolveProviderName applies harness provider precedence: an explicit name +// wins, then AGENTFIELD_HARNESS_PROVIDER, then DefaultProvider ("aforge"). +// Blank / whitespace-only values are treated as unset. +func ResolveProviderName(name string) string { + if trimmed := strings.TrimSpace(name); trimmed != "" { + return trimmed + } + if envName := strings.TrimSpace(os.Getenv(ProviderEnvVar)); envName != "" { + return envName + } + return DefaultProvider +} // BuildProvider creates a Provider instance for the given provider name. // Supported providers: "aforge", "claude-code", "codex", "gemini", "opencode". func BuildProvider(name string, binPath string) (Provider, error) { + name = ResolveProviderName(name) switch name { case ProviderAforge: return NewAforgeProvider(binPath), nil diff --git a/sdk/go/harness/factory_test.go b/sdk/go/harness/factory_test.go new file mode 100644 index 000000000..be480ed4b --- /dev/null +++ b/sdk/go/harness/factory_test.go @@ -0,0 +1,48 @@ +package harness + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResolveProviderName(t *testing.T) { + tests := []struct { + name string + explicit string + env string + want string + }{ + {name: "default", want: ProviderAforge}, + {name: "environment", env: ProviderCodex, want: ProviderCodex}, + {name: "explicit wins", explicit: ProviderGemini, env: ProviderCodex, want: ProviderGemini}, + {name: "blank explicit", explicit: " ", want: ProviderAforge}, + {name: "blank environment", env: " ", want: ProviderAforge}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv(ProviderEnvVar, tt.env) + assert.Equal(t, tt.want, ResolveProviderName(tt.explicit)) + }) + } +} + +func TestBuildProvider_DefaultsToAforge(t *testing.T) { + t.Setenv(ProviderEnvVar, "") + provider, err := BuildProvider("", "") + require.NoError(t, err) + require.NotNil(t, provider) + _, ok := provider.(*AforgeProvider) + assert.True(t, ok) +} + +func TestBuildProvider_RejectsUnknownName(t *testing.T) { + provider, err := BuildProvider("nope", "") + assert.Nil(t, provider) + require.Error(t, err) + assert.Contains(t, err.Error(), "nope") + assert.True(t, strings.Contains(err.Error(), "supported: aforge, claude-code, codex, gemini, opencode")) +} diff --git a/sdk/go/harness/provider.go b/sdk/go/harness/provider.go index 2e7b499f5..f986f2240 100644 --- a/sdk/go/harness/provider.go +++ b/sdk/go/harness/provider.go @@ -15,6 +15,17 @@ const ( ProviderGemini = "gemini" ) +const ( + // DefaultProvider is the harness provider used when no provider is + // configured explicitly and AGENTFIELD_HARNESS_PROVIDER is unset. + // AForge is AgentField's native coding harness. + DefaultProvider = ProviderAforge + + // ProviderEnvVar overrides the default provider process-wide. An + // explicitly configured provider still wins over it. + ProviderEnvVar = "AGENTFIELD_HARNESS_PROVIDER" +) + // Provider is the interface that CLI-based harness providers implement. // Each provider knows how to invoke a specific coding agent (opencode, // claude-code, etc.) and return a RawResult. @@ -25,7 +36,9 @@ type Provider interface { // Options control a single harness invocation. Fields are optional; // zero values mean "use default". type Options struct { - // Provider name: "aforge", "opencode", "claude-code", "codex", or "gemini". + // Provider name: "aforge", "opencode", "claude-code", "codex", or + // "gemini". An explicit value wins over AGENTFIELD_HARNESS_PROVIDER; + // when both are empty, the provider defaults to "aforge". Provider string // Model identifier passed to the coding agent. It may carry a diff --git a/sdk/go/harness/runner.go b/sdk/go/harness/runner.go index cb4f8b7ba..23a71f01b 100644 --- a/sdk/go/harness/runner.go +++ b/sdk/go/harness/runner.go @@ -52,12 +52,7 @@ type schemaAware interface { // is non-nil, the validated output is unmarshalled into it. func (r *Runner) Run(ctx context.Context, prompt string, schema map[string]any, dest any, overrides Options) (*Result, error) { opts := r.mergeOptions(overrides) - - if opts.Provider == "" { - return nil, fmt.Errorf( - "no harness provider specified: set Provider in runner defaults or pass it to Run()", - ) - } + opts.Provider = ResolveProviderName(opts.Provider) provider, err := r.buildProvider(opts) if err != nil { diff --git a/sdk/go/harness/runner_invariant_test.go b/sdk/go/harness/runner_invariant_test.go index 11251f4f6..ca138c1d7 100644 --- a/sdk/go/harness/runner_invariant_test.go +++ b/sdk/go/harness/runner_invariant_test.go @@ -198,7 +198,6 @@ func TestInvariant_Runner_ProviderFactoryExhaustiveness(t *testing.T) { // unknown provider names return an error with a non-nil error value. func TestInvariant_Runner_ProviderFactoryUnknownReturnsError(t *testing.T) { unknownNames := []string{ - "", "nonexistent", "gpt-4", "anthropic", diff --git a/sdk/go/harness/runner_test.go b/sdk/go/harness/runner_test.go index bf95c5b39..f027bd4bb 100644 --- a/sdk/go/harness/runner_test.go +++ b/sdk/go/harness/runner_test.go @@ -37,11 +37,15 @@ func TestRunner_Run_NoSchema(t *testing.T) { runner := NewRunner(Options{Provider: "opencode"}) // We can't easily test with real providers, so test the merge/validation logic - t.Run("missing provider", func(t *testing.T) { - r := NewRunner(Options{}) - _, err := r.Run(context.Background(), "test", nil, nil, Options{}) - assert.Error(t, err) - assert.Contains(t, err.Error(), "no harness provider") + t.Run("empty provider defaults to aforge", func(t *testing.T) { + t.Setenv(ProviderEnvVar, "") + r := NewRunner(Options{BinPath: filepath.Join(t.TempDir(), "missing-aforge")}) + result, err := r.Run(context.Background(), "test", nil, nil, Options{}) + require.NoError(t, err) + require.NotNil(t, result) + assert.True(t, result.IsError) + assert.Contains(t, result.ErrorMessage, "missing-aforge") + assert.NotContains(t, result.ErrorMessage, "no harness provider specified") }) t.Run("unknown provider", func(t *testing.T) { From 76479fbeba2fa3ccf872e4bd56d089180765e36d Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Mon, 17 Aug 2026 11:21:41 -0400 Subject: [PATCH 08/11] docs(harness): lead with AForge as the default harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrites the harness-providers lead so the zero-setup path is the headline: AForge is the default, `af aforge ensure` installs it alongside `af`, and picking Claude Code / Codex / Gemini CLI / OpenCode is an override of one option rather than a prerequisite. Documents the provider precedence chain (explicit > AGENTFIELD_HARNESS_PROVIDER > aforge) and that an unset model means the provider's own default. Drops the "go build -o aforge ./cmd/aforge" instruction — that repo is not public and is no longer how anyone gets the binary. Adds the grok row (Python SDK only) so the install table matches the supported provider set. Co-Authored-By: Claude Fable 5 --- README.md | 11 ++-- docs/harness-providers.md | 131 ++++++++++++++++++++++++++------------ 2 files changed, 97 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index e65e6d51d..e5cd11712 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ @@ -251,7 +251,7 @@ Two examples already run at this load. The [deep-research engine](https://agentf - **[Reasoners & Skills](https://agentfield.ai/docs/build/building-blocks/reasoners?utm_source=github-readme&utm_campaign=github-readme&utm_id=github-readme-reasoners)** - `@app.reasoner()` for AI judgment, `@app.skill()` for deterministic code - **[Structured AI](https://agentfield.ai/docs/reference/sdks/python?utm_source=github-readme&utm_campaign=github-readme&utm_id=github-readme-structured-ai)** - `app.ai(schema=MyModel)` → typed Pydantic/Zod output from any LLM -- **[Harness](https://agentfield.ai/docs/build/intelligence/harness?utm_source=github-readme&utm_campaign=github-readme&utm_id=github-readme-harness)** - `app.harness("Fix the bug")` dispatches multi-turn tasks to Claude Code, Codex, Gemini CLI, or OpenCode +- **[Harness](https://agentfield.ai/docs/build/intelligence/harness?utm_source=github-readme&utm_campaign=github-readme&utm_id=github-readme-harness)** - `app.harness("Fix the bug")` dispatches multi-turn tasks to AForge, AgentField's own coding harness — no setup. Add `provider="claude-code"` (or `codex`, `gemini`, `opencode`) to orchestrate someone else's. - **[Cross-Agent Calls](https://agentfield.ai/docs/build/coordination/cross-agent-calls?utm_source=github-readme&utm_campaign=github-readme&utm_id=github-readme-cross-agent-calls)** - `app.call("other-agent.func")` routes through the control plane with full tracing - **[Discovery](https://agentfield.ai/docs/reference/sdks/python?utm_source=github-readme&utm_campaign=github-readme&utm_id=github-readme-discovery)** - `app.discover(tags=["ml*"])` finds agents and capabilities across the mesh. `tools="discover"` lets LLMs auto-invoke them. - **[Memory](https://agentfield.ai/docs/build/coordination/shared-memory?utm_source=github-readme&utm_campaign=github-readme&utm_id=github-readme-memory)** - `app.memory.set()` / `.get()` / `.similarity_search()` - KV + vector search, four scopes, no Redis needed @@ -283,7 +283,8 @@ Two examples already run at this load. The [deep-research engine](https://agentf | Feature | How | |---|---| | Structured output (Pydantic/Zod) | `app.ai(schema=MyModel)` | -| Multi-turn coding agents | `app.harness("task", provider="claude-code")` | +| Multi-turn coding agents | `app.harness("task")` — AForge by default | +| Orchestrate another harness | `app.harness("task", provider="claude-code")` | | LLM auto-discovers agents and tools | `app.ai(tools="discover")` | | Multimodal (text, image, audio) | `app.ai("Describe", image_url="...")` | | Streaming responses | `app.ai("...", stream=True)` | @@ -379,7 +380,9 @@ Two examples already run at this load. The [deep-research engine](https://agentf | Feature | How | |---|---| -| 4 providers | Claude Code, Codex, Gemini CLI, OpenCode | +| Zero-setup default harness | AForge (`aforge`), installed alongside `af` | +| Swap the worker, keep the loop | `provider="claude-code"` \| `"codex"` \| `"gemini"` \| `"opencode"` | +| Fleet-wide default override | `AGENTFIELD_HARNESS_PROVIDER=codex` | | Schema-constrained output | `schema=ResultModel` (Pydantic/Zod) | | Cost capping | `max_budget_usd=3.0` | | Turn limiting | `max_turns=100` | diff --git a/docs/harness-providers.md b/docs/harness-providers.md index a39a89f90..4598292b7 100644 --- a/docs/harness-providers.md +++ b/docs/harness-providers.md @@ -1,18 +1,83 @@ # Harness providers -AgentField harness providers run external coding agents. Install the provider -wrapper you need, install its CLI when required, and verify the runtime before -starting a workflow. +`app.harness()` hands a task to a coding agent — a multi-turn worker that reads, +writes, and edits files, then reports back through the same structured-output +contract as `app.ai()`. AgentField ships its own harness, **AForge**, and it is +the default: a call with no provider set runs `aforge`. Naming a different +provider swaps the worker without changing the surrounding loop, which is how +you orchestrate Claude Code, Codex, Gemini CLI, or OpenCode from a reasoner. + +## Default: AForge + +The `aforge` binary is provisioned alongside the `af` CLI — the curl installer, +the desktop app, and the published Docker images all ship it. To install or +repair it on demand: + +```bash +af aforge ensure +``` + +Set `OPENROUTER_API_KEY`, then call the harness with nothing else configured: + +```python +result = await app.harness("Fix the failing test in tests/test_auth.py", schema=Report) +``` + +```go +result, err := agent.Harness(ctx, task, schema, &dest, harness.Options{Cwd: repoRoot}) +``` + +```ts +const result = await app.harness(task, { schema }); +``` + +The model defaults to AForge's own default. Set `AFORGE_MODEL` to change it +process-wide, or pass `model=` per call. + +Verify the runtime before a paid run: + +```bash +af harness doctor --provider aforge +``` + +## Choosing a different provider + +Provider selection follows one precedence chain: + +| Order | Source | Example | +| --- | --- | --- | +| 1 | Explicit value on the call or in the agent's harness config | `app.harness(task, provider="codex")` | +| 2 | `AGENTFIELD_HARNESS_PROVIDER` environment variable | `AGENTFIELD_HARNESS_PROVIDER=claude-code` | +| 3 | Default | `aforge` | + +Same loop code, different worker: + +```python +# AForge — nothing to configure +report = await app.harness(task, schema=Report) + +# Orchestrate Claude Code instead +report = await app.harness(task, schema=Report, provider="claude-code") + +# ...or Codex, Gemini CLI, OpenCode +report = await app.harness(task, schema=Report, provider="codex") +``` + +The same override exists in every SDK — `harness.Options{Provider: harness.ProviderCodex}` +in Go, `{ provider: 'codex' }` in TypeScript — and an agent-wide default can be +set once on the agent's harness config (`HarnessConfig(provider="codex")` in +Python, `agent.HarnessConfig{Provider: "codex"}` in Go). ## Install -| Provider | Python extra | Required CLI | Authentication | -| --- | --- | --- | --- | -| `aforge` | None | `aforge` (`af aforge ensure`) | `OPENROUTER_API_KEY` | -| Claude Code | `agentfield[harness-claude]` | Bundled by `claude-agent-sdk` | Claude login or `ANTHROPIC_API_KEY` | -| Codex | `agentfield[harness-codex]` | `codex` | Codex login or `OPENAI_API_KEY` | -| Gemini | None | `gemini` | Gemini login, `GEMINI_API_KEY`, or `GOOGLE_API_KEY` | -| OpenCode | `agentfield[harness-opencode]` | `opencode` | Provider credentials configured in OpenCode | +| Provider | Install | Python extra | Required CLI | Authentication | +| --- | --- | --- | --- | --- | +| `aforge` (default) | `af aforge ensure` (shipped with `af`) | None | `aforge` | `OPENROUTER_API_KEY` | +| `claude-code` | `pip install 'agentfield[harness-claude]'` | `agentfield[harness-claude]` | Bundled by `claude-agent-sdk` | Claude login or `ANTHROPIC_API_KEY` | +| `codex` | `npm install -g @openai/codex` | `agentfield[harness-codex]` | `codex` | Codex login or `OPENAI_API_KEY` | +| `gemini` | `npm install -g @google/gemini-cli` | None | `gemini` | Gemini login, `GEMINI_API_KEY`, or `GOOGLE_API_KEY` | +| `opencode` | `curl -fsSL https://opencode.ai/install \| bash` | `agentfield[harness-opencode]` | `opencode` | Provider credentials configured in OpenCode | +| `grok` | Install the Grok Build CLI, then `grok login` | None | `grok` | `XAI_API_KEY` | Install every Python wrapper with: @@ -34,47 +99,31 @@ The pinned build, its download host and the opt-out are documented under [docs/ENVIRONMENT_VARIABLES.md](ENVIRONMENT_VARIABLES.md). The extras install Python wrappers. They do not replace the runtime preflight: -Aforge and Gemini are CLI-only, and Codex or OpenCode may still require a -separately available executable depending on the wrapper and platform. +AForge and Gemini are CLI-only, and Codex or OpenCode may still require a +separately available executable depending on the wrapper and platform. `grok` +is available in the Python SDK only. -### Aforge drop-in usage +### AForge adapter contract -Aforge is registered as `aforge` in the Python, Go, and TypeScript SDKs. The +AForge is registered as `aforge` in the Python, Go, and TypeScript SDKs. The adapters default to the direct non-interactive contract, `aforge exec --json`, -send the task over stdin, and map Aforge's usage ledger into AgentField turns, +send the task over stdin, and map AForge's usage ledger into AgentField turns, token counts, and cost metrics. Set `AGENTFIELD_AFORGE_COMMAND=do` to opt into -the routed `aforge do --json --yes-spend` workflow instead. Install a current -aforge-v2 binary on `PATH`, set `OPENROUTER_API_KEY`, then select the provider: - -```python -result = await app.harness(task, provider="aforge", model="openrouter/z-ai/glm-5.2") -``` - -```go -result, err := harness.NewRunner(harness.Options{Provider: harness.ProviderAforge}).Run( - ctx, task, nil, nil, harness.Options{Cwd: repoRoot}, -) -``` - -```ts -const result = await new HarnessRunner().run(task, { - provider: 'aforge', - model: 'openrouter/z-ai/glm-5.2', - cwd: repoRoot, -}); -``` +the routed `aforge do --json --yes-spend` workflow instead. -Set `AFORGE_MAX_CONCURRENT` to cap simultaneous Aforge subprocesses. The +Set `AFORGE_MAX_CONCURRENT` to cap simultaneous AForge subprocesses. The default is 8. `AGENTFIELD_HARNESS_TIMEOUT_SECONDS` is the outer watchdog; each -adapter gives Aforge a five-second landing window to emit its exit-2 timeout +adapter gives AForge a five-second landing window to emit its exit-2 timeout envelope. Schema runs use a unique output directory per invocation so parallel -jobs can safely share a checkout. Set `AFORGE_BIN` to an absolute -path when the unreleased binary is not installed on `PATH`. +jobs can safely share a checkout. Set `AFORGE_BIN` to an absolute path when the +binary is installed somewhere off `PATH`. ## Model selection and reasoning-effort variants -Every provider accepts a `model` option on `.harness()` calls. The model string -may carry a reasoning-effort variant after a `#` separator: +Every provider accepts a `model` option on `.harness()` calls. Leaving it unset +uses the provider's own default — AForge picks its own model, `claude-code` +keeps using `sonnet`. The model string may carry a reasoning-effort variant +after a `#` separator: ```python result = await app.harness( From 2cb7e311f4190d385c8d28eec3335626f3d07a8f Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Mon, 17 Aug 2026 11:23:22 -0400 Subject: [PATCH 09/11] docs(harness): correct the v2 design doc's "provider is required" rule The design doc still asserted that HarnessConfig.provider has no implicit default and that a call without one raises. Both are false now: provider resolves through explicit > AGENTFIELD_HARNESS_PROVIDER > "aforge", and model defaults to the provider's own rather than "sonnet". Co-Authored-By: Claude Fable 5 --- docs/design/harness-v2-design.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/design/harness-v2-design.md b/docs/design/harness-v2-design.md index f3e8fbd90..dd2b998b3 100644 --- a/docs/design/harness-v2-design.md +++ b/docs/design/harness-v2-design.md @@ -26,8 +26,8 @@ app = Agent( node_id="my-agent", ai_config=AIConfig(model="openai/gpt-4o"), harness_config=HarnessConfig( - provider="claude-code", # Required — no implicit default - model="sonnet", + provider="claude-code", # Optional — defaults to "aforge" + model="sonnet", # Optional — defaults to the provider's own ), ) ``` @@ -38,8 +38,8 @@ import { Agent } from '@agentfield/sdk'; const agent = new Agent({ nodeId: 'my-agent', harnessConfig: { - provider: 'claude-code', // Required - model: 'sonnet', + provider: 'claude-code', // Optional — defaults to 'aforge' + model: 'sonnet', // Optional — defaults to the provider's own }, }); ``` @@ -112,7 +112,7 @@ app = Agent(node_id="minimal-agent") result = await app.harness( "Fix the bug", - provider="gemini", # Required when no harness_config + provider="gemini", # Optional — omit to use the default, "aforge" model="flash", cwd="/my/project", ) @@ -350,12 +350,12 @@ Layer 4: Full retry (expensive, last resort class HarnessConfig(BaseModel): """Configuration for coding agent harness calls. - Provider is required — there is no implicit default. - All other fields have sensible defaults that can be overridden per-call. + Provider defaults to "aforge", AgentField's native harness. + All fields have sensible defaults that can be overridden per-call. """ - # Provider selection (required) - provider: str # "claude-code" | "codex" | "gemini" | "opencode" - model: str = "sonnet" + # Provider selection: explicit > AGENTFIELD_HARNESS_PROVIDER > "aforge" + provider: str = "aforge" # | "claude-code" | "codex" | "gemini" | "opencode" + model: Optional[str] = None # None → the provider's own default # Execution limits max_turns: int = 30 @@ -426,7 +426,8 @@ interface HarnessConfig { 1. HarnessConfig defaults (set at agent construction) 2. Per-call overrides (passed to .harness() method) → Per-call values win over HarnessConfig defaults - → If no HarnessConfig AND no per-call provider → raise error + → If no HarnessConfig AND no per-call provider → AGENTFIELD_HARNESS_PROVIDER, + then the default provider "aforge" ``` --- From eaaea0e45e0ffb0fb3b19e9fcb3d9c40a89c0c93 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Mon, 17 Aug 2026 11:24:10 -0400 Subject: [PATCH 10/11] feat(harness): make aforge the default provider in the TypeScript SDK HarnessRunner.run no longer throws when nothing sets a provider. Both the runner and buildProvider route the name through resolveProviderName, which applies the same precedence as the Python and Go SDKs: explicit value > AGENTFIELD_HARNESS_PROVIDER > DEFAULT_HARNESS_PROVIDER ("aforge"). The resolved name is written back onto the options so providers and error messages see the real provider. HarnessConfig.provider becomes optional to match, so `{}` is a complete config. Co-Authored-By: Claude Fable 5 --- .../src/harness/providers/factory.ts | 34 ++++++++--- sdk/typescript/src/harness/runner.ts | 9 +-- sdk/typescript/src/harness/types.ts | 14 ++++- .../tests/harness_provider_factory.test.ts | 58 +++++++++++++++++++ sdk/typescript/tests/harness_runner.test.ts | 23 +++++++- 5 files changed, 121 insertions(+), 17 deletions(-) create mode 100644 sdk/typescript/tests/harness_provider_factory.test.ts diff --git a/sdk/typescript/src/harness/providers/factory.ts b/sdk/typescript/src/harness/providers/factory.ts index 53a2a9d74..ebf9d97c9 100644 --- a/sdk/typescript/src/harness/providers/factory.ts +++ b/sdk/typescript/src/harness/providers/factory.ts @@ -2,32 +2,50 @@ import type { HarnessProvider } from './base.js'; import type { HarnessConfig } from '../types.js'; export const SUPPORTED_PROVIDERS = new Set(['aforge', 'claude-code', 'codex', 'gemini', 'opencode']); +export const DEFAULT_HARNESS_PROVIDER = 'aforge'; +export const HARNESS_PROVIDER_ENV_VAR = 'AGENTFIELD_HARNESS_PROVIDER'; + +/** + * Applies harness provider precedence: an explicit name wins, then + * AGENTFIELD_HARNESS_PROVIDER, then DEFAULT_HARNESS_PROVIDER ("aforge"). + * Blank / whitespace-only values are treated as unset. + */ +export function resolveProviderName(explicit?: string): string { + const explicitName = explicit?.trim(); + if (explicitName) { + return explicitName; + } + + const envName = process.env[HARNESS_PROVIDER_ENV_VAR]?.trim(); + return envName || DEFAULT_HARNESS_PROVIDER; +} export async function buildProvider(config: HarnessConfig): Promise { - if (!SUPPORTED_PROVIDERS.has(config.provider)) { + const provider = resolveProviderName(config.provider); + if (!SUPPORTED_PROVIDERS.has(provider)) { throw new Error( - `Unknown harness provider: "${config.provider}". Supported: ${[...SUPPORTED_PROVIDERS].sort().join(', ')}` + `Unknown harness provider: "${provider}". Supported: ${[...SUPPORTED_PROVIDERS].sort().join(', ')}` ); } - if (config.provider === 'aforge') { + if (provider === 'aforge') { const { AforgeProvider } = await import('./aforge.js'); return new AforgeProvider(config.aforgeBin ?? 'aforge'); } - if (config.provider === 'claude-code') { + if (provider === 'claude-code') { const { ClaudeCodeProvider } = await import('./claude.js'); return new ClaudeCodeProvider(); } - if (config.provider === 'codex') { + if (provider === 'codex') { const { CodexProvider } = await import('./codex.js'); return new CodexProvider(config.codexBin ?? 'codex'); } - if (config.provider === 'gemini') { + if (provider === 'gemini') { const { GeminiProvider } = await import('./gemini.js'); return new GeminiProvider(config.geminiBin ?? 'gemini'); } - if (config.provider === 'opencode') { + if (provider === 'opencode') { const { OpenCodeProvider } = await import('./opencode.js'); return new OpenCodeProvider(config.opencodeBin ?? 'opencode'); } - throw new Error(`Provider "${config.provider}" is not yet implemented.`); + throw new Error(`Provider "${provider}" is not yet implemented.`); } diff --git a/sdk/typescript/src/harness/runner.ts b/sdk/typescript/src/harness/runner.ts index edcb3fd08..722ed80b1 100644 --- a/sdk/typescript/src/harness/runner.ts +++ b/sdk/typescript/src/harness/runner.ts @@ -1,7 +1,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { buildPromptSuffix, cleanupTempFiles, getOutputPath, parseAndValidate } from './schema.js'; -import { buildProvider } from './providers/factory.js'; +import { buildProvider, resolveProviderName } from './providers/factory.js'; import type { HarnessProvider } from './providers/base.js'; import { createHarnessResult, @@ -56,10 +56,7 @@ export class HarnessRunner { public async run(prompt: string, options: HarnessOptions = {}) { const { schema, ...rest } = options; const resolved = this.resolveOptions(this.config, rest); - - if (!resolved.provider) { - throw new Error("No harness provider specified. Set 'provider' in HarnessConfig or pass it to .harness() call."); - } + resolved.provider = resolveProviderName(resolved.provider); const provider = await this.buildProvider(resolved.provider, resolved); const cwd = resolved.cwd ?? '.'; @@ -222,7 +219,7 @@ export class HarnessRunner { private async buildProvider(providerName: string, options: RunnerOptions): Promise { const { provider: _, ...rest } = options; - return buildProvider({ provider: providerName as HarnessConfig['provider'], ...rest }); + return buildProvider({ provider: providerName as NonNullable, ...rest }); } private computeBackoffDelay( diff --git a/sdk/typescript/src/harness/types.ts b/sdk/typescript/src/harness/types.ts index a04f98f35..e1385e8fa 100644 --- a/sdk/typescript/src/harness/types.ts +++ b/sdk/typescript/src/harness/types.ts @@ -1,5 +1,11 @@ export interface HarnessConfig { - provider: 'aforge' | 'claude-code' | 'codex' | 'gemini' | 'opencode'; + /** + * Coding agent provider. Defaults to `aforge`, AgentField's native harness. + * When unset, `AGENTFIELD_HARNESS_PROVIDER` is consulted before the default. + * An explicit value always wins. + */ + provider?: 'aforge' | 'claude-code' | 'codex' | 'gemini' | 'opencode'; + /** Model identifier. Empty means the provider's own default. */ model?: string; /** * Provider-specific reasoning-effort variant (e.g. `high`, `minimal`). @@ -25,7 +31,13 @@ export interface HarnessConfig { } export interface HarnessOptions { + /** + * Coding agent provider. Defaults to `aforge`, AgentField's native harness. + * When unset, `AGENTFIELD_HARNESS_PROVIDER` is consulted before the default. + * An explicit value always wins. + */ provider?: string; + /** Model identifier. Empty means the provider's own default. */ model?: string; /** * Provider-specific reasoning-effort variant (e.g. `high`, `minimal`). diff --git a/sdk/typescript/tests/harness_provider_factory.test.ts b/sdk/typescript/tests/harness_provider_factory.test.ts new file mode 100644 index 000000000..236af0d35 --- /dev/null +++ b/sdk/typescript/tests/harness_provider_factory.test.ts @@ -0,0 +1,58 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { AforgeProvider } from '../src/harness/providers/aforge.js'; +import { + buildProvider, + HARNESS_PROVIDER_ENV_VAR, + resolveProviderName, +} from '../src/harness/providers/factory.js'; +import type { HarnessConfig } from '../src/harness/types.js'; + +let originalProviderEnv: string | undefined; + +beforeEach(() => { + originalProviderEnv = process.env[HARNESS_PROVIDER_ENV_VAR]; + delete process.env[HARNESS_PROVIDER_ENV_VAR]; +}); + +afterEach(() => { + if (originalProviderEnv === undefined) { + delete process.env[HARNESS_PROVIDER_ENV_VAR]; + } else { + process.env[HARNESS_PROVIDER_ENV_VAR] = originalProviderEnv; + } +}); + +describe('harness provider factory', () => { + it('defaults to aforge', () => { + expect(resolveProviderName(undefined)).toBe('aforge'); + }); + + it('honours the environment fallback', () => { + process.env[HARNESS_PROVIDER_ENV_VAR] = 'codex'; + expect(resolveProviderName(undefined)).toBe('codex'); + }); + + it('prefers an explicit provider over the environment', () => { + process.env[HARNESS_PROVIDER_ENV_VAR] = 'codex'; + expect(resolveProviderName('gemini')).toBe('gemini'); + }); + + it('treats blank explicit and environment values as unset', () => { + expect(resolveProviderName(' ')).toBe('aforge'); + + process.env[HARNESS_PROVIDER_ENV_VAR] = ' '; + expect(resolveProviderName(undefined)).toBe('aforge'); + }); + + it('builds the aforge provider when no provider is configured', async () => { + await expect(buildProvider({})).resolves.toBeInstanceOf(AforgeProvider); + }); + + it('rejects genuinely unknown providers with the supported list', async () => { + const config = { provider: 'nope' } as unknown as HarnessConfig; + await expect(buildProvider(config)).rejects.toThrow( + 'Unknown harness provider: "nope". Supported: aforge, claude-code, codex, gemini, opencode' + ); + }); +}); diff --git a/sdk/typescript/tests/harness_runner.test.ts b/sdk/typescript/tests/harness_runner.test.ts index cf3a903d3..304ce35b4 100644 --- a/sdk/typescript/tests/harness_runner.test.ts +++ b/sdk/typescript/tests/harness_runner.test.ts @@ -21,6 +21,7 @@ function makeTempDir(): string { afterEach(() => { vi.restoreAllMocks(); + vi.unstubAllEnvs(); for (const dir of tempDirs.splice(0, tempDirs.length)) { fs.rmSync(dir, { recursive: true, force: true }); } @@ -208,9 +209,27 @@ describe('harness runner', () => { expect(fs.existsSync(getOutputPath(cwd))).toBe(false); }); - it('run throws when no provider is configured', async () => { + it('run defaults to aforge when no provider is configured', async () => { + vi.stubEnv('AGENTFIELD_HARNESS_PROVIDER', ''); + const provider = new MockProvider(); + const buildProviderSpy = vi.spyOn(factory, 'buildProvider').mockResolvedValue(provider); const runner = new HarnessRunner(); - await expect(runner.run('hello', {})).rejects.toThrow(/No harness provider specified/); + await runner.run('hello', {}); + + expect(buildProviderSpy).toHaveBeenCalledWith(expect.objectContaining({ provider: 'aforge' })); + expect(provider.lastOptions?.provider).toBe('aforge'); + }); + + it('per-call provider wins over the runner default and environment', async () => { + vi.stubEnv('AGENTFIELD_HARNESS_PROVIDER', 'gemini'); + const provider = new MockProvider(); + const buildProviderSpy = vi.spyOn(factory, 'buildProvider').mockResolvedValue(provider); + const runner = new HarnessRunner({ provider: 'codex' }); + + await runner.run('hello', { provider: 'opencode' }); + + expect(buildProviderSpy).toHaveBeenCalledWith(expect.objectContaining({ provider: 'opencode' })); + expect(provider.lastOptions?.provider).toBe('opencode'); }); it('retries on transient error then succeeds', async () => { From dbb19b2d96c85689af0c04e746cb41afc7f08651 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Mon, 17 Aug 2026 11:26:26 -0400 Subject: [PATCH 11/11] fix(harness): stop pointing users at a private repo to get aforge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Go and TypeScript adapters told anyone hitting a missing binary to "Build it from https://github.com/Agent-Field/aforge-v2" — a repo they cannot open. AForge ships with `af`, so the message now names `af aforge ensure` and the AFORGE_BIN escape hatch, matching the Python install hint. Co-Authored-By: Claude Fable 5 --- sdk/go/harness/aforge.go | 2 +- sdk/typescript/src/harness/providers/aforge.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/go/harness/aforge.go b/sdk/go/harness/aforge.go index 8a85e1359..f23df3577 100644 --- a/sdk/go/harness/aforge.go +++ b/sdk/go/harness/aforge.go @@ -241,7 +241,7 @@ func (p *AforgeProvider) Execute(ctx context.Context, prompt string, options Opt if isExecNotFound(err) { return &RawResult{ IsError: true, - ErrorMessage: fmt.Sprintf("Aforge binary not found at '%s'. Build it from https://github.com/Agent-Field/aforge-v2", p.BinPath), + ErrorMessage: fmt.Sprintf("AForge binary not found at '%s'. Install it with `af aforge ensure`, or set AFORGE_BIN to its path.", p.BinPath), FailureType: FailureCrash, Metrics: Metrics{DurationAPIMS: apiMS}, }, nil diff --git a/sdk/typescript/src/harness/providers/aforge.ts b/sdk/typescript/src/harness/providers/aforge.ts index ef8e0ef48..5aa8667b5 100644 --- a/sdk/typescript/src/harness/providers/aforge.ts +++ b/sdk/typescript/src/harness/providers/aforge.ts @@ -269,7 +269,7 @@ export class AforgeProvider implements HarnessProvider { if (message.includes('ENOENT')) { return createRawResult({ isError: true, - errorMessage: `Aforge binary not found at '${this.bin}'. Build it from https://github.com/Agent-Field/aforge-v2`, + errorMessage: `AForge binary not found at '${this.bin}'. Install it with \`af aforge ensure\`, or set AFORGE_BIN to its path.`, failureType: 'crash', metrics: createMetrics({ durationApiMs: Date.now() - startApi }), });