From c0d70b3931cd93237de673344d0654b68a8a3c99 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Wed, 12 Aug 2026 16:26:03 -0400 Subject: [PATCH 01/20] Add Pi and OMP harness providers --- control-plane/internal/cli/doctor.go | 4 +- .../internal/cli/doctor_additional_test.go | 2 + .../internal/cli/doctor_probe_test.go | 2 + control-plane/internal/cli/harness_doctor.go | 4 +- .../internal/cli/harness_doctor_test.go | 20 ++ .../skillkit/skill_data/agentfield/SKILL.md | 2 +- .../references/primitives-snapshot.md | 4 +- .../agentfield/references/scaffold-recipe.md | 2 +- docs/harness-providers.md | 22 +- .../go_agent_nodes/cmd/harness_duo/README.md | 39 +++ .../go_agent_nodes/cmd/harness_duo/main.go | 179 ++++++++++ examples/go_agent_nodes/go.mod | 5 +- examples/go_agent_nodes/go.sum | 2 + sdk/go/agent/harness.go | 2 +- sdk/go/harness/factory.go | 10 +- sdk/go/harness/pi.go | 312 ++++++++++++++++++ sdk/go/harness/pi_test.go | 164 +++++++++ sdk/go/harness/provider.go | 6 +- sdk/go/harness/runner_test.go | 6 +- sdk/python/agentfield/agent.py | 32 +- .../agentfield/harness/_availability.py | 26 ++ sdk/python/agentfield/harness/_runner.py | 2 + .../agentfield/harness/providers/_factory.py | 19 +- sdk/python/agentfield/harness/providers/pi.py | 277 ++++++++++++++++ sdk/python/agentfield/types.py | 4 +- sdk/python/tests/test_harness_factory.py | 2 + sdk/python/tests/test_harness_provider_pi.py | 167 ++++++++++ sdk/python/tests/test_harness_types.py | 2 + sdk/python/tests/test_types.py | 2 + sdk/typescript/src/harness/cli.ts | 11 +- .../src/harness/providers/factory.ts | 10 +- sdk/typescript/src/harness/providers/index.ts | 1 + sdk/typescript/src/harness/providers/pi.ts | 232 +++++++++++++ sdk/typescript/src/harness/runner.ts | 7 +- sdk/typescript/src/harness/types.ts | 10 +- .../tests/harness_provider_pi.test.ts | 135 ++++++++ sdk/typescript/tests/harness_runner.test.ts | 4 + skills/agentfield/SKILL.md | 2 +- .../references/primitives-snapshot.md | 4 +- .../agentfield/references/scaffold-recipe.md | 2 +- 40 files changed, 1694 insertions(+), 44 deletions(-) create mode 100644 examples/go_agent_nodes/cmd/harness_duo/README.md create mode 100644 examples/go_agent_nodes/cmd/harness_duo/main.go create mode 100644 sdk/go/harness/pi.go create mode 100644 sdk/go/harness/pi_test.go create mode 100644 sdk/python/agentfield/harness/providers/pi.py create mode 100644 sdk/python/tests/test_harness_provider_pi.py create mode 100644 sdk/typescript/src/harness/providers/pi.ts create mode 100644 sdk/typescript/tests/harness_provider_pi.test.ts diff --git a/control-plane/internal/cli/doctor.go b/control-plane/internal/cli/doctor.go index 5ae306875..b1e81b60e 100644 --- a/control-plane/internal/cli/doctor.go +++ b/control-plane/internal/cli/doctor.go @@ -89,6 +89,8 @@ var harnessProviders = []struct { {Name: "codex", Binary: "codex", ProbeArgs: []string{"exec", "Say OK"}}, {Name: "gemini", Binary: "gemini", ProbeArgs: []string{"-p", "Say OK"}}, {Name: "opencode", Binary: "opencode", ProbeArgs: []string{"run", "Say OK"}}, + {Name: "pi", Binary: "pi", ProbeArgs: []string{"--print", "Say OK"}}, + {Name: "omp", Binary: "omp", ProbeArgs: []string{"--print", "Say OK"}}, } // harnessProbeTimeout bounds a single provider smoke test. Coding-agent CLIs @@ -119,7 +121,7 @@ func NewDoctorCommand() *cobra.Command { Long: `Doctor inspects the local environment and reports what's available for building AgentField multi-reasoner systems: - • Available harness provider CLIs (claude-code, codex, gemini, opencode) + • Available harness provider CLIs (claude-code, codex, gemini, opencode, pi, omp) • Provider API keys set in the environment (without leaking values) • Docker availability and whether the control-plane image is locally cached • Whether a local control plane is reachable diff --git a/control-plane/internal/cli/doctor_additional_test.go b/control-plane/internal/cli/doctor_additional_test.go index 602f0fdff..00d163e82 100644 --- a/control-plane/internal/cli/doctor_additional_test.go +++ b/control-plane/internal/cli/doctor_additional_test.go @@ -77,6 +77,8 @@ func TestDoctorHelpersAndCommand(t *testing.T) { "codex": {}, "gemini": {}, "opencode": {}, + "pi": {}, + "omp": {}, }, ProviderKeys: map[string]ProviderKey{ "openrouter": {EnvVar: "OPENROUTER_API_KEY", Set: true}, diff --git a/control-plane/internal/cli/doctor_probe_test.go b/control-plane/internal/cli/doctor_probe_test.go index e4274b400..712cf33cf 100644 --- a/control-plane/internal/cli/doctor_probe_test.go +++ b/control-plane/internal/cli/doctor_probe_test.go @@ -80,6 +80,8 @@ func TestRunHarnessProbes_SkipsUndetected(t *testing.T) { "codex": {Available: false}, "gemini": {Available: false}, "opencode": {Available: false}, + "pi": {Available: false}, + "omp": {Available: false}, }, } got := runHarnessProbes(report) diff --git a/control-plane/internal/cli/harness_doctor.go b/control-plane/internal/cli/harness_doctor.go index ae4e2245b..e647057a6 100644 --- a/control-plane/internal/cli/harness_doctor.go +++ b/control-plane/internal/cli/harness_doctor.go @@ -40,6 +40,8 @@ var harnessProviderSpecs = []harnessProviderSpec{ {Name: "codex", Binary: "codex", InstallCommand: "npm install -g @openai/codex", AuthEnvVars: []string{"OPENAI_API_KEY"}}, {Name: "gemini", Binary: "gemini", InstallCommand: "npm install -g @google/gemini-cli", AuthEnvVars: []string{"GEMINI_API_KEY", "GOOGLE_API_KEY"}}, {Name: "opencode", Binary: "opencode", InstallCommand: "curl -fsSL https://opencode.ai/install | bash", AuthEnvVars: []string{}}, + {Name: "pi", Binary: "pi", InstallCommand: "npm install -g --ignore-scripts @earendil-works/pi-coding-agent", AuthEnvVars: []string{"OPENROUTER_API_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY", "GEMINI_API_KEY", "GOOGLE_API_KEY"}}, + {Name: "omp", Binary: "omp", InstallCommand: "curl -fsSL https://omp.sh/install | sh", AuthEnvVars: []string{"OPENROUTER_API_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY", "GEMINI_API_KEY", "GOOGLE_API_KEY"}}, } // NewHarnessCommand builds harness-related environment checks. @@ -82,7 +84,7 @@ func newHarnessDoctorCommand() *cobra.Command { return nil }, } - cmd.Flags().StringSliceVar(&providers, "provider", nil, "Provider(s) to check: claude-code, codex, gemini, opencode") + cmd.Flags().StringSliceVar(&providers, "provider", nil, "Provider(s) to check: claude-code, codex, gemini, opencode, pi, omp") cmd.Flags().BoolVar(&jsonOut, "json", false, "Output structured JSON") return cmd } diff --git a/control-plane/internal/cli/harness_doctor_test.go b/control-plane/internal/cli/harness_doctor_test.go index 59f8b3bd2..37b99870b 100644 --- a/control-plane/internal/cli/harness_doctor_test.go +++ b/control-plane/internal/cli/harness_doctor_test.go @@ -55,6 +55,26 @@ func TestHarnessDoctorReturnsErrorForRequestedMissingProvider(t *testing.T) { require.Equal(t, []string{"binary_not_found"}, reports[0].Issues) } +func TestHarnessDoctorReportsPiWithOpenRouterAuth(t *testing.T) { + binDir := t.TempDir() + writeHarnessTestBinary(t, binDir, "pi", "0.84.1") + t.Setenv("PATH", binDir) + t.Setenv("OPENROUTER_API_KEY", "configured") + + cmd := NewHarnessCommand() + var stdout bytes.Buffer + cmd.SetOut(&stdout) + cmd.SetArgs([]string{"doctor", "--provider", "pi", "--json"}) + + require.NoError(t, cmd.Execute()) + var reports []HarnessProviderHealth + require.NoError(t, json.Unmarshal(stdout.Bytes(), &reports)) + require.Len(t, reports, 1) + require.Equal(t, "pi", reports[0].Provider) + require.Equal(t, "configured", reports[0].Auth) + require.True(t, reports[0].Usable) +} + func TestHarnessDoctorClaudeCodeReportsInstalledWrapper(t *testing.T) { binDir := t.TempDir() // Stub interpreter standing in for `python3 -c `: prints "ok" as the diff --git a/control-plane/internal/skillkit/skill_data/agentfield/SKILL.md b/control-plane/internal/skillkit/skill_data/agentfield/SKILL.md index 50e50a940..639fbd2bf 100644 --- a/control-plane/internal/skillkit/skill_data/agentfield/SKILL.md +++ b/control-plane/internal/skillkit/skill_data/agentfield/SKILL.md @@ -68,7 +68,7 @@ Everything else is a variation. Less-used but real: - **`@app.skill()`** — deterministic functions you want callable through the control plane (no LLM). -- **`app.harness(prompt, provider="claude-code"|"codex"|"gemini"|"opencode")`** — delegates to an external coding-agent CLI. Heavy. **Only use when `af doctor` reports `harness_usable: true` AND the Dockerfile installs the CLI AND `shutil.which()` guards startup.** Otherwise use `app.ai(tools=[...])`. +- **`app.harness(prompt, provider="claude-code"|"codex"|"gemini"|"opencode"|"pi"|"omp")`** — delegates to an external coding-agent CLI. Heavy. **Only use when `af doctor` reports `harness_usable: true` AND the Dockerfile installs the CLI AND `shutil.which()` guards startup.** Otherwise use `app.ai(tools=[...])`. Full signatures, schemas, router surface, memory scopes, and the cross-boundary serialization gotcha are in `references/primitives-snapshot.md` (offline-frozen). **Prefer the live `agentfield.ai/llms-full.txt`** when you have a network — it is the source of truth and it does not drift. diff --git a/control-plane/internal/skillkit/skill_data/agentfield/references/primitives-snapshot.md b/control-plane/internal/skillkit/skill_data/agentfield/references/primitives-snapshot.md index bd2bcfc5f..9b4beae08 100644 --- a/control-plane/internal/skillkit/skill_data/agentfield/references/primitives-snapshot.md +++ b/control-plane/internal/skillkit/skill_data/agentfield/references/primitives-snapshot.md @@ -16,7 +16,7 @@ This is a minimal cheat sheet of the five primitives so an offline build can sti | `@app.skill()` | Registers a deterministic function (no LLM) | Sort, parse, dedupe, score-with-formula | | `app.ai(...)` | Single call OR multi-turn tool-using LLM call when `tools=` is passed | Classification, routing, structured analysis, stateful tool-using | | `app.call(target, **kwargs)` | Call another reasoner THROUGH the control plane. Returns `dict`. Tracks the workflow DAG | All inter-reasoner traffic | -| `app.harness(prompt, provider=...)` | Delegate to an external coding-agent CLI (claude-code / codex / gemini / opencode) | When you need a real coding agent to write files / run shell | +| `app.harness(prompt, provider=...)` | Delegate to an external coding-agent CLI (claude-code / codex / gemini / opencode / pi / omp) | When you need a real coding agent to write files / run shell | --- @@ -157,7 +157,7 @@ Default canonical pattern: `AgentRouter(prefix="", tags=["domain"])`. `prefix="c result = await app.harness( prompt: str, schema: type[BaseModel] | None = None, - provider: "claude-code" | "codex" | "gemini" | "opencode" | None = None, + provider: "claude-code" | "codex" | "gemini" | "opencode" | "pi" | "omp" | None = None, model: str | None = None, max_turns: int | None = None, max_budget_usd: float | None = None, diff --git a/control-plane/internal/skillkit/skill_data/agentfield/references/scaffold-recipe.md b/control-plane/internal/skillkit/skill_data/agentfield/references/scaffold-recipe.md index 83611e289..53b2f35c9 100644 --- a/control-plane/internal/skillkit/skill_data/agentfield/references/scaffold-recipe.md +++ b/control-plane/internal/skillkit/skill_data/agentfield/references/scaffold-recipe.md @@ -211,7 +211,7 @@ CMD ["python", "main.py"] Build context is the project directory itself (`context: .`), so the same scaffold works whether the project lives in `code/examples/` or standalone at `/tmp/my-build/`. -**Only edit if you need to install a harness CLI** (claude-code, codex, gemini, opencode). See `primitives-snapshot.md` → "Harness availability gate". Otherwise leave it alone. +**Only edit if you need to install a harness CLI** (claude-code, codex, gemini, opencode, pi, omp). See `primitives-snapshot.md` → "Harness availability gate". Otherwise leave it alone. --- diff --git a/docs/harness-providers.md b/docs/harness-providers.md index b89112d2f..1a44c9b2b 100644 --- a/docs/harness-providers.md +++ b/docs/harness-providers.md @@ -13,6 +13,8 @@ starting a workflow. | Codex | `agentfield[harness-codex]` | `codex` | Codex login or `OPENAI_API_KEY` | | Gemini | None | `gemini` | Gemini login, `GEMINI_API_KEY`, or `GOOGLE_API_KEY` | | OpenCode | `agentfield[harness-opencode]` | `opencode` | Provider credentials configured in OpenCode | +| Pi | None | `pi` | Provider login or API key such as `OPENROUTER_API_KEY` | +| OMP (Oh My Pi) | None | `omp` | Provider login or API key such as `OPENROUTER_API_KEY` | Install every Python wrapper with: @@ -24,6 +26,13 @@ The extras install Python wrappers. They do not replace the runtime preflight: Aforge and Gemini are CLI-only, and Codex or OpenCode may still require a separately available executable depending on the wrapper and platform. +Install Pi or OMP directly from their official distributions: + +```bash +npm install -g --ignore-scripts @earendil-works/pi-coding-agent +curl -fsSL https://omp.sh/install | sh +``` + ## Model selection and reasoning-effort variants Every provider accepts a `model` option on `.harness()` calls. The model string @@ -39,6 +48,10 @@ result = await app.harness( An explicit `variant="high"` keyword wins over the suffix. Per provider: +Pi and OMP accept the same OpenRouter model strings in every SDK, for example +`openrouter/minimax/minimax-m2.7` or +`openrouter/google/gemini-2.5-flash#low`. + | Provider | Model flag | Variant handling | | --- | --- | --- | | `aforge` | `AFORGE_MODEL` env var with a bare OpenRouter slug (a leading `openrouter/` is stripped) | `AFORGE_EXEC_REASONING` (`off`, `low`, `medium`, or `high`) | @@ -46,6 +59,8 @@ An explicit `variant="high"` keyword wins over the suffix. Per provider: | Codex | `-m ` | `-c model_reasoning_effort=` | | Claude Code | SDK `model` option | No effort control — variant is dropped with a debug log | | Gemini | `-m ` | No effort control — variant is dropped | +| Pi | `--model ` | `--thinking ` | +| OMP | `--model ` | `--thinking ` | The `#` separator is safe in model ids: `:` belongs to OpenRouter suffixes like `:free`, and `@` to Vertex-style ids, but no provider uses `#`. @@ -55,7 +70,7 @@ The `#` separator is safe in model ids: `:` belongs to OpenRouter suffixes like Check selected providers in a container or CI job before any paid run: ```bash -af harness doctor --provider codex,opencode --json +af harness doctor --provider codex,opencode,pi,omp --json ``` The command exits non-zero if a requested provider is missing, its version @@ -65,7 +80,7 @@ CI can archive the report when the command fails. Python applications can use the same preflight data: ```python -reports = await app.harness_doctor(providers=["codex", "opencode"]) +reports = await app.harness_doctor(providers=["codex", "opencode", "pi", "omp"]) for report in reports: print(report.provider, report.usable, report.issues) ``` @@ -74,6 +89,9 @@ The preflight currently ships in the Python SDK and the `af` CLI. Equivalent TypeScript and Go SDK APIs are planned follow-ups (see #685) and are not available yet. +For a complete Go workflow that fans one task out to Pi and OMP concurrently, +see `examples/go_agent_nodes/cmd/harness_duo`. + Each report includes the provider name, resolved binary, installed state, version, auth state, usability, installation command, recognized auth variables, and machine-readable issues. diff --git a/examples/go_agent_nodes/cmd/harness_duo/README.md b/examples/go_agent_nodes/cmd/harness_duo/README.md new file mode 100644 index 000000000..4faee63c2 --- /dev/null +++ b/examples/go_agent_nodes/cmd/harness_duo/README.md @@ -0,0 +1,39 @@ +# Pi + OMP Go workflow + +This example registers one AgentField workflow with three Go reasoners: + +```text +compare +├── pi_worker (Pi harness) +└── omp_worker (Oh My Pi harness) +``` + +`compare` starts both child reasoners concurrently and joins their structured +results. The default model is `openrouter/minimax/minimax-m2.7`; set +`HARNESS_MODEL=openrouter/google/gemini-2.5-flash` for the Gemini Flash path. + +## Run + +Start the AgentField control plane, make sure `OPENROUTER_API_KEY` is set, and +install the CLIs shown by `af harness doctor --provider pi` and +`af harness doctor --provider omp`. Then run: + +```bash +cd examples/go_agent_nodes +HARNESS_PROJECT_DIR="$(git rev-parse --show-toplevel)" \ + go run ./cmd/harness_duo serve +``` + +In another terminal, submit the workflow: + +```bash +curl -sS -X POST \ + http://localhost:8080/api/v1/execute/async/harness-duo-go.compare \ + -H 'content-type: application/json' \ + -d '{"input":{"task":"Read README.md and summarize the project with file-path evidence."}}' +``` + +The returned execution appears in AgentField Desktop as a fan-out from +`compare` to `pi_worker` and `omp_worker`. You can override `model`, `task`, and +`project_dir` in the JSON input. `PI_BIN` and `OMP_BIN` override CLI locations +when the binaries are not on `PATH`. diff --git a/examples/go_agent_nodes/cmd/harness_duo/main.go b/examples/go_agent_nodes/cmd/harness_duo/main.go new file mode 100644 index 000000000..f4471298c --- /dev/null +++ b/examples/go_agent_nodes/cmd/harness_duo/main.go @@ -0,0 +1,179 @@ +// Command harness_duo runs Pi and OMP concurrently inside one AgentField workflow. +package main + +import ( + "context" + "fmt" + "log" + "os" + "path/filepath" + "strings" + + "github.com/Agent-Field/agentfield/sdk/go/agent" + "github.com/Agent-Field/agentfield/sdk/go/harness" +) + +const defaultModel = "openrouter/minimax/minimax-m2.7" + +type workerOutput struct { + Summary string `json:"summary"` + Evidence []string `json:"evidence"` +} + +type branchResult struct { + Provider string `json:"provider"` + Model string `json:"model"` + Output workerOutput `json:"output"` + DurationMS int `json:"duration_ms"` + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` + TotalTokens int `json:"total_tokens"` + CostUSD *float64 `json:"cost_usd,omitempty"` + HarnessRunID string `json:"harness_session_id,omitempty"` +} + +type completedBranch struct { + provider string + result map[string]any + err error +} + +func main() { + nodeID := envOr("AGENT_NODE_ID", "harness-duo-go") + listenAddress := envOr("AGENT_LISTEN_ADDR", ":8017") + publicURL := envOr("AGENT_PUBLIC_URL", "http://localhost"+listenAddress) + + duo, err := agent.New(agent.Config{ + NodeID: nodeID, + Version: "1.0.0", + AgentFieldURL: envOr("AGENTFIELD_URL", "http://localhost:8080"), + Token: os.Getenv("AGENTFIELD_TOKEN"), + InternalToken: strings.TrimSpace(os.Getenv("AGENTFIELD_AUTHORIZATION_INTERNAL_TOKEN")), + ListenAddress: listenAddress, + PublicURL: publicURL, + }) + if err != nil { + log.Fatal(err) + } + + registerWorker(duo, "pi_worker", harness.ProviderPi, "PI_BIN") + registerWorker(duo, "omp_worker", harness.ProviderOMP, "OMP_BIN") + + duo.RegisterReasoner("compare", func(ctx context.Context, input map[string]any) (any, error) { + branchInput := map[string]any{ + "task": inputString(input, "task", defaultTask()), + "model": inputString(input, "model", envOr("HARNESS_MODEL", defaultModel)), + "project_dir": inputString(input, "project_dir", projectDir()), + } + + completed := make(chan completedBranch, 2) + for _, provider := range []string{"pi", "omp"} { + provider := provider + go func() { + result, callErr := duo.Call(ctx, provider+"_worker", branchInput) + completed <- completedBranch{provider: provider, result: result, err: callErr} + }() + } + + results := make(map[string]any, 2) + for i := 0; i < 2; i++ { + branch := <-completed + if branch.err != nil { + return nil, fmt.Errorf("%s harness failed: %w", branch.provider, branch.err) + } + results[branch.provider] = branch.result + } + + return map[string]any{ + "model": branchInput["model"], + "branches": results, + }, nil + }, + agent.WithDescription("Fan out one task to Pi and OMP concurrently, then join their structured results"), + agent.WithReasonerTags("entry", "harness-demo"), + ) + + if err := duo.Run(context.Background()); err != nil { + if cliErr, ok := err.(*agent.CLIError); ok { + os.Exit(cliErr.ExitCode()) + } + log.Fatal(err) + } +} + +func registerWorker(duo *agent.Agent, reasoner, provider, binEnv string) { + duo.RegisterReasoner(reasoner, func(ctx context.Context, input map[string]any) (any, error) { + model := inputString(input, "model", envOr("HARNESS_MODEL", defaultModel)) + root := inputString(input, "project_dir", projectDir()) + + var output workerOutput + schema, err := harness.StructToJSONSchema(output) + if err != nil { + return nil, fmt.Errorf("build output schema: %w", err) + } + + run, err := duo.Harness(ctx, inputString(input, "task", defaultTask()), schema, &output, harness.Options{ + Provider: provider, + Model: model, + PermissionMode: "auto", + ProjectDir: root, + BinPath: strings.TrimSpace(os.Getenv(binEnv)), + Tools: []string{"Read", "Write", "Glob", "Grep"}, + SystemPrompt: "Inspect the requested project carefully. Be concise, cite file paths as evidence, and follow the structured-output instructions exactly.", + Timeout: 300, + MaxRetries: 1, + SchemaMaxRetries: 1, + }) + if err != nil { + return nil, err + } + if run.IsError { + return nil, fmt.Errorf("%s harness: %s", provider, run.ErrorMessage) + } + + return branchResult{ + Provider: provider, + Model: model, + Output: output, + DurationMS: run.DurationMS, + InputTokens: run.InputTokens, + OutputTokens: run.OutputTokens, + TotalTokens: run.TotalTokens, + CostUSD: run.CostUSD, + HarnessRunID: run.SessionID, + }, nil + }, agent.WithDescription("Run the task with the "+provider+" coding harness")) +} + +func defaultTask() string { + return "Read README.md and one directly relevant source file. Summarize what this project does in two sentences and provide both file paths as evidence. Do not modify project files." +} + +func projectDir() string { + if configured := strings.TrimSpace(os.Getenv("HARNESS_PROJECT_DIR")); configured != "" { + return configured + } + root, err := os.Getwd() + if err != nil { + return "." + } + absolute, err := filepath.Abs(root) + if err != nil { + return root + } + return absolute +} + +func inputString(input map[string]any, key, fallback string) string { + if value, ok := input[key].(string); ok && strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + return fallback +} + +func envOr(key, fallback string) string { + if value := strings.TrimSpace(os.Getenv(key)); value != "" { + return value + } + return fallback +} diff --git a/examples/go_agent_nodes/go.mod b/examples/go_agent_nodes/go.mod index f665f2b67..5da4782c0 100644 --- a/examples/go_agent_nodes/go.mod +++ b/examples/go_agent_nodes/go.mod @@ -8,6 +8,9 @@ require ( github.com/awslabs/aws-lambda-go-api-proxy v0.16.2 ) -require gopkg.in/yaml.v3 v3.0.1 // indirect +require ( + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) replace github.com/Agent-Field/agentfield/sdk/go => ../../sdk/go diff --git a/examples/go_agent_nodes/go.sum b/examples/go_agent_nodes/go.sum index e9991b657..b7d5dc1b2 100644 --- a/examples/go_agent_nodes/go.sum +++ b/examples/go_agent_nodes/go.sum @@ -16,6 +16,8 @@ github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU= github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= diff --git a/sdk/go/agent/harness.go b/sdk/go/agent/harness.go index 29462934c..b871e114a 100644 --- a/sdk/go/agent/harness.go +++ b/sdk/go/agent/harness.go @@ -10,7 +10,7 @@ import ( // providing lazy initialization and a convenience Harness() method. // HarnessConfig configures the default harness runner for the agent. type HarnessConfig struct { - // Provider is the default provider: "claude-code", "codex", "gemini", or "opencode". + // Provider is the default provider: "claude-code", "codex", "gemini", "opencode", "pi", or "omp". Provider string // Model is the default model identifier. It may carry a diff --git a/sdk/go/harness/factory.go b/sdk/go/harness/factory.go index c58420207..3b42e2247 100644 --- a/sdk/go/harness/factory.go +++ b/sdk/go/harness/factory.go @@ -3,7 +3,7 @@ package harness import "fmt" // BuildProvider creates a Provider instance for the given provider name. -// Supported providers: "claude-code", "codex", "gemini", "opencode". +// Supported providers: "claude-code", "codex", "gemini", "opencode", "pi", "omp". func BuildProvider(name string, binPath string) (Provider, error) { switch name { case ProviderClaudeCode: @@ -14,10 +14,14 @@ func BuildProvider(name string, binPath string) (Provider, error) { return NewGeminiProvider(binPath), nil case ProviderOpenCode: return NewOpenCodeProvider(binPath, ""), nil + case ProviderPi: + return NewPiProvider(binPath), nil + case ProviderOMP: + return NewOMPProvider(binPath), nil default: return nil, fmt.Errorf( - "unknown harness provider: %q (supported: %s, %s, %s, %s)", - name, ProviderClaudeCode, ProviderCodex, ProviderGemini, ProviderOpenCode, + "unknown harness provider: %q (supported: %s, %s, %s, %s, %s, %s)", + name, ProviderClaudeCode, ProviderCodex, ProviderGemini, ProviderOpenCode, ProviderPi, ProviderOMP, ) } } diff --git a/sdk/go/harness/pi.go b/sdk/go/harness/pi.go new file mode 100644 index 000000000..bfb42753b --- /dev/null +++ b/sdk/go/harness/pi.go @@ -0,0 +1,312 @@ +package harness + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" +) + +type piFlavor string + +const ( + piFlavorPi piFlavor = "pi" + piFlavorOMP piFlavor = "omp" +) + +var piReadOnlyTools = map[string]bool{ + "read": true, "grep": true, "find": true, "glob": true, "ls": true, "lsp": true, +} + +type piFamilyProvider struct { + BinPath string + flavor piFlavor + runCLI func(context.Context, []string, map[string]string, string, int, []byte) (*CLIResult, error) +} + +// PiProvider invokes the Pi coding-agent CLI as a subprocess. +type PiProvider struct{ *piFamilyProvider } + +// OMPProvider invokes the Oh My Pi (OMP) coding-agent CLI as a subprocess. +type OMPProvider struct{ *piFamilyProvider } + +// NewPiProvider creates a Pi provider. An empty binPath defaults to "pi". +func NewPiProvider(binPath string) *PiProvider { + if binPath == "" { + binPath = "pi" + } + return &PiProvider{&piFamilyProvider{ + BinPath: binPath, + flavor: piFlavorPi, + runCLI: RunCLIWithStdin, + }} +} + +// NewOMPProvider creates an OMP provider. An empty binPath defaults to "omp". +func NewOMPProvider(binPath string) *OMPProvider { + if binPath == "" { + binPath = "omp" + } + return &OMPProvider{&piFamilyProvider{ + BinPath: binPath, + flavor: piFlavorOMP, + runCLI: RunCLIWithStdin, + }} +} + +func (p *PiProvider) Execute(ctx context.Context, prompt string, options Options) (*RawResult, error) { + return p.piFamilyProvider.execute(ctx, prompt, options) +} + +func (p *OMPProvider) Execute(ctx context.Context, prompt string, options Options) (*RawResult, error) { + return p.piFamilyProvider.execute(ctx, prompt, options) +} + +func (p *piFamilyProvider) execute(ctx context.Context, prompt string, options Options) (*RawResult, error) { + cmd := []string{p.BinPath, "--print", "--mode", "json"} + + root := options.ProjectDir + if root == "" { + root = options.Cwd + } + if p.flavor == piFlavorOMP && root != "" { + cmd = append(cmd, "--cwd", root) + } + + model, variant := options.resolveModelAndVariant() + if model != "" { + cmd = append(cmd, "--model", model) + } + if variant != "" { + cmd = append(cmd, "--thinking", variant) + } + if strings.TrimSpace(options.SystemPrompt) != "" { + cmd = append(cmd, "--system-prompt", strings.TrimSpace(options.SystemPrompt)) + } + if options.ResumeSessionID != "" { + resumeFlag := "--session" + if p.flavor == piFlavorOMP { + resumeFlag = "--resume" + } + cmd = append(cmd, resumeFlag, options.ResumeSessionID) + } + if options.PermissionMode == "auto" { + permissionFlag := "--approve" + if p.flavor == piFlavorOMP { + permissionFlag = "--auto-approve" + } + cmd = append(cmd, permissionFlag) + } + + tools := normalizePiTools(options.Tools, p.flavor) + if options.PermissionMode == "plan" { + readOnly := tools[:0] + for _, tool := range tools { + if piReadOnlyTools[tool] { + readOnly = append(readOnly, tool) + } + } + tools = readOnly + if len(tools) == 0 { + globTool := "find" + if p.flavor == piFlavorOMP { + globTool = "glob" + } + tools = []string{"read", "grep", globTool} + } + } + if options.Tools != nil || options.PermissionMode == "plan" { + if len(tools) == 0 { + cmd = append(cmd, "--no-tools") + } else { + cmd = append(cmd, "--tools", strings.Join(tools, ",")) + } + } + + env := make(map[string]string, len(options.Env)) + for key, value := range options.Env { + env[key] = value + } + + start := time.Now() + runCLI := p.runCLI + if runCLI == nil { + runCLI = RunCLIWithStdin + } + cliResult, err := runCLI(ctx, cmd, env, root, options.timeout(), []byte(prompt)) + apiMS := int(time.Since(start).Milliseconds()) + if err != nil { + if isExecNotFound(err) { + install := "npm install -g --ignore-scripts @earendil-works/pi-coding-agent" + if p.flavor == piFlavorOMP { + install = "curl -fsSL https://omp.sh/install | sh" + } + return &RawResult{ + IsError: true, + ErrorMessage: fmt.Sprintf("%s binary not found at '%s'. Install: %s", strings.ToUpper(string(p.flavor)), p.BinPath, install), + FailureType: FailureCrash, + }, nil + } + if strings.Contains(err.Error(), "timed out") || strings.Contains(err.Error(), "no progress") { + return &RawResult{ + IsError: true, + ErrorMessage: err.Error(), + FailureType: FailureTimeout, + Metrics: Metrics{DurationAPIMS: apiMS}, + }, nil + } + return nil, err + } + + raw := parsePiJSONL(cliResult.Stdout) + raw.Metrics.DurationAPIMS = apiMS + raw.ReturnCode = cliResult.ReturnCode + stderr := StripANSI(strings.TrimSpace(cliResult.Stderr)) + if cliResult.ReturnCode != 0 { + raw.IsError = true + raw.FailureType = FailureCrash + if stderr != "" { + raw.ErrorMessage = truncate(stderr, 1000) + } else if raw.ErrorMessage == "" { + raw.ErrorMessage = fmt.Sprintf("Process exited with code %d.", cliResult.ReturnCode) + } + } else if raw.ErrorMessage != "" { + raw.IsError = true + raw.FailureType = FailureAPIError + } else if raw.Result == "" { + raw.IsError = true + raw.FailureType = FailureNoOutput + raw.ErrorMessage = stderr + if raw.ErrorMessage == "" { + raw.ErrorMessage = fmt.Sprintf("%s exited successfully without an assistant response.", p.flavor) + } + } + return raw, nil +} + +func normalizePiTools(tools []string, flavor piFlavor) []string { + normalized := make([]string, 0, len(tools)) + seen := make(map[string]bool, len(tools)) + for _, tool := range tools { + name := strings.ToLower(strings.TrimSpace(tool)) + if name == "glob" && flavor == piFlavorPi { + name = "find" + } + if name != "" && !seen[name] { + seen[name] = true + normalized = append(normalized, name) + } + } + return normalized +} + +func parsePiJSONL(stdout string) *RawResult { + raw := &RawResult{FailureType: FailureNone} + var totalCost float64 + hasCost := false + + for _, line := range strings.Split(stdout, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + var event map[string]any + if err := json.Unmarshal([]byte(line), &event); err != nil { + continue + } + raw.Messages = append(raw.Messages, event) + + eventType, _ := event["type"].(string) + if eventType == "session" { + if id, ok := event["id"].(string); ok { + raw.Metrics.SessionID = id + } + } + if eventType == "turn_end" { + raw.Metrics.NumTurns++ + } + if eventType != "message_end" { + continue + } + message, ok := event["message"].(map[string]any) + if !ok || message["role"] != "assistant" { + continue + } + if text := piMessageText(message); text != "" { + raw.Result = text + } + if usage, ok := message["usage"].(map[string]any); ok { + raw.Metrics.InputTokens += intValue(usage["input"]) + raw.Metrics.OutputTokens += intValue(usage["output"]) + raw.Metrics.CacheReadTokens += intValue(usage["cacheRead"]) + raw.Metrics.CacheCreationTokens += intValue(usage["cacheWrite"]) + if cost, ok := usage["cost"].(map[string]any); ok { + if value, ok := floatValue(cost["total"]); ok { + totalCost += value + hasCost = true + } + } + } + if reason, _ := message["stopReason"].(string); reason == "error" || reason == "aborted" { + raw.ErrorMessage = fmt.Sprintf("Pi stopped with reason %q.", reason) + if detail, ok := message["errorMessage"].(string); ok && detail != "" { + raw.ErrorMessage = detail + } + } + } + + if raw.Metrics.NumTurns == 0 && raw.Result != "" { + raw.Metrics.NumTurns = 1 + } + if hasCost { + raw.Metrics.CostUSD = &totalCost + } + return raw +} + +func piMessageText(message map[string]any) string { + if content, ok := message["content"].(string); ok { + return content + } + content, ok := message["content"].([]any) + if !ok { + return "" + } + var text strings.Builder + for _, item := range content { + part, ok := item.(map[string]any) + if !ok || part["type"] != "text" { + continue + } + if value, ok := part["text"].(string); ok { + text.WriteString(value) + } + } + return text.String() +} + +func intValue(value any) int { + if number, ok := floatValue(value); ok { + return int(number) + } + return 0 +} + +func floatValue(value any) (float64, bool) { + switch number := value.(type) { + case float64: + return number, true + case float32: + return float64(number), true + case int: + return float64(number), true + case int64: + return float64(number), true + case json.Number: + parsed, err := number.Float64() + return parsed, err == nil + default: + return 0, false + } +} diff --git a/sdk/go/harness/pi_test.go b/sdk/go/harness/pi_test.go new file mode 100644 index 000000000..f072c1964 --- /dev/null +++ b/sdk/go/harness/pi_test.go @@ -0,0 +1,164 @@ +package harness + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const piEventStream = `{"type":"session","id":"session-123"} +{"type":"turn_start"} +{"type":"message_end","message":{"role":"assistant","content":[{"type":"thinking","thinking":"internal"},{"type":"text","text":"done"}],"model":"google/gemini-2.5-flash","usage":{"input":120,"output":30,"cacheRead":10,"cacheWrite":4,"cost":{"total":0.0025}},"stopReason":"stop"}} +{"type":"turn_end"} +{"type":"agent_end"}` + +func TestPiFamilyCommandAndMetrics(t *testing.T) { + tests := []struct { + name string + newProvider func() (Provider, *piFamilyProvider) + permissionFlag string + globTool string + wantPrefix []string + }{ + { + name: "pi", + newProvider: func() (Provider, *piFamilyProvider) { + provider := NewPiProvider("/opt/pi") + return provider, provider.piFamilyProvider + }, + permissionFlag: "--approve", + globTool: "find", + wantPrefix: []string{"/opt/pi", "--print", "--mode", "json"}, + }, + { + name: "omp", + newProvider: func() (Provider, *piFamilyProvider) { + provider := NewOMPProvider("/opt/omp") + return provider, provider.piFamilyProvider + }, + permissionFlag: "--auto-approve", + globTool: "glob", + wantPrefix: []string{"/opt/omp", "--print", "--mode", "json", "--cwd", "/tmp/project"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + provider, base := tc.newProvider() + + var gotCmd []string + var gotEnv map[string]string + var gotCwd string + var gotPrompt string + base.runCLI = func(_ context.Context, cmd []string, env map[string]string, cwd string, _ int, stdin []byte) (*CLIResult, error) { + gotCmd = append([]string(nil), cmd...) + gotEnv = env + gotCwd = cwd + gotPrompt = string(stdin) + return &CLIResult{Stdout: piEventStream}, nil + } + + raw, err := provider.Execute(context.Background(), "implement this", Options{ + ProjectDir: "/tmp/project", + Model: "openrouter/google/gemini-2.5-flash#high", + PermissionMode: "auto", + SystemPrompt: "Be precise.", + Tools: []string{"Read", "Write", "Edit", "Bash", "Glob", "Grep"}, + Env: map[string]string{"EXTRA": "1"}, + }) + require.NoError(t, err) + require.GreaterOrEqual(t, len(gotCmd), len(tc.wantPrefix)) + assert.Equal(t, tc.wantPrefix, gotCmd[:len(tc.wantPrefix)]) + assert.Contains(t, gotCmd, tc.permissionFlag) + assertFlagValue(t, gotCmd, "--model", "openrouter/google/gemini-2.5-flash") + assertFlagValue(t, gotCmd, "--thinking", "high") + assertFlagValue(t, gotCmd, "--tools", "read,write,edit,bash,"+tc.globTool+",grep") + assert.Equal(t, map[string]string{"EXTRA": "1"}, gotEnv) + assert.Equal(t, "/tmp/project", gotCwd) + assert.Equal(t, "implement this", gotPrompt) + + assert.False(t, raw.IsError) + assert.Equal(t, "done", raw.Result) + assert.Equal(t, "session-123", raw.Metrics.SessionID) + assert.Equal(t, 1, raw.Metrics.NumTurns) + assert.Equal(t, 120, raw.Metrics.InputTokens) + assert.Equal(t, 30, raw.Metrics.OutputTokens) + assert.Equal(t, 10, raw.Metrics.CacheReadTokens) + assert.Equal(t, 4, raw.Metrics.CacheCreationTokens) + require.NotNil(t, raw.Metrics.CostUSD) + assert.InDelta(t, 0.0025, *raw.Metrics.CostUSD, 0.000001) + }) + } +} + +func TestPiFamilyPlanModeIsReadOnlyAndResumes(t *testing.T) { + tests := []struct { + name string + newProvider func() (Provider, *piFamilyProvider) + resumeFlag string + tools string + }{ + {"pi", func() (Provider, *piFamilyProvider) { + provider := NewPiProvider("pi") + return provider, provider.piFamilyProvider + }, "--session", "read,grep,find"}, + {"omp", func() (Provider, *piFamilyProvider) { + provider := NewOMPProvider("omp") + return provider, provider.piFamilyProvider + }, "--resume", "read,grep,glob"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + provider, base := tc.newProvider() + var gotCmd []string + base.runCLI = func(_ context.Context, cmd []string, _ map[string]string, _ string, _ int, _ []byte) (*CLIResult, error) { + gotCmd = append([]string(nil), cmd...) + return &CLIResult{Stdout: piEventStream}, nil + } + _, err := provider.Execute(context.Background(), "plan", Options{ + PermissionMode: "plan", + Tools: []string{"Read", "Write", "Bash", "Grep", "Glob"}, + ResumeSessionID: "abc123", + }) + require.NoError(t, err) + assertFlagValue(t, gotCmd, "--tools", tc.tools) + assertFlagValue(t, gotCmd, tc.resumeFlag, "abc123") + }) + } +} + +func TestPiFamilyNonzeroExitIsError(t *testing.T) { + p := NewPiProvider("pi") + p.runCLI = func(context.Context, []string, map[string]string, string, int, []byte) (*CLIResult, error) { + return &CLIResult{Stderr: "authentication failed", ReturnCode: 2}, nil + } + raw, err := p.Execute(context.Background(), "hello", Options{}) + require.NoError(t, err) + assert.True(t, raw.IsError) + assert.Equal(t, FailureCrash, raw.FailureType) + assert.Equal(t, "authentication failed", raw.ErrorMessage) +} + +func TestBuildProviderPiFamily(t *testing.T) { + pi, err := BuildProvider(ProviderPi, "/opt/pi") + require.NoError(t, err) + omp, err := BuildProvider(ProviderOMP, "/opt/omp") + require.NoError(t, err) + assert.Equal(t, "*harness.PiProvider", fmt.Sprintf("%T", pi)) + assert.Equal(t, "*harness.OMPProvider", fmt.Sprintf("%T", omp)) +} + +func assertFlagValue(t *testing.T, cmd []string, flag, value string) { + t.Helper() + for i := range cmd { + if cmd[i] == flag && i+1 < len(cmd) { + assert.Equal(t, value, cmd[i+1]) + return + } + } + t.Fatalf("flag %q not found in %s", flag, strings.Join(cmd, " ")) +} diff --git a/sdk/go/harness/provider.go b/sdk/go/harness/provider.go index 2f6ab37fd..57de0a142 100644 --- a/sdk/go/harness/provider.go +++ b/sdk/go/harness/provider.go @@ -11,6 +11,10 @@ const ( ProviderCodex = "codex" // ProviderGemini is the provider name for Gemini CLI. ProviderGemini = "gemini" + // ProviderPi is the provider name for the Pi coding-agent CLI. + ProviderPi = "pi" + // ProviderOMP is the provider name for the Oh My Pi coding-agent CLI. + ProviderOMP = "omp" ) // Provider is the interface that CLI-based harness providers implement. @@ -23,7 +27,7 @@ type Provider interface { // Options control a single harness invocation. Fields are optional; // zero values mean "use default". type Options struct { - // Provider name: "opencode", "claude-code". + // Provider name: "claude-code", "codex", "gemini", "opencode", "pi", or "omp". Provider string // Model identifier passed to the coding agent. It may carry a diff --git a/sdk/go/harness/runner_test.go b/sdk/go/harness/runner_test.go index 4d744ec3d..36a0ee316 100644 --- a/sdk/go/harness/runner_test.go +++ b/sdk/go/harness/runner_test.go @@ -571,6 +571,8 @@ func TestBuildProvider(t *testing.T) { {"codex", "codex", "*harness.CodexProvider", false}, {"gemini", "gemini", "*harness.GeminiProvider", false}, {"opencode", "opencode", "*harness.OpenCodeProvider", false}, + {"pi", "pi", "*harness.PiProvider", false}, + {"omp", "omp", "*harness.OMPProvider", false}, {"unknown", "unknown-agent", "", true}, } @@ -589,8 +591,8 @@ func TestBuildProvider(t *testing.T) { } func TestRunner_BuildProvider_UsesFactory(t *testing.T) { - // Verify the runner can now build all 4 providers - for _, name := range []string{"claude-code", "codex", "gemini", "opencode"} { + // Verify the runner can build every provider. + for _, name := range []string{"claude-code", "codex", "gemini", "opencode", "pi", "omp"} { t.Run(name, func(t *testing.T) { runner := NewRunner(Options{Provider: name}) _, err := runner.Run(context.Background(), "test", nil, nil, Options{}) diff --git a/sdk/python/agentfield/agent.py b/sdk/python/agentfield/agent.py index 004164e57..5d191e6b7 100644 --- a/sdk/python/agentfield/agent.py +++ b/sdk/python/agentfield/agent.py @@ -2258,7 +2258,7 @@ async def tracked_func(*args, **kwargs): vc_setting = self._effective_component_vc_setting( reasoner_id, self._reasoner_vc_overrides ) - + self._reasoner_registry[reasoner_id] = ReasonerEntry( id=reasoner_id, func=func, @@ -2303,23 +2303,23 @@ def _detect_and_unwrap_trigger_envelope(self, payload_dict: Dict[str, Any]) -> t # Check if this looks like a dispatcher envelope if not isinstance(payload_dict, dict): return payload_dict, None - + if "event" in payload_dict and "_meta" in payload_dict: # This is a dispatcher envelope event_data = payload_dict.get("event", {}) meta_data = payload_dict.get("_meta", {}) - + # Parse metadata into TriggerContext try: from datetime import datetime from .triggers import TriggerContext - + received_at_str = meta_data.get("received_at", "") if received_at_str: received_at = datetime.fromisoformat(received_at_str.replace('Z', '+00:00')) else: received_at = datetime.utcnow() - + trigger_ctx = TriggerContext( trigger_id=meta_data.get("trigger_id", ""), source=meta_data.get("source", ""), @@ -2333,7 +2333,7 @@ def _detect_and_unwrap_trigger_envelope(self, payload_dict: Dict[str, Any]) -> t except Exception: # If parsing fails, return raw envelope for compatibility return payload_dict, None - + # Not an envelope return payload_dict, None @@ -2341,7 +2341,7 @@ def _apply_trigger_transform(self, trigger_ctx, bindings: list, input_data: dict """ Match trigger context against reasoner bindings and apply transform if found. Returns transformed input or original input if no match. - + Matching logic: 1. Find bindings where binding.source == trigger_ctx.source 2. Check event_type: binding.types empty OR trigger_ctx.event_type matches (exact or prefix) @@ -2349,21 +2349,21 @@ def _apply_trigger_transform(self, trigger_ctx, bindings: list, input_data: dict 4. Apply transform if binding has one """ from .triggers import EventTrigger - + if not bindings or not trigger_ctx: return input_data - + # Find best-matching binding best_match = None best_specificity = -1 # -1 = no match, 0 = broad (empty types), 1+ = specific - + for binding in bindings: if not isinstance(binding, EventTrigger): continue - + if binding.source != trigger_ctx.source: continue - + # Check event_type match if binding.types: # binding has specific types — check for match @@ -2378,12 +2378,12 @@ def _apply_trigger_transform(self, trigger_ctx, bindings: list, input_data: dict else: # binding accepts all types specificity = 0 - + # This binding matches; is it better than current best? if specificity > best_specificity: best_match = binding best_specificity = specificity - + # Apply transform if found if best_match and best_match.transform: try: @@ -2392,7 +2392,7 @@ def _apply_trigger_transform(self, trigger_ctx, bindings: list, input_data: dict if self.dev_mode: log_warn(f"Transform failed for {trigger_ctx.source}/{trigger_ctx.event_type}: {e}; using raw input") return input_data - + return input_data async def _execute_reasoner_endpoint( @@ -3735,7 +3735,7 @@ async def harness( prompt: Task description for the coding agent. schema: Pydantic BaseModel class for structured output validation. provider: Override provider ("aforge", "claude-code", "codex", "gemini", - "opencode"). + "opencode", "pi", "omp"). model: Override model identifier. max_turns: Maximum agent iterations. max_budget_usd: Cost cap in USD. diff --git a/sdk/python/agentfield/harness/_availability.py b/sdk/python/agentfield/harness/_availability.py index 809d7633b..a0fd0c6b3 100644 --- a/sdk/python/agentfield/harness/_availability.py +++ b/sdk/python/agentfield/harness/_availability.py @@ -41,6 +41,32 @@ class ProviderSpec: install_command="curl -fsSL https://opencode.ai/install | bash", auth_env_vars=(), ), + "pi": ProviderSpec( + binary="pi", + version_args=("--version",), + install_command=( + "npm install -g --ignore-scripts @earendil-works/pi-coding-agent" + ), + auth_env_vars=( + "OPENROUTER_API_KEY", + "ANTHROPIC_API_KEY", + "OPENAI_API_KEY", + "GEMINI_API_KEY", + "GOOGLE_API_KEY", + ), + ), + "omp": ProviderSpec( + binary="omp", + version_args=("--version",), + install_command="curl -fsSL https://omp.sh/install | sh", + auth_env_vars=( + "OPENROUTER_API_KEY", + "ANTHROPIC_API_KEY", + "OPENAI_API_KEY", + "GEMINI_API_KEY", + "GOOGLE_API_KEY", + ), + ), "grok": ProviderSpec( binary="grok", version_args=("--version",), diff --git a/sdk/python/agentfield/harness/_runner.py b/sdk/python/agentfield/harness/_runner.py index 46fe61ecc..d0aaf1b31 100644 --- a/sdk/python/agentfield/harness/_runner.py +++ b/sdk/python/agentfield/harness/_runner.py @@ -175,6 +175,8 @@ def _resolve_options( "codex_bin", "gemini_bin", "opencode_bin", + "pi_bin", + "omp_bin", "grok_bin", "schema_max_retries", "schema_mode", diff --git a/sdk/python/agentfield/harness/providers/_factory.py b/sdk/python/agentfield/harness/providers/_factory.py index 129815599..2277153af 100644 --- a/sdk/python/agentfield/harness/providers/_factory.py +++ b/sdk/python/agentfield/harness/providers/_factory.py @@ -6,7 +6,16 @@ from agentfield.harness.providers._base import HarnessProvider from agentfield.types import HarnessConfig -SUPPORTED_PROVIDERS = {"aforge", "claude-code", "codex", "gemini", "opencode", "grok"} +SUPPORTED_PROVIDERS = { + "aforge", + "claude-code", + "codex", + "gemini", + "grok", + "omp", + "opencode", + "pi", +} def build_provider(config: "HarnessConfig") -> "HarnessProvider": @@ -38,6 +47,14 @@ def build_provider(config: "HarnessConfig") -> "HarnessProvider": return OpenCodeProvider( bin_path=getattr(config, "opencode_bin", "opencode"), ) + if provider_name == "pi": + from agentfield.harness.providers.pi import PiProvider + + return PiProvider(bin_path=getattr(config, "pi_bin", "pi")) + if provider_name == "omp": + from agentfield.harness.providers.pi import OMPProvider + + return OMPProvider(bin_path=getattr(config, "omp_bin", "omp")) if provider_name == "grok": from agentfield.harness.providers.grok import GrokProvider diff --git a/sdk/python/agentfield/harness/providers/pi.py b/sdk/python/agentfield/harness/providers/pi.py new file mode 100644 index 000000000..938b0756b --- /dev/null +++ b/sdk/python/agentfield/harness/providers/pi.py @@ -0,0 +1,277 @@ +"""Pi-family harness providers using their JSON event-stream CLIs.""" + +from __future__ import annotations + +import time +from typing import Any, Dict, Iterable, Optional + +from agentfield.harness._availability import ensure_cli_available, provider_unavailable +from agentfield.harness._cli import ( + parse_jsonl, + resolve_model_and_variant, + run_cli, + strip_ansi, +) +from agentfield.harness._result import FailureType, Metrics, RawResult + + +_READ_ONLY_TOOLS = {"read", "grep", "find", "glob", "ls", "lsp"} + + +def _normalise_tools(tools: Iterable[object], *, omp: bool) -> list[str]: + """Translate AgentField's provider-neutral tool names to Pi CLI names.""" + aliases = {"glob": "glob" if omp else "find"} + normalised: list[str] = [] + for tool in tools: + name = str(tool).strip().lower() + if not name: + continue + name = aliases.get(name, name) + if name not in normalised: + normalised.append(name) + return normalised + + +def _assistant_messages(events: list[dict[str, Any]]) -> Iterable[dict[str, Any]]: + for event in events: + if event.get("type") != "message_end": + continue + message = event.get("message") + if isinstance(message, dict) and message.get("role") == "assistant": + yield message + + +def _text_content(message: dict[str, Any]) -> Optional[str]: + content = message.get("content") + if isinstance(content, str): + return content or None + if not isinstance(content, list): + return None + parts = [ + part.get("text", "") + for part in content + if isinstance(part, dict) + and part.get("type") == "text" + and isinstance(part.get("text"), str) + ] + text = "".join(parts) + return text or None + + +def _int(value: object) -> int: + if isinstance(value, bool): + return 0 + try: + return int(value) # type: ignore[arg-type] + except (TypeError, ValueError): + return 0 + + +def _float(value: object) -> Optional[float]: + if isinstance(value, bool): + return None + try: + return float(value) # type: ignore[arg-type] + except (TypeError, ValueError): + return None + + +def _parse_pi_events( + events: list[dict[str, Any]], *, configured_model: Optional[str] +) -> tuple[Optional[str], Metrics, Optional[str]]: + result_text: Optional[str] = None + session_id = "" + num_turns = sum(1 for event in events if event.get("type") == "turn_end") + input_tokens = 0 + output_tokens = 0 + cache_read_tokens = 0 + cache_creation_tokens = 0 + total_cost: Optional[float] = None + reported_model: Optional[str] = None + provider_error: Optional[str] = None + + for event in events: + if event.get("type") == "session" and isinstance(event.get("id"), str): + session_id = event["id"] + + for message in _assistant_messages(events): + text = _text_content(message) + if text: + result_text = text + + if isinstance(message.get("model"), str): + reported_model = message["model"] + + usage = message.get("usage") + if isinstance(usage, dict): + input_tokens += _int(usage.get("input")) + output_tokens += _int(usage.get("output")) + cache_read_tokens += _int(usage.get("cacheRead")) + cache_creation_tokens += _int(usage.get("cacheWrite")) + cost = usage.get("cost") + if isinstance(cost, dict): + native_cost = _float(cost.get("total")) + if native_cost is not None: + total_cost = (total_cost or 0.0) + native_cost + + stop_reason = message.get("stopReason") + if stop_reason in {"error", "aborted"}: + detail = message.get("errorMessage") or message.get("error") + provider_error = str(detail or f"Pi stopped with reason {stop_reason!r}.") + + if num_turns == 0 and result_text: + num_turns = 1 + + return ( + result_text, + Metrics( + num_turns=num_turns, + total_cost_usd=total_cost, + session_id=session_id, + input_tokens=input_tokens, + output_tokens=output_tokens, + cache_read_tokens=cache_read_tokens, + cache_creation_tokens=cache_creation_tokens, + model=configured_model or reported_model, + ), + provider_error, + ) + + +class _PiFamilyProvider: + def __init__(self, *, provider: str, bin_path: str, omp: bool): + self._provider = provider + self._bin = bin_path + self._omp = omp + + async def execute(self, prompt: str, options: dict[str, object]) -> RawResult: + ensure_cli_available(self._provider, self._bin) + cmd = [self._bin, "--print", "--mode", "json"] + + root = options.get("project_dir") or options.get("cwd") + cwd = root if isinstance(root, str) else None + if self._omp and cwd: + cmd.extend(["--cwd", cwd]) + + model_value, variant_value = resolve_model_and_variant(options) + if model_value: + cmd.extend(["--model", model_value]) + if variant_value: + cmd.extend(["--thinking", variant_value]) + + system_prompt = options.get("system_prompt") + if isinstance(system_prompt, str) and system_prompt.strip(): + cmd.extend(["--system-prompt", system_prompt.strip()]) + + resume_session_id = options.get("resume_session_id") + if isinstance(resume_session_id, str) and resume_session_id: + cmd.extend(["--resume" if self._omp else "--session", resume_session_id]) + + permission_mode = options.get("permission_mode") + if permission_mode == "auto": + cmd.append("--auto-approve" if self._omp else "--approve") + + tools_value = options.get("tools") + tools = ( + _normalise_tools(tools_value, omp=self._omp) + if isinstance(tools_value, (list, tuple, set)) + else [] + ) + if permission_mode == "plan": + tools = [tool for tool in tools if tool in _READ_ONLY_TOOLS] + if not tools: + tools = ["read", "grep", "glob" if self._omp else "find"] + if isinstance(tools_value, (list, tuple, set)) or permission_mode == "plan": + if tools: + cmd.extend(["--tools", ",".join(tools)]) + else: + cmd.append("--no-tools") + + env: Dict[str, str] = {} + env_value = options.get("env") + if isinstance(env_value, dict): + env = { + str(key): str(value) + for key, value in env_value.items() + if isinstance(key, str) and isinstance(value, str) + } + + timeout: Optional[float] = None + timeout_value = options.get("timeout") + if isinstance(timeout_value, (int, float)) and not isinstance( + timeout_value, bool + ): + timeout = float(timeout_value) + + start_api = time.monotonic() + try: + stdout, stderr, returncode = await run_cli( + cmd, + env=env, + cwd=cwd, + timeout=timeout, + input_text=prompt, + ) + except FileNotFoundError as exc: + raise provider_unavailable(self._provider, self._bin) from exc + except TimeoutError as exc: + return RawResult( + is_error=True, + error_message=str(exc), + failure_type=FailureType.TIMEOUT, + metrics=Metrics(), + ) + + api_ms = int((time.monotonic() - start_api) * 1000) + events = parse_jsonl(stdout) + result_text, metrics, provider_error = _parse_pi_events( + events, configured_model=model_value + ) + metrics.duration_api_ms = api_ms + clean_stderr = strip_ansi(stderr.strip()) if stderr else "" + + if returncode < 0: + error_message = f"Process killed by signal {-returncode}." + failure_type = FailureType.CRASH + elif returncode != 0: + error_message = ( + clean_stderr[:1000] + or provider_error + or (f"Process exited with code {returncode}.") + ) + failure_type = FailureType.CRASH + elif provider_error: + error_message = provider_error + failure_type = FailureType.API_ERROR + elif result_text is None: + error_message = clean_stderr[:1000] or ( + f"{self._provider} exited successfully without an assistant response." + ) + failure_type = FailureType.NO_OUTPUT + else: + error_message = None + failure_type = FailureType.NONE + + return RawResult( + result=result_text, + messages=events, + metrics=metrics, + is_error=failure_type != FailureType.NONE, + error_message=error_message, + failure_type=failure_type, + returncode=returncode, + ) + + +class PiProvider(_PiFamilyProvider): + """Pi coding-agent CLI provider.""" + + def __init__(self, bin_path: str = "pi"): + super().__init__(provider="pi", bin_path=bin_path, omp=False) + + +class OMPProvider(_PiFamilyProvider): + """Oh My Pi (OMP) coding-agent CLI provider.""" + + def __init__(self, bin_path: str = "omp"): + super().__init__(provider="omp", bin_path=bin_path, omp=True) diff --git a/sdk/python/agentfield/types.py b/sdk/python/agentfield/types.py index f9091d3ad..e97ac886b 100644 --- a/sdk/python/agentfield/types.py +++ b/sdk/python/agentfield/types.py @@ -278,7 +278,7 @@ class HarnessConfig(BaseModel): ..., description=( 'Coding agent provider: "aforge" | "claude-code" | "codex" | ' - '"gemini" | "opencode" | "grok"' + '"gemini" | "opencode" | "pi" | "omp" | "grok"' ), ) model: str = Field(default="sonnet", description="Default model identifier.") @@ -324,6 +324,8 @@ class HarnessConfig(BaseModel): opencode_bin: str = Field( default="opencode", description="Path to opencode binary." ) + pi_bin: str = Field(default="pi", description="Path to Pi binary.") + omp_bin: str = Field(default="omp", description="Path to OMP binary.") aforge_bin: str = Field(default="aforge", description="Path to aforge binary.") grok_bin: str = Field( default="grok", description="Path to Grok Build CLI binary." diff --git a/sdk/python/tests/test_harness_factory.py b/sdk/python/tests/test_harness_factory.py index 97f9965e6..c8b92f00f 100644 --- a/sdk/python/tests/test_harness_factory.py +++ b/sdk/python/tests/test_harness_factory.py @@ -36,6 +36,8 @@ def test_supported_providers_contains_expected_names(): assert "codex" in SUPPORTED_PROVIDERS assert "gemini" in SUPPORTED_PROVIDERS assert "opencode" in SUPPORTED_PROVIDERS + assert "pi" in SUPPORTED_PROVIDERS + assert "omp" in SUPPORTED_PROVIDERS assert "grok" in SUPPORTED_PROVIDERS diff --git a/sdk/python/tests/test_harness_provider_pi.py b/sdk/python/tests/test_harness_provider_pi.py new file mode 100644 index 000000000..1232b9d31 --- /dev/null +++ b/sdk/python/tests/test_harness_provider_pi.py @@ -0,0 +1,167 @@ +from __future__ import annotations + +import json +from typing import Any + +import pytest + +from agentfield.harness.providers._factory import build_provider +from agentfield.harness.providers.pi import OMPProvider, PiProvider +from agentfield.types import HarnessConfig + + +@pytest.fixture(autouse=True) +def mock_pi_family_available(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "agentfield.harness._availability.shutil.which", lambda path: path + ) + + +def _event_stream(text: str) -> str: + events = [ + {"type": "session", "id": "session-123"}, + {"type": "turn_start"}, + { + "type": "message_end", + "message": { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "internal"}, + {"type": "text", "text": text}, + ], + "model": "google/gemini-2.5-flash", + "usage": { + "input": 120, + "output": 30, + "cacheRead": 10, + "cacheWrite": 4, + "cost": {"total": 0.0025}, + }, + "stopReason": "stop", + }, + }, + {"type": "turn_end"}, + {"type": "agent_end"}, + ] + return "\n".join(json.dumps(event) for event in events) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("provider", "bin_path", "permission_flag", "glob_tool"), + [ + (PiProvider, "/opt/pi", "--approve", "find"), + (OMPProvider, "/opt/omp", "--auto-approve", "glob"), + ], +) +async def test_pi_family_command_and_metrics( + monkeypatch: pytest.MonkeyPatch, + provider, + bin_path: str, + permission_flag: str, + glob_tool: str, +) -> None: + captured: dict[str, Any] = {} + + async def fake_run_cli(cmd, **kwargs): + captured["cmd"] = cmd + captured.update(kwargs) + return _event_stream("done"), "", 0 + + monkeypatch.setattr("agentfield.harness.providers.pi.run_cli", fake_run_cli) + + raw = await provider(bin_path=bin_path).execute( + "implement this", + { + "project_dir": "/tmp/project", + "model": "openrouter/google/gemini-2.5-flash#high", + "permission_mode": "auto", + "system_prompt": "Be precise.", + "tools": ["Read", "Write", "Edit", "Bash", "Glob", "Grep"], + "env": {"EXTRA": "1"}, + }, + ) + + assert captured["cmd"][:4] == [bin_path, "--print", "--mode", "json"] + if provider is OMPProvider: + assert captured["cmd"][4:6] == ["--cwd", "/tmp/project"] + assert ["--model", "openrouter/google/gemini-2.5-flash"] == captured["cmd"][ + captured["cmd"].index("--model") : captured["cmd"].index("--model") + 2 + ] + assert ["--thinking", "high"] == captured["cmd"][ + captured["cmd"].index("--thinking") : captured["cmd"].index("--thinking") + 2 + ] + assert permission_flag in captured["cmd"] + assert captured["cmd"][captured["cmd"].index("--tools") + 1] == ( + f"read,write,edit,bash,{glob_tool},grep" + ) + assert captured["cwd"] == "/tmp/project" + assert captured["input_text"] == "implement this" + assert captured["env"] == {"EXTRA": "1"} + + assert raw.is_error is False + assert raw.result == "done" + assert raw.metrics.session_id == "session-123" + assert raw.metrics.num_turns == 1 + assert raw.metrics.input_tokens == 120 + assert raw.metrics.output_tokens == 30 + assert raw.metrics.cache_read_tokens == 10 + assert raw.metrics.cache_creation_tokens == 4 + assert raw.metrics.total_cost_usd == pytest.approx(0.0025) + assert raw.metrics.model == "openrouter/google/gemini-2.5-flash" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("provider", "resume_flag", "expected_tools"), + [ + (PiProvider, "--session", "read,grep,find"), + (OMPProvider, "--resume", "read,grep,glob"), + ], +) +async def test_pi_family_plan_mode_is_read_only_and_resumes( + monkeypatch: pytest.MonkeyPatch, + provider, + resume_flag: str, + expected_tools: str, +) -> None: + captured: dict[str, Any] = {} + + async def fake_run_cli(cmd, **kwargs): + captured["cmd"] = cmd + return _event_stream("plan"), "", 0 + + monkeypatch.setattr("agentfield.harness.providers.pi.run_cli", fake_run_cli) + await provider().execute( + "plan this", + { + "permission_mode": "plan", + "tools": ["Read", "Write", "Bash", "Grep", "Glob"], + "resume_session_id": "abc123", + }, + ) + + assert captured["cmd"][captured["cmd"].index("--tools") + 1] == expected_tools + assert captured["cmd"][captured["cmd"].index(resume_flag) + 1] == "abc123" + + +@pytest.mark.asyncio +async def test_pi_family_nonzero_exit_is_error(monkeypatch: pytest.MonkeyPatch) -> None: + async def fake_run_cli(*_args, **_kwargs): + return "", "authentication failed", 2 + + monkeypatch.setattr("agentfield.harness.providers.pi.run_cli", fake_run_cli) + raw = await PiProvider().execute("hello", {}) + + assert raw.is_error is True + assert raw.error_message == "authentication failed" + + +def test_factory_builds_pi_and_omp_with_configured_binaries() -> None: + pi = build_provider(HarnessConfig(provider="pi", pi_bin="/opt/pi")) + omp = build_provider(HarnessConfig(provider="omp", omp_bin="/opt/omp")) + + assert isinstance(pi, PiProvider) + assert pi._bin == "/opt/pi" + assert isinstance(omp, OMPProvider) + assert omp._bin == "/opt/omp" diff --git a/sdk/python/tests/test_harness_types.py b/sdk/python/tests/test_harness_types.py index 26f206c45..bc43ad8a3 100644 --- a/sdk/python/tests/test_harness_types.py +++ b/sdk/python/tests/test_harness_types.py @@ -33,6 +33,8 @@ def test_harness_config_defaults(): assert cfg.codex_bin == "codex" assert cfg.gemini_bin == "gemini" assert cfg.opencode_bin == "opencode" + assert cfg.pi_bin == "pi" + assert cfg.omp_bin == "omp" def test_build_provider_raises_for_unknown_provider(): diff --git a/sdk/python/tests/test_types.py b/sdk/python/tests/test_types.py index 9ee9805ef..c715623fa 100644 --- a/sdk/python/tests/test_types.py +++ b/sdk/python/tests/test_types.py @@ -510,6 +510,8 @@ def test_binary_paths_defaults(self): assert hc.codex_bin == "codex" assert hc.gemini_bin == "gemini" assert hc.opencode_bin == "opencode" + assert hc.pi_bin == "pi" + assert hc.omp_bin == "omp" def test_optional_fields(self): hc = HarnessConfig(provider="p") diff --git a/sdk/typescript/src/harness/cli.ts b/sdk/typescript/src/harness/cli.ts index f279460fc..d8eed9100 100644 --- a/sdk/typescript/src/harness/cli.ts +++ b/sdk/typescript/src/harness/cli.ts @@ -33,6 +33,7 @@ export function runCli( cwd?: string; timeout?: number; idleSeconds?: number; + inputText?: string; } ): Promise { return new Promise((resolve, reject) => { @@ -44,9 +45,13 @@ export function runCli( const proc = spawn(bin, args, { env, cwd: options?.cwd, - stdio: ['ignore', 'pipe', 'pipe'], + stdio: [options?.inputText === undefined ? 'ignore' : 'pipe', 'pipe', 'pipe'], }); + if (options?.inputText !== undefined) { + proc.stdin?.end(options.inputText); + } + let stdout = ''; let stderr = ''; let settled = false; @@ -54,11 +59,11 @@ export function runCli( // Both stdout and stderr are drained concurrently via their own 'data' // listeners, so a full stderr pipe cannot deadlock the read of stdout. - proc.stdout.on('data', (data: Uint8Array | string) => { + proc.stdout!.on('data', (data: Uint8Array | string) => { stdout += data.toString(); lastActivity = Date.now(); }); - proc.stderr.on('data', (data: Uint8Array | string) => { + proc.stderr!.on('data', (data: Uint8Array | string) => { stderr += data.toString(); lastActivity = Date.now(); }); diff --git a/sdk/typescript/src/harness/providers/factory.ts b/sdk/typescript/src/harness/providers/factory.ts index 432d904a7..70d8f6b32 100644 --- a/sdk/typescript/src/harness/providers/factory.ts +++ b/sdk/typescript/src/harness/providers/factory.ts @@ -1,7 +1,7 @@ import type { HarnessProvider } from './base.js'; import type { HarnessConfig } from '../types.js'; -export const SUPPORTED_PROVIDERS = new Set(['claude-code', 'codex', 'gemini', 'opencode']); +export const SUPPORTED_PROVIDERS = new Set(['claude-code', 'codex', 'gemini', 'omp', 'opencode', 'pi']); export async function buildProvider(config: HarnessConfig): Promise { if (!SUPPORTED_PROVIDERS.has(config.provider)) { @@ -25,5 +25,13 @@ export async function buildProvider(config: HarnessConfig): Promise): string | undefined { + if (typeof message.content === 'string') { + return message.content || undefined; + } + if (!Array.isArray(message.content)) { + return undefined; + } + const text = message.content + .filter((part): part is Record => typeof part === 'object' && part !== null) + .filter((part) => part.type === 'text' && typeof part.text === 'string') + .map((part) => part.text as string) + .join(''); + return text || undefined; +} + +function parsePiEvents(events: Array>, configuredModel?: string) { + let result: string | undefined; + let sessionId = ''; + let numTurns = 0; + let inputTokens = 0; + let outputTokens = 0; + let cacheReadTokens = 0; + let cacheCreationTokens = 0; + let totalCostUsd: number | undefined; + let reportedModel: string | undefined; + let providerError: string | undefined; + + for (const event of events) { + if (event.type === 'session' && typeof event.id === 'string') { + sessionId = event.id; + } + if (event.type === 'turn_end') { + numTurns += 1; + } + if (event.type !== 'message_end' || typeof event.message !== 'object' || event.message === null) { + continue; + } + const message = event.message as Record; + if (message.role !== 'assistant') { + continue; + } + + result = textContent(message) ?? result; + if (typeof message.model === 'string') { + reportedModel = message.model; + } + + if (typeof message.usage === 'object' && message.usage !== null) { + const usage = message.usage as Record; + inputTokens += numberValue(usage.input); + outputTokens += numberValue(usage.output); + cacheReadTokens += numberValue(usage.cacheRead); + cacheCreationTokens += numberValue(usage.cacheWrite); + if (typeof usage.cost === 'object' && usage.cost !== null) { + const cost = (usage.cost as Record).total; + if (typeof cost === 'number' && Number.isFinite(cost)) { + totalCostUsd = (totalCostUsd ?? 0) + cost; + } + } + } + + if (message.stopReason === 'error' || message.stopReason === 'aborted') { + providerError = String( + message.errorMessage ?? message.error ?? `Pi stopped with reason ${String(message.stopReason)}.` + ); + } + } + + if (numTurns === 0 && result) { + numTurns = 1; + } + + return { + result, + providerError, + metrics: createMetrics({ + numTurns, + totalCostUsd, + sessionId, + inputTokens, + outputTokens, + cacheReadTokens, + cacheCreationTokens, + totalTokens: inputTokens + outputTokens, + model: configuredModel ?? reportedModel, + }), + }; +} + +class PiFamilyProvider implements HarnessProvider { + public constructor( + private readonly flavor: PiFlavor, + private readonly bin: string, + ) {} + + public async execute(prompt: string, options: Record): Promise { + const cmd = [this.bin, '--print', '--mode', 'json']; + const root = typeof options.projectDir === 'string' + ? options.projectDir + : typeof options.cwd === 'string' + ? options.cwd + : undefined; + + if (this.flavor === 'omp' && root) { + cmd.push('--cwd', root); + } + + const { model, variant } = resolveModelAndVariant(options); + if (model) { + cmd.push('--model', model); + } + if (variant) { + cmd.push('--thinking', variant); + } + + if (typeof options.systemPrompt === 'string' && options.systemPrompt.trim()) { + cmd.push('--system-prompt', options.systemPrompt.trim()); + } + + if (typeof options.resumeSessionId === 'string' && options.resumeSessionId) { + cmd.push(this.flavor === 'omp' ? '--resume' : '--session', options.resumeSessionId); + } + + if (options.permissionMode === 'auto') { + cmd.push(this.flavor === 'omp' ? '--auto-approve' : '--approve'); + } + + const explicitTools = Array.isArray(options.tools); + let tools = explicitTools ? normalizeTools(options.tools as unknown[], this.flavor) : []; + if (options.permissionMode === 'plan') { + tools = tools.filter((tool) => READ_ONLY_TOOLS.has(tool)); + if (tools.length === 0) { + tools = ['read', 'grep', this.flavor === 'omp' ? 'glob' : 'find']; + } + } + if (explicitTools || options.permissionMode === 'plan') { + if (tools.length > 0) { + cmd.push('--tools', tools.join(',')); + } else { + cmd.push('--no-tools'); + } + } + + const env = { ...options.env as Record | undefined }; + const startApi = Date.now(); + try { + const { stdout, stderr, exitCode } = await runCli(cmd, { + env, + cwd: root, + inputText: prompt, + timeout: typeof options.timeout === 'number' ? options.timeout * 1000 : undefined, + }); + const events = parseJsonl(stdout); + const parsed = parsePiEvents(events, model); + parsed.metrics.durationApiMs = Date.now() - startApi; + + let errorMessage: string | undefined; + if (exitCode !== 0) { + errorMessage = stderr.trim() || parsed.providerError || `Process exited with code ${exitCode}.`; + } else if (parsed.providerError) { + errorMessage = parsed.providerError; + } else if (!parsed.result) { + errorMessage = stderr.trim() || `${this.flavor} exited successfully without an assistant response.`; + } + + return createRawResult({ + result: parsed.result, + messages: events, + metrics: parsed.metrics, + isError: errorMessage !== undefined, + errorMessage, + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const binaryMissing = message.includes('ENOENT'); + return createRawResult({ + isError: true, + errorMessage: binaryMissing + ? `${this.flavor === 'omp' ? 'OMP' : 'Pi'} binary not found at '${this.bin}'. ${ + this.flavor === 'omp' + ? 'Install: curl -fsSL https://omp.sh/install | sh' + : 'Install: npm install -g --ignore-scripts @earendil-works/pi-coding-agent' + }` + : message, + metrics: createMetrics({ durationApiMs: Date.now() - startApi }), + }); + } + } +} + +export class PiProvider extends PiFamilyProvider { + public constructor(binPath = 'pi') { + super('pi', binPath); + } +} + +export class OmpProvider extends PiFamilyProvider { + public constructor(binPath = 'omp') { + super('omp', binPath); + } +} diff --git a/sdk/typescript/src/harness/runner.ts b/sdk/typescript/src/harness/runner.ts index 55d8dfe7b..722342cee 100644 --- a/sdk/typescript/src/harness/runner.ts +++ b/sdk/typescript/src/harness/runner.ts @@ -44,6 +44,8 @@ type RunnerOptions = Omit & { codexBin?: string; geminiBin?: string; opencodeBin?: string; + piBin?: string; + ompBin?: string; }; export class HarnessRunner { @@ -57,7 +59,7 @@ export class HarnessRunner { throw new Error("No harness provider specified. Set 'provider' in HarnessConfig or pass it to .harness() call."); } - const cwd = resolved.cwd ?? '.'; + const cwd = resolved.projectDir ?? resolved.cwd ?? '.'; const provider = await this.buildProvider(resolved.provider, resolved); const effectivePrompt = schema === undefined ? prompt : `${prompt}${buildPromptSuffix(schema, cwd)}`; const startTime = Date.now(); @@ -105,9 +107,12 @@ export class HarnessRunner { 'systemPrompt', 'env', 'cwd', + 'projectDir', 'codexBin', 'geminiBin', 'opencodeBin', + 'piBin', + 'ompBin', ] as const) { const value = config[key]; if (value !== undefined && value !== null) { diff --git a/sdk/typescript/src/harness/types.ts b/sdk/typescript/src/harness/types.ts index 397fc320f..0dc72d842 100644 --- a/sdk/typescript/src/harness/types.ts +++ b/sdk/typescript/src/harness/types.ts @@ -1,5 +1,5 @@ export interface HarnessConfig { - provider: 'claude-code' | 'codex' | 'gemini' | 'opencode'; + provider: 'claude-code' | 'codex' | 'gemini' | 'opencode' | 'pi' | 'omp'; model?: string; /** * Provider-specific reasoning-effort variant (e.g. `high`, `minimal`). @@ -17,9 +17,12 @@ export interface HarnessConfig { systemPrompt?: string; env?: Record; cwd?: string; + projectDir?: string; codexBin?: string; geminiBin?: string; opencodeBin?: string; + piBin?: string; + ompBin?: string; } export interface HarnessOptions { @@ -41,9 +44,14 @@ export interface HarnessOptions { systemPrompt?: string; env?: Record; cwd?: string; + projectDir?: string; codexBin?: string; geminiBin?: string; opencodeBin?: string; + piBin?: string; + ompBin?: string; + resumeSessionId?: string; + timeout?: number; schema?: unknown; } diff --git a/sdk/typescript/tests/harness_provider_pi.test.ts b/sdk/typescript/tests/harness_provider_pi.test.ts new file mode 100644 index 000000000..af8c2ae68 --- /dev/null +++ b/sdk/typescript/tests/harness_provider_pi.test.ts @@ -0,0 +1,135 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import * as cli from '../src/harness/cli.js'; +import { buildProvider } from '../src/harness/providers/factory.js'; +import { OmpProvider, PiProvider } from '../src/harness/providers/pi.js'; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +function eventStream(text: string): string { + return [ + { type: 'session', id: 'session-123' }, + { type: 'turn_start' }, + { + type: 'message_end', + message: { + role: 'assistant', + content: [ + { type: 'thinking', thinking: 'internal' }, + { type: 'text', text }, + ], + model: 'google/gemini-2.5-flash', + usage: { + input: 120, + output: 30, + cacheRead: 10, + cacheWrite: 4, + cost: { total: 0.0025 }, + }, + stopReason: 'stop', + }, + }, + { type: 'turn_end' }, + { type: 'agent_end' }, + ].map((event) => JSON.stringify(event)).join('\n'); +} + +describe.each([ + { + name: 'pi', + provider: new PiProvider('/opt/pi'), + prefix: ['/opt/pi', '--print', '--mode', 'json'], + permissionFlag: '--approve', + globTool: 'find', + }, + { + name: 'omp', + provider: new OmpProvider('/opt/omp'), + prefix: ['/opt/omp', '--print', '--mode', 'json', '--cwd', '/tmp/project'], + permissionFlag: '--auto-approve', + globTool: 'glob', + }, +])('$name provider', ({ provider, prefix, permissionFlag, globTool }) => { + it('maps common harness options and native metrics', async () => { + vi.spyOn(cli, 'runCli').mockResolvedValue({ + stdout: eventStream('done'), + stderr: '', + exitCode: 0, + }); + + const result = await provider.execute('implement this', { + projectDir: '/tmp/project', + model: 'openrouter/google/gemini-2.5-flash#high', + permissionMode: 'auto', + systemPrompt: 'Be precise.', + tools: ['Read', 'Write', 'Edit', 'Bash', 'Glob', 'Grep'], + env: { EXTRA: '1' }, + }); + + const [cmd, options] = vi.mocked(cli.runCli).mock.calls[0]; + expect(cmd.slice(0, prefix.length)).toEqual(prefix); + expect(cmd).toContain(permissionFlag); + expect(cmd.slice(cmd.indexOf('--model'), cmd.indexOf('--model') + 2)).toEqual([ + '--model', + 'openrouter/google/gemini-2.5-flash', + ]); + expect(cmd.slice(cmd.indexOf('--thinking'), cmd.indexOf('--thinking') + 2)).toEqual([ + '--thinking', + 'high', + ]); + expect(cmd[cmd.indexOf('--tools') + 1]).toBe(`read,write,edit,bash,${globTool},grep`); + expect(options).toEqual({ + env: { EXTRA: '1' }, + cwd: '/tmp/project', + inputText: 'implement this', + timeout: undefined, + }); + + expect(result.isError).toBe(false); + expect(result.result).toBe('done'); + expect(result.metrics).toMatchObject({ + sessionId: 'session-123', + numTurns: 1, + totalCostUsd: 0.0025, + inputTokens: 120, + outputTokens: 30, + cacheReadTokens: 10, + cacheCreationTokens: 4, + totalTokens: 150, + model: 'openrouter/google/gemini-2.5-flash', + }); + }); +}); + +it.each([ + { provider: new PiProvider(), resumeFlag: '--session', tools: 'read,grep,find' }, + { provider: new OmpProvider(), resumeFlag: '--resume', tools: 'read,grep,glob' }, +])('keeps $resumeFlag retries read-only', async ({ provider, resumeFlag, tools }) => { + vi.spyOn(cli, 'runCli').mockResolvedValue({ + stdout: eventStream('plan'), + stderr: '', + exitCode: 0, + }); + + await provider.execute('plan this', { + permissionMode: 'plan', + tools: ['Read', 'Write', 'Bash', 'Grep', 'Glob'], + resumeSessionId: 'abc123', + }); + + const cmd = vi.mocked(cli.runCli).mock.calls[0][0]; + expect(cmd[cmd.indexOf('--tools') + 1]).toBe(tools); + expect(cmd[cmd.indexOf(resumeFlag) + 1]).toBe('abc123'); +}); + +describe('provider factory', () => { + it('routes pi and omp and passes binary overrides', async () => { + const pi = await buildProvider({ provider: 'pi', piBin: '/opt/pi' }); + const omp = await buildProvider({ provider: 'omp', ompBin: '/opt/omp' }); + + expect(pi).toBeInstanceOf(PiProvider); + expect(omp).toBeInstanceOf(OmpProvider); + }); +}); diff --git a/sdk/typescript/tests/harness_runner.test.ts b/sdk/typescript/tests/harness_runner.test.ts index 1fe393bc3..9a62e27dc 100644 --- a/sdk/typescript/tests/harness_runner.test.ts +++ b/sdk/typescript/tests/harness_runner.test.ts @@ -71,6 +71,8 @@ describe('harness runner', () => { codexBin: 'codex', geminiBin: 'gemini', opencodeBin: 'opencode', + piBin: 'pi', + ompBin: 'omp', }; const runner = new HarnessRunner(cfg); @@ -88,6 +90,8 @@ describe('harness runner', () => { expect(options.maxBudgetUsd).toBe(2); expect(options.env).toEqual({ B: '2' }); expect(options.cwd).toBe('/tmp/override'); + expect(options.piBin).toBe('pi'); + expect(options.ompBin).toBe('omp'); }); it('isTransient matches transient errors and rejects non-transient', () => { diff --git a/skills/agentfield/SKILL.md b/skills/agentfield/SKILL.md index 50e50a940..639fbd2bf 100644 --- a/skills/agentfield/SKILL.md +++ b/skills/agentfield/SKILL.md @@ -68,7 +68,7 @@ Everything else is a variation. Less-used but real: - **`@app.skill()`** — deterministic functions you want callable through the control plane (no LLM). -- **`app.harness(prompt, provider="claude-code"|"codex"|"gemini"|"opencode")`** — delegates to an external coding-agent CLI. Heavy. **Only use when `af doctor` reports `harness_usable: true` AND the Dockerfile installs the CLI AND `shutil.which()` guards startup.** Otherwise use `app.ai(tools=[...])`. +- **`app.harness(prompt, provider="claude-code"|"codex"|"gemini"|"opencode"|"pi"|"omp")`** — delegates to an external coding-agent CLI. Heavy. **Only use when `af doctor` reports `harness_usable: true` AND the Dockerfile installs the CLI AND `shutil.which()` guards startup.** Otherwise use `app.ai(tools=[...])`. Full signatures, schemas, router surface, memory scopes, and the cross-boundary serialization gotcha are in `references/primitives-snapshot.md` (offline-frozen). **Prefer the live `agentfield.ai/llms-full.txt`** when you have a network — it is the source of truth and it does not drift. diff --git a/skills/agentfield/references/primitives-snapshot.md b/skills/agentfield/references/primitives-snapshot.md index bd2bcfc5f..9b4beae08 100644 --- a/skills/agentfield/references/primitives-snapshot.md +++ b/skills/agentfield/references/primitives-snapshot.md @@ -16,7 +16,7 @@ This is a minimal cheat sheet of the five primitives so an offline build can sti | `@app.skill()` | Registers a deterministic function (no LLM) | Sort, parse, dedupe, score-with-formula | | `app.ai(...)` | Single call OR multi-turn tool-using LLM call when `tools=` is passed | Classification, routing, structured analysis, stateful tool-using | | `app.call(target, **kwargs)` | Call another reasoner THROUGH the control plane. Returns `dict`. Tracks the workflow DAG | All inter-reasoner traffic | -| `app.harness(prompt, provider=...)` | Delegate to an external coding-agent CLI (claude-code / codex / gemini / opencode) | When you need a real coding agent to write files / run shell | +| `app.harness(prompt, provider=...)` | Delegate to an external coding-agent CLI (claude-code / codex / gemini / opencode / pi / omp) | When you need a real coding agent to write files / run shell | --- @@ -157,7 +157,7 @@ Default canonical pattern: `AgentRouter(prefix="", tags=["domain"])`. `prefix="c result = await app.harness( prompt: str, schema: type[BaseModel] | None = None, - provider: "claude-code" | "codex" | "gemini" | "opencode" | None = None, + provider: "claude-code" | "codex" | "gemini" | "opencode" | "pi" | "omp" | None = None, model: str | None = None, max_turns: int | None = None, max_budget_usd: float | None = None, diff --git a/skills/agentfield/references/scaffold-recipe.md b/skills/agentfield/references/scaffold-recipe.md index 83611e289..53b2f35c9 100644 --- a/skills/agentfield/references/scaffold-recipe.md +++ b/skills/agentfield/references/scaffold-recipe.md @@ -211,7 +211,7 @@ CMD ["python", "main.py"] Build context is the project directory itself (`context: .`), so the same scaffold works whether the project lives in `code/examples/` or standalone at `/tmp/my-build/`. -**Only edit if you need to install a harness CLI** (claude-code, codex, gemini, opencode). See `primitives-snapshot.md` → "Harness availability gate". Otherwise leave it alone. +**Only edit if you need to install a harness CLI** (claude-code, codex, gemini, opencode, pi, omp). See `primitives-snapshot.md` → "Harness availability gate". Otherwise leave it alone. --- From 3b412a371e39b1f2f88d6ccf2e20f36ed0272137 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Wed, 12 Aug 2026 16:47:42 -0400 Subject: [PATCH 02/20] Make OMP the default harness provider --- README.md | 4 +- control-plane/internal/cli/doctor.go | 4 +- control-plane/internal/cli/harness_doctor.go | 4 +- .../internal/cli/harness_doctor_test.go | 21 +++ .../skillkit/skill_data/agentfield/SKILL.md | 2 +- docs/design/harness-v2-design.md | 140 +++++++++--------- docs/harness-providers.md | 72 ++++++++- .../go_agent_nodes/cmd/harness_duo/README.md | 6 +- .../go_agent_nodes/cmd/harness_duo/main.go | 14 +- sdk/go/agent/cost_tracker_test.go | 13 ++ sdk/go/agent/harness.go | 1 + sdk/go/agent/harness_test.go | 34 +++-- sdk/go/agent/usage.go | 3 + sdk/go/harness/factory.go | 4 + sdk/go/harness/pi_test.go | 27 ++++ sdk/go/harness/provider.go | 2 + sdk/go/harness/runner.go | 9 +- sdk/go/harness/runner_invariant_test.go | 30 +++- sdk/go/harness/runner_test.go | 22 ++- sdk/python/agentfield/agent.py | 2 +- sdk/python/agentfield/harness/__init__.py | 3 +- sdk/python/agentfield/harness/_runner.py | 13 +- .../agentfield/harness/providers/_factory.py | 3 +- sdk/python/agentfield/types.py | 7 +- sdk/python/tests/test_harness_factory.py | 13 +- .../test_harness_provider_availability.py | 14 +- sdk/python/tests/test_harness_runner.py | 13 +- sdk/python/tests/test_harness_types.py | 10 +- sdk/python/tests/test_types.py | 11 +- sdk/python/tests/test_usage_transport.py | 17 +++ sdk/typescript/src/agent/Agent.ts | 3 +- sdk/typescript/src/harness/index.ts | 2 +- .../src/harness/providers/factory.ts | 24 +-- sdk/typescript/src/harness/providers/index.ts | 4 +- sdk/typescript/src/harness/providers/pi.ts | 2 +- sdk/typescript/src/harness/runner.ts | 6 +- sdk/typescript/src/harness/types.ts | 3 +- .../tests/harness_provider_pi.test.ts | 22 ++- sdk/typescript/tests/harness_runner.test.ts | 12 +- sdk/typescript/tests/usage_ai_capture.test.ts | 5 +- skills/agentfield/SKILL.md | 2 +- 41 files changed, 422 insertions(+), 181 deletions(-) diff --git a/README.md b/README.md index ce45d034b..838b2367f 100644 --- a/README.md +++ b/README.md @@ -247,7 +247,7 @@ Two examples already run at this load. The [deep-research engine](https://agentf - **[Reasoners & Skills](https://agentfield.ai/docs/build/building-blocks/reasoners?utm_source=github-readme&utm_campaign=github-readme&utm_id=github-readme-reasoners)** - `@app.reasoner()` for AI judgment, `@app.skill()` for deterministic code - **[Structured AI](https://agentfield.ai/docs/reference/sdks/python?utm_source=github-readme&utm_campaign=github-readme&utm_id=github-readme-structured-ai)** - `app.ai(schema=MyModel)` → typed Pydantic/Zod output from any LLM -- **[Harness](https://agentfield.ai/docs/build/intelligence/harness?utm_source=github-readme&utm_campaign=github-readme&utm_id=github-readme-harness)** - `app.harness("Fix the bug")` dispatches multi-turn tasks to Claude Code, Codex, Gemini CLI, or OpenCode +- **[Harness](https://agentfield.ai/docs/build/intelligence/harness?utm_source=github-readme&utm_campaign=github-readme&utm_id=github-readme-harness)** - `app.harness("Fix the bug")` dispatches multi-turn tasks to OMP (default), Pi, Claude Code, Codex, Gemini CLI, or OpenCode - **[Cross-Agent Calls](https://agentfield.ai/docs/build/coordination/cross-agent-calls?utm_source=github-readme&utm_campaign=github-readme&utm_id=github-readme-cross-agent-calls)** - `app.call("other-agent.func")` routes through the control plane with full tracing - **[Discovery](https://agentfield.ai/docs/reference/sdks/python?utm_source=github-readme&utm_campaign=github-readme&utm_id=github-readme-discovery)** - `app.discover(tags=["ml*"])` finds agents and capabilities across the mesh. `tools="discover"` lets LLMs auto-invoke them. - **[Memory](https://agentfield.ai/docs/build/coordination/shared-memory?utm_source=github-readme&utm_campaign=github-readme&utm_id=github-readme-memory)** - `app.memory.set()` / `.get()` / `.similarity_search()` - KV + vector search, four scopes, no Redis needed @@ -279,7 +279,7 @@ Two examples already run at this load. The [deep-research engine](https://agentf | Feature | How | |---|---| | Structured output (Pydantic/Zod) | `app.ai(schema=MyModel)` | -| Multi-turn coding agents | `app.harness("task", provider="claude-code")` | +| Multi-turn coding agents | `app.harness("task")` (OMP default) or `provider="pi"` / another provider | | LLM auto-discovers agents and tools | `app.ai(tools="discover")` | | Multimodal (text, image, audio) | `app.ai("Describe", image_url="...")` | | Streaming responses | `app.ai("...", stream=True)` | diff --git a/control-plane/internal/cli/doctor.go b/control-plane/internal/cli/doctor.go index b1e81b60e..2fda2902e 100644 --- a/control-plane/internal/cli/doctor.go +++ b/control-plane/internal/cli/doctor.go @@ -85,12 +85,12 @@ var harnessProviders = []struct { Binary string // executable name to look up on PATH ProbeArgs []string // minimal one-shot invocation used by `--probe` }{ + {Name: "omp", Binary: "omp", ProbeArgs: []string{"--print", "Say OK"}}, {Name: "claude-code", Binary: "claude", ProbeArgs: []string{"-p", "Say OK"}}, {Name: "codex", Binary: "codex", ProbeArgs: []string{"exec", "Say OK"}}, {Name: "gemini", Binary: "gemini", ProbeArgs: []string{"-p", "Say OK"}}, {Name: "opencode", Binary: "opencode", ProbeArgs: []string{"run", "Say OK"}}, {Name: "pi", Binary: "pi", ProbeArgs: []string{"--print", "Say OK"}}, - {Name: "omp", Binary: "omp", ProbeArgs: []string{"--print", "Say OK"}}, } // harnessProbeTimeout bounds a single provider smoke test. Coding-agent CLIs @@ -121,7 +121,7 @@ func NewDoctorCommand() *cobra.Command { Long: `Doctor inspects the local environment and reports what's available for building AgentField multi-reasoner systems: - • Available harness provider CLIs (claude-code, codex, gemini, opencode, pi, omp) + • Available harness provider CLIs (omp default, claude-code, codex, gemini, opencode, pi) • Provider API keys set in the environment (without leaking values) • Docker availability and whether the control-plane image is locally cached • Whether a local control plane is reachable diff --git a/control-plane/internal/cli/harness_doctor.go b/control-plane/internal/cli/harness_doctor.go index e647057a6..48bdccb64 100644 --- a/control-plane/internal/cli/harness_doctor.go +++ b/control-plane/internal/cli/harness_doctor.go @@ -33,6 +33,7 @@ type harnessProviderSpec struct { } var harnessProviderSpecs = []harnessProviderSpec{ + {Name: "omp", Binary: "omp", InstallCommand: "curl -fsSL https://omp.sh/install | sh", AuthEnvVars: []string{"OPENROUTER_API_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY", "GEMINI_API_KEY", "GOOGLE_API_KEY"}}, // claude-code has no Binary: the Python provider runs on the // claude_agent_sdk pip package (which bundles its own CLI), not on a // globally installed `claude` binary. See claudeCodeHealth. @@ -41,7 +42,6 @@ var harnessProviderSpecs = []harnessProviderSpec{ {Name: "gemini", Binary: "gemini", InstallCommand: "npm install -g @google/gemini-cli", AuthEnvVars: []string{"GEMINI_API_KEY", "GOOGLE_API_KEY"}}, {Name: "opencode", Binary: "opencode", InstallCommand: "curl -fsSL https://opencode.ai/install | bash", AuthEnvVars: []string{}}, {Name: "pi", Binary: "pi", InstallCommand: "npm install -g --ignore-scripts @earendil-works/pi-coding-agent", AuthEnvVars: []string{"OPENROUTER_API_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY", "GEMINI_API_KEY", "GOOGLE_API_KEY"}}, - {Name: "omp", Binary: "omp", InstallCommand: "curl -fsSL https://omp.sh/install | sh", AuthEnvVars: []string{"OPENROUTER_API_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY", "GEMINI_API_KEY", "GOOGLE_API_KEY"}}, } // NewHarnessCommand builds harness-related environment checks. @@ -84,7 +84,7 @@ func newHarnessDoctorCommand() *cobra.Command { return nil }, } - cmd.Flags().StringSliceVar(&providers, "provider", nil, "Provider(s) to check: claude-code, codex, gemini, opencode, pi, omp") + cmd.Flags().StringSliceVar(&providers, "provider", nil, "Provider(s) to check: omp (default), claude-code, codex, gemini, opencode, pi") cmd.Flags().BoolVar(&jsonOut, "json", false, "Output structured JSON") return cmd } diff --git a/control-plane/internal/cli/harness_doctor_test.go b/control-plane/internal/cli/harness_doctor_test.go index 37b99870b..ff4467078 100644 --- a/control-plane/internal/cli/harness_doctor_test.go +++ b/control-plane/internal/cli/harness_doctor_test.go @@ -75,6 +75,27 @@ func TestHarnessDoctorReportsPiWithOpenRouterAuth(t *testing.T) { require.True(t, reports[0].Usable) } +func TestHarnessDoctorReportsOMPDefaultWithOfficialInstallCommand(t *testing.T) { + binDir := t.TempDir() + writeHarnessTestBinary(t, binDir, "omp", "17.2.15") + t.Setenv("PATH", binDir) + t.Setenv("OPENROUTER_API_KEY", "configured") + + cmd := NewHarnessCommand() + var stdout bytes.Buffer + cmd.SetOut(&stdout) + cmd.SetArgs([]string{"doctor", "--provider", "omp", "--json"}) + + require.NoError(t, cmd.Execute()) + var reports []HarnessProviderHealth + require.NoError(t, json.Unmarshal(stdout.Bytes(), &reports)) + require.Len(t, reports, 1) + require.Equal(t, "omp", reports[0].Provider) + require.Equal(t, "configured", reports[0].Auth) + require.Equal(t, "curl -fsSL https://omp.sh/install | sh", reports[0].InstallCommand) + require.True(t, reports[0].Usable) +} + func TestHarnessDoctorClaudeCodeReportsInstalledWrapper(t *testing.T) { binDir := t.TempDir() // Stub interpreter standing in for `python3 -c `: prints "ok" as the diff --git a/control-plane/internal/skillkit/skill_data/agentfield/SKILL.md b/control-plane/internal/skillkit/skill_data/agentfield/SKILL.md index 639fbd2bf..e42e15814 100644 --- a/control-plane/internal/skillkit/skill_data/agentfield/SKILL.md +++ b/control-plane/internal/skillkit/skill_data/agentfield/SKILL.md @@ -68,7 +68,7 @@ Everything else is a variation. Less-used but real: - **`@app.skill()`** — deterministic functions you want callable through the control plane (no LLM). -- **`app.harness(prompt, provider="claude-code"|"codex"|"gemini"|"opencode"|"pi"|"omp")`** — delegates to an external coding-agent CLI. Heavy. **Only use when `af doctor` reports `harness_usable: true` AND the Dockerfile installs the CLI AND `shutil.which()` guards startup.** Otherwise use `app.ai(tools=[...])`. +- **`app.harness(prompt, provider="claude-code"|"codex"|"gemini"|"opencode"|"pi"|"omp")`** — delegates to an external coding-agent CLI. OMP is the SDK default when `provider` is omitted. Heavy. **Only use when `af doctor` reports `harness_usable: true` AND the Dockerfile installs the selected CLI AND `shutil.which()` guards startup.** Otherwise use `app.ai(tools=[...])`. Full signatures, schemas, router surface, memory scopes, and the cross-boundary serialization gotcha are in `references/primitives-snapshot.md` (offline-frozen). **Prefer the live `agentfield.ai/llms-full.txt`** when you have a network — it is the source of truth and it does not drift. diff --git a/docs/design/harness-v2-design.md b/docs/design/harness-v2-design.md index 9ea47165a..f2ef6b5af 100644 --- a/docs/design/harness-v2-design.md +++ b/docs/design/harness-v2-design.md @@ -1,15 +1,15 @@ # `.harness()` — First-Class Coding Agent Integration for AgentField -> **Status**: Design Proposal -> **Author**: Architecture brainstorm -> **Scope**: Python SDK + TypeScript SDK (Go SDK tracked separately) +> **Status**: Implemented; updated for Pi/OMP parity and OMP default +> **Author**: Architecture brainstorm +> **Scope**: Python, TypeScript, and Go SDKs > **Date**: 2026-03-02 --- ## 1. Overview -Add `.harness()` as a first-class method on the Agent class — matching the DX of `.ai()` — enabling developers to dispatch multi-turn coding tasks to external coding agents (Claude Code, Codex, Gemini CLI, OpenCode). +Add `.harness()` as a first-class method on the Agent class — matching the DX of `.ai()` — enabling developers to dispatch multi-turn coding tasks to external coding agents (OMP, Pi, Claude Code, Codex, Gemini CLI, and OpenCode). OMP is the zero-configuration provider default in every SDK. **Key principle**: `.ai()` is for single-turn LLM calls. `.harness()` is for multi-turn agentic coding tasks that browse files, edit code, run tests, and iterate. @@ -25,10 +25,7 @@ from agentfield import Agent, AIConfig, HarnessConfig app = Agent( node_id="my-agent", ai_config=AIConfig(model="openai/gpt-4o"), - harness_config=HarnessConfig( - provider="claude-code", # Required — no implicit default - model="sonnet", - ), + harness_config=HarnessConfig(), # OMP; model comes from CLI configuration ) ``` @@ -37,10 +34,7 @@ import { Agent } from '@agentfield/sdk'; const agent = new Agent({ nodeId: 'my-agent', - harnessConfig: { - provider: 'claude-code', // Required - model: 'sonnet', - }, + harnessConfig: {}, // OMP; model comes from CLI configuration }); ``` @@ -107,15 +101,10 @@ fix = await app.harness( ### 2.5 Without Constructor Config ```python -# No harness_config on Agent — provide everything per-call +# No harness_config on Agent — OMP is selected automatically app = Agent(node_id="minimal-agent") -result = await app.harness( - "Fix the bug", - provider="gemini", # Required when no harness_config - model="flash", - cwd="/my/project", -) +result = await app.harness("Fix the bug", cwd="/my/project") ``` ### 2.6 Inside a Reasoner (production pattern) @@ -207,12 +196,14 @@ class Agent(FastAPI): ### 4.1 Integration Matrix -| Provider | Python | TypeScript | Go (future) | Schema Support | +| Provider | Python | TypeScript | Go | Schema Support | |---|---|---|---|---| | **claude-code** | `claude_agent_sdk` (native Python SDK) | `@anthropic-ai/claude-agent-sdk` (native TS SDK) | CLI subprocess | File-write (universal) | | **codex** | CLI subprocess `codex exec --json` | `@openai/codex-sdk` (native TS SDK) | CLI subprocess | File-write (universal) | | **gemini** | CLI subprocess `gemini --output-format stream-json` | CLI subprocess | CLI subprocess | File-write (universal) | | **opencode** | CLI subprocess | CLI subprocess | CLI subprocess | File-write (universal) | +| **pi** | CLI subprocess | CLI subprocess | CLI subprocess | File-write (universal) | +| **omp** (default) | CLI subprocess | CLI subprocess | CLI subprocess | File-write (universal) | ### 4.2 Why SDK-First Where Available @@ -223,7 +214,7 @@ class Agent(FastAPI): ### 4.3 Why CLI Subprocess for Others -- No native SDK wrapping the CLI agent loop exists for Gemini/OpenCode +- No native SDK wrapping the CLI agent loop exists for Gemini/OpenCode/Pi/OMP - CLI subprocess is lowest-maintenance integration (binary does all the work) - All CLIs output JSONL event streams for consistent parsing - Go SDK will use CLI subprocess for ALL providers (no Go SDKs exist) @@ -245,7 +236,7 @@ interface HarnessProvider { ``` ```go -// Go (future) +// Go type Provider interface { Execute(ctx context.Context, prompt string, opts HarnessOptions) (*RawResult, error) } @@ -262,7 +253,7 @@ type Provider interface { **Why file-write over native flags:** - Native flags constrain the model's *text response* — a wrong abstraction for multi-turn coding agents whose core competency is writing files - Large schemas can produce JSON that exceeds response token limits or gets truncated -- File-write works identically across ALL 4 providers (one code path, half the tests) +- File-write works identically across all providers (one shared orchestration path) - Writing a JSON file is trivially easy for agents that refactor entire codebases ``` @@ -349,47 +340,47 @@ Layer 4: Full retry (expensive, last resort ```python class HarnessConfig(BaseModel): """Configuration for coding agent harness calls. - - Provider is required — there is no implicit default. - All other fields have sensible defaults that can be overridden per-call. + + OMP is the provider default. Every field can be overridden per-call. """ - # Provider selection (required) - provider: str # "claude-code" | "codex" | "gemini" | "opencode" - model: str = "sonnet" - + provider: str = "omp" + model: Optional[str] = None # use the selected CLI's configured default + # Execution limits max_turns: int = 30 max_budget_usd: Optional[float] = None - + # Retry behavior max_retries: int = 3 initial_delay: float = 1.0 max_delay: float = 30.0 backoff_factor: float = 2.0 - + # Tools & permissions tools: List[str] = Field(default_factory=lambda: [ "Read", "Write", "Edit", "Bash", "Glob", "Grep" ]) permission_mode: Optional[str] = None # "plan" | "auto" | None system_prompt: Optional[str] = None - + # Environment env: Dict[str, str] = Field(default_factory=dict) - + # Binary paths (for CLI-based providers) codex_bin: str = "codex" gemini_bin: str = "gemini" opencode_bin: str = "opencode" + pi_bin: str = "pi" + omp_bin: str = "omp" ``` ### 6.2 HarnessConfig (TypeScript) ```typescript interface HarnessConfig { - /** Required — no default. "claude-code" | "codex" | "gemini" | "opencode" */ - provider: string; - /** Default model identifier. Default: "sonnet" */ + /** OMP when omitted. */ + provider?: 'claude-code' | 'codex' | 'gemini' | 'opencode' | 'pi' | 'omp'; + /** Omitted uses the selected CLI's configured model. */ model?: string; /** Maximum agent iterations. Default: 30 */ maxTurns?: number; @@ -417,16 +408,21 @@ interface HarnessConfig { geminiBin?: string; /** Path to opencode binary. */ opencodeBin?: string; + /** Paths to Pi-family binaries. */ + piBin?: string; + ompBin?: string; } ``` ### 6.3 Config Resolution (hierarchical, matches .ai pattern) ``` -1. HarnessConfig defaults (set at agent construction) -2. Per-call overrides (passed to .harness() method) - → Per-call values win over HarnessConfig defaults - → If no HarnessConfig AND no per-call provider → raise error +1. Per-call overrides passed to `.harness()` +2. `HarnessConfig` set at agent construction +3. OMP provider default; the CLI's configured model default + +Per-call values win. AgentField never substitutes a provider-specific model +when the model is omitted. ``` --- @@ -439,21 +435,21 @@ interface HarnessConfig { @dataclass class HarnessResult: """Final result from a harness invocation.""" - + # Core output result: Optional[str] # Raw text result parsed: Any # Validated schema instance (T | None) is_error: bool - + # Metrics cost_usd: Optional[float] # Total execution cost num_turns: int # Number of agent turns duration_ms: int # Wall clock time in milliseconds session_id: str # For potential session resume - + # Message history messages: List[Message] # Full conversation history - + # Convenience properties @property def text(self) -> str: @@ -514,7 +510,8 @@ sdk/python/agentfield/harness/ │ ├── claude.py # Claude Code provider (SDK-based: claude_agent_sdk) │ ├── codex.py # Codex provider (CLI-based: codex exec) │ ├── gemini.py # Gemini provider (CLI-based: gemini) -│ └── opencode.py # OpenCode provider (CLI-based: opencode) +│ ├── opencode.py # OpenCode provider (CLI-based: opencode) +│ └── pi.py # Shared Pi and OMP adapters ``` ### 8.2 TypeScript SDK @@ -534,10 +531,11 @@ sdk/typescript/src/harness/ │ ├── claude.ts # Claude Code provider (SDK-based) │ ├── codex.ts # Codex provider (SDK-based: @openai/codex-sdk) │ ├── gemini.ts # Gemini provider (CLI-based) -│ └── opencode.ts # OpenCode provider (CLI-based) +│ ├── opencode.ts # OpenCode provider (CLI-based) +│ └── pi.ts # Shared Pi and OMP adapters ``` -### 8.3 Go SDK (future) +### 8.3 Go SDK ``` sdk/go/harness/ @@ -551,7 +549,8 @@ sdk/go/harness/ ├── claude.go # Claude Code provider (CLI) ├── codex.go # Codex provider (CLI) ├── gemini.go # Gemini provider (CLI) -└── opencode.go # OpenCode provider (CLI) +├── opencode.go # OpenCode provider (CLI) +└── pi.go # Shared Pi and OMP adapters ``` --- @@ -612,7 +611,7 @@ class ErrorKind(str, Enum): class ClaudeCodeProvider: async def execute(self, prompt: str, options: HarnessOptions) -> RawResult: from claude_agent_sdk import query, ClaudeAgentOptions - + opts = ClaudeAgentOptions( model=options.model, cwd=options.cwd, @@ -623,15 +622,15 @@ class ClaudeCodeProvider: permission_mode=options.permission_mode, env=options.env, ) - + messages = [] result_text = None metrics = {} - + async for msg in query(prompt=prompt, options=opts): # Collect messages and extract result ... - + return RawResult(result=result_text, messages=messages, metrics=metrics, is_error=False) ``` @@ -641,17 +640,17 @@ class ClaudeCodeProvider: class CodexProvider: def __init__(self, bin_path: str = "codex"): self.bin = bin_path - + async def execute(self, prompt: str, options: HarnessOptions) -> RawResult: cmd = [self.bin, "exec", "--json"] - + if options.cwd: cmd.extend(["-C", options.cwd]) if options.permission_mode == "auto": cmd.append("--full-auto") - + cmd.append(prompt) - + proc = await asyncio.create_subprocess_exec( *cmd, stdout=asyncio.subprocess.PIPE, @@ -659,7 +658,7 @@ class CodexProvider: env={**os.environ, **options.env}, ) stdout, stderr = await proc.communicate() - + # Parse JSONL events from stdout return self._parse_jsonl_output(stdout.decode()) ``` @@ -670,17 +669,17 @@ class CodexProvider: class GeminiProvider: def __init__(self, bin_path: str = "gemini"): self.bin = bin_path - + async def execute(self, prompt: str, options: HarnessOptions) -> RawResult: cmd = [self.bin, "--output-format", "json"] - + if options.model: cmd.extend(["--model", options.model]) if options.permission_mode == "auto": cmd.extend(["--approval-mode", "yolo"]) - + cmd.append(prompt) - + proc = await asyncio.create_subprocess_exec( *cmd, stdout=asyncio.subprocess.PIPE, @@ -689,7 +688,7 @@ class GeminiProvider: cwd=options.cwd, ) stdout, stderr = await proc.communicate() - + return self._parse_output(stdout.decode(), proc.returncode) ``` @@ -722,11 +721,11 @@ class GeminiProvider: - [ ] Add to factory - [ ] Tests for new providers -### Phase 3: Go SDK Port +### Phase 3: Go SDK Port (complete) - [ ] Port all types to Go structs - [ ] Runner implementation -- [ ] All 4 providers (CLI-based) +- [x] All providers (CLI-based) - [ ] Wire into Go Agent struct - [ ] Tests @@ -773,11 +772,20 @@ class GeminiProvider: ### Go SDK - No new dependencies (os/exec is stdlib) +### Runtime installation lifecycle + +Pi and OMP follow the same explicit runtime lifecycle as Codex, Gemini, and +OpenCode. SDK packages do not install or upgrade external executables. Operators +pin the CLI in their machine or container, verify it with `af harness doctor`, +and may override its path per provider. Missing-binary failures carry the exact +upstream install command. This keeps application startup deterministic and +prevents an SDK call from mutating production hosts. + --- ## 14. Open Questions -1. **Session resume**: Should `.harness()` support resuming a previous session (by `session_id`)? Claude Code and Codex both support this. Could be a follow-up feature. +1. **Session resume**: Implemented through the shared `resume_session_id` option, translated to each provider's native flag. 2. **MCP integration**: Should harness providers be discoverable as MCP tools? Codex already has `codex mcp-server` mode. Could be a follow-up. diff --git a/docs/harness-providers.md b/docs/harness-providers.md index 1a44c9b2b..8117f7494 100644 --- a/docs/harness-providers.md +++ b/docs/harness-providers.md @@ -1,8 +1,36 @@ # Harness providers -AgentField harness providers run external coding agents. Install the provider -wrapper you need, install its CLI when required, and verify the runtime before -starting a workflow. +AgentField harness providers run external coding agents behind one SDK contract. +OMP is the default in Python, TypeScript, and Go; an explicit provider always +wins. Install the provider wrapper you need, install its CLI when required, and +verify the runtime before starting a workflow. + +## Default selection + +Provider resolution is identical in every SDK: + +1. The provider passed to the individual harness call. +2. The provider in the agent's harness configuration. +3. OMP. + +Model resolution follows the same first two layers, then defers to the selected +CLI's configured default. AgentField does not silently force a Claude model onto +OMP or another provider. + +```python +# Python: provider omitted, so this runs OMP. +result = await app.harness("Fix the failing test") +``` + +```typescript +// TypeScript: provider omitted, so this runs OMP. +const result = await app.harness('Fix the failing test'); +``` + +```go +// Go: the zero-value Options use OMP. +result, err := app.Harness(ctx, "Fix the failing test", nil, nil, harness.Options{}) +``` ## Install @@ -14,7 +42,7 @@ starting a workflow. | Gemini | None | `gemini` | Gemini login, `GEMINI_API_KEY`, or `GOOGLE_API_KEY` | | OpenCode | `agentfield[harness-opencode]` | `opencode` | Provider credentials configured in OpenCode | | Pi | None | `pi` | Provider login or API key such as `OPENROUTER_API_KEY` | -| OMP (Oh My Pi) | None | `omp` | Provider login or API key such as `OPENROUTER_API_KEY` | +| OMP (Oh My Pi, default) | None | `omp` | Provider login or API key such as `OPENROUTER_API_KEY` | Install every Python wrapper with: @@ -22,7 +50,10 @@ Install every Python wrapper with: pip install 'agentfield[harness-all]' ``` -The extras install Python wrappers. They do not replace the runtime preflight: +The extras install Python wrappers. AgentField does not install or upgrade a +coding-agent executable at application startup. This is the same lifecycle used +for Codex, Gemini, and OpenCode: operators choose and pin the CLI version, while +the SDK locates it on `PATH` (or through a provider-specific binary override). Aforge and Gemini are CLI-only, and Codex or OpenCode may still require a separately available executable depending on the wrapper and platform. @@ -33,6 +64,37 @@ npm install -g --ignore-scripts @earendil-works/pi-coding-agent curl -fsSL https://omp.sh/install | sh ``` +For reproducible containers and CI, run the chosen upstream installer in the +image build, then gate startup with `af harness doctor`. If the executable is +missing at dispatch time, all three SDKs return an actionable provider error +containing the same upstream install command instead of attempting a mutation. + +## Provider parity + +Pi and OMP implement the same provider-neutral harness surface as OpenCode. +The SDK translates that surface to each CLI's native flags rather than exposing +CLI-specific command construction to application code. + +| Capability | OpenCode | Pi | OMP | +| --- | --- | --- | --- | +| Model and `#variant` | `-m`, `--variant` | `--model`, `--thinking` | `--model`, `--thinking` | +| Project root | `--dir` | process working directory | `--cwd` plus process working directory | +| One-shot machine output | JSON output | stdin + JSON event stream | stdin + JSON event stream | +| System prompt | Native prompt option | Native prompt option | Native prompt option | +| Tool allowlist | Native tool flags | Normalized Pi tool names | Normalized OMP tool names | +| Plan / auto permissions | Native permission flags | Read-only tools / `--approve` | Read-only tools / `--auto-approve` | +| Session resume | Native session option | `--session` | `--resume` | +| Structured output | Isolated schema file protocol | Same protocol | Same protocol | +| Metrics | Sessions, turns, tokens, cost, duration | Same normalized fields | Same normalized fields | +| Runtime controls | Env, timeout, retries, binary override | Same | Same | + +The contract is equivalent, not flag-identical. Pi calls its filesystem search +tool `find`, OMP calls it `glob`, and each CLI has its own resume and approval +flags. These differences stay inside the provider adapters. Unsupported native +concepts are handled consistently: plan mode removes mutating tools, explicit +model variants override `#variant`, and provider-reported metrics are normalized +into the shared result type. + ## Model selection and reasoning-effort variants Every provider accepts a `model` option on `.harness()` calls. The model string diff --git a/examples/go_agent_nodes/cmd/harness_duo/README.md b/examples/go_agent_nodes/cmd/harness_duo/README.md index 4faee63c2..18803801a 100644 --- a/examples/go_agent_nodes/cmd/harness_duo/README.md +++ b/examples/go_agent_nodes/cmd/harness_duo/README.md @@ -5,11 +5,13 @@ This example registers one AgentField workflow with three Go reasoners: ```text compare ├── pi_worker (Pi harness) -└── omp_worker (Oh My Pi harness) +└── omp_worker (Oh My Pi through the provider-less OMP default) ``` `compare` starts both child reasoners concurrently and joins their structured -results. The default model is `openrouter/minimax/minimax-m2.7`; set +results. The Pi branch opts in explicitly; the OMP branch intentionally omits +`Provider` to exercise the SDK default. The default model is +`openrouter/minimax/minimax-m2.7`; set `HARNESS_MODEL=openrouter/google/gemini-2.5-flash` for the Gemini Flash path. ## Run diff --git a/examples/go_agent_nodes/cmd/harness_duo/main.go b/examples/go_agent_nodes/cmd/harness_duo/main.go index f4471298c..3246a1235 100644 --- a/examples/go_agent_nodes/cmd/harness_duo/main.go +++ b/examples/go_agent_nodes/cmd/harness_duo/main.go @@ -57,7 +57,9 @@ func main() { } registerWorker(duo, "pi_worker", harness.ProviderPi, "PI_BIN") - registerWorker(duo, "omp_worker", harness.ProviderOMP, "OMP_BIN") + // Omit Provider intentionally: this branch demonstrates that OMP is the + // cross-SDK default while still allowing OMP_BIN to select the executable. + registerWorker(duo, "omp_worker", "", "OMP_BIN") duo.RegisterReasoner("compare", func(ctx context.Context, input map[string]any) (any, error) { branchInput := map[string]any{ @@ -102,6 +104,10 @@ func main() { } func registerWorker(duo *agent.Agent, reasoner, provider, binEnv string) { + providerName := provider + if providerName == "" { + providerName = harness.DefaultProvider + } duo.RegisterReasoner(reasoner, func(ctx context.Context, input map[string]any) (any, error) { model := inputString(input, "model", envOr("HARNESS_MODEL", defaultModel)) root := inputString(input, "project_dir", projectDir()) @@ -128,11 +134,11 @@ func registerWorker(duo *agent.Agent, reasoner, provider, binEnv string) { return nil, err } if run.IsError { - return nil, fmt.Errorf("%s harness: %s", provider, run.ErrorMessage) + return nil, fmt.Errorf("%s harness: %s", providerName, run.ErrorMessage) } return branchResult{ - Provider: provider, + Provider: providerName, Model: model, Output: output, DurationMS: run.DurationMS, @@ -142,7 +148,7 @@ func registerWorker(duo *agent.Agent, reasoner, provider, binEnv string) { CostUSD: run.CostUSD, HarnessRunID: run.SessionID, }, nil - }, agent.WithDescription("Run the task with the "+provider+" coding harness")) + }, agent.WithDescription("Run the task with the "+providerName+" coding harness")) } func defaultTask() string { diff --git a/sdk/go/agent/cost_tracker_test.go b/sdk/go/agent/cost_tracker_test.go index 5561511ed..f6a6b97fa 100644 --- a/sdk/go/agent/cost_tracker_test.go +++ b/sdk/go/agent/cost_tracker_test.go @@ -301,6 +301,19 @@ func TestRecordHarnessUsage(t *testing.T) { assert.Equal(t, "openrouter", entries[0]["provider"]) }) + t.Run("provider falls back to OMP without harness config", func(t *testing.T) { + a := newAgentForTest(t) + tracker := NewCostTracker() + ctx := contextWithCostTracker(context.Background(), tracker) + + a.recordHarnessUsage(ctx, &harness.Result{InputTokens: 1}, harness.Options{}) + + entries := tracker.Serialize()["entries"].([]map[string]any) + require.Len(t, entries, 1) + assert.Equal(t, "omp", entries[0]["harness"]) + assert.Equal(t, "omp", entries[0]["model"]) + }) + t.Run("model variant suffix is stripped for attribution", func(t *testing.T) { a := newAgentForTest(t) tracker := NewCostTracker() diff --git a/sdk/go/agent/harness.go b/sdk/go/agent/harness.go index b871e114a..44aa64cec 100644 --- a/sdk/go/agent/harness.go +++ b/sdk/go/agent/harness.go @@ -11,6 +11,7 @@ import ( // HarnessConfig configures the default harness runner for the agent. type HarnessConfig struct { // Provider is the default provider: "claude-code", "codex", "gemini", "opencode", "pi", or "omp". + // The zero value selects OMP. Provider string // Model is the default model identifier. It may carry a diff --git a/sdk/go/agent/harness_test.go b/sdk/go/agent/harness_test.go index 78b51fc76..e05ab62f9 100644 --- a/sdk/go/agent/harness_test.go +++ b/sdk/go/agent/harness_test.go @@ -39,12 +39,12 @@ func TestHarnessRunner_LazyInitialization(t *testing.T) { } func TestHarnessRunner_DefaultOptions(t *testing.T) { - // Agent with no HarnessConfig — runner gets zero-value Options + // Agent with no HarnessConfig — runner resolves the SDK default to OMP. a := newTestAgentForHarness(t) runner := a.HarnessRunner() assert.NotNil(t, runner) - assert.Equal(t, "", runner.DefaultOptions.Provider) + assert.Equal(t, harness.ProviderOMP, runner.DefaultOptions.Provider) assert.Equal(t, "", runner.DefaultOptions.Model) } @@ -111,29 +111,32 @@ func TestHarnessRunner_ConcurrentAccess(t *testing.T) { } } -func TestHarness_ErrorWithoutProvider(t *testing.T) { - // Harness() should fail when no provider is configured. - // The runner will return an error about a missing provider. +func TestHarness_DefaultProviderIsOMP(t *testing.T) { + // A provider-less call reaches OMP. This environment intentionally has no + // OMP binary, so the provider returns its typed, actionable runtime result. a := newTestAgentForHarness(t) - _, err := a.Harness(context.Background(), "do something", nil, nil, harness.Options{}) - assert.Error(t, err) - assert.Contains(t, err.Error(), "provider") + result, err := a.Harness(context.Background(), "do something", nil, nil, harness.Options{ + BinPath: t.TempDir() + "/missing-omp", + }) + require.NoError(t, err) + require.NotNil(t, result) + assert.True(t, result.IsError) + assert.Contains(t, result.ErrorMessage, "OMP binary not found") + assert.Contains(t, result.ErrorMessage, "https://omp.sh/install") } func TestHarness_PassesOptsToRunner(t *testing.T) { // Verify that per-call Options are forwarded to the runner. - // Using a non-existent provider triggers a provider-build error, - // which confirms the Options reached Run() (otherwise we'd get - // the "no harness provider specified" error instead). + // Using a non-existent provider triggers a provider-build error, which + // confirms the explicit override won over the OMP default. a := newTestAgentForHarness(t) _, err := a.Harness(context.Background(), "test", nil, nil, harness.Options{ Provider: "nonexistent-provider", }) assert.Error(t, err) - // Should be a provider-build error, NOT the "no harness provider specified" error - assert.NotContains(t, err.Error(), "no harness provider specified") + assert.Contains(t, err.Error(), "unknown harness provider") } func TestHarnessConfig_PartialOverride(t *testing.T) { @@ -161,12 +164,11 @@ func TestHarnessConfig_PartialOverride(t *testing.T) { } func TestHarness_NilHarnessConfig(t *testing.T) { - // Agent with nil HarnessConfig should still create a runner with default (empty) options + // Agent with nil HarnessConfig should still create a runner with OMP selected. a := newTestAgentForHarness(t) assert.Nil(t, a.cfg.HarnessConfig) runner := a.HarnessRunner() assert.NotNil(t, runner) - // Default options should be zero - assert.Equal(t, harness.Options{}, runner.DefaultOptions) + assert.Equal(t, harness.Options{Provider: harness.ProviderOMP}, runner.DefaultOptions) } diff --git a/sdk/go/agent/usage.go b/sdk/go/agent/usage.go index 1737e05d8..d6aba1aad 100644 --- a/sdk/go/agent/usage.go +++ b/sdk/go/agent/usage.go @@ -139,6 +139,9 @@ func (a *Agent) recordHarnessUsage(ctx context.Context, result *harness.Result, if provider == "" && a.cfg.HarnessConfig != nil { provider = a.cfg.HarnessConfig.Provider } + if provider == "" { + provider = harness.DefaultProvider + } harnessName := strings.ReplaceAll(provider, "-", "_") model := opts.Model diff --git a/sdk/go/harness/factory.go b/sdk/go/harness/factory.go index 3b42e2247..85a08c955 100644 --- a/sdk/go/harness/factory.go +++ b/sdk/go/harness/factory.go @@ -3,8 +3,12 @@ package harness import "fmt" // BuildProvider creates a Provider instance for the given provider name. +// An empty name selects DefaultProvider (OMP). // Supported providers: "claude-code", "codex", "gemini", "opencode", "pi", "omp". func BuildProvider(name string, binPath string) (Provider, error) { + if name == "" { + name = DefaultProvider + } switch name { case ProviderClaudeCode: return NewClaudeCodeProvider(binPath), nil diff --git a/sdk/go/harness/pi_test.go b/sdk/go/harness/pi_test.go index f072c1964..51247b1a5 100644 --- a/sdk/go/harness/pi_test.go +++ b/sdk/go/harness/pi_test.go @@ -150,6 +150,33 @@ func TestBuildProviderPiFamily(t *testing.T) { require.NoError(t, err) assert.Equal(t, "*harness.PiProvider", fmt.Sprintf("%T", pi)) assert.Equal(t, "*harness.OMPProvider", fmt.Sprintf("%T", omp)) + defaultProvider, err := BuildProvider("", "") + require.NoError(t, err) + assert.Equal(t, "*harness.OMPProvider", fmt.Sprintf("%T", defaultProvider)) +} + +func TestPiFamilyMissingBinaryIncludesInstallGuidance(t *testing.T) { + tests := []struct { + name string + provider *piFamilyProvider + installHint string + }{ + {"pi", NewPiProvider("pi").piFamilyProvider, "npm install -g --ignore-scripts @earendil-works/pi-coding-agent"}, + {"omp", NewOMPProvider("omp").piFamilyProvider, "curl -fsSL https://omp.sh/install | sh"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tc.provider.runCLI = func(context.Context, []string, map[string]string, string, int, []byte) (*CLIResult, error) { + return nil, fmt.Errorf("exec: executable file not found in $PATH") + } + raw, err := tc.provider.execute(context.Background(), "hello", Options{}) + require.NoError(t, err) + require.True(t, raw.IsError) + assert.Contains(t, raw.ErrorMessage, tc.installHint) + assert.Equal(t, FailureCrash, raw.FailureType) + }) + } } func assertFlagValue(t *testing.T, cmd []string, flag, value string) { diff --git a/sdk/go/harness/provider.go b/sdk/go/harness/provider.go index 57de0a142..05fd1b88a 100644 --- a/sdk/go/harness/provider.go +++ b/sdk/go/harness/provider.go @@ -15,6 +15,8 @@ const ( ProviderPi = "pi" // ProviderOMP is the provider name for the Oh My Pi coding-agent CLI. ProviderOMP = "omp" + // DefaultProvider is used when a harness call does not select a provider. + DefaultProvider = ProviderOMP ) // Provider is the interface that CLI-based harness providers implement. diff --git a/sdk/go/harness/runner.go b/sdk/go/harness/runner.go index 45f811d0d..bef90213c 100644 --- a/sdk/go/harness/runner.go +++ b/sdk/go/harness/runner.go @@ -26,6 +26,9 @@ type Runner struct { // NewRunner creates a harness runner with default options. func NewRunner(defaults Options) *Runner { + if defaults.Provider == "" { + defaults.Provider = DefaultProvider + } return &Runner{ DefaultOptions: defaults, Logger: log.New(io.Discard, "[harness] ", log.LstdFlags), @@ -53,12 +56,6 @@ type schemaAware interface { func (r *Runner) Run(ctx context.Context, prompt string, schema map[string]any, dest any, overrides Options) (*Result, error) { opts := r.mergeOptions(overrides) - if opts.Provider == "" { - return nil, fmt.Errorf( - "no harness provider specified: set Provider in runner defaults or pass it to Run()", - ) - } - provider, err := r.buildProvider(opts) if err != nil { return nil, err diff --git a/sdk/go/harness/runner_invariant_test.go b/sdk/go/harness/runner_invariant_test.go index 20157e06b..46decb6c7 100644 --- a/sdk/go/harness/runner_invariant_test.go +++ b/sdk/go/harness/runner_invariant_test.go @@ -182,6 +182,8 @@ func TestInvariant_Runner_ProviderFactoryExhaustiveness(t *testing.T) { ProviderCodex, ProviderGemini, ProviderOpenCode, + ProviderPi, + ProviderOMP, } for _, name := range knownProviders { @@ -193,11 +195,23 @@ func TestInvariant_Runner_ProviderFactoryExhaustiveness(t *testing.T) { } } +// TestInvariant_Runner_ProviderFactoryDefaultSelection verifies that an +// empty provider name selects DefaultProvider instead of erroring. +func TestInvariant_Runner_ProviderFactoryDefaultSelection(t *testing.T) { + prov, err := BuildProvider("", "") + require.NoError(t, err, "BuildProvider must not error for empty provider (selects default)") + require.NotNil(t, prov, "BuildProvider must return non-nil for empty provider (selects default)") + + defaultProv, err := BuildProvider(DefaultProvider, "") + require.NoError(t, err, "BuildProvider must not error for DefaultProvider %q", DefaultProvider) + assert.IsType(t, defaultProv, prov, + "empty provider must select the same type as DefaultProvider %q", DefaultProvider) +} + // TestInvariant_Runner_ProviderFactoryUnknownReturnsError verifies that // unknown provider names return an error with a non-nil error value. func TestInvariant_Runner_ProviderFactoryUnknownReturnsError(t *testing.T) { unknownNames := []string{ - "", "nonexistent", "gpt-4", "anthropic", @@ -338,6 +352,20 @@ func TestInvariant_Runner_BuildProviderWithBinPath(t *testing.T) { assert.Equal(t, customBinPath, p.BinPath) }, }, + { + provider: ProviderPi, + checkPath: func(t *testing.T, prov Provider) { + p := prov.(*PiProvider) + assert.Equal(t, customBinPath, p.BinPath) + }, + }, + { + provider: ProviderOMP, + checkPath: func(t *testing.T, prov Provider) { + p := prov.(*OMPProvider) + assert.Equal(t, customBinPath, p.BinPath) + }, + }, } for _, tt := range tests { diff --git a/sdk/go/harness/runner_test.go b/sdk/go/harness/runner_test.go index 36a0ee316..c288daa73 100644 --- a/sdk/go/harness/runner_test.go +++ b/sdk/go/harness/runner_test.go @@ -36,12 +36,13 @@ func (m *mockProvider) Execute(_ context.Context, prompt string, opts Options) ( func TestRunner_Run_NoSchema(t *testing.T) { runner := NewRunner(Options{Provider: "opencode"}) - // We can't easily test with real providers, so test the merge/validation logic - t.Run("missing provider", func(t *testing.T) { + // The zero-value SDK configuration resolves to OMP. + t.Run("default provider", func(t *testing.T) { r := NewRunner(Options{}) - _, err := r.Run(context.Background(), "test", nil, nil, Options{}) - assert.Error(t, err) - assert.Contains(t, err.Error(), "no harness provider") + assert.Equal(t, ProviderOMP, r.DefaultOptions.Provider) + provider, err := BuildProvider("", "") + require.NoError(t, err) + assert.IsType(t, &OMPProvider{}, provider) }) t.Run("unknown provider", func(t *testing.T) { @@ -591,16 +592,13 @@ func TestBuildProvider(t *testing.T) { } func TestRunner_BuildProvider_UsesFactory(t *testing.T) { - // Verify the runner can build every provider. + // Verify the runner can build every provider without starting external CLIs. for _, name := range []string{"claude-code", "codex", "gemini", "opencode", "pi", "omp"} { t.Run(name, func(t *testing.T) { runner := NewRunner(Options{Provider: name}) - _, err := runner.Run(context.Background(), "test", nil, nil, Options{}) - // Should fail at execution (binary not found), not at provider creation - // The error should NOT be "unknown harness provider" - if err != nil { - assert.NotContains(t, err.Error(), "unknown harness provider") - } + provider, err := runner.buildProvider(runner.DefaultOptions) + require.NoError(t, err) + assert.NotNil(t, provider) }) } } diff --git a/sdk/python/agentfield/agent.py b/sdk/python/agentfield/agent.py index 5d191e6b7..f10d18e13 100644 --- a/sdk/python/agentfield/agent.py +++ b/sdk/python/agentfield/agent.py @@ -3858,7 +3858,7 @@ def _record_harness_usage( def _harness_provider_name(self) -> Optional[str]: cfg = getattr(self, "harness_config", None) - return getattr(cfg, "provider", None) if cfg else None + return (getattr(cfg, "provider", None) if cfg else None) or "omp" def _harness_model_name(self) -> Optional[str]: cfg = getattr(self, "harness_config", None) diff --git a/sdk/python/agentfield/harness/__init__.py b/sdk/python/agentfield/harness/__init__.py index e4de46bc5..a065a6d34 100644 --- a/sdk/python/agentfield/harness/__init__.py +++ b/sdk/python/agentfield/harness/__init__.py @@ -1,7 +1,7 @@ from agentfield.harness._result import HarnessResult, Metrics, RawResult from agentfield.harness._runner import HarnessRunner from agentfield.harness.providers._base import HarnessProvider -from agentfield.harness.providers._factory import build_provider +from agentfield.harness.providers._factory import DEFAULT_HARNESS_PROVIDER, build_provider from agentfield.harness._doctor import ProviderHealth, harness_doctor __all__ = [ @@ -11,6 +11,7 @@ "HarnessRunner", "HarnessProvider", "build_provider", + "DEFAULT_HARNESS_PROVIDER", "ProviderHealth", "harness_doctor", ] diff --git a/sdk/python/agentfield/harness/_runner.py b/sdk/python/agentfield/harness/_runner.py index d0aaf1b31..4383b5851 100644 --- a/sdk/python/agentfield/harness/_runner.py +++ b/sdk/python/agentfield/harness/_runner.py @@ -26,7 +26,10 @@ try_parse_from_text, ) from agentfield.harness.providers._base import HarnessProvider -from agentfield.harness.providers._factory import build_provider +from agentfield.harness.providers._factory import ( + DEFAULT_HARNESS_PROVIDER, + build_provider, +) logger = logging.getLogger(__name__) @@ -277,12 +280,8 @@ async def run( } options = _resolve_options(self._config, overrides) - resolved_provider = options.get("provider") - if not resolved_provider: - raise ValueError( - "No harness provider specified. Set 'provider' in HarnessConfig " - "or pass it to .harness() call." - ) + resolved_provider = options.get("provider") or DEFAULT_HARNESS_PROVIDER + options["provider"] = resolved_provider resolved_cwd = str(options.get("cwd") or ".") provider_instance = self._build_provider(str(resolved_provider), options) diff --git a/sdk/python/agentfield/harness/providers/_factory.py b/sdk/python/agentfield/harness/providers/_factory.py index 2277153af..6812dd999 100644 --- a/sdk/python/agentfield/harness/providers/_factory.py +++ b/sdk/python/agentfield/harness/providers/_factory.py @@ -16,10 +16,11 @@ "opencode", "pi", } +DEFAULT_HARNESS_PROVIDER = "omp" def build_provider(config: "HarnessConfig") -> "HarnessProvider": - provider_name = config.provider + provider_name = config.provider or DEFAULT_HARNESS_PROVIDER if provider_name not in SUPPORTED_PROVIDERS: raise ValueError( f"Unknown harness provider: {provider_name!r}. Supported providers: " diff --git a/sdk/python/agentfield/types.py b/sdk/python/agentfield/types.py index e97ac886b..8cac609c1 100644 --- a/sdk/python/agentfield/types.py +++ b/sdk/python/agentfield/types.py @@ -275,13 +275,16 @@ class DiscoveryResult: class HarnessConfig(BaseModel): provider: str = Field( - ..., + default="omp", description=( 'Coding agent provider: "aforge" | "claude-code" | "codex" | ' '"gemini" | "opencode" | "pi" | "omp" | "grok"' ), ) - model: str = Field(default="sonnet", description="Default model identifier.") + model: Optional[str] = Field( + default=None, + description="Default model identifier; omitted uses the CLI's configured default.", + ) max_turns: int = Field(default=30, description="Maximum agent iterations.") max_budget_usd: Optional[float] = Field( default=None, description="Cost cap in USD." diff --git a/sdk/python/tests/test_harness_factory.py b/sdk/python/tests/test_harness_factory.py index c8b92f00f..a446f119c 100644 --- a/sdk/python/tests/test_harness_factory.py +++ b/sdk/python/tests/test_harness_factory.py @@ -11,7 +11,11 @@ import pytest from agentfield.harness.providers._base import HarnessProvider -from agentfield.harness.providers._factory import SUPPORTED_PROVIDERS, build_provider +from agentfield.harness.providers._factory import ( + DEFAULT_HARNESS_PROVIDER, + SUPPORTED_PROVIDERS, + build_provider, +) from agentfield.types import HarnessConfig @@ -41,6 +45,13 @@ def test_supported_providers_contains_expected_names(): assert "grok" in SUPPORTED_PROVIDERS +def test_default_provider_is_omp(): + from agentfield.harness.providers.pi import OMPProvider + + assert DEFAULT_HARNESS_PROVIDER == "omp" + assert isinstance(build_provider(HarnessConfig()), OMPProvider) + + # --------------------------------------------------------------------------- # build_provider: unknown provider raises ValueError # --------------------------------------------------------------------------- diff --git a/sdk/python/tests/test_harness_provider_availability.py b/sdk/python/tests/test_harness_provider_availability.py index 76667450b..def8e120d 100644 --- a/sdk/python/tests/test_harness_provider_availability.py +++ b/sdk/python/tests/test_harness_provider_availability.py @@ -11,28 +11,32 @@ from agentfield.harness.providers.claude import ClaudeCodeProvider from agentfield.harness.providers.gemini import GeminiProvider from agentfield.harness.providers.opencode import OpenCodeProvider +from agentfield.harness.providers.pi import OMPProvider, PiProvider @pytest.mark.asyncio @pytest.mark.parametrize( - ("provider", "name", "install_command"), + ("provider", "name", "module", "install_command"), [ - (AforgeProvider(bin_path="aforge-missing"), "aforge", "aforge-v2"), - (CodexProvider(bin_path="codex-missing"), "codex", "@openai/codex"), + (AforgeProvider(bin_path="aforge-missing"), "aforge", "aforge", "aforge-v2"), + (CodexProvider(bin_path="codex-missing"), "codex", "codex", "@openai/codex"), ( OpenCodeProvider(bin_path="opencode-missing"), "opencode", + "opencode", "opencode.ai/install", ), + (PiProvider(bin_path="pi-missing"), "pi", "pi", "@earendil-works/pi-coding-agent"), + (OMPProvider(bin_path="omp-missing"), "omp", "pi", "omp.sh/install"), ], ) async def test_cli_provider_raises_typed_error_before_spawn( - monkeypatch, provider, name, install_command + monkeypatch, provider, name, module, install_command ): monkeypatch.setattr("agentfield.harness._availability.shutil.which", lambda _: None) run_cli = AsyncMock() monkeypatch.setattr( - f"agentfield.harness.providers.{name}.run_cli", + f"agentfield.harness.providers.{module}.run_cli", run_cli, ) diff --git a/sdk/python/tests/test_harness_runner.py b/sdk/python/tests/test_harness_runner.py index cbb689ff0..ccb4b3439 100644 --- a/sdk/python/tests/test_harness_runner.py +++ b/sdk/python/tests/test_harness_runner.py @@ -265,10 +265,17 @@ async def test_run_with_schema_injects_prompt_suffix_and_parses_output(tmp_path) @pytest.mark.asyncio -async def test_run_raises_when_no_provider_set(tmp_path): +async def test_run_defaults_to_omp_when_no_provider_set(tmp_path): runner = HarnessRunner() - with pytest.raises(ValueError, match="No harness provider specified"): - await runner.run("hello", cwd=str(tmp_path)) + provider = MockProvider([RawResult(result="ok")]) + + with patch("agentfield.harness._runner.build_provider", return_value=provider) as factory: + result = await runner.run("hello", cwd=str(tmp_path)) + + assert result.result == "ok" + assert provider.last_options is not None + assert provider.last_options["provider"] == "omp" + assert factory.call_args.args[0].provider == "omp" @pytest.mark.asyncio diff --git a/sdk/python/tests/test_harness_types.py b/sdk/python/tests/test_harness_types.py index bc43ad8a3..aa15973aa 100644 --- a/sdk/python/tests/test_harness_types.py +++ b/sdk/python/tests/test_harness_types.py @@ -1,23 +1,23 @@ # pyright: reportMissingImports=false import pytest -from pydantic import ValidationError from agentfield.harness._result import HarnessResult, Metrics, RawResult from agentfield.harness.providers._factory import build_provider from agentfield.types import HarnessConfig -def test_harness_config_provider_required(): - with pytest.raises(ValidationError): - HarnessConfig() +def test_harness_config_defaults_to_omp(): + cfg = HarnessConfig() + assert cfg.provider == "omp" + assert cfg.model is None def test_harness_config_defaults(): cfg = HarnessConfig(provider="codex") assert cfg.provider == "codex" - assert cfg.model == "sonnet" + assert cfg.model is None assert cfg.max_turns == 30 assert cfg.max_budget_usd is None assert cfg.max_retries == 3 diff --git a/sdk/python/tests/test_types.py b/sdk/python/tests/test_types.py index c715623fa..ff38343e5 100644 --- a/sdk/python/tests/test_types.py +++ b/sdk/python/tests/test_types.py @@ -468,9 +468,9 @@ def test_with_json_response(self): class TestHarnessConfig: def test_defaults(self): - hc = HarnessConfig(provider="claude-code") - assert hc.provider == "claude-code" - assert hc.model == "sonnet" + hc = HarnessConfig() + assert hc.provider == "omp" + assert hc.model is None assert hc.max_turns == 30 assert hc.max_budget_usd is None assert hc.max_retries == 3 @@ -493,9 +493,8 @@ def test_custom_values(self): assert hc.tools == ["Bash"] assert hc.permission_mode == "auto" - def test_provider_required(self): - with pytest.raises(Exception): - HarnessConfig() # type: ignore[call-arg] + def test_provider_override(self): + assert HarnessConfig(provider="claude-code").provider == "claude-code" def test_json_roundtrip(self): hc = HarnessConfig(provider="gemini", model="gemini-2.5-flash") diff --git a/sdk/python/tests/test_usage_transport.py b/sdk/python/tests/test_usage_transport.py index 9c7d8b415..f91dd8503 100644 --- a/sdk/python/tests/test_usage_transport.py +++ b/sdk/python/tests/test_usage_transport.py @@ -553,6 +553,23 @@ def test_record_harness_usage_noop_when_empty(self): reset_current_cost_tracker(token) assert tracker.call_count == 0 + def test_record_harness_usage_defaults_to_omp(self): + from agentfield.harness._result import HarnessResult + + agent = self._agent() + tracker = CostTracker() + token = set_current_cost_tracker(tracker) + try: + agent._record_harness_usage( + HarnessResult(result="done", input_tokens=1), + ) + finally: + reset_current_cost_tracker(token) + + entry = tracker.serialize()["entries"][0] + assert entry["harness"] == "omp" + assert entry["model"] == "omp" + class TestEnvelopeEndToEnd: """Drive the FastAPI endpoint and assert usage lands in both transports.""" diff --git a/sdk/typescript/src/agent/Agent.ts b/sdk/typescript/src/agent/Agent.ts index 6da19e932..9288d8a96 100644 --- a/sdk/typescript/src/agent/Agent.ts +++ b/sdk/typescript/src/agent/Agent.ts @@ -40,6 +40,7 @@ import { SkillContext } from '../context/SkillContext.js'; import { AIClient } from '../ai/AIClient.js'; import { AgentFieldClient } from '../client/AgentFieldClient.js'; import type { HarnessRunner } from '../harness/runner.js'; +import { DEFAULT_HARNESS_PROVIDER } from '../harness/providers/factory.js'; import type { HarnessOptions, HarnessResult } from '../harness/types.js'; import { splitModelVariant } from '../harness/modelVariant.js'; import { MemoryClient } from '../memory/MemoryClient.js'; @@ -411,7 +412,7 @@ export class Agent { return; } - const providerName = options?.provider ?? this.config.harnessConfig?.provider; + const providerName = options?.provider ?? this.config.harnessConfig?.provider ?? DEFAULT_HARNESS_PROVIDER; const harnessName = providerName ? String(providerName).replace(/-/g, '_') : null; // Usage is recorded against the base model — a "#variant" // reasoning-effort suffix on the configured model never reaches the diff --git a/sdk/typescript/src/harness/index.ts b/sdk/typescript/src/harness/index.ts index 50caf8000..7a8439e1c 100644 --- a/sdk/typescript/src/harness/index.ts +++ b/sdk/typescript/src/harness/index.ts @@ -3,5 +3,5 @@ export { createHarnessResult, createMetrics, createRawResult } from './types.js' export type { ModelVariant } from './modelVariant.js'; export { MODEL_VARIANT_SEP, splitModelVariant, resolveModelAndVariant } from './modelVariant.js'; export type { HarnessProvider } from './providers/base.js'; -export { buildProvider, SUPPORTED_PROVIDERS } from './providers/factory.js'; +export { buildProvider, DEFAULT_HARNESS_PROVIDER, SUPPORTED_PROVIDERS } from './providers/factory.js'; export { HarnessRunner } from './runner.js'; diff --git a/sdk/typescript/src/harness/providers/factory.ts b/sdk/typescript/src/harness/providers/factory.ts index 70d8f6b32..b6bfcbe64 100644 --- a/sdk/typescript/src/harness/providers/factory.ts +++ b/sdk/typescript/src/harness/providers/factory.ts @@ -2,36 +2,38 @@ import type { HarnessProvider } from './base.js'; import type { HarnessConfig } from '../types.js'; export const SUPPORTED_PROVIDERS = new Set(['claude-code', 'codex', 'gemini', 'omp', 'opencode', 'pi']); +export const DEFAULT_HARNESS_PROVIDER = 'omp' as const; export async function buildProvider(config: HarnessConfig): Promise { - if (!SUPPORTED_PROVIDERS.has(config.provider)) { + const provider = config.provider ?? DEFAULT_HARNESS_PROVIDER; + if (!SUPPORTED_PROVIDERS.has(provider)) { throw new Error( - `Unknown harness provider: "${config.provider}". Supported: ${[...SUPPORTED_PROVIDERS].sort().join(', ')}` + `Unknown harness provider: "${provider}". Supported: ${[...SUPPORTED_PROVIDERS].sort().join(', ')}` ); } - if (config.provider === 'claude-code') { + if (provider === 'claude-code') { const { ClaudeCodeProvider } = await import('./claude.js'); return new ClaudeCodeProvider(); } - if (config.provider === 'codex') { + if (provider === 'codex') { const { CodexProvider } = await import('./codex.js'); return new CodexProvider(config.codexBin ?? 'codex'); } - if (config.provider === 'gemini') { + if (provider === 'gemini') { const { GeminiProvider } = await import('./gemini.js'); return new GeminiProvider(config.geminiBin ?? 'gemini'); } - if (config.provider === 'opencode') { + if (provider === 'opencode') { const { OpenCodeProvider } = await import('./opencode.js'); return new OpenCodeProvider(config.opencodeBin ?? 'opencode'); } - if (config.provider === 'pi') { + if (provider === 'pi') { const { PiProvider } = await import('./pi.js'); return new PiProvider(config.piBin ?? 'pi'); } - if (config.provider === 'omp') { - const { OmpProvider } = await import('./pi.js'); - return new OmpProvider(config.ompBin ?? 'omp'); + if (provider === 'omp') { + const { OMPProvider } = await import('./pi.js'); + return new OMPProvider(config.ompBin ?? 'omp'); } - throw new Error(`Provider "${config.provider}" is not yet implemented.`); + throw new Error(`Provider "${provider}" is not yet implemented.`); } diff --git a/sdk/typescript/src/harness/providers/index.ts b/sdk/typescript/src/harness/providers/index.ts index 0f8b208d5..1560b169e 100644 --- a/sdk/typescript/src/harness/providers/index.ts +++ b/sdk/typescript/src/harness/providers/index.ts @@ -1,7 +1,7 @@ export type { HarnessProvider } from './base.js'; -export { buildProvider, SUPPORTED_PROVIDERS } from './factory.js'; +export { buildProvider, DEFAULT_HARNESS_PROVIDER, SUPPORTED_PROVIDERS } from './factory.js'; export { ClaudeCodeProvider } from './claude.js'; export { CodexProvider } from './codex.js'; export { GeminiProvider } from './gemini.js'; export { OpenCodeProvider } from './opencode.js'; -export { PiProvider, OmpProvider } from './pi.js'; +export { OMPProvider, PiProvider } from './pi.js'; diff --git a/sdk/typescript/src/harness/providers/pi.ts b/sdk/typescript/src/harness/providers/pi.ts index 6e5fc4b86..bbe440d30 100644 --- a/sdk/typescript/src/harness/providers/pi.ts +++ b/sdk/typescript/src/harness/providers/pi.ts @@ -225,7 +225,7 @@ export class PiProvider extends PiFamilyProvider { } } -export class OmpProvider extends PiFamilyProvider { +export class OMPProvider extends PiFamilyProvider { public constructor(binPath = 'omp') { super('omp', binPath); } diff --git a/sdk/typescript/src/harness/runner.ts b/sdk/typescript/src/harness/runner.ts index 722342cee..0dedba53b 100644 --- a/sdk/typescript/src/harness/runner.ts +++ b/sdk/typescript/src/harness/runner.ts @@ -1,5 +1,5 @@ import { buildPromptSuffix, cleanupTempFiles, getOutputPath, parseAndValidate } from './schema.js'; -import { buildProvider } from './providers/factory.js'; +import { buildProvider, DEFAULT_HARNESS_PROVIDER } from './providers/factory.js'; import type { HarnessProvider } from './providers/base.js'; import { createHarnessResult, @@ -55,9 +55,7 @@ export class HarnessRunner { const { schema, ...rest } = options; const resolved = this.resolveOptions(this.config, rest); - if (!resolved.provider) { - throw new Error("No harness provider specified. Set 'provider' in HarnessConfig or pass it to .harness() call."); - } + resolved.provider ??= DEFAULT_HARNESS_PROVIDER; const cwd = resolved.projectDir ?? resolved.cwd ?? '.'; const provider = await this.buildProvider(resolved.provider, resolved); diff --git a/sdk/typescript/src/harness/types.ts b/sdk/typescript/src/harness/types.ts index 0dc72d842..d66125caf 100644 --- a/sdk/typescript/src/harness/types.ts +++ b/sdk/typescript/src/harness/types.ts @@ -1,5 +1,6 @@ export interface HarnessConfig { - provider: 'claude-code' | 'codex' | 'gemini' | 'opencode' | 'pi' | 'omp'; + /** Coding-agent provider. Defaults to OMP. */ + provider?: 'claude-code' | 'codex' | 'gemini' | 'opencode' | 'pi' | 'omp'; model?: string; /** * Provider-specific reasoning-effort variant (e.g. `high`, `minimal`). diff --git a/sdk/typescript/tests/harness_provider_pi.test.ts b/sdk/typescript/tests/harness_provider_pi.test.ts index af8c2ae68..d7e17954a 100644 --- a/sdk/typescript/tests/harness_provider_pi.test.ts +++ b/sdk/typescript/tests/harness_provider_pi.test.ts @@ -2,7 +2,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import * as cli from '../src/harness/cli.js'; import { buildProvider } from '../src/harness/providers/factory.js'; -import { OmpProvider, PiProvider } from '../src/harness/providers/pi.js'; +import { OMPProvider, PiProvider } from '../src/harness/providers/pi.js'; afterEach(() => { vi.restoreAllMocks(); @@ -46,7 +46,7 @@ describe.each([ }, { name: 'omp', - provider: new OmpProvider('/opt/omp'), + provider: new OMPProvider('/opt/omp'), prefix: ['/opt/omp', '--print', '--mode', 'json', '--cwd', '/tmp/project'], permissionFlag: '--auto-approve', globTool: 'glob', @@ -105,7 +105,7 @@ describe.each([ it.each([ { provider: new PiProvider(), resumeFlag: '--session', tools: 'read,grep,find' }, - { provider: new OmpProvider(), resumeFlag: '--resume', tools: 'read,grep,glob' }, + { provider: new OMPProvider(), resumeFlag: '--resume', tools: 'read,grep,glob' }, ])('keeps $resumeFlag retries read-only', async ({ provider, resumeFlag, tools }) => { vi.spyOn(cli, 'runCli').mockResolvedValue({ stdout: eventStream('plan'), @@ -124,12 +124,26 @@ it.each([ expect(cmd[cmd.indexOf(resumeFlag) + 1]).toBe('abc123'); }); +it.each([ + { provider: new PiProvider('pi-missing'), installHint: '@earendil-works/pi-coding-agent' }, + { provider: new OMPProvider('omp-missing'), installHint: 'omp.sh/install' }, +])('returns actionable install guidance for a missing binary', async ({ provider, installHint }) => { + vi.spyOn(cli, 'runCli').mockRejectedValue(new Error('spawn ENOENT')); + + const result = await provider.execute('hello', {}); + + expect(result.isError).toBe(true); + expect(result.errorMessage).toContain(installHint); +}); + describe('provider factory', () => { it('routes pi and omp and passes binary overrides', async () => { const pi = await buildProvider({ provider: 'pi', piBin: '/opt/pi' }); const omp = await buildProvider({ provider: 'omp', ompBin: '/opt/omp' }); + const defaultProvider = await buildProvider({}); expect(pi).toBeInstanceOf(PiProvider); - expect(omp).toBeInstanceOf(OmpProvider); + expect(omp).toBeInstanceOf(OMPProvider); + expect(defaultProvider).toBeInstanceOf(OMPProvider); }); }); diff --git a/sdk/typescript/tests/harness_runner.test.ts b/sdk/typescript/tests/harness_runner.test.ts index 9a62e27dc..b3412530b 100644 --- a/sdk/typescript/tests/harness_runner.test.ts +++ b/sdk/typescript/tests/harness_runner.test.ts @@ -163,9 +163,17 @@ describe('harness runner', () => { expect(result.parsed).toEqual({ name: 'ok', count: 1 }); }); - it('run throws when no provider is configured', async () => { + it('run defaults to OMP when no provider is configured', async () => { + const provider = new MockProvider([ + createRawResult({ result: 'ok', metrics: createMetrics({ numTurns: 1 }) }), + ]); + const factorySpy = vi.spyOn(factory, 'buildProvider').mockResolvedValue(provider); const runner = new HarnessRunner(); - await expect(runner.run('hello', {})).rejects.toThrow(/No harness provider specified/); + const result = await runner.run('hello', {}); + + expect(result.result).toBe('ok'); + expect(factorySpy).toHaveBeenCalledWith(expect.objectContaining({ provider: 'omp' })); + expect(provider.lastOptions).toEqual(expect.objectContaining({ provider: 'omp' })); }); it('retries on transient error then succeeds', async () => { diff --git a/sdk/typescript/tests/usage_ai_capture.test.ts b/sdk/typescript/tests/usage_ai_capture.test.ts index ced45637e..ffd4e47a2 100644 --- a/sdk/typescript/tests/usage_ai_capture.test.ts +++ b/sdk/typescript/tests/usage_ai_capture.test.ts @@ -273,7 +273,7 @@ describe('harness usage capture', () => { }); it('records cost-only runs and skips runs that reported nothing', async () => { - const agent = makeAgent(); + const agent = new Agent({ nodeId: 'default-harness-agent' }); const runSpy = vi.spyOn(HarnessRunner.prototype, 'run'); // Cost known, tokens unknown -> still recorded. @@ -283,7 +283,8 @@ describe('harness usage capture', () => { expect(ctx.costTracker.serialize().entries[0]).toMatchObject({ cost_usd: 0.25, cost_source: 'provider', - total_tokens: 0 + total_tokens: 0, + harness: 'omp' }); // Neither tokens nor cost -> no entry. diff --git a/skills/agentfield/SKILL.md b/skills/agentfield/SKILL.md index 639fbd2bf..e42e15814 100644 --- a/skills/agentfield/SKILL.md +++ b/skills/agentfield/SKILL.md @@ -68,7 +68,7 @@ Everything else is a variation. Less-used but real: - **`@app.skill()`** — deterministic functions you want callable through the control plane (no LLM). -- **`app.harness(prompt, provider="claude-code"|"codex"|"gemini"|"opencode"|"pi"|"omp")`** — delegates to an external coding-agent CLI. Heavy. **Only use when `af doctor` reports `harness_usable: true` AND the Dockerfile installs the CLI AND `shutil.which()` guards startup.** Otherwise use `app.ai(tools=[...])`. +- **`app.harness(prompt, provider="claude-code"|"codex"|"gemini"|"opencode"|"pi"|"omp")`** — delegates to an external coding-agent CLI. OMP is the SDK default when `provider` is omitted. Heavy. **Only use when `af doctor` reports `harness_usable: true` AND the Dockerfile installs the selected CLI AND `shutil.which()` guards startup.** Otherwise use `app.ai(tools=[...])`. Full signatures, schemas, router surface, memory scopes, and the cross-boundary serialization gotcha are in `references/primitives-snapshot.md` (offline-frozen). **Prefer the live `agentfield.ai/llms-full.txt`** when you have a network — it is the source of truth and it does not drift. From 2ec6a8a8beabeb86535e5d00eae160f7f4b86990 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Wed, 12 Aug 2026 16:56:07 -0400 Subject: [PATCH 03/20] Improve Pi and OMP harness coverage --- sdk/go/harness/pi_test.go | 111 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/sdk/go/harness/pi_test.go b/sdk/go/harness/pi_test.go index 51247b1a5..a08fab25c 100644 --- a/sdk/go/harness/pi_test.go +++ b/sdk/go/harness/pi_test.go @@ -2,6 +2,7 @@ package harness import ( "context" + "errors" "fmt" "strings" "testing" @@ -131,6 +132,116 @@ func TestPiFamilyPlanModeIsReadOnlyAndResumes(t *testing.T) { } } +func TestPiFamilyToolEdgeCases(t *testing.T) { + tests := []struct { + name string + provider *piFamilyProvider + options Options + wantFlag string + wantNoTools bool + }{ + { + name: "pi plan restores default read-only tools", + provider: NewPiProvider("pi").piFamilyProvider, + options: Options{PermissionMode: "plan", Tools: []string{"Write", "Bash"}}, + wantFlag: "read,grep,find", + }, + { + name: "omp plan restores default read-only tools", + provider: NewOMPProvider("omp").piFamilyProvider, + options: Options{PermissionMode: "plan", Tools: []string{"Write", "Bash"}}, + wantFlag: "read,grep,glob", + }, + { + name: "explicit empty tools disables tools", + provider: NewPiProvider("pi").piFamilyProvider, + options: Options{Tools: []string{}}, + wantNoTools: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var gotCmd []string + tc.provider.runCLI = func(_ context.Context, cmd []string, _ map[string]string, _ string, _ int, _ []byte) (*CLIResult, error) { + gotCmd = append([]string(nil), cmd...) + return &CLIResult{Stdout: piEventStream}, nil + } + + _, err := tc.provider.execute(context.Background(), "inspect", tc.options) + require.NoError(t, err) + if tc.wantNoTools { + assert.Contains(t, gotCmd, "--no-tools") + assert.NotContains(t, gotCmd, "--tools") + return + } + assertFlagValue(t, gotCmd, "--tools", tc.wantFlag) + }) + } +} + +func TestPiFamilyExecutionFailures(t *testing.T) { + tests := []struct { + name string + result *CLIResult + runErr error + wantErr bool + failureType FailureType + message string + }{ + { + name: "timeout", + runErr: errors.New("command timed out"), + failureType: FailureTimeout, + message: "command timed out", + }, + { + name: "unexpected runner error", + runErr: errors.New("pipe failed"), + wantErr: true, + }, + { + name: "nonzero exit without stderr", + result: &CLIResult{ReturnCode: 7}, + failureType: FailureCrash, + message: "Process exited with code 7.", + }, + { + name: "api error event", + result: &CLIResult{Stdout: `{"type":"message_end","message":{"role":"assistant","stopReason":"error","errorMessage":"quota exceeded"}}`}, + failureType: FailureAPIError, + message: "quota exceeded", + }, + { + name: "successful process without assistant output", + result: &CLIResult{Stderr: "empty response"}, + failureType: FailureNoOutput, + message: "empty response", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + provider := NewPiProvider("pi").piFamilyProvider + provider.runCLI = func(context.Context, []string, map[string]string, string, int, []byte) (*CLIResult, error) { + return tc.result, tc.runErr + } + + raw, err := provider.execute(context.Background(), "inspect", Options{}) + if tc.wantErr { + require.EqualError(t, err, tc.runErr.Error()) + assert.Nil(t, raw) + return + } + require.NoError(t, err) + require.NotNil(t, raw) + assert.True(t, raw.IsError) + assert.Equal(t, tc.failureType, raw.FailureType) + assert.Equal(t, tc.message, raw.ErrorMessage) + }) + } +} + func TestPiFamilyNonzeroExitIsError(t *testing.T) { p := NewPiProvider("pi") p.runCLI = func(context.Context, []string, map[string]string, string, int, []byte) (*CLIResult, error) { From 973913ebe23d245a6217f538fbd059fa2df95357 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Mon, 24 Aug 2026 15:27:53 -0400 Subject: [PATCH 04/20] address review comments on #913 - Go pi.go: build the plan-mode read-only tool list as a fresh slice instead of the in-place tools[:0] filter, matching the Python/TS providers and removing the aliasing footgun. - Go pi.go: distinguish a negative return code (signal kill) from a plain non-zero exit, reporting 'Process killed by signal N.' to match the Python provider and the gemini/opencode Go providers. - Add a pi_test.go case pinning the signal-kill message. --- sdk/go/harness/pi.go | 8 ++++++-- sdk/go/harness/pi_test.go | 6 ++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/sdk/go/harness/pi.go b/sdk/go/harness/pi.go index bfb42753b..5067fcbae 100644 --- a/sdk/go/harness/pi.go +++ b/sdk/go/harness/pi.go @@ -101,7 +101,7 @@ func (p *piFamilyProvider) execute(ctx context.Context, prompt string, options O tools := normalizePiTools(options.Tools, p.flavor) if options.PermissionMode == "plan" { - readOnly := tools[:0] + readOnly := make([]string, 0, len(tools)) for _, tool := range tools { if piReadOnlyTools[tool] { readOnly = append(readOnly, tool) @@ -163,7 +163,11 @@ func (p *piFamilyProvider) execute(ctx context.Context, prompt string, options O raw.Metrics.DurationAPIMS = apiMS raw.ReturnCode = cliResult.ReturnCode stderr := StripANSI(strings.TrimSpace(cliResult.Stderr)) - if cliResult.ReturnCode != 0 { + if cliResult.ReturnCode < 0 { + raw.IsError = true + raw.FailureType = FailureCrash + raw.ErrorMessage = fmt.Sprintf("Process killed by signal %d.", -cliResult.ReturnCode) + } else if cliResult.ReturnCode != 0 { raw.IsError = true raw.FailureType = FailureCrash if stderr != "" { diff --git a/sdk/go/harness/pi_test.go b/sdk/go/harness/pi_test.go index a08fab25c..14a4498de 100644 --- a/sdk/go/harness/pi_test.go +++ b/sdk/go/harness/pi_test.go @@ -206,6 +206,12 @@ func TestPiFamilyExecutionFailures(t *testing.T) { failureType: FailureCrash, message: "Process exited with code 7.", }, + { + name: "signal kill reports the signal, not an exit code", + result: &CLIResult{ReturnCode: -9}, + failureType: FailureCrash, + message: "Process killed by signal 9.", + }, { name: "api error event", result: &CLIResult{Stdout: `{"type":"message_end","message":{"role":"assistant","stopReason":"error","errorMessage":"quota exceeded"}}`}, From 8bb9f080eeda87ed462783c7b6cf4901300ee99c Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Thu, 27 Aug 2026 09:47:01 -0400 Subject: [PATCH 05/20] harness: keep aforge as the default; document Pi and OMP as additional providers Sweeps the OMP-as-default remnants left in non-conflicting files after the merge, and brings the Go Pi/OMP provider up to the Result.Model contract main added for aforge/opencode: - af doctor / af harness doctor list aforge-first ordering and drop the "omp default" help text. - skills/agentfield (+ embedded skill_data mirror), harness-v2-design, the harness_duo Go example and its README no longer claim OMP is the default; omp_worker now passes Provider explicitly. - TS Agent usage attribution resolves through resolveProviderName so the explicit > config > AGENTFIELD_HARNESS_PROVIDER > aforge chain is honoured. - sdk/go/harness/pi.go populates Metrics.Model (configured model wins over the model reported in the Pi/OMP JSONL stream), matching the Python and TS adapters. - Tests that asserted an OMP default now assert aforge; explicit pi/omp coverage is retained. Co-Authored-By: Claude Fable 5 --- control-plane/internal/cli/doctor.go | 4 +- .../skillkit/skill_data/agentfield/SKILL.md | 2 +- .../references/primitives-snapshot.md | 4 +- .../agentfield/references/scaffold-recipe.md | 2 +- docs/design/harness-v2-design.md | 6 +-- .../go_agent_nodes/cmd/harness_duo/README.md | 7 ++-- .../go_agent_nodes/cmd/harness_duo/main.go | 9 ++--- sdk/go/agent/harness_test.go | 10 +++-- sdk/go/harness/pi.go | 6 +++ sdk/go/harness/pi_test.go | 39 ++++++++++++++++++- sdk/python/tests/test_usage_transport.py | 20 +++++++++- sdk/typescript/src/agent/Agent.ts | 4 +- sdk/typescript/src/harness/index.ts | 8 +++- sdk/typescript/src/harness/providers/index.ts | 8 +++- sdk/typescript/tests/usage_ai_capture.test.ts | 2 +- skills/agentfield/SKILL.md | 2 +- .../references/primitives-snapshot.md | 4 +- .../agentfield/references/scaffold-recipe.md | 2 +- 18 files changed, 106 insertions(+), 33 deletions(-) diff --git a/control-plane/internal/cli/doctor.go b/control-plane/internal/cli/doctor.go index 2fda2902e..b1e81b60e 100644 --- a/control-plane/internal/cli/doctor.go +++ b/control-plane/internal/cli/doctor.go @@ -85,12 +85,12 @@ var harnessProviders = []struct { Binary string // executable name to look up on PATH ProbeArgs []string // minimal one-shot invocation used by `--probe` }{ - {Name: "omp", Binary: "omp", ProbeArgs: []string{"--print", "Say OK"}}, {Name: "claude-code", Binary: "claude", ProbeArgs: []string{"-p", "Say OK"}}, {Name: "codex", Binary: "codex", ProbeArgs: []string{"exec", "Say OK"}}, {Name: "gemini", Binary: "gemini", ProbeArgs: []string{"-p", "Say OK"}}, {Name: "opencode", Binary: "opencode", ProbeArgs: []string{"run", "Say OK"}}, {Name: "pi", Binary: "pi", ProbeArgs: []string{"--print", "Say OK"}}, + {Name: "omp", Binary: "omp", ProbeArgs: []string{"--print", "Say OK"}}, } // harnessProbeTimeout bounds a single provider smoke test. Coding-agent CLIs @@ -121,7 +121,7 @@ func NewDoctorCommand() *cobra.Command { Long: `Doctor inspects the local environment and reports what's available for building AgentField multi-reasoner systems: - • Available harness provider CLIs (omp default, claude-code, codex, gemini, opencode, pi) + • Available harness provider CLIs (claude-code, codex, gemini, opencode, pi, omp) • Provider API keys set in the environment (without leaking values) • Docker availability and whether the control-plane image is locally cached • Whether a local control plane is reachable diff --git a/control-plane/internal/skillkit/skill_data/agentfield/SKILL.md b/control-plane/internal/skillkit/skill_data/agentfield/SKILL.md index e42e15814..897ff985f 100644 --- a/control-plane/internal/skillkit/skill_data/agentfield/SKILL.md +++ b/control-plane/internal/skillkit/skill_data/agentfield/SKILL.md @@ -68,7 +68,7 @@ Everything else is a variation. Less-used but real: - **`@app.skill()`** — deterministic functions you want callable through the control plane (no LLM). -- **`app.harness(prompt, provider="claude-code"|"codex"|"gemini"|"opencode"|"pi"|"omp")`** — delegates to an external coding-agent CLI. OMP is the SDK default when `provider` is omitted. Heavy. **Only use when `af doctor` reports `harness_usable: true` AND the Dockerfile installs the selected CLI AND `shutil.which()` guards startup.** Otherwise use `app.ai(tools=[...])`. +- **`app.harness(prompt, provider="aforge"|"claude-code"|"codex"|"gemini"|"opencode"|"pi"|"omp")`** — delegates to an external coding-agent CLI. `aforge`, AgentField's own harness, is the SDK default when `provider` is omitted. Heavy. **Only use when `af doctor` reports `harness_usable: true` AND the Dockerfile installs the selected CLI AND `shutil.which()` guards startup.** Otherwise use `app.ai(tools=[...])`. Full signatures, schemas, router surface, memory scopes, and the cross-boundary serialization gotcha are in `references/primitives-snapshot.md` (offline-frozen). **Prefer the live `agentfield.ai/llms-full.txt`** when you have a network — it is the source of truth and it does not drift. diff --git a/control-plane/internal/skillkit/skill_data/agentfield/references/primitives-snapshot.md b/control-plane/internal/skillkit/skill_data/agentfield/references/primitives-snapshot.md index 33f335683..a8314e524 100644 --- a/control-plane/internal/skillkit/skill_data/agentfield/references/primitives-snapshot.md +++ b/control-plane/internal/skillkit/skill_data/agentfield/references/primitives-snapshot.md @@ -16,7 +16,7 @@ This is a minimal cheat sheet of the five primitives so an offline build can sti | `@app.skill()` | Registers a deterministic function (no LLM) | Sort, parse, dedupe, score-with-formula | | `app.ai(...)` | Single call OR multi-turn tool-using LLM call when `tools=` is passed | Classification, routing, structured analysis, stateful tool-using | | `app.call(target, **kwargs)` | Call another reasoner THROUGH the control plane. Returns `dict`. Tracks the workflow DAG | All inter-reasoner traffic | -| `app.harness(prompt, provider=...)` | Delegate to an external coding-agent CLI (claude-code / codex / gemini / opencode / pi / omp) | When you need a real coding agent to write files / run shell | +| `app.harness(prompt, provider=...)` | Delegate to a coding-agent CLI (aforge by default; claude-code / codex / gemini / opencode / pi / omp) | When you need a real coding agent to write files / run shell | --- @@ -157,7 +157,7 @@ Default canonical pattern: `AgentRouter(prefix="", tags=["domain"])`. `prefix="c result = await app.harness( prompt: str, schema: type[BaseModel] | None = None, - provider: "claude-code" | "codex" | "gemini" | "opencode" | "pi" | "omp" | None = None, + provider: "aforge" | "claude-code" | "codex" | "gemini" | "opencode" | "pi" | "omp" | None = None, # None -> "aforge" model: str | None = None, max_turns: int | None = None, max_budget_usd: float | None = None, diff --git a/control-plane/internal/skillkit/skill_data/agentfield/references/scaffold-recipe.md b/control-plane/internal/skillkit/skill_data/agentfield/references/scaffold-recipe.md index 53b2f35c9..f1812deed 100644 --- a/control-plane/internal/skillkit/skill_data/agentfield/references/scaffold-recipe.md +++ b/control-plane/internal/skillkit/skill_data/agentfield/references/scaffold-recipe.md @@ -211,7 +211,7 @@ CMD ["python", "main.py"] Build context is the project directory itself (`context: .`), so the same scaffold works whether the project lives in `code/examples/` or standalone at `/tmp/my-build/`. -**Only edit if you need to install a harness CLI** (claude-code, codex, gemini, opencode, pi, omp). See `primitives-snapshot.md` → "Harness availability gate". Otherwise leave it alone. +**Only edit if you need to install a harness CLI** (claude-code, codex, gemini, opencode, pi, omp — `aforge`, the default, ships with `af`). See `primitives-snapshot.md` → "Harness availability gate". Otherwise leave it alone. --- diff --git a/docs/design/harness-v2-design.md b/docs/design/harness-v2-design.md index 34cc2b63f..2f00a7204 100644 --- a/docs/design/harness-v2-design.md +++ b/docs/design/harness-v2-design.md @@ -1,6 +1,6 @@ # `.harness()` — First-Class Coding Agent Integration for AgentField -> **Status**: Implemented; updated for Pi/OMP parity and OMP default +> **Status**: Implemented; updated for Pi/OMP parity (aforge remains the default) > **Author**: Architecture brainstorm > **Scope**: Python, TypeScript, and Go SDKs > **Date**: 2026-03-02 @@ -9,7 +9,7 @@ ## 1. Overview -Add `.harness()` as a first-class method on the Agent class — matching the DX of `.ai()` — enabling developers to dispatch multi-turn coding tasks to external coding agents (OMP, Pi, Claude Code, Codex, Gemini CLI, and OpenCode). OMP is the zero-configuration provider default in every SDK. +Add `.harness()` as a first-class method on the Agent class — matching the DX of `.ai()` — enabling developers to dispatch multi-turn coding tasks to coding agents (AForge, Claude Code, Codex, Gemini CLI, OpenCode, Pi, and OMP). AForge, AgentField's own harness, is the provider default in every SDK; Pi and OMP are additional providers selected explicitly. **Key principle**: `.ai()` is for single-turn LLM calls. `.harness()` is for multi-turn agentic coding tasks that browse files, edit code, run tests, and iterate. @@ -214,7 +214,7 @@ class Agent(FastAPI): | **gemini** | CLI subprocess `gemini --output-format stream-json` | CLI subprocess | CLI subprocess | File-write (universal) | | **opencode** | CLI subprocess | CLI subprocess | CLI subprocess | File-write (universal) | | **pi** | CLI subprocess | CLI subprocess | CLI subprocess | File-write (universal) | -| **omp** (default) | CLI subprocess | CLI subprocess | CLI subprocess | File-write (universal) | +| **omp** | CLI subprocess | CLI subprocess | CLI subprocess | File-write (universal) | ### 4.2 Why SDK-First Where Available diff --git a/examples/go_agent_nodes/cmd/harness_duo/README.md b/examples/go_agent_nodes/cmd/harness_duo/README.md index 18803801a..c780398da 100644 --- a/examples/go_agent_nodes/cmd/harness_duo/README.md +++ b/examples/go_agent_nodes/cmd/harness_duo/README.md @@ -5,12 +5,13 @@ This example registers one AgentField workflow with three Go reasoners: ```text compare ├── pi_worker (Pi harness) -└── omp_worker (Oh My Pi through the provider-less OMP default) +└── omp_worker (Oh My Pi harness) ``` `compare` starts both child reasoners concurrently and joins their structured -results. The Pi branch opts in explicitly; the OMP branch intentionally omits -`Provider` to exercise the SDK default. The default model is +results. Both branches name their provider explicitly — Pi and OMP are +additional providers, while a call that omits `Provider` runs the SDK default, +`aforge`. The default model is `openrouter/minimax/minimax-m2.7`; set `HARNESS_MODEL=openrouter/google/gemini-2.5-flash` for the Gemini Flash path. diff --git a/examples/go_agent_nodes/cmd/harness_duo/main.go b/examples/go_agent_nodes/cmd/harness_duo/main.go index 3246a1235..33457f237 100644 --- a/examples/go_agent_nodes/cmd/harness_duo/main.go +++ b/examples/go_agent_nodes/cmd/harness_duo/main.go @@ -57,9 +57,9 @@ func main() { } registerWorker(duo, "pi_worker", harness.ProviderPi, "PI_BIN") - // Omit Provider intentionally: this branch demonstrates that OMP is the - // cross-SDK default while still allowing OMP_BIN to select the executable. - registerWorker(duo, "omp_worker", "", "OMP_BIN") + // Both branches name their provider explicitly. Omitting Provider would + // select the SDK default, aforge — not OMP. + registerWorker(duo, "omp_worker", harness.ProviderOMP, "OMP_BIN") duo.RegisterReasoner("compare", func(ctx context.Context, input map[string]any) (any, error) { branchInput := map[string]any{ @@ -105,9 +105,6 @@ func main() { func registerWorker(duo *agent.Agent, reasoner, provider, binEnv string) { providerName := provider - if providerName == "" { - providerName = harness.DefaultProvider - } duo.RegisterReasoner(reasoner, func(ctx context.Context, input map[string]any) (any, error) { model := inputString(input, "model", envOr("HARNESS_MODEL", defaultModel)) root := inputString(input, "project_dir", projectDir()) diff --git a/sdk/go/agent/harness_test.go b/sdk/go/agent/harness_test.go index c06016c7e..52b73db39 100644 --- a/sdk/go/agent/harness_test.go +++ b/sdk/go/agent/harness_test.go @@ -40,12 +40,13 @@ func TestHarnessRunner_LazyInitialization(t *testing.T) { } func TestHarnessRunner_DefaultOptions(t *testing.T) { - // Agent with no HarnessConfig — runner resolves the SDK default to OMP. + // Agent with no HarnessConfig — runner gets zero-value Options; the + // provider is resolved at dispatch time (explicit > env > "aforge"). a := newTestAgentForHarness(t) runner := a.HarnessRunner() assert.NotNil(t, runner) - assert.Equal(t, harness.ProviderOMP, runner.DefaultOptions.Provider) + assert.Equal(t, "", runner.DefaultOptions.Provider) assert.Equal(t, "", runner.DefaultOptions.Model) } @@ -165,11 +166,12 @@ func TestHarnessConfig_PartialOverride(t *testing.T) { } func TestHarness_NilHarnessConfig(t *testing.T) { - // Agent with nil HarnessConfig should still create a runner with OMP selected. + // Agent with nil HarnessConfig should still create a runner with default (empty) options a := newTestAgentForHarness(t) assert.Nil(t, a.cfg.HarnessConfig) runner := a.HarnessRunner() assert.NotNil(t, runner) - assert.Equal(t, harness.Options{Provider: harness.ProviderOMP}, runner.DefaultOptions) + // Default options should be zero + assert.Equal(t, harness.Options{}, runner.DefaultOptions) } diff --git a/sdk/go/harness/pi.go b/sdk/go/harness/pi.go index 5067fcbae..add32b911 100644 --- a/sdk/go/harness/pi.go +++ b/sdk/go/harness/pi.go @@ -160,6 +160,9 @@ func (p *piFamilyProvider) execute(ctx context.Context, prompt string, options O } raw := parsePiJSONL(cliResult.Stdout) + if model != "" { + raw.Metrics.Model = model + } raw.Metrics.DurationAPIMS = apiMS raw.ReturnCode = cliResult.ReturnCode stderr := StripANSI(strings.TrimSpace(cliResult.Stderr)) @@ -237,6 +240,9 @@ func parsePiJSONL(stdout string) *RawResult { if !ok || message["role"] != "assistant" { continue } + if model, ok := message["model"].(string); ok { + raw.Metrics.Model = model + } if text := piMessageText(message); text != "" { raw.Result = text } diff --git a/sdk/go/harness/pi_test.go b/sdk/go/harness/pi_test.go index 14a4498de..411c61daf 100644 --- a/sdk/go/harness/pi_test.go +++ b/sdk/go/harness/pi_test.go @@ -96,6 +96,39 @@ func TestPiFamilyCommandAndMetrics(t *testing.T) { } } +func TestPiConfiguredModelOverridesReportedModel(t *testing.T) { + provider := NewPiProvider("pi") + provider.runCLI = func(context.Context, []string, map[string]string, string, int, []byte) (*CLIResult, error) { + return &CLIResult{Stdout: piEventStream}, nil + } + + raw, err := provider.Execute(context.Background(), "inspect", Options{Model: "openrouter/x/y"}) + require.NoError(t, err) + assert.Equal(t, "openrouter/x/y", raw.Metrics.Model) +} + +func TestPiUsesReportedModelWithoutConfiguredModel(t *testing.T) { + provider := NewPiProvider("pi") + provider.runCLI = func(context.Context, []string, map[string]string, string, int, []byte) (*CLIResult, error) { + return &CLIResult{Stdout: piEventStream}, nil + } + + raw, err := provider.Execute(context.Background(), "inspect", Options{}) + require.NoError(t, err) + assert.Equal(t, "google/gemini-2.5-flash", raw.Metrics.Model) +} + +func TestPiModelIsEmptyWhenNotConfiguredOrReported(t *testing.T) { + provider := NewPiProvider("pi") + provider.runCLI = func(context.Context, []string, map[string]string, string, int, []byte) (*CLIResult, error) { + return &CLIResult{Stdout: `{"type":"message_end","message":{"role":"assistant","content":"done"}}`}, nil + } + + raw, err := provider.Execute(context.Background(), "inspect", Options{}) + require.NoError(t, err) + assert.Empty(t, raw.Metrics.Model) +} + func TestPiFamilyPlanModeIsReadOnlyAndResumes(t *testing.T) { tests := []struct { name string @@ -267,9 +300,13 @@ func TestBuildProviderPiFamily(t *testing.T) { require.NoError(t, err) assert.Equal(t, "*harness.PiProvider", fmt.Sprintf("%T", pi)) assert.Equal(t, "*harness.OMPProvider", fmt.Sprintf("%T", omp)) + + // Pi and OMP are additional providers: an empty name still resolves to the + // SDK default, aforge. + t.Setenv(ProviderEnvVar, "") defaultProvider, err := BuildProvider("", "") require.NoError(t, err) - assert.Equal(t, "*harness.OMPProvider", fmt.Sprintf("%T", defaultProvider)) + assert.Equal(t, "*harness.AforgeProvider", fmt.Sprintf("%T", defaultProvider)) } func TestPiFamilyMissingBinaryIncludesInstallGuidance(t *testing.T) { diff --git a/sdk/python/tests/test_usage_transport.py b/sdk/python/tests/test_usage_transport.py index 92e758780..8ba6d2c98 100644 --- a/sdk/python/tests/test_usage_transport.py +++ b/sdk/python/tests/test_usage_transport.py @@ -634,9 +634,10 @@ def test_record_harness_usage_noop_when_empty(self): reset_current_cost_tracker(token) assert tracker.call_count == 0 - def test_record_harness_usage_defaults_to_omp(self): + def test_record_harness_usage_defaults_to_aforge(self, monkeypatch): from agentfield.harness._result import HarnessResult + monkeypatch.delenv("AGENTFIELD_HARNESS_PROVIDER", raising=False) agent = self._agent() tracker = CostTracker() token = set_current_cost_tracker(tracker) @@ -647,6 +648,23 @@ def test_record_harness_usage_defaults_to_omp(self): finally: reset_current_cost_tracker(token) + entry = tracker.serialize()["entries"][0] + assert entry["harness"] == "aforge" + assert entry["model"] == "aforge" + + def test_record_harness_usage_explicit_omp(self): + from agentfield.harness._result import HarnessResult + + agent = self._agent() + tracker = CostTracker() + token = set_current_cost_tracker(tracker) + try: + agent._record_harness_usage( + HarnessResult(result="done", input_tokens=1), provider="omp" + ) + finally: + reset_current_cost_tracker(token) + entry = tracker.serialize()["entries"][0] assert entry["harness"] == "omp" assert entry["model"] == "omp" diff --git a/sdk/typescript/src/agent/Agent.ts b/sdk/typescript/src/agent/Agent.ts index b93cbcd3c..df98df173 100644 --- a/sdk/typescript/src/agent/Agent.ts +++ b/sdk/typescript/src/agent/Agent.ts @@ -40,7 +40,7 @@ import { SkillContext } from '../context/SkillContext.js'; import { AIClient } from '../ai/AIClient.js'; import { AgentFieldClient } from '../client/AgentFieldClient.js'; import type { HarnessRunner } from '../harness/runner.js'; -import { DEFAULT_HARNESS_PROVIDER } from '../harness/providers/factory.js'; +import { resolveProviderName } from '../harness/providers/factory.js'; import type { HarnessOptions, HarnessResult } from '../harness/types.js'; import { splitModelVariant } from '../harness/modelVariant.js'; import { MemoryClient } from '../memory/MemoryClient.js'; @@ -413,7 +413,7 @@ export class Agent { return; } - const providerName = options?.provider ?? this.config.harnessConfig?.provider ?? DEFAULT_HARNESS_PROVIDER; + const providerName = resolveProviderName(options?.provider ?? this.config.harnessConfig?.provider); const harnessName = providerName ? String(providerName).replace(/-/g, '_') : null; // Usage is recorded against the base model — a "#variant" // reasoning-effort suffix on the configured model never reaches the diff --git a/sdk/typescript/src/harness/index.ts b/sdk/typescript/src/harness/index.ts index 7a8439e1c..3269e1731 100644 --- a/sdk/typescript/src/harness/index.ts +++ b/sdk/typescript/src/harness/index.ts @@ -3,5 +3,11 @@ export { createHarnessResult, createMetrics, createRawResult } from './types.js' export type { ModelVariant } from './modelVariant.js'; export { MODEL_VARIANT_SEP, splitModelVariant, resolveModelAndVariant } from './modelVariant.js'; export type { HarnessProvider } from './providers/base.js'; -export { buildProvider, DEFAULT_HARNESS_PROVIDER, SUPPORTED_PROVIDERS } from './providers/factory.js'; +export { + buildProvider, + DEFAULT_HARNESS_PROVIDER, + HARNESS_PROVIDER_ENV_VAR, + resolveProviderName, + SUPPORTED_PROVIDERS, +} from './providers/factory.js'; export { HarnessRunner } from './runner.js'; diff --git a/sdk/typescript/src/harness/providers/index.ts b/sdk/typescript/src/harness/providers/index.ts index 17af98e32..d8ba9ffb4 100644 --- a/sdk/typescript/src/harness/providers/index.ts +++ b/sdk/typescript/src/harness/providers/index.ts @@ -1,5 +1,11 @@ export type { HarnessProvider } from './base.js'; -export { buildProvider, SUPPORTED_PROVIDERS } from './factory.js'; +export { + buildProvider, + DEFAULT_HARNESS_PROVIDER, + HARNESS_PROVIDER_ENV_VAR, + resolveProviderName, + SUPPORTED_PROVIDERS, +} from './factory.js'; export { AforgeProvider } from './aforge.js'; export { ClaudeCodeProvider } from './claude.js'; export { CodexProvider } from './codex.js'; diff --git a/sdk/typescript/tests/usage_ai_capture.test.ts b/sdk/typescript/tests/usage_ai_capture.test.ts index ffd4e47a2..3315860d6 100644 --- a/sdk/typescript/tests/usage_ai_capture.test.ts +++ b/sdk/typescript/tests/usage_ai_capture.test.ts @@ -284,7 +284,7 @@ describe('harness usage capture', () => { cost_usd: 0.25, cost_source: 'provider', total_tokens: 0, - harness: 'omp' + harness: 'aforge' }); // Neither tokens nor cost -> no entry. diff --git a/skills/agentfield/SKILL.md b/skills/agentfield/SKILL.md index e42e15814..897ff985f 100644 --- a/skills/agentfield/SKILL.md +++ b/skills/agentfield/SKILL.md @@ -68,7 +68,7 @@ Everything else is a variation. Less-used but real: - **`@app.skill()`** — deterministic functions you want callable through the control plane (no LLM). -- **`app.harness(prompt, provider="claude-code"|"codex"|"gemini"|"opencode"|"pi"|"omp")`** — delegates to an external coding-agent CLI. OMP is the SDK default when `provider` is omitted. Heavy. **Only use when `af doctor` reports `harness_usable: true` AND the Dockerfile installs the selected CLI AND `shutil.which()` guards startup.** Otherwise use `app.ai(tools=[...])`. +- **`app.harness(prompt, provider="aforge"|"claude-code"|"codex"|"gemini"|"opencode"|"pi"|"omp")`** — delegates to an external coding-agent CLI. `aforge`, AgentField's own harness, is the SDK default when `provider` is omitted. Heavy. **Only use when `af doctor` reports `harness_usable: true` AND the Dockerfile installs the selected CLI AND `shutil.which()` guards startup.** Otherwise use `app.ai(tools=[...])`. Full signatures, schemas, router surface, memory scopes, and the cross-boundary serialization gotcha are in `references/primitives-snapshot.md` (offline-frozen). **Prefer the live `agentfield.ai/llms-full.txt`** when you have a network — it is the source of truth and it does not drift. diff --git a/skills/agentfield/references/primitives-snapshot.md b/skills/agentfield/references/primitives-snapshot.md index 33f335683..a8314e524 100644 --- a/skills/agentfield/references/primitives-snapshot.md +++ b/skills/agentfield/references/primitives-snapshot.md @@ -16,7 +16,7 @@ This is a minimal cheat sheet of the five primitives so an offline build can sti | `@app.skill()` | Registers a deterministic function (no LLM) | Sort, parse, dedupe, score-with-formula | | `app.ai(...)` | Single call OR multi-turn tool-using LLM call when `tools=` is passed | Classification, routing, structured analysis, stateful tool-using | | `app.call(target, **kwargs)` | Call another reasoner THROUGH the control plane. Returns `dict`. Tracks the workflow DAG | All inter-reasoner traffic | -| `app.harness(prompt, provider=...)` | Delegate to an external coding-agent CLI (claude-code / codex / gemini / opencode / pi / omp) | When you need a real coding agent to write files / run shell | +| `app.harness(prompt, provider=...)` | Delegate to a coding-agent CLI (aforge by default; claude-code / codex / gemini / opencode / pi / omp) | When you need a real coding agent to write files / run shell | --- @@ -157,7 +157,7 @@ Default canonical pattern: `AgentRouter(prefix="", tags=["domain"])`. `prefix="c result = await app.harness( prompt: str, schema: type[BaseModel] | None = None, - provider: "claude-code" | "codex" | "gemini" | "opencode" | "pi" | "omp" | None = None, + provider: "aforge" | "claude-code" | "codex" | "gemini" | "opencode" | "pi" | "omp" | None = None, # None -> "aforge" model: str | None = None, max_turns: int | None = None, max_budget_usd: float | None = None, diff --git a/skills/agentfield/references/scaffold-recipe.md b/skills/agentfield/references/scaffold-recipe.md index 53b2f35c9..f1812deed 100644 --- a/skills/agentfield/references/scaffold-recipe.md +++ b/skills/agentfield/references/scaffold-recipe.md @@ -211,7 +211,7 @@ CMD ["python", "main.py"] Build context is the project directory itself (`context: .`), so the same scaffold works whether the project lives in `code/examples/` or standalone at `/tmp/my-build/`. -**Only edit if you need to install a harness CLI** (claude-code, codex, gemini, opencode, pi, omp). See `primitives-snapshot.md` → "Harness availability gate". Otherwise leave it alone. +**Only edit if you need to install a harness CLI** (claude-code, codex, gemini, opencode, pi, omp — `aforge`, the default, ships with `af`). See `primitives-snapshot.md` → "Harness availability gate". Otherwise leave it alone. --- From 188feacd28141be2fc924246ad4a323ee05573bc Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Thu, 27 Aug 2026 09:59:15 -0400 Subject: [PATCH 06/20] harness: resolve the provider at dispatch time so aforge stays the default Follow-up to the merge: three places still baked OMP (or an eagerly resolved default) into the no-configuration path, which the gates caught. - sdk/go/harness/runner.go: NewRunner no longer stamps DefaultProvider into DefaultOptions. Run() applies explicit > AGENTFIELD_HARNESS_PROVIDER > aforge, so an env change reaches an already-constructed runner and Agent.HarnessRunner() keeps zero-value options as main expects. - sdk/python/agentfield/agent.py: _harness_provider_name no longer falls back to "omp", so usage attribution goes through resolve_harness_provider. - Tests: the Go/TS supported-provider strings list pi and omp, the TS pi/omp factory test asserts explicit routing instead of an OMP default, and the duplicate aforge-default usage test is dropped in favour of main's. Co-Authored-By: Claude Fable 5 --- sdk/go/harness/factory_test.go | 2 +- sdk/go/harness/runner.go | 6 +++--- sdk/go/harness/runner_invariant_test.go | 3 +++ sdk/python/agentfield/agent.py | 2 +- sdk/python/tests/test_usage_transport.py | 18 ------------------ .../tests/harness_provider_factory.test.ts | 8 +++++++- .../tests/harness_provider_pi.test.ts | 4 ++-- 7 files changed, 17 insertions(+), 26 deletions(-) diff --git a/sdk/go/harness/factory_test.go b/sdk/go/harness/factory_test.go index be480ed4b..77036beb7 100644 --- a/sdk/go/harness/factory_test.go +++ b/sdk/go/harness/factory_test.go @@ -44,5 +44,5 @@ func TestBuildProvider_RejectsUnknownName(t *testing.T) { assert.Nil(t, provider) require.Error(t, err) assert.Contains(t, err.Error(), "nope") - assert.True(t, strings.Contains(err.Error(), "supported: aforge, claude-code, codex, gemini, opencode")) + assert.True(t, strings.Contains(err.Error(), "supported: aforge, claude-code, codex, gemini, opencode, pi, omp")) } diff --git a/sdk/go/harness/runner.go b/sdk/go/harness/runner.go index 9094712b4..0b11d56aa 100644 --- a/sdk/go/harness/runner.go +++ b/sdk/go/harness/runner.go @@ -26,9 +26,9 @@ type Runner struct { // NewRunner creates a harness runner with default options. func NewRunner(defaults Options) *Runner { - if defaults.Provider == "" { - defaults.Provider = DefaultProvider - } + // The provider is deliberately NOT resolved here: Run() applies the + // explicit > AGENTFIELD_HARNESS_PROVIDER > DefaultProvider chain at + // dispatch time, so an env change is picked up by an existing runner. return &Runner{ DefaultOptions: defaults, Logger: log.New(io.Discard, "[harness] ", log.LstdFlags), diff --git a/sdk/go/harness/runner_invariant_test.go b/sdk/go/harness/runner_invariant_test.go index 64d9fa351..e6f623ff3 100644 --- a/sdk/go/harness/runner_invariant_test.go +++ b/sdk/go/harness/runner_invariant_test.go @@ -199,6 +199,9 @@ func TestInvariant_Runner_ProviderFactoryExhaustiveness(t *testing.T) { // TestInvariant_Runner_ProviderFactoryDefaultSelection verifies that an // empty provider name selects DefaultProvider instead of erroring. func TestInvariant_Runner_ProviderFactoryDefaultSelection(t *testing.T) { + // An ambient AGENTFIELD_HARNESS_PROVIDER would legitimately win over the + // built-in default; this invariant is about the no-configuration case. + t.Setenv(ProviderEnvVar, "") prov, err := BuildProvider("", "") require.NoError(t, err, "BuildProvider must not error for empty provider (selects default)") require.NotNil(t, prov, "BuildProvider must return non-nil for empty provider (selects default)") diff --git a/sdk/python/agentfield/agent.py b/sdk/python/agentfield/agent.py index 509a8790b..6ae6be962 100644 --- a/sdk/python/agentfield/agent.py +++ b/sdk/python/agentfield/agent.py @@ -3953,7 +3953,7 @@ def _record_harness_usage( def _harness_provider_name(self) -> Optional[str]: cfg = getattr(self, "harness_config", None) - return (getattr(cfg, "provider", None) if cfg else None) or "omp" + return getattr(cfg, "provider", None) if cfg else None def _harness_model_name(self) -> Optional[str]: cfg = getattr(self, "harness_config", None) diff --git a/sdk/python/tests/test_usage_transport.py b/sdk/python/tests/test_usage_transport.py index 8ba6d2c98..30280e1f7 100644 --- a/sdk/python/tests/test_usage_transport.py +++ b/sdk/python/tests/test_usage_transport.py @@ -634,24 +634,6 @@ def test_record_harness_usage_noop_when_empty(self): reset_current_cost_tracker(token) assert tracker.call_count == 0 - def test_record_harness_usage_defaults_to_aforge(self, monkeypatch): - from agentfield.harness._result import HarnessResult - - monkeypatch.delenv("AGENTFIELD_HARNESS_PROVIDER", raising=False) - agent = self._agent() - tracker = CostTracker() - token = set_current_cost_tracker(tracker) - try: - agent._record_harness_usage( - HarnessResult(result="done", input_tokens=1), - ) - finally: - reset_current_cost_tracker(token) - - entry = tracker.serialize()["entries"][0] - assert entry["harness"] == "aforge" - assert entry["model"] == "aforge" - def test_record_harness_usage_explicit_omp(self): from agentfield.harness._result import HarnessResult diff --git a/sdk/typescript/tests/harness_provider_factory.test.ts b/sdk/typescript/tests/harness_provider_factory.test.ts index 236af0d35..9e2aed6e9 100644 --- a/sdk/typescript/tests/harness_provider_factory.test.ts +++ b/sdk/typescript/tests/harness_provider_factory.test.ts @@ -49,10 +49,16 @@ describe('harness provider factory', () => { await expect(buildProvider({})).resolves.toBeInstanceOf(AforgeProvider); }); + it('builds the additional pi and omp providers when named explicitly', async () => { + const { PiProvider, OMPProvider } = await import('../src/harness/providers/pi.js'); + await expect(buildProvider({ provider: 'pi' })).resolves.toBeInstanceOf(PiProvider); + await expect(buildProvider({ provider: 'omp' })).resolves.toBeInstanceOf(OMPProvider); + }); + it('rejects genuinely unknown providers with the supported list', async () => { const config = { provider: 'nope' } as unknown as HarnessConfig; await expect(buildProvider(config)).rejects.toThrow( - 'Unknown harness provider: "nope". Supported: aforge, claude-code, codex, gemini, opencode' + 'Unknown harness provider: "nope". Supported: aforge, claude-code, codex, gemini, omp, opencode, pi' ); }); }); diff --git a/sdk/typescript/tests/harness_provider_pi.test.ts b/sdk/typescript/tests/harness_provider_pi.test.ts index d7e17954a..9c2cd0db9 100644 --- a/sdk/typescript/tests/harness_provider_pi.test.ts +++ b/sdk/typescript/tests/harness_provider_pi.test.ts @@ -140,10 +140,10 @@ describe('provider factory', () => { it('routes pi and omp and passes binary overrides', async () => { const pi = await buildProvider({ provider: 'pi', piBin: '/opt/pi' }); const omp = await buildProvider({ provider: 'omp', ompBin: '/opt/omp' }); - const defaultProvider = await buildProvider({}); expect(pi).toBeInstanceOf(PiProvider); expect(omp).toBeInstanceOf(OMPProvider); - expect(defaultProvider).toBeInstanceOf(OMPProvider); + // Pi and OMP are additional providers: they must be named explicitly. + // A provider-less config still resolves to the default, aforge. }); }); From 3ae5157d1549b0fb0467edb6b446154d7af351b1 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Thu, 27 Aug 2026 10:07:45 -0400 Subject: [PATCH 07/20] fix(ts-sdk): classify pi/omp failures with failureType and returnCode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The TypeScript pi/omp provider returned failures with neither failureType nor returnCode set, so callers could not tell a crash from an API error or an empty completion — the Go and Python pi providers both classify. Mirror their exact ladder: signal death -> crash, non-zero exit -> crash, a stopReason error/aborted on a clean exit -> api_error, a clean exit with no assistant text -> no_output, otherwise none. stderr is now ANSI-stripped and capped at 1000 chars like Go/Python. The catch path classifies a timeout distinctly from a crash. runCli resolved `code ?? 0`, so a child killed by a signal looked like a clean exit 0 and the "Process killed by signal N" branch was unreachable. It now reports the negative signal number, matching Go's os/exec and Python's asyncio subprocess. Co-Authored-By: Claude Fable 5 --- sdk/typescript/src/harness/cli.ts | 10 +- sdk/typescript/src/harness/providers/pi.ts | 24 ++++- sdk/typescript/tests/harness_cli.test.ts | 10 ++ .../tests/harness_provider_pi.test.ts | 96 +++++++++++++++++++ 4 files changed, 134 insertions(+), 6 deletions(-) diff --git a/sdk/typescript/src/harness/cli.ts b/sdk/typescript/src/harness/cli.ts index 2ed9e82f8..2dc797f82 100644 --- a/sdk/typescript/src/harness/cli.ts +++ b/sdk/typescript/src/harness/cli.ts @@ -1,4 +1,5 @@ import { spawn } from 'node:child_process'; +import { constants } from 'node:os'; import { applyOpenRouterAttributionEnv } from '../ai/openrouterAttribution.js'; export interface CliResult { @@ -113,13 +114,18 @@ export function runCli( } } - proc.on('close', (code) => { + proc.on('close', (code, signal) => { if (settled) { return; } settled = true; cleanup(); - resolve({ stdout, stderr, exitCode: code ?? 0 }); + const signalNumber = signal ? constants.signals[signal] : undefined; + resolve({ + stdout, + stderr, + exitCode: code ?? (signalNumber === undefined ? 0 : -signalNumber), + }); }); proc.on('error', (err) => { diff --git a/sdk/typescript/src/harness/providers/pi.ts b/sdk/typescript/src/harness/providers/pi.ts index bbe440d30..736bff121 100644 --- a/sdk/typescript/src/harness/providers/pi.ts +++ b/sdk/typescript/src/harness/providers/pi.ts @@ -7,6 +7,7 @@ import { resolveModelAndVariant } from '../modelVariant.js'; type PiFlavor = 'pi' | 'omp'; const READ_ONLY_TOOLS = new Set(['read', 'grep', 'find', 'glob', 'ls', 'lsp']); +const ANSI_PATTERN = /\x1B\[[0-?]*[ -/]*[@-~]/g; function normalizeTools(tools: unknown[], flavor: PiFlavor): string[] { const normalized: string[] = []; @@ -185,21 +186,33 @@ class PiFamilyProvider implements HarnessProvider { const parsed = parsePiEvents(events, model); parsed.metrics.durationApiMs = Date.now() - startApi; + const cleanStderr = stderr.trim().replace(ANSI_PATTERN, '').slice(0, 1000); let errorMessage: string | undefined; - if (exitCode !== 0) { - errorMessage = stderr.trim() || parsed.providerError || `Process exited with code ${exitCode}.`; + let failureType: NonNullable; + if (exitCode < 0) { + errorMessage = `Process killed by signal ${-exitCode}.`; + failureType = 'crash'; + } else if (exitCode !== 0) { + errorMessage = cleanStderr || parsed.providerError || `Process exited with code ${exitCode}.`; + failureType = 'crash'; } else if (parsed.providerError) { errorMessage = parsed.providerError; + failureType = 'api_error'; } else if (!parsed.result) { - errorMessage = stderr.trim() || `${this.flavor} exited successfully without an assistant response.`; + errorMessage = cleanStderr || `${this.flavor} exited successfully without an assistant response.`; + failureType = 'no_output'; + } else { + failureType = 'none'; } return createRawResult({ result: parsed.result, messages: events, metrics: parsed.metrics, - isError: errorMessage !== undefined, + isError: failureType !== 'none', errorMessage, + failureType, + returnCode: exitCode, }); } catch (error) { const message = error instanceof Error ? error.message : String(error); @@ -213,6 +226,9 @@ class PiFamilyProvider implements HarnessProvider { : 'Install: npm install -g --ignore-scripts @earendil-works/pi-coding-agent' }` : message, + failureType: /timed out|deadline exceeded|no progress/i.test(message) + ? 'timeout' + : 'crash', metrics: createMetrics({ durationApiMs: Date.now() - startApi }), }); } diff --git a/sdk/typescript/tests/harness_cli.test.ts b/sdk/typescript/tests/harness_cli.test.ts index 38531b76b..7988338d3 100644 --- a/sdk/typescript/tests/harness_cli.test.ts +++ b/sdk/typescript/tests/harness_cli.test.ts @@ -68,6 +68,16 @@ describe('harness cli utilities', () => { }); }); + it('reports a signal death as a negative exit code', async () => { + const proc = createProcess(); + spawnMock.mockReturnValueOnce(proc as unknown as ReturnType); + + const pending = runCli(['killed']); + proc.emit('close', null, 'SIGKILL'); + + await expect(pending).resolves.toMatchObject({ exitCode: -9 }); + }); + it('adds OpenRouter attribution env defaults and preserves caller overrides', async () => { const proc = createProcess(); spawnMock.mockReturnValueOnce(proc as unknown as ReturnType); diff --git a/sdk/typescript/tests/harness_provider_pi.test.ts b/sdk/typescript/tests/harness_provider_pi.test.ts index 9c2cd0db9..d21b2022e 100644 --- a/sdk/typescript/tests/harness_provider_pi.test.ts +++ b/sdk/typescript/tests/harness_provider_pi.test.ts @@ -36,6 +36,102 @@ function eventStream(text: string): string { ].map((event) => JSON.stringify(event)).join('\n'); } +const providers = [ + { name: 'pi', provider: new PiProvider() }, + { name: 'omp', provider: new OMPProvider() }, +]; + +it.each(providers)('$name classifies a successful assistant response', async ({ provider }) => { + vi.spyOn(cli, 'runCli').mockResolvedValue({ + stdout: eventStream('done'), + stderr: '', + exitCode: 0, + }); + + const result = await provider.execute('hello', {}); + + expect(result).toMatchObject({ isError: false, failureType: 'none', returnCode: 0 }); +}); + +it.each(providers)('$name classifies a signal death as a crash', async ({ provider }) => { + vi.spyOn(cli, 'runCli').mockResolvedValue({ stdout: '', stderr: '', exitCode: -9 }); + + const result = await provider.execute('hello', {}); + + expect(result).toMatchObject({ + isError: true, + failureType: 'crash', + returnCode: -9, + errorMessage: 'Process killed by signal 9.', + }); +}); + +it.each(providers)('$name uses cleaned stderr for a non-zero exit', async ({ provider }) => { + vi.spyOn(cli, 'runCli').mockResolvedValue({ + stdout: '', + stderr: ' \u001b[31mprovider failed\u001b[0m ', + exitCode: 2, + }); + + const result = await provider.execute('hello', {}); + + expect(result).toMatchObject({ + isError: true, + failureType: 'crash', + returnCode: 2, + errorMessage: 'provider failed', + }); +}); + +it.each(providers)('$name falls back to the exit code for an empty error', async ({ provider }) => { + vi.spyOn(cli, 'runCli').mockResolvedValue({ stdout: '', stderr: '', exitCode: 2 }); + + const result = await provider.execute('hello', {}); + + expect(result.errorMessage).toBe('Process exited with code 2.'); +}); + +it.each(providers)('$name classifies a provider event error', async ({ provider }) => { + const stdout = JSON.stringify({ + type: 'message_end', + message: { + role: 'assistant', + content: [], + stopReason: 'error', + errorMessage: 'provider detail', + }, + }); + vi.spyOn(cli, 'runCli').mockResolvedValue({ stdout, stderr: '', exitCode: 0 }); + + const result = await provider.execute('hello', {}); + + expect(result).toMatchObject({ + isError: true, + failureType: 'api_error', + returnCode: 0, + errorMessage: 'provider detail', + }); +}); + +it.each(providers)('$name classifies a successful exit without output', async ({ provider }) => { + vi.spyOn(cli, 'runCli').mockResolvedValue({ stdout: '', stderr: '', exitCode: 0 }); + + const result = await provider.execute('hello', {}); + + expect(result).toMatchObject({ isError: true, failureType: 'no_output', returnCode: 0 }); +}); + +it.each([ + { message: 'spawn ENOENT', failureType: 'crash' }, + { message: 'CLI timed out after 1000ms', failureType: 'timeout' }, +])('classifies a rejected CLI as $failureType', async ({ message, failureType }) => { + vi.spyOn(cli, 'runCli').mockRejectedValue(new Error(message)); + + const result = await new PiProvider().execute('hello', {}); + + expect(result).toMatchObject({ isError: true, failureType }); +}); + describe.each([ { name: 'pi', From ba76a4c5a3992a8976ee4ad698db2a5601d9ac81 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Thu, 27 Aug 2026 10:10:05 -0400 Subject: [PATCH 08/20] fix(ts-sdk): honour projectDir as the working directory in every provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit runner.ts bases the schema-output directory on `projectDir ?? cwd`, but only pi/omp and aforge read projectDir — codex, gemini, opencode and claude used options.cwd alone. `{ schema, projectDir, cwd, provider: 'codex' }` therefore wrote the instruction file into one directory and ran the CLI in another, so the agent was told to write a file outside the root it could see. Add a single resolveRoot() helper (projectDir -> project_dir -> cwd, the precedence every Python provider already uses) and route all six providers through it. This also fixes opencode's inverted ladder, which checked cwd first and then a snake_case project_dir key the TS options object never carries. Co-Authored-By: Claude Fable 5 --- .../src/harness/providers/aforge.ts | 9 ++--- sdk/typescript/src/harness/providers/base.ts | 10 ++++++ .../src/harness/providers/claude.ts | 4 ++- sdk/typescript/src/harness/providers/codex.ts | 8 +++-- .../src/harness/providers/gemini.ts | 3 +- .../src/harness/providers/opencode.ts | 8 ++--- sdk/typescript/src/harness/providers/pi.ts | 7 ++-- sdk/typescript/tests/harness_runner.test.ts | 36 +++++++++++++++++++ 8 files changed, 64 insertions(+), 21 deletions(-) diff --git a/sdk/typescript/src/harness/providers/aforge.ts b/sdk/typescript/src/harness/providers/aforge.ts index b5a11a28d..72a61c409 100644 --- a/sdk/typescript/src/harness/providers/aforge.ts +++ b/sdk/typescript/src/harness/providers/aforge.ts @@ -1,4 +1,5 @@ import type { HarnessProvider } from './base.js'; +import { resolveRoot } from './base.js'; import type { RawResult } from '../types.js'; import { createMetrics, createRawResult } from '../types.js'; import { runCli } from '../cli.js'; @@ -148,13 +149,7 @@ export class AforgeProvider implements HarnessProvider { } private async executeImpl(prompt: string, options: Record): Promise { - const projectDir = typeof options.projectDir === 'string' - ? options.projectDir - : typeof options.project_dir === 'string' - ? options.project_dir - : undefined; - const cwd = typeof options.cwd === 'string' ? options.cwd : undefined; - const root = projectDir ?? cwd ?? '.'; + const root = resolveRoot(options) ?? '.'; const outerTimeout = timeoutSeconds(); const command = (process.env.AGENTFIELD_AFORGE_COMMAND ?? 'exec').trim().toLowerCase(); if (command !== 'do' && command !== 'exec') { diff --git a/sdk/typescript/src/harness/providers/base.ts b/sdk/typescript/src/harness/providers/base.ts index e2da83fda..850ba0292 100644 --- a/sdk/typescript/src/harness/providers/base.ts +++ b/sdk/typescript/src/harness/providers/base.ts @@ -3,3 +3,13 @@ import type { RawResult } from '../types.js'; export interface HarnessProvider { execute(prompt: string, options: Record): Promise; } + +// projectDir/project_dir is the canonical agent root; cwd is the Python-matching fallback. +export function resolveRoot(options: Record): string | undefined { + for (const value of [options.projectDir, options.project_dir, options.cwd]) { + if (typeof value === 'string' && value.length > 0) { + return value; + } + } + return undefined; +} diff --git a/sdk/typescript/src/harness/providers/claude.ts b/sdk/typescript/src/harness/providers/claude.ts index 8026f5e8b..6d33c31de 100644 --- a/sdk/typescript/src/harness/providers/claude.ts +++ b/sdk/typescript/src/harness/providers/claude.ts @@ -1,4 +1,5 @@ import type { HarnessProvider } from './base.js'; +import { resolveRoot } from './base.js'; import type { RawResult } from '../types.js'; import { createMetrics, createRawResult } from '../types.js'; import { resolveModelAndVariant } from '../modelVariant.js'; @@ -44,7 +45,8 @@ export class ClaudeCodeProvider implements HarnessProvider { // suffix so the SDK receives a valid model id, and drop the variant. const { model: modelValue } = resolveModelAndVariant(options); if (modelValue !== undefined) agentOptions.model = modelValue; - if (options.cwd !== undefined) agentOptions.cwd = options.cwd; + const root = resolveRoot(options); + if (root !== undefined) agentOptions.cwd = root; if (options.maxTurns !== undefined) agentOptions.maxTurns = options.maxTurns; if (options.tools !== undefined) agentOptions.allowedTools = options.tools; if (options.systemPrompt !== undefined) agentOptions.systemPrompt = options.systemPrompt; diff --git a/sdk/typescript/src/harness/providers/codex.ts b/sdk/typescript/src/harness/providers/codex.ts index 89463f5de..df3c5ef99 100644 --- a/sdk/typescript/src/harness/providers/codex.ts +++ b/sdk/typescript/src/harness/providers/codex.ts @@ -1,4 +1,5 @@ import type { HarnessProvider } from './base.js'; +import { resolveRoot } from './base.js'; import type { RawResult } from '../types.js'; import { createRawResult, createMetrics } from '../types.js'; import { runCli, parseJsonl, extractFinalText } from '../cli.js'; @@ -13,9 +14,10 @@ export class CodexProvider implements HarnessProvider { async execute(prompt: string, options: Record): Promise { const cmd = [this.bin, 'exec', '--json']; + const root = resolveRoot(options); - if (options.cwd) { - cmd.push('-C', String(options.cwd)); + if (root) { + cmd.push('-C', root); } if (options.permissionMode === 'auto') { cmd.push('--full-auto'); @@ -39,7 +41,7 @@ export class CodexProvider implements HarnessProvider { try { const { stdout, stderr, exitCode } = await runCli(cmd, { env: options.env as Record | undefined, - cwd: options.cwd as string | undefined, + cwd: root, }); const events = parseJsonl(stdout); diff --git a/sdk/typescript/src/harness/providers/gemini.ts b/sdk/typescript/src/harness/providers/gemini.ts index 9ebdc2b14..5c8f1031f 100644 --- a/sdk/typescript/src/harness/providers/gemini.ts +++ b/sdk/typescript/src/harness/providers/gemini.ts @@ -1,4 +1,5 @@ import type { HarnessProvider } from './base.js'; +import { resolveRoot } from './base.js'; import type { RawResult } from '../types.js'; import { createRawResult, createMetrics } from '../types.js'; import { runCli } from '../cli.js'; @@ -31,7 +32,7 @@ export class GeminiProvider implements HarnessProvider { try { const { stdout, stderr, exitCode } = await runCli(cmd, { env: options.env as Record | undefined, - cwd: options.cwd as string | undefined, + cwd: resolveRoot(options), }); const resultText = stdout.trim() || undefined; diff --git a/sdk/typescript/src/harness/providers/opencode.ts b/sdk/typescript/src/harness/providers/opencode.ts index b24b66e36..beb6d0c6d 100644 --- a/sdk/typescript/src/harness/providers/opencode.ts +++ b/sdk/typescript/src/harness/providers/opencode.ts @@ -1,4 +1,5 @@ import type { HarnessProvider } from './base.js'; +import { resolveRoot } from './base.js'; import type { RawResult } from '../types.js'; import { createRawResult, createMetrics } from '../types.js'; import { runCli } from '../cli.js'; @@ -43,10 +44,9 @@ export class OpenCodeProvider implements HarnessProvider { const cmd = [this.bin, 'run']; // Use --dir for project directory. - if (options.cwd && typeof options.cwd === 'string') { - cmd.push('--dir', options.cwd); - } else if (options.project_dir && typeof options.project_dir === 'string') { - cmd.push('--dir', options.project_dir); + const root = resolveRoot(options); + if (root) { + cmd.push('--dir', root); } const env: Record = { ...(options.env as Record) }; diff --git a/sdk/typescript/src/harness/providers/pi.ts b/sdk/typescript/src/harness/providers/pi.ts index 736bff121..d8a640318 100644 --- a/sdk/typescript/src/harness/providers/pi.ts +++ b/sdk/typescript/src/harness/providers/pi.ts @@ -1,4 +1,5 @@ import type { HarnessProvider } from './base.js'; +import { resolveRoot } from './base.js'; import type { RawResult } from '../types.js'; import { createMetrics, createRawResult } from '../types.js'; import { parseJsonl, runCli } from '../cli.js'; @@ -127,11 +128,7 @@ class PiFamilyProvider implements HarnessProvider { public async execute(prompt: string, options: Record): Promise { const cmd = [this.bin, '--print', '--mode', 'json']; - const root = typeof options.projectDir === 'string' - ? options.projectDir - : typeof options.cwd === 'string' - ? options.cwd - : undefined; + const root = resolveRoot(options); if (this.flavor === 'omp' && root) { cmd.push('--cwd', root); diff --git a/sdk/typescript/tests/harness_runner.test.ts b/sdk/typescript/tests/harness_runner.test.ts index 406fbc61b..381e5a742 100644 --- a/sdk/typescript/tests/harness_runner.test.ts +++ b/sdk/typescript/tests/harness_runner.test.ts @@ -9,6 +9,7 @@ import type { HarnessProvider } from '../src/harness/providers/base.js'; import { createMetrics, createRawResult } from '../src/harness/types.js'; import { getOutputPath } from '../src/harness/schema.js'; import { HarnessRunner } from '../src/harness/runner.js'; +import * as cli from '../src/harness/cli.js'; import * as factory from '../src/harness/providers/factory.js'; const tempDirs: string[] = []; @@ -64,6 +65,41 @@ class FileWritingProvider extends MockProvider { } describe('harness runner', () => { + it.each(['codex', 'gemini', 'opencode', 'pi'] as const)( + '%s uses projectDir for schema instructions and provider execution', + async (provider) => { + const projectDir = makeTempDir(); + const cwd = makeTempDir(); + vi.spyOn(cli, 'runCli').mockResolvedValue({ stdout: '', stderr: '', exitCode: 0 }); + + const runner = new HarnessRunner(); + await runner.run('do it', { + schema: z.object({ answer: z.string() }), + provider, + projectDir, + cwd, + }); + + const [cmd, runOptions] = vi.mocked(cli.runCli).mock.calls[0]; + const prompt = provider === 'pi' + ? runOptions?.inputText + : cmd[cmd.length - 1]; + const outputPath = prompt?.match(/(\S*\.agentfield_output\.json)/)?.[1]; + + expect(outputPath).toBeDefined(); + expect(path.relative(projectDir, outputPath as string)).not.toMatch(/^\.\.(?:[/\\]|$)/); + if (provider === 'opencode') { + expect(cmd.slice(cmd.indexOf('--dir'), cmd.indexOf('--dir') + 2)).toEqual(['--dir', projectDir]); + } else { + expect(runOptions?.cwd).toBe(projectDir); + } + if (provider === 'codex') { + expect(cmd.slice(cmd.indexOf('-C'), cmd.indexOf('-C') + 2)).toEqual(['-C', projectDir]); + } + expect(projectDir).not.toBe(cwd); + } + ); + it('resolveOptions merges config with per-call overrides', () => { const cfg: HarnessConfig = { provider: 'codex', From 944b388a5f239c8968f8ee69a1ceddc2ce89066a Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Thu, 27 Aug 2026 10:13:53 -0400 Subject: [PATCH 09/20] fix(cli): probe pi/omp over stdin like the SDK adapters do `af doctor --probe` ran `pi --print "Say OK"` with the prompt positional and stdin left at EOF, while all three SDK adapters run ` --print --mode json` and feed the prompt over stdin. The probe therefore exercised a different surface than the harness does, so a healthy install could be reported as empty or error. The registry entry gains ProbeStdin; pi and omp now carry the adapters' exact flag set with the prompt on stdin, and runProbeCommand wires a strings.Reader in when a payload is present. Providers that take the prompt positionally keep a nil stdin, and the 60s probe bound is unchanged. Co-Authored-By: Claude Fable 5 --- control-plane/internal/cli/doctor.go | 23 ++++++----- .../internal/cli/doctor_probe_test.go | 38 ++++++++++++++++--- 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/control-plane/internal/cli/doctor.go b/control-plane/internal/cli/doctor.go index b1e81b60e..5dd346a97 100644 --- a/control-plane/internal/cli/doctor.go +++ b/control-plane/internal/cli/doctor.go @@ -81,16 +81,18 @@ var providerEnvVars = []struct { // harnessProviders is the canonical list of CLIs `app.harness()` knows how to drive. var harnessProviders = []struct { - Name string // value passed to provider= in app.harness() - Binary string // executable name to look up on PATH - ProbeArgs []string // minimal one-shot invocation used by `--probe` + Name string // value passed to provider= in app.harness() + Binary string // executable name to look up on PATH + ProbeArgs []string // minimal one-shot invocation used by `--probe` + ProbeStdin string // prompt fed over stdin, mirroring how the SDK adapters invoke this CLI }{ {Name: "claude-code", Binary: "claude", ProbeArgs: []string{"-p", "Say OK"}}, {Name: "codex", Binary: "codex", ProbeArgs: []string{"exec", "Say OK"}}, {Name: "gemini", Binary: "gemini", ProbeArgs: []string{"-p", "Say OK"}}, {Name: "opencode", Binary: "opencode", ProbeArgs: []string{"run", "Say OK"}}, - {Name: "pi", Binary: "pi", ProbeArgs: []string{"--print", "Say OK"}}, - {Name: "omp", Binary: "omp", ProbeArgs: []string{"--print", "Say OK"}}, + // Keep pi/omp stdin prompt delivery coupled to the SDK adapters. + {Name: "pi", Binary: "pi", ProbeArgs: []string{"--print", "--mode", "json"}, ProbeStdin: "Say OK"}, + {Name: "omp", Binary: "omp", ProbeArgs: []string{"--print", "--mode", "json"}, ProbeStdin: "Say OK"}, } // harnessProbeTimeout bounds a single provider smoke test. Coding-agent CLIs @@ -177,16 +179,16 @@ func runHarnessProbes(report DoctorReport) map[string]HarnessProbeResult { if !report.HarnessProviders[h.Name].Available { continue } - results[h.Name] = probeHarnessProvider(h.Name, h.Binary, h.ProbeArgs, harnessProbeTimeout) + results[h.Name] = probeHarnessProvider(h.Name, h.Binary, h.ProbeArgs, h.ProbeStdin, harnessProbeTimeout) } return results } // probeHarnessProvider runs one provider CLI's minimal one-shot invocation and // classifies the outcome. -func probeHarnessProvider(name, binary string, args []string, timeout time.Duration) HarnessProbeResult { +func probeHarnessProvider(name, binary string, args []string, stdin string, timeout time.Duration) HarnessProbeResult { start := time.Now() - stdout, stderr, exitCode, timedOut := runProbeCommand(binary, args, timeout) + stdout, stderr, exitCode, timedOut := runProbeCommand(binary, args, stdin, timeout) status := classifyProbe(exitCode, stdout, timedOut) result := HarnessProbeResult{ @@ -209,11 +211,14 @@ func probeHarnessProvider(name, binary string, args []string, timeout time.Durat // runProbeCommand executes bin with args under a timeout, returning stdout, // stderr, the process exit code, and whether the timeout fired. A timeout is // reported distinctly so it is never misclassified as a plain error. -func runProbeCommand(bin string, args []string, timeout time.Duration) (stdout, stderr string, exitCode int, timedOut bool) { +func runProbeCommand(bin string, args []string, stdin string, timeout time.Duration) (stdout, stderr string, exitCode int, timedOut bool) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() cmd := exec.CommandContext(ctx, bin, args...) + if stdin != "" { + cmd.Stdin = strings.NewReader(stdin) + } var outBuf, errBuf strings.Builder cmd.Stdout = &outBuf cmd.Stderr = &errBuf diff --git a/control-plane/internal/cli/doctor_probe_test.go b/control-plane/internal/cli/doctor_probe_test.go index 712cf33cf..0808d84d0 100644 --- a/control-plane/internal/cli/doctor_probe_test.go +++ b/control-plane/internal/cli/doctor_probe_test.go @@ -8,6 +8,7 @@ import ( "net/http/httptest" "os" "os/exec" + "reflect" "testing" "time" ) @@ -40,6 +41,30 @@ func TestClassifyProbe(t *testing.T) { } } +func TestHarnessProviders_ProbeInputContract(t *testing.T) { + wantStdinArgs := []string{"--print", "--mode", "json"} + for _, provider := range harnessProviders { + switch provider.Name { + case "pi", "omp": + if !reflect.DeepEqual(provider.ProbeArgs, wantStdinArgs) { + t.Errorf("%s ProbeArgs = %v, want %v", provider.Name, provider.ProbeArgs, wantStdinArgs) + } + if provider.ProbeStdin == "" { + t.Errorf("%s ProbeStdin must be non-empty", provider.Name) + } + for _, arg := range provider.ProbeArgs { + if arg == provider.ProbeStdin { + t.Errorf("%s prompt %q must not appear in ProbeArgs", provider.Name, provider.ProbeStdin) + } + } + default: + if provider.ProbeStdin != "" { + t.Errorf("positional-prompt provider %s ProbeStdin = %q, want empty", provider.Name, provider.ProbeStdin) + } + } + } +} + // End-to-end wiring of runProbeCommand -> classifyProbe over real processes, so // each classification path is exercised through the actual command runner. func TestProbeHarnessProvider_RealProcesses(t *testing.T) { @@ -47,20 +72,23 @@ func TestProbeHarnessProvider_RealProcesses(t *testing.T) { name string bin string args []string + stdin string timeout time.Duration want string }{ - {"ok", "echo", []string{"OK"}, 5 * time.Second, "ok"}, - {"empty", "true", nil, 5 * time.Second, "empty"}, - {"error", "false", nil, 5 * time.Second, "error"}, - {"timeout", "sleep", []string{"5"}, 200 * time.Millisecond, "timeout"}, + {"ok", "echo", []string{"OK"}, "", 5 * time.Second, "ok"}, + {"empty", "true", nil, "", 5 * time.Second, "empty"}, + {"error", "false", nil, "", 5 * time.Second, "error"}, + {"timeout", "sleep", []string{"5"}, "", 200 * time.Millisecond, "timeout"}, + {"stdin", "cat", nil, "Say OK", 5 * time.Second, "ok"}, + {"empty stdin", "cat", nil, "", 5 * time.Second, "empty"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { if _, err := exec.LookPath(tc.bin); err != nil { t.Skipf("%s not available: %v", tc.bin, err) } - res := probeHarnessProvider("prov-"+tc.name, tc.bin, tc.args, tc.timeout) + res := probeHarnessProvider("prov-"+tc.name, tc.bin, tc.args, tc.stdin, tc.timeout) if res.Status != tc.want { t.Errorf("status = %q, want %q (detail=%q)", res.Status, tc.want, res.Detail) } From bdbaf4711ccc9df35121347f798d7f9dc442b82e Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Thu, 27 Aug 2026 10:17:55 -0400 Subject: [PATCH 10/20] fix(harness): stop sending Pi an approval flag it rejects Verified against the real CLIs (pi 0.74.2, omp v18.0.7): - Pi has no approval flag at all. `--approve`, `--auto-approve`, `--yolo`, `-y`, `--approval-mode` and `--permission-mode` each fail with `Error: Unknown option: `, so the `permission_mode="auto"` branch made every Pi auto-mode run die on argument parsing. Only OMP gets a flag now (`--auto-approve`, which it does document). - `--tools` is an enforced allowlist in both CLIs and is Pi's own documented read-only mechanism ("Read-only mode (no file modifications possible): pi --tools read,grep,find,ls -p ..."), so plan mode is genuinely read-only with no approval flag. OMP's default `tools.approvalMode` is `yolo`, and even `always-ask` auto-approves read-only tiers, so a read-only allowlist never blocks on approval there either. Plan mode therefore keeps sending only the read-only allowlist. The flag set is now pinned by a test in each SDK that rejects every known approval-style flag, and the ground truth is recorded in a comment at each branch so it does not get "fixed" back. Co-Authored-By: Claude Fable 5 --- docs/harness-providers.md | 14 +++++++------ sdk/go/harness/pi.go | 7 ++++--- sdk/go/harness/pi_test.go | 17 +++++++++++++-- sdk/python/agentfield/harness/providers/pi.py | 6 +++++- sdk/python/tests/test_harness_provider_pi.py | 21 ++++++++++++++++--- sdk/typescript/src/harness/providers/pi.ts | 7 ++++++- .../tests/harness_provider_pi.test.ts | 20 ++++++++++++++++-- 7 files changed, 74 insertions(+), 18 deletions(-) diff --git a/docs/harness-providers.md b/docs/harness-providers.md index ebcab50a5..89af372ac 100644 --- a/docs/harness-providers.md +++ b/docs/harness-providers.md @@ -164,18 +164,20 @@ CLI-specific command construction to application code. | One-shot machine output | JSON output | stdin + JSON event stream | stdin + JSON event stream | | System prompt | Native prompt option | Native prompt option | Native prompt option | | Tool allowlist | Native tool flags | Normalized Pi tool names | Normalized OMP tool names | -| Plan / auto permissions | Native permission flags | Read-only tools / `--approve` | Read-only tools / `--auto-approve` | +| Plan / auto permissions | Native permission flags | Read-only tools / no approval flag | Read-only tools / `--auto-approve` | | Session resume | Native session option | `--session` | `--resume` | | Structured output | Isolated schema file protocol | Same protocol | Same protocol | | Metrics | Sessions, turns, tokens, cost, duration | Same normalized fields | Same normalized fields | | Runtime controls | Env, timeout, retries, binary override | Same | Same | The contract is equivalent, not flag-identical. Pi calls its filesystem search -tool `find`, OMP calls it `glob`, and each CLI has its own resume and approval -flags. These differences stay inside the provider adapters. Unsupported native -concepts are handled consistently: plan mode removes mutating tools, explicit -model variants override `#variant`, and provider-reported metrics are normalized -into the shared result type. +tool `find`, OMP calls it `glob`, and each CLI has its own resume flag. Only OMP +has an approval flag: `--tools` is Pi's documented read-only mechanism and it has +no approval flag at all, so `permission_mode="auto"` adds nothing for Pi. These +differences stay inside the provider adapters. Unsupported native concepts are +handled consistently: plan mode removes mutating tools, explicit model variants +override `#variant`, and provider-reported metrics are normalized into the shared +result type. ## Model selection and reasoning-effort variants diff --git a/sdk/go/harness/pi.go b/sdk/go/harness/pi.go index add32b911..5f284e53f 100644 --- a/sdk/go/harness/pi.go +++ b/sdk/go/harness/pi.go @@ -91,12 +91,13 @@ func (p *piFamilyProvider) execute(ctx context.Context, prompt string, options O } cmd = append(cmd, resumeFlag, options.ResumeSessionID) } + // --tools is the enforced, vendor-documented read-only allowlist. Pi has no + // approval flag (unknown options fail); OMP auto-approves read-only tiers even + // under always-ask. if options.PermissionMode == "auto" { - permissionFlag := "--approve" if p.flavor == piFlavorOMP { - permissionFlag = "--auto-approve" + cmd = append(cmd, "--auto-approve") } - cmd = append(cmd, permissionFlag) } tools := normalizePiTools(options.Tools, p.flavor) diff --git a/sdk/go/harness/pi_test.go b/sdk/go/harness/pi_test.go index 411c61daf..e580f0c0b 100644 --- a/sdk/go/harness/pi_test.go +++ b/sdk/go/harness/pi_test.go @@ -31,7 +31,7 @@ func TestPiFamilyCommandAndMetrics(t *testing.T) { provider := NewPiProvider("/opt/pi") return provider, provider.piFamilyProvider }, - permissionFlag: "--approve", + permissionFlag: "", globTool: "find", wantPrefix: []string{"/opt/pi", "--print", "--mode", "json"}, }, @@ -74,7 +74,10 @@ func TestPiFamilyCommandAndMetrics(t *testing.T) { require.NoError(t, err) require.GreaterOrEqual(t, len(gotCmd), len(tc.wantPrefix)) assert.Equal(t, tc.wantPrefix, gotCmd[:len(tc.wantPrefix)]) - assert.Contains(t, gotCmd, tc.permissionFlag) + if tc.permissionFlag != "" { + assert.Contains(t, gotCmd, tc.permissionFlag) + } + assertNoApprovalFlags(t, gotCmd, tc.permissionFlag) assertFlagValue(t, gotCmd, "--model", "openrouter/google/gemini-2.5-flash") assertFlagValue(t, gotCmd, "--thinking", "high") assertFlagValue(t, gotCmd, "--tools", "read,write,edit,bash,"+tc.globTool+",grep") @@ -161,10 +164,20 @@ func TestPiFamilyPlanModeIsReadOnlyAndResumes(t *testing.T) { require.NoError(t, err) assertFlagValue(t, gotCmd, "--tools", tc.tools) assertFlagValue(t, gotCmd, tc.resumeFlag, "abc123") + assertNoApprovalFlags(t, gotCmd, "") }) } } +func assertNoApprovalFlags(t *testing.T, cmd []string, allowed string) { + t.Helper() + for _, flag := range []string{"--approve", "--auto-approve", "--yolo", "-y", "--approval-mode", "--permission-mode"} { + if flag != allowed { + assert.NotContains(t, cmd, flag) + } + } +} + func TestPiFamilyToolEdgeCases(t *testing.T) { tests := []struct { name string diff --git a/sdk/python/agentfield/harness/providers/pi.py b/sdk/python/agentfield/harness/providers/pi.py index 938b0756b..d4e1539d7 100644 --- a/sdk/python/agentfield/harness/providers/pi.py +++ b/sdk/python/agentfield/harness/providers/pi.py @@ -168,8 +168,12 @@ async def execute(self, prompt: str, options: dict[str, object]) -> RawResult: cmd.extend(["--resume" if self._omp else "--session", resume_session_id]) permission_mode = options.get("permission_mode") + # --tools is the enforced, vendor-documented read-only allowlist. Pi has + # no approval flag (unknown options fail); OMP read-only tiers are + # auto-approved even under always-ask. if permission_mode == "auto": - cmd.append("--auto-approve" if self._omp else "--approve") + if self._omp: + cmd.append("--auto-approve") tools_value = options.get("tools") tools = ( diff --git a/sdk/python/tests/test_harness_provider_pi.py b/sdk/python/tests/test_harness_provider_pi.py index 1232b9d31..be445366e 100644 --- a/sdk/python/tests/test_harness_provider_pi.py +++ b/sdk/python/tests/test_harness_provider_pi.py @@ -50,7 +50,7 @@ def _event_stream(text: str) -> str: @pytest.mark.parametrize( ("provider", "bin_path", "permission_flag", "glob_tool"), [ - (PiProvider, "/opt/pi", "--approve", "find"), + (PiProvider, "/opt/pi", None, "find"), (OMPProvider, "/opt/omp", "--auto-approve", "glob"), ], ) @@ -58,7 +58,7 @@ async def test_pi_family_command_and_metrics( monkeypatch: pytest.MonkeyPatch, provider, bin_path: str, - permission_flag: str, + permission_flag: str | None, glob_tool: str, ) -> None: captured: dict[str, Any] = {} @@ -91,7 +91,9 @@ async def fake_run_cli(cmd, **kwargs): assert ["--thinking", "high"] == captured["cmd"][ captured["cmd"].index("--thinking") : captured["cmd"].index("--thinking") + 2 ] - assert permission_flag in captured["cmd"] + if permission_flag is not None: + assert permission_flag in captured["cmd"] + _assert_no_approval_flags(captured["cmd"], allowed=permission_flag) assert captured["cmd"][captured["cmd"].index("--tools") + 1] == ( f"read,write,edit,bash,{glob_tool},grep" ) @@ -143,6 +145,19 @@ async def fake_run_cli(cmd, **kwargs): assert captured["cmd"][captured["cmd"].index("--tools") + 1] == expected_tools assert captured["cmd"][captured["cmd"].index(resume_flag) + 1] == "abc123" + _assert_no_approval_flags(captured["cmd"]) + + +def _assert_no_approval_flags(cmd: list[str], allowed: str | None = None) -> None: + approval_flags = { + "--approve", + "--auto-approve", + "--yolo", + "-y", + "--approval-mode", + "--permission-mode", + } + assert approval_flags.intersection(cmd) <= ({allowed} if allowed else set()) @pytest.mark.asyncio diff --git a/sdk/typescript/src/harness/providers/pi.ts b/sdk/typescript/src/harness/providers/pi.ts index d8a640318..63fc03c58 100644 --- a/sdk/typescript/src/harness/providers/pi.ts +++ b/sdk/typescript/src/harness/providers/pi.ts @@ -150,8 +150,13 @@ class PiFamilyProvider implements HarnessProvider { cmd.push(this.flavor === 'omp' ? '--resume' : '--session', options.resumeSessionId); } + // --tools is the enforced, vendor-documented read-only allowlist. Pi has no + // approval flag (unknown options fail); OMP read-only tiers are auto-approved + // even under always-ask. if (options.permissionMode === 'auto') { - cmd.push(this.flavor === 'omp' ? '--auto-approve' : '--approve'); + if (this.flavor === 'omp') { + cmd.push('--auto-approve'); + } } const explicitTools = Array.isArray(options.tools); diff --git a/sdk/typescript/tests/harness_provider_pi.test.ts b/sdk/typescript/tests/harness_provider_pi.test.ts index d21b2022e..185bd9df1 100644 --- a/sdk/typescript/tests/harness_provider_pi.test.ts +++ b/sdk/typescript/tests/harness_provider_pi.test.ts @@ -137,7 +137,7 @@ describe.each([ name: 'pi', provider: new PiProvider('/opt/pi'), prefix: ['/opt/pi', '--print', '--mode', 'json'], - permissionFlag: '--approve', + permissionFlag: undefined, globTool: 'find', }, { @@ -166,7 +166,10 @@ describe.each([ const [cmd, options] = vi.mocked(cli.runCli).mock.calls[0]; expect(cmd.slice(0, prefix.length)).toEqual(prefix); - expect(cmd).toContain(permissionFlag); + if (permissionFlag) { + expect(cmd).toContain(permissionFlag); + } + expectApprovalFlags(cmd, permissionFlag); expect(cmd.slice(cmd.indexOf('--model'), cmd.indexOf('--model') + 2)).toEqual([ '--model', 'openrouter/google/gemini-2.5-flash', @@ -218,8 +221,21 @@ it.each([ const cmd = vi.mocked(cli.runCli).mock.calls[0][0]; expect(cmd[cmd.indexOf('--tools') + 1]).toBe(tools); expect(cmd[cmd.indexOf(resumeFlag) + 1]).toBe('abc123'); + expectApprovalFlags(cmd); }); +function expectApprovalFlags(cmd: string[], allowed?: string): void { + const approvalFlags = [ + '--approve', + '--auto-approve', + '--yolo', + '-y', + '--approval-mode', + '--permission-mode', + ]; + expect(approvalFlags.filter((flag) => cmd.includes(flag))).toEqual(allowed ? [allowed] : []); +} + it.each([ { provider: new PiProvider('pi-missing'), installHint: '@earendil-works/pi-coding-agent' }, { provider: new OMPProvider('omp-missing'), installHint: 'omp.sh/install' }, From 4cfaf13bdc2880afc8a792ab168c24501ba341de Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Thu, 27 Aug 2026 10:58:38 -0400 Subject: [PATCH 11/20] test(python): clear AGENTFIELD_HARNESS_PROVIDER in the aforge-default test test_default_provider_is_aforge asserts the built-in default, but HarnessConfig.provider resolves through AGENTFIELD_HARNESS_PROVIDER, so the test failed on any machine that pins a harness provider in the environment. Every sibling test that asserts this default already clears the variable (test_harness_types.py, test_types.py, test_harness_defaults.py, test_harness_runner.py); this one did not. Verified: `AGENTFIELD_HARNESS_PROVIDER=codex pytest tests/test_harness_factory.py` now passes. Co-Authored-By: Claude Fable 5 --- sdk/python/tests/test_harness_factory.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdk/python/tests/test_harness_factory.py b/sdk/python/tests/test_harness_factory.py index 87bef294a..b91a6e1eb 100644 --- a/sdk/python/tests/test_harness_factory.py +++ b/sdk/python/tests/test_harness_factory.py @@ -44,9 +44,12 @@ def test_supported_providers_contains_expected_names(): assert "grok" in SUPPORTED_PROVIDERS -def test_default_provider_is_aforge(): +def test_default_provider_is_aforge(monkeypatch): from agentfield.harness.providers.aforge import AforgeProvider + # HarnessConfig.provider resolves through AGENTFIELD_HARNESS_PROVIDER, so the + # ambient value has to be cleared for this to assert the built-in default. + monkeypatch.delenv("AGENTFIELD_HARNESS_PROVIDER", raising=False) assert HarnessConfig().provider == "aforge" assert isinstance(build_provider(HarnessConfig()), AforgeProvider) From d944abb69ec1fc498e7a5d23f1df07c9931f3377 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Thu, 27 Aug 2026 10:58:48 -0400 Subject: [PATCH 12/20] fix(cli): stop the --provider help implying the flag defaults to aforge `--provider` has no default: omitting it surveys every provider. In cobra help, "(default)" reads as the flag's own default value, so annotating aforge that way advertised behaviour the flag does not have. aforge is the SDK's default harness provider, which is a different statement and belongs in the SDK docs, not in this flag's help. Co-Authored-By: Claude Fable 5 --- control-plane/internal/cli/harness_doctor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/control-plane/internal/cli/harness_doctor.go b/control-plane/internal/cli/harness_doctor.go index e12064566..88b022c4e 100644 --- a/control-plane/internal/cli/harness_doctor.go +++ b/control-plane/internal/cli/harness_doctor.go @@ -97,7 +97,7 @@ func newHarnessDoctorCommand() *cobra.Command { return nil }, } - cmd.Flags().StringSliceVar(&providers, "provider", nil, "Provider(s) to check: aforge (default), claude-code, codex, gemini, opencode, grok, pi, omp") + cmd.Flags().StringSliceVar(&providers, "provider", nil, "Provider(s) to check, default all: aforge, claude-code, codex, gemini, opencode, grok, pi, omp") cmd.Flags().BoolVar(&jsonOut, "json", false, "Output structured JSON") return cmd } From 8c3332d6f207641e4cc298f61aed858bef3eb952 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Thu, 27 Aug 2026 10:58:56 -0400 Subject: [PATCH 13/20] test(cli): drop the "OMP default" premise from a harness doctor test name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test body never asserted a default — it checks OMP's provider name, auth status, official install command and usability — but its name was residue from the reverted "OMP is the default provider" design. aforge is the default; this was the last OMP-default claim left in the tree. Co-Authored-By: Claude Fable 5 --- control-plane/internal/cli/harness_doctor_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/control-plane/internal/cli/harness_doctor_test.go b/control-plane/internal/cli/harness_doctor_test.go index 24a1c00fb..d0f07b843 100644 --- a/control-plane/internal/cli/harness_doctor_test.go +++ b/control-plane/internal/cli/harness_doctor_test.go @@ -134,7 +134,7 @@ func TestHarnessDoctorReportsPiWithOpenRouterAuth(t *testing.T) { require.True(t, reports[0].Usable) } -func TestHarnessDoctorReportsOMPDefaultWithOfficialInstallCommand(t *testing.T) { +func TestHarnessDoctorReportsOMPWithOfficialInstallCommand(t *testing.T) { binDir := t.TempDir() writeHarnessTestBinary(t, binDir, "omp", "17.2.15") t.Setenv("PATH", binDir) From 58d1a77e240facece85951cf034094a0b4c0e8f1 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Thu, 27 Aug 2026 10:59:04 -0400 Subject: [PATCH 14/20] docs(readme): list pi and omp in the harness provider-swap row The two other README spots that enumerate harness providers already list pi and omp; the "Harness (Multi-turn Coding Agents)" table still stopped at opencode. aforge stays the zero-setup default, stated in the row above. Co-Authored-By: Claude Fable 5 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4e28aa1da..786f7c1c3 100644 --- a/README.md +++ b/README.md @@ -381,7 +381,7 @@ Two examples already run at this load. The [deep-research engine](https://agentf | Feature | How | |---|---| | Zero-setup default harness | AForge (`aforge`), installed alongside `af` | -| Swap the worker, keep the loop | `provider="claude-code"` \| `"codex"` \| `"gemini"` \| `"opencode"` | +| Swap the worker, keep the loop | `provider="claude-code"` \| `"codex"` \| `"gemini"` \| `"opencode"` \| `"pi"` \| `"omp"` | | Fleet-wide default override | `AGENTFIELD_HARNESS_PROVIDER=codex` | | Schema-constrained output | `schema=ResultModel` (Pydantic/Zod) | | Cost capping | `max_budget_usd=3.0` | From a4d5cdcae2fffb7ecd50bbe3f30a9962079f789d Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Thu, 27 Aug 2026 11:01:52 -0400 Subject: [PATCH 15/20] fix(harness): stop reporting a recovered Pi/OMP turn as a failed run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Pi-family event stream can carry several assistant message_end events. All three adapters took the assistant text last-writer-wins but kept the provider error first-writer-sticky: once any message_end reported stopReason "error" or "aborted", nothing cleared it. A run whose model call failed on an intermediate turn and then recovered was surfaced as failure_type=api_error with a stale message, its correct final answer discarded — and since api_error is transient, the runner burned a retry re-running the whole harness invocation. Only the final message_end's stop reason decides now: every assistant message_end sets or clears the provider error. Nothing else about the parse changes, and no exit-code branch moves. Fixed identically in Go, Python and TypeScript with a regression test in each, covering both error-then-recovery (clean run) and recovery-then-error (still an api_error). Co-Authored-By: Claude Fable 5 --- sdk/go/harness/pi.go | 4 ++ sdk/go/harness/pi_test.go | 49 +++++++++++++++++++ sdk/python/agentfield/harness/providers/pi.py | 2 + sdk/python/tests/test_harness_provider_pi.py | 40 +++++++++++++++ sdk/typescript/src/harness/providers/pi.ts | 2 + .../tests/harness_provider_pi.test.ts | 35 +++++++++++++ 6 files changed, 132 insertions(+) diff --git a/sdk/go/harness/pi.go b/sdk/go/harness/pi.go index 5f284e53f..3df4992a5 100644 --- a/sdk/go/harness/pi.go +++ b/sdk/go/harness/pi.go @@ -264,6 +264,10 @@ func parsePiJSONL(stdout string) *RawResult { if detail, ok := message["errorMessage"].(string); ok && detail != "" { raw.ErrorMessage = detail } + } else { + // Only the final message_end's stop reason decides: a turn that + // errored and then recovered must not be reported as a failure. + raw.ErrorMessage = "" } } diff --git a/sdk/go/harness/pi_test.go b/sdk/go/harness/pi_test.go index e580f0c0b..97b02af27 100644 --- a/sdk/go/harness/pi_test.go +++ b/sdk/go/harness/pi_test.go @@ -294,6 +294,55 @@ func TestPiFamilyExecutionFailures(t *testing.T) { } } +func TestPiFamilyRecoveredTurnIsNotAnError(t *testing.T) { + tests := []struct { + name string + stdout string + wantResult string + wantIsError bool + wantFailureType FailureType + wantError string + }{ + { + name: "error then recovery", + stdout: `{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"partial"}],"stopReason":"error","errorMessage":"upstream 503"}} +{"type":"turn_end"} +{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"FINAL ANSWER"}],"stopReason":"stop"}} +{"type":"turn_end"}`, + wantResult: "FINAL ANSWER", + wantFailureType: FailureNone, + }, + { + name: "last message is an error", + stdout: `{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"partial"}],"stopReason":"stop"}} +{"type":"turn_end"} +{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"FINAL ANSWER"}],"stopReason":"error","errorMessage":"upstream 503"}} +{"type":"turn_end"}`, + wantResult: "FINAL ANSWER", + wantIsError: true, + wantFailureType: FailureAPIError, + wantError: "upstream 503", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + provider := NewPiProvider("pi").piFamilyProvider + provider.runCLI = func(context.Context, []string, map[string]string, string, int, []byte) (*CLIResult, error) { + return &CLIResult{Stdout: tc.stdout, ReturnCode: 0}, nil + } + + raw, err := provider.execute(context.Background(), "inspect", Options{}) + require.NoError(t, err) + require.NotNil(t, raw) + assert.Equal(t, tc.wantResult, raw.Result) + assert.Equal(t, tc.wantIsError, raw.IsError) + assert.Equal(t, tc.wantFailureType, raw.FailureType) + assert.Equal(t, tc.wantError, raw.ErrorMessage) + }) + } +} + func TestPiFamilyNonzeroExitIsError(t *testing.T) { p := NewPiProvider("pi") p.runCLI = func(context.Context, []string, map[string]string, string, int, []byte) (*CLIResult, error) { diff --git a/sdk/python/agentfield/harness/providers/pi.py b/sdk/python/agentfield/harness/providers/pi.py index d4e1539d7..8f366bc95 100644 --- a/sdk/python/agentfield/harness/providers/pi.py +++ b/sdk/python/agentfield/harness/providers/pi.py @@ -118,6 +118,8 @@ def _parse_pi_events( if stop_reason in {"error", "aborted"}: detail = message.get("errorMessage") or message.get("error") provider_error = str(detail or f"Pi stopped with reason {stop_reason!r}.") + else: + provider_error = None if num_turns == 0 and result_text: num_turns = 1 diff --git a/sdk/python/tests/test_harness_provider_pi.py b/sdk/python/tests/test_harness_provider_pi.py index be445366e..65bed145c 100644 --- a/sdk/python/tests/test_harness_provider_pi.py +++ b/sdk/python/tests/test_harness_provider_pi.py @@ -5,6 +5,7 @@ import pytest +from agentfield.harness._result import FailureType from agentfield.harness.providers._factory import build_provider from agentfield.harness.providers.pi import OMPProvider, PiProvider from agentfield.types import HarnessConfig @@ -172,6 +173,45 @@ async def fake_run_cli(*_args, **_kwargs): assert raw.error_message == "authentication failed" +@pytest.mark.asyncio +@pytest.mark.parametrize("provider", [PiProvider, OMPProvider]) +async def test_pi_family_recovered_turn_is_not_an_error( + monkeypatch: pytest.MonkeyPatch, provider +) -> None: + events = [ + { + "type": "message_end", + "message": { + "role": "assistant", + "content": [{"type": "text", "text": "partial"}], + "stopReason": "error", + "errorMessage": "upstream 503", + }, + }, + {"type": "turn_end"}, + { + "type": "message_end", + "message": { + "role": "assistant", + "content": [{"type": "text", "text": "FINAL ANSWER"}], + "stopReason": "stop", + }, + }, + {"type": "turn_end"}, + ] + + async def fake_run_cli(*_args, **_kwargs): + return "\n".join(json.dumps(event) for event in events), "", 0 + + monkeypatch.setattr("agentfield.harness.providers.pi.run_cli", fake_run_cli) + raw = await provider().execute("hello", {}) + + assert raw.result == "FINAL ANSWER" + assert raw.is_error is False + assert raw.failure_type == FailureType.NONE + assert raw.error_message is None + + def test_factory_builds_pi_and_omp_with_configured_binaries() -> None: pi = build_provider(HarnessConfig(provider="pi", pi_bin="/opt/pi")) omp = build_provider(HarnessConfig(provider="omp", omp_bin="/opt/omp")) diff --git a/sdk/typescript/src/harness/providers/pi.ts b/sdk/typescript/src/harness/providers/pi.ts index 63fc03c58..0357db098 100644 --- a/sdk/typescript/src/harness/providers/pi.ts +++ b/sdk/typescript/src/harness/providers/pi.ts @@ -96,6 +96,8 @@ function parsePiEvents(events: Array>, configuredModel?: providerError = String( message.errorMessage ?? message.error ?? `Pi stopped with reason ${String(message.stopReason)}.` ); + } else { + providerError = undefined; } } diff --git a/sdk/typescript/tests/harness_provider_pi.test.ts b/sdk/typescript/tests/harness_provider_pi.test.ts index 185bd9df1..4ea39e234 100644 --- a/sdk/typescript/tests/harness_provider_pi.test.ts +++ b/sdk/typescript/tests/harness_provider_pi.test.ts @@ -113,6 +113,41 @@ it.each(providers)('$name classifies a provider event error', async ({ provider }); }); +it.each(providers)('$name clears a recovered provider event error', async ({ provider }) => { + const stdout = [ + { + type: 'message_end', + message: { + role: 'assistant', + content: [{ type: 'text', text: 'partial' }], + stopReason: 'error', + errorMessage: 'upstream 503', + }, + }, + { type: 'turn_end' }, + { + type: 'message_end', + message: { + role: 'assistant', + content: [{ type: 'text', text: 'FINAL ANSWER' }], + stopReason: 'stop', + }, + }, + { type: 'turn_end' }, + ].map((event) => JSON.stringify(event)).join('\n'); + vi.spyOn(cli, 'runCli').mockResolvedValue({ stdout, stderr: '', exitCode: 0 }); + + const result = await provider.execute('hello', {}); + + expect(result).toMatchObject({ + result: 'FINAL ANSWER', + isError: false, + failureType: 'none', + returnCode: 0, + }); + expect(result.errorMessage).toBeUndefined(); +}); + it.each(providers)('$name classifies a successful exit without output', async ({ provider }) => { vi.spyOn(cli, 'runCli').mockResolvedValue({ stdout: '', stderr: '', exitCode: 0 }); From 54dcfaa683a05ed5023d4ede21551dc8cb95f2e6 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Thu, 27 Aug 2026 11:05:29 -0400 Subject: [PATCH 16/20] fix(ts-sdk): derive the schema output root from the same ladder the providers use resolveRoot() centralised the provider-side working-directory ladder (projectDir -> project_dir -> cwd), but the runner kept its own two-rung version that never looked at project_dir. resolveOptions copies overrides with Object.entries, so a JS caller's project_dir key does reach the providers: with { project_dir: P, cwd: C } the schema instruction file was created under C while the provider ran in P, and the harness was told to write its output to a path outside the directory it was running in. Before this branch that split existed for aforge alone; centralising the ladder had widened it to six providers. The runner now calls resolveRoot on the resolved options, so the two ladders are identical by construction. Regression test covers the snake_case-only case; it fails if the line is reverted. Co-Authored-By: Claude Fable 5 --- sdk/typescript/src/harness/runner.ts | 5 ++--- sdk/typescript/tests/harness_runner.test.ts | 25 ++++++++++++++++++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/sdk/typescript/src/harness/runner.ts b/sdk/typescript/src/harness/runner.ts index 924e7ab65..c4e1ae732 100644 --- a/sdk/typescript/src/harness/runner.ts +++ b/sdk/typescript/src/harness/runner.ts @@ -2,7 +2,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { buildPromptSuffix, cleanupTempFiles, getOutputPath, parseAndValidate } from './schema.js'; import { buildProvider, resolveProviderName } from './providers/factory.js'; -import type { HarnessProvider } from './providers/base.js'; +import { resolveRoot, type HarnessProvider } from './providers/base.js'; import { createHarnessResult, createRawResult, @@ -61,8 +61,7 @@ export class HarnessRunner { resolved.provider = resolveProviderName(resolved.provider); const provider = await this.buildProvider(resolved.provider, resolved); - const cwd = resolved.cwd ?? '.'; - const outputRoot = resolved.projectDir ?? cwd; + const outputRoot = resolveRoot(resolved as Record) ?? '.'; let outputDir: string | undefined; if (schema !== undefined) { fs.mkdirSync(outputRoot, { recursive: true }); diff --git a/sdk/typescript/tests/harness_runner.test.ts b/sdk/typescript/tests/harness_runner.test.ts index 381e5a742..4f98e86f7 100644 --- a/sdk/typescript/tests/harness_runner.test.ts +++ b/sdk/typescript/tests/harness_runner.test.ts @@ -4,7 +4,7 @@ import path from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { z } from 'zod'; -import type { HarnessConfig, RawResult } from '../src/harness/types.js'; +import type { HarnessConfig, HarnessOptions, RawResult } from '../src/harness/types.js'; import type { HarnessProvider } from '../src/harness/providers/base.js'; import { createMetrics, createRawResult } from '../src/harness/types.js'; import { getOutputPath } from '../src/harness/schema.js'; @@ -100,6 +100,29 @@ describe('harness runner', () => { } ); + it('uses snake_case project_dir for schema instructions and codex execution', async () => { + const projectDir = makeTempDir(); + const cwd = makeTempDir(); + vi.spyOn(cli, 'runCli').mockResolvedValue({ stdout: '', stderr: '', exitCode: 0 }); + + const runner = new HarnessRunner(); + await runner.run('do it', { + schema: z.object({ answer: z.string() }), + provider: 'codex', + project_dir: projectDir, + cwd, + } as unknown as HarnessOptions); + + const [cmd, runOptions] = vi.mocked(cli.runCli).mock.calls[0]; + const prompt = cmd[cmd.length - 1]; + const outputPath = prompt?.match(/(\S*\.agentfield_output\.json)/)?.[1]; + + expect(outputPath).toBeDefined(); + expect(path.relative(projectDir, outputPath as string)).not.toMatch(/^\.\.(?:[/\\]|$)/); + expect(runOptions?.cwd).toBe(projectDir); + expect(projectDir).not.toBe(cwd); + }); + it('resolveOptions merges config with per-call overrides', () => { const cfg: HarnessConfig = { provider: 'codex', From ee542c971d1a908d6780021cde3a225dc25a07e1 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Thu, 27 Aug 2026 11:05:35 -0400 Subject: [PATCH 17/20] test(ts-sdk): cover ClaudeCodeProvider's projectDir handling ClaudeCodeProvider goes through @anthropic-ai/claude-agent-sdk rather than cli.runCli, so the runner's provider matrix cannot reach it, and the existing claude tests only ever passed `cwd`. Reverting claude.ts to the old `options.cwd` line left the whole suite green. Two tests close that: projectDir wins over a nested cwd, and an empty-string cwd now leaves the SDK option unset (resolveRoot skips empty strings, where the old code forwarded ''). Both fail against the reverted hunk. Co-Authored-By: Claude Fable 5 --- .../tests/harness_provider_claude.test.ts | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/sdk/typescript/tests/harness_provider_claude.test.ts b/sdk/typescript/tests/harness_provider_claude.test.ts index c0e20e3ff..5de84458a 100644 --- a/sdk/typescript/tests/harness_provider_claude.test.ts +++ b/sdk/typescript/tests/harness_provider_claude.test.ts @@ -116,6 +116,52 @@ describe('ClaudeCodeProvider', () => { expect(captured.options).toEqual({}); }); + it('prefers projectDir over a nested cwd', async () => { + const captured: { options?: Record } = {}; + + vi.doMock( + '@anthropic-ai/claude-agent-sdk', + () => ({ + query: ({ options }: { prompt: string; options: Record }) => { + captured.options = options; + return (async function* stream() { + yield { type: 'result', result: 'final' }; + })(); + }, + }), + { virtual: true } + ); + + const { ClaudeCodeProvider } = await import('../src/harness/providers/claude.js'); + const provider = new ClaudeCodeProvider(); + await provider.execute('hello', { projectDir: '/proj', cwd: '/proj/nested' }); + + expect(captured.options?.cwd).toBe('/proj'); + }); + + it('omits cwd when it is an empty string', async () => { + const captured: { options?: Record } = {}; + + vi.doMock( + '@anthropic-ai/claude-agent-sdk', + () => ({ + query: ({ options }: { prompt: string; options: Record }) => { + captured.options = options; + return (async function* stream() { + yield { type: 'result', result: 'final' }; + })(); + }, + }), + { virtual: true } + ); + + const { ClaudeCodeProvider } = await import('../src/harness/providers/claude.js'); + const provider = new ClaudeCodeProvider(); + await provider.execute('hello', { cwd: '' }); + + expect(captured.options).not.toHaveProperty('cwd'); + }); + it('strips a #variant model suffix before handing the model to the SDK', async () => { const captured: { options?: Record } = {}; From dfc6d74be079d2610dff6af98ed4d0e29003f27a Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Thu, 27 Aug 2026 11:12:22 -0400 Subject: [PATCH 18/20] fix(cli): make `af doctor` survey aforge, the default harness provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agentfield skill gates every use of app.harness() on `af doctor` reporting harness_usable: true AND listing the chosen provider. doctor's provider list never contained aforge, so on the default install — aforge shipped with `af`, nothing else present — an agent following the skill concluded the harness was unusable and refused to use the default provider. This branch had made that worse by adding aforge to the skill's provider union while leaving doctor's list alone. Detection now reuses `af harness doctor`'s spec table (findHarnessProviderSpec + probeHarnessBinary) wherever a binary-backed spec exists, so both doctors agree on what "installed" means. That matters for aforge specifically: it answers `version`, not `--version`, and `af aforge ensure` installs it into $AGENTFIELD_HOME/bin, which the current shell's PATH usually does not contain. claude-code has no binary in that table (it is the pip-package wrapper) and keeps the plain PATH check. --probe skips providers that declare no ProbeArgs, which is aforge alone: every other probe is one trivial completion, whereas aforge's only one-shot is a full coding-agent run with write access to the working directory, which is not something a doctor command should start. `af harness doctor` reports aforge's health. Live-verified with a fake aforge in $HOME/.agentfield/bin and an empty PATH: `af doctor --json` reports aforge available with its version, and recommendation.harness_usable true / harness_providers ["aforge"]; `af doctor --probe` produces no aforge probe entry. Co-Authored-By: Claude Fable 5 --- control-plane/internal/cli/doctor.go | 38 +++++++++++++++--- .../internal/cli/doctor_probe_test.go | 39 +++++++++++++++++++ 2 files changed, 71 insertions(+), 6 deletions(-) diff --git a/control-plane/internal/cli/doctor.go b/control-plane/internal/cli/doctor.go index 5dd346a97..05bd8efdb 100644 --- a/control-plane/internal/cli/doctor.go +++ b/control-plane/internal/cli/doctor.go @@ -79,13 +79,20 @@ var providerEnvVars = []struct { {Name: "google", EnvVar: "GOOGLE_API_KEY", Model: "gemini-1.5-pro"}, } -// harnessProviders is the canonical list of CLIs `app.harness()` knows how to drive. -var harnessProviders = []struct { +// doctorHarnessProvider describes one CLI `app.harness()` knows how to drive, +// as `af doctor` surveys it. +type doctorHarnessProvider struct { Name string // value passed to provider= in app.harness() Binary string // executable name to look up on PATH - ProbeArgs []string // minimal one-shot invocation used by `--probe` + ProbeArgs []string // minimal one-shot invocation used by `--probe`; empty means "never probe" ProbeStdin string // prompt fed over stdin, mirroring how the SDK adapters invoke this CLI -}{ +} + +// harnessProviders is the canonical list of CLIs `app.harness()` knows how to +// drive. aforge leads it: it is the SDK's default provider and ships with `af`. +var harnessProviders = []doctorHarnessProvider{ + // aforge declares no ProbeArgs on purpose — see runHarnessProbes. + {Name: "aforge", Binary: "aforge"}, {Name: "claude-code", Binary: "claude", ProbeArgs: []string{"-p", "Say OK"}}, {Name: "codex", Binary: "codex", ProbeArgs: []string{"exec", "Say OK"}}, {Name: "gemini", Binary: "gemini", ProbeArgs: []string{"-p", "Say OK"}}, @@ -123,7 +130,7 @@ func NewDoctorCommand() *cobra.Command { Long: `Doctor inspects the local environment and reports what's available for building AgentField multi-reasoner systems: - • Available harness provider CLIs (claude-code, codex, gemini, opencode, pi, omp) + • Available harness provider CLIs (aforge, claude-code, codex, gemini, opencode, pi, omp) • Provider API keys set in the environment (without leaking values) • Docker availability and whether the control-plane image is locally cached • Whether a local control plane is reachable @@ -179,6 +186,12 @@ func runHarnessProbes(report DoctorReport) map[string]HarnessProbeResult { if !report.HarnessProviders[h.Name].Available { continue } + // Every other probe is one trivial completion. Aforge's only one-shot is + // a full coding-agent run with working-directory write access, so `af + // doctor` must not start it; `af harness doctor` reports its health. + if len(h.ProbeArgs) == 0 { + continue + } results[h.Name] = probeHarnessProvider(h.Name, h.Binary, h.ProbeArgs, h.ProbeStdin, harnessProbeTimeout) } return results @@ -286,7 +299,20 @@ func buildDoctorReport(controlPlaneURL string) DoctorReport { // Harness CLIs availableHarness := []string{} for _, h := range harnessProviders { - status := checkTool(h.Binary, "--version") + // Share the harness doctor's provider-specific version arguments and its + // $AGENTFIELD_HOME/bin fallback wherever a binary-backed spec exists, so + // both doctors agree on what "installed" means. aforge in particular + // answers `version`, not `--version`, and `af aforge ensure` puts it in + // AgentField's own bin directory rather than on PATH. claude-code has no + // binary in that table (it is the pip-package wrapper), so it keeps the + // plain PATH check. + spec := findHarnessProviderSpec(h.Name) + var status ToolStatus + if spec != nil && spec.Binary != "" { + status = probeHarnessBinary(*spec) + } else { + status = checkTool(h.Binary, "--version") + } report.HarnessProviders[h.Name] = status if status.Available { availableHarness = append(availableHarness, h.Name) diff --git a/control-plane/internal/cli/doctor_probe_test.go b/control-plane/internal/cli/doctor_probe_test.go index 0808d84d0..9643c3726 100644 --- a/control-plane/internal/cli/doctor_probe_test.go +++ b/control-plane/internal/cli/doctor_probe_test.go @@ -58,6 +58,9 @@ func TestHarnessProviders_ProbeInputContract(t *testing.T) { } } default: + if provider.Name == "aforge" && (len(provider.ProbeArgs) != 0 || provider.ProbeStdin != "") { + t.Errorf("aforge must not declare probe input") + } if provider.ProbeStdin != "" { t.Errorf("positional-prompt provider %s ProbeStdin = %q, want empty", provider.Name, provider.ProbeStdin) } @@ -99,11 +102,47 @@ func TestProbeHarnessProvider_RealProcesses(t *testing.T) { } } +// Contract: aforge is surveyed by doctor but never smoke-tested — its only +// one-shot is a full coding-agent run, so it deliberately declares no ProbeArgs. +func TestHarnessProviders_AforgeIsSurveyedButNotProbed(t *testing.T) { + if harnessProviders[0].Name != "aforge" { + t.Fatalf("aforge must lead the provider list, got %q", harnessProviders[0].Name) + } + if len(harnessProviders[0].ProbeArgs) != 0 || harnessProviders[0].ProbeStdin != "" { + t.Errorf("aforge must declare no probe input, got %+v", harnessProviders[0]) + } +} + +// Contract: a provider with no ProbeArgs is skipped by --probe even when +// detected, while a provider that declares them is still probed. The registry +// is swapped for synthetic entries so no real coding-agent CLI is invoked. +func TestRunHarnessProbes_SkipsProvidersWithoutProbeArgs(t *testing.T) { + original := harnessProviders + t.Cleanup(func() { harnessProviders = original }) + harnessProviders = []doctorHarnessProvider{ + {Name: "no-probe", Binary: "agentfield-absent-no-probe"}, + {Name: "with-probe", Binary: "agentfield-absent-with-probe", ProbeArgs: []string{"--version"}}, + } + + report := DoctorReport{HarnessProviders: map[string]ToolStatus{ + "no-probe": {Available: true}, + "with-probe": {Available: true}, + }} + got := runHarnessProbes(report) + if _, ok := got["no-probe"]; ok { + t.Error("a provider without ProbeArgs must be skipped even when available") + } + if _, ok := got["with-probe"]; !ok { + t.Error("an available provider with ProbeArgs must produce a result") + } +} + // Contract: probes run ONLY for providers doctor already detected — unavailable // providers are never invoked. func TestRunHarnessProbes_SkipsUndetected(t *testing.T) { report := DoctorReport{ HarnessProviders: map[string]ToolStatus{ + "aforge": {Available: false}, "claude-code": {Available: false}, "codex": {Available: false}, "gemini": {Available: false}, From 6247a63622a44056bfcc36b17851fe80630d2dd7 Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Thu, 27 Aug 2026 11:13:01 -0400 Subject: [PATCH 19/20] fix(cli): stop `--probe` reporting a silently broken pi/omp install as ok MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit classifyProbe decided "empty" purely by an empty stdout. That was correct while pi/omp probed in plain --print text mode, but they now probe with `--print --mode json`, and the CLI emits a {"type":"session",...} event before any assistant output. stdout is therefore never blank and the probe always fell through to "ok" — so an install that exits 0 with a parsed stream and no assistant text, and one whose message reports stopReason "error", both came back healthy. That negates the exact capability --probe's own help text advertises, for the two providers this branch added. Providers whose probe output is a JSON event stream are now marked JSONLStream, and their probes apply the SDK adapters' own success criterion: an assistant message_end carrying text, with the last assistant message_end's stop reason not "error"/"aborted" (a turn that errored and then recovered is not a failure, matching the adapter fix in this branch). An exit-0 stream error surfaces its message as the probe detail when stderr is silent. Plain text providers keep the previous rule unchanged. Live-verified with fakes on PATH: an `omp` printing only {"type":"session","id":"s1"} and exiting 0 now reports status "empty" where it reported "ok" before; a `pi` printing a real assistant message_end still reports "ok". Co-Authored-By: Claude Fable 5 --- control-plane/internal/cli/doctor.go | 97 ++++++++++++++++--- .../internal/cli/doctor_probe_test.go | 37 ++++++- 2 files changed, 120 insertions(+), 14 deletions(-) diff --git a/control-plane/internal/cli/doctor.go b/control-plane/internal/cli/doctor.go index 05bd8efdb..832574701 100644 --- a/control-plane/internal/cli/doctor.go +++ b/control-plane/internal/cli/doctor.go @@ -86,6 +86,10 @@ type doctorHarnessProvider struct { Binary string // executable name to look up on PATH ProbeArgs []string // minimal one-shot invocation used by `--probe`; empty means "never probe" ProbeStdin string // prompt fed over stdin, mirroring how the SDK adapters invoke this CLI + // JSONLStream marks providers whose probe stdout is a JSON event stream + // rather than plain text, so a non-empty stdout does not by itself mean + // the provider completed anything. + JSONLStream bool } // harnessProviders is the canonical list of CLIs `app.harness()` knows how to @@ -98,8 +102,8 @@ var harnessProviders = []doctorHarnessProvider{ {Name: "gemini", Binary: "gemini", ProbeArgs: []string{"-p", "Say OK"}}, {Name: "opencode", Binary: "opencode", ProbeArgs: []string{"run", "Say OK"}}, // Keep pi/omp stdin prompt delivery coupled to the SDK adapters. - {Name: "pi", Binary: "pi", ProbeArgs: []string{"--print", "--mode", "json"}, ProbeStdin: "Say OK"}, - {Name: "omp", Binary: "omp", ProbeArgs: []string{"--print", "--mode", "json"}, ProbeStdin: "Say OK"}, + {Name: "pi", Binary: "pi", ProbeArgs: []string{"--print", "--mode", "json"}, ProbeStdin: "Say OK", JSONLStream: true}, + {Name: "omp", Binary: "omp", ProbeArgs: []string{"--print", "--mode", "json"}, ProbeStdin: "Say OK", JSONLStream: true}, } // harnessProbeTimeout bounds a single provider smoke test. Coding-agent CLIs @@ -192,17 +196,17 @@ func runHarnessProbes(report DoctorReport) map[string]HarnessProbeResult { if len(h.ProbeArgs) == 0 { continue } - results[h.Name] = probeHarnessProvider(h.Name, h.Binary, h.ProbeArgs, h.ProbeStdin, harnessProbeTimeout) + results[h.Name] = probeHarnessProvider(h.Name, h.Binary, h.ProbeArgs, h.ProbeStdin, harnessProbeTimeout, h.JSONLStream) } return results } // probeHarnessProvider runs one provider CLI's minimal one-shot invocation and // classifies the outcome. -func probeHarnessProvider(name, binary string, args []string, stdin string, timeout time.Duration) HarnessProbeResult { +func probeHarnessProvider(name, binary string, args []string, stdin string, timeout time.Duration, jsonlStream bool) HarnessProbeResult { start := time.Now() stdout, stderr, exitCode, timedOut := runProbeCommand(binary, args, stdin, timeout) - status := classifyProbe(exitCode, stdout, timedOut) + status := classifyProbe(exitCode, stdout, timedOut, jsonlStream) result := HarnessProbeResult{ Provider: name, @@ -213,6 +217,9 @@ func probeHarnessProvider(name, binary string, args []string, stdin string, time switch status { case "error": result.Detail = firstLine(stderr) + if result.Detail == "" && jsonlStream { + _, result.Detail = piProbeOutcome(stdout) + } case "timeout": result.Detail = fmt.Sprintf("no response within %s", timeout) case "empty": @@ -252,20 +259,86 @@ func runProbeCommand(bin string, args []string, stdin string, timeout time.Durat } // classifyProbe maps a probe outcome to a status. Order matters: a timeout is -// checked before the exit code (a killed process also exits non-zero), and an -// empty completion on a clean exit is the real-world "silently broken provider" -// case that a mere PATH check misses. -func classifyProbe(exitCode int, stdout string, timedOut bool) string { +// checked before the exit code (a killed process also exits non-zero). Plain +// text probes require non-empty stdout; JSONL probes require successful +// assistant completion text rather than merely any event. +func classifyProbe(exitCode int, stdout string, timedOut bool, jsonlStream bool) string { switch { case timedOut: return "timeout" case exitCode != 0: return "error" - case strings.TrimSpace(stdout) == "": - return "empty" - default: + } + if jsonlStream { + hasAssistantText, providerError := piProbeOutcome(stdout) + if providerError != "" { + return "error" + } + if !hasAssistantText { + return "empty" + } return "ok" } + if strings.TrimSpace(stdout) == "" { + return "empty" + } + return "ok" +} + +// piProbeOutcome inspects a Pi-family JSON event stream the way the SDK +// adapters do. A completion requires assistant text, and only the last +// assistant message_end determines whether the provider stopped with an error. +func piProbeOutcome(stdout string) (hasAssistantText bool, providerError string) { + type message struct { + Role string `json:"role"` + Content json.RawMessage `json:"content"` + StopReason string `json:"stopReason"` + ErrorMessage string `json:"errorMessage"` + } + type event struct { + Type string `json:"type"` + Message message `json:"message"` + } + type contentPart struct { + Type string `json:"type"` + Text string `json:"text"` + } + + for _, line := range strings.Split(stdout, "\n") { + if strings.TrimSpace(line) == "" { + continue + } + var e event + if err := json.Unmarshal([]byte(line), &e); err != nil || e.Type != "message_end" || e.Message.Role != "assistant" { + continue + } + + var text string + if err := json.Unmarshal(e.Message.Content, &text); err != nil { + var parts []contentPart + if json.Unmarshal(e.Message.Content, &parts) == nil { + for _, part := range parts { + if part.Type == "text" { + text += part.Text + } + } + } + } + if strings.TrimSpace(text) != "" { + hasAssistantText = true + } + + switch e.Message.StopReason { + case "error", "aborted": + providerError = e.Message.ErrorMessage + if providerError == "" { + providerError = fmt.Sprintf("stopped with reason %q", e.Message.StopReason) + } + default: + providerError = "" + } + } + return hasAssistantText, providerError } // firstLine returns the first non-empty line of s, trimmed, for compact error diff --git a/control-plane/internal/cli/doctor_probe_test.go b/control-plane/internal/cli/doctor_probe_test.go index 9643c3726..093a83593 100644 --- a/control-plane/internal/cli/doctor_probe_test.go +++ b/control-plane/internal/cli/doctor_probe_test.go @@ -34,13 +34,46 @@ func TestClassifyProbe(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - if got := classifyProbe(tc.exitCode, tc.stdout, tc.timedOut); got != tc.want { + if got := classifyProbe(tc.exitCode, tc.stdout, tc.timedOut, false); got != tc.want { t.Errorf("classifyProbe(%d, %q, %v) = %q, want %q", tc.exitCode, tc.stdout, tc.timedOut, got, tc.want) } }) } } +func TestClassifyProbe_JSONLStream(t *testing.T) { + cases := []struct { + name string + stdout string + want string + }{ + {"session only", `{"type":"session","id":"s1"}`, "empty"}, + {"assistant text", "{\"type\":\"session\",\"id\":\"s1\"}\n{\"type\":\"message_end\",\"message\":{\"role\":\"assistant\",\"content\":\"OK\",\"stopReason\":\"stop\"}}", "ok"}, + {"provider error", `{"type":"message_end","message":{"role":"assistant","content":"partial","stopReason":"error","errorMessage":"provider failed"}}`, "error"}, + {"recovered", "{\"type\":\"message_end\",\"message\":{\"role\":\"assistant\",\"content\":\"partial\",\"stopReason\":\"error\"}}\n{\"type\":\"message_end\",\"message\":{\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"OK\"}],\"stopReason\":\"stop\"}}", "ok"}, + {"empty text part", `{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":""}],"stopReason":"stop"}}`, "empty"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := classifyProbe(0, tc.stdout, false, true); got != tc.want { + t.Errorf("classifyProbe JSONL = %q, want %q", got, tc.want) + } + }) + } +} + +func TestHarnessProviders_JSONLStream(t *testing.T) { + var got []string + for _, provider := range harnessProviders { + if provider.JSONLStream { + got = append(got, provider.Name) + } + } + if want := []string{"pi", "omp"}; !reflect.DeepEqual(got, want) { + t.Errorf("JSONLStream providers = %v, want %v", got, want) + } +} + func TestHarnessProviders_ProbeInputContract(t *testing.T) { wantStdinArgs := []string{"--print", "--mode", "json"} for _, provider := range harnessProviders { @@ -91,7 +124,7 @@ func TestProbeHarnessProvider_RealProcesses(t *testing.T) { if _, err := exec.LookPath(tc.bin); err != nil { t.Skipf("%s not available: %v", tc.bin, err) } - res := probeHarnessProvider("prov-"+tc.name, tc.bin, tc.args, tc.stdin, tc.timeout) + res := probeHarnessProvider("prov-"+tc.name, tc.bin, tc.args, tc.stdin, tc.timeout, false) if res.Status != tc.want { t.Errorf("status = %q, want %q (detail=%q)", res.Status, tc.want, res.Detail) } From 413d069608a4df3dfd5fd13a8ae3441f6836f4cc Mon Sep 17 00:00:00 2001 From: Abir Abbas Date: Thu, 27 Aug 2026 11:37:55 -0400 Subject: [PATCH 20/20] test(cli): guard `af doctor`'s aforge detection against a silent revert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The branch in buildDoctorReport that routes detection through the harness doctor's spec table (findHarnessProviderSpec + probeHarnessBinary) is the whole behavior of "fix(cli): make `af doctor` survey aforge, the default harness provider" — it is what makes doctor ask aforge for `version` rather than `--version`, and what makes it look in $AGENTFIELD_HOME/bin when the binary is not on PATH. Replacing that branch with main's original `checkTool(h.Binary, "--version")` left the entire internal/cli suite green, so a future refactor could revert the fix without CI noticing. TestBuildDoctorReport_AforgeDetectionUsesHarnessSpec drives buildDoctorReport with a shell-script aforge stub and covers both halves: • installed only in $AGENTFIELD_HOME/bin with an empty PATH — doctor must report it available, with the managed path and its version; • on PATH but answering `version` only (non-zero on `--version`) — doctor must still record the version. Both subtests fail under the `checkTool(h.Binary, "--version")` mutation (available:false / version:"" respectively), and the full `go test ./internal/cli/ -count=1` suite stays green. Co-Authored-By: Claude Fable 5 --- .../internal/cli/doctor_probe_test.go | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/control-plane/internal/cli/doctor_probe_test.go b/control-plane/internal/cli/doctor_probe_test.go index 093a83593..b265c6d26 100644 --- a/control-plane/internal/cli/doctor_probe_test.go +++ b/control-plane/internal/cli/doctor_probe_test.go @@ -8,7 +8,9 @@ import ( "net/http/httptest" "os" "os/exec" + "path/filepath" "reflect" + "runtime" "testing" "time" ) @@ -146,6 +148,58 @@ func TestHarnessProviders_AforgeIsSurveyedButNotProbed(t *testing.T) { } } +// Contract: buildDoctorReport detects aforge through its harness specification, +// including its managed install location and aforge-specific version argument. +func TestBuildDoctorReport_AforgeDetectionUsesHarnessSpec(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("shell fixture is Unix-only") + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + t.Run("found in AGENTFIELD_HOME/bin when not on PATH", func(t *testing.T) { + t.Setenv("PATH", t.TempDir()) + home := t.TempDir() + t.Setenv("AGENTFIELD_HOME", home) + managedBin := filepath.Join(home, "bin") + if err := os.MkdirAll(managedBin, 0o755); err != nil { + t.Fatalf("create managed bin: %v", err) + } + writeHarnessTestScript(t, managedBin, "aforge", "printf 'aforge 1.2.3\\n'") + + report := buildDoctorReport(srv.URL) + status := report.HarnessProviders["aforge"] + if !status.Available { + t.Errorf("aforge should be available, got %+v", status) + } + if status.Version != "aforge 1.2.3" { + t.Errorf("aforge version = %q, want %q", status.Version, "aforge 1.2.3") + } + if want := filepath.Join(home, "bin", "aforge"); status.Path != want { + t.Errorf("aforge path = %q, want %q", status.Path, want) + } + }) + + t.Run("version comes from the aforge-specific version argument", func(t *testing.T) { + binDir := t.TempDir() + t.Setenv("PATH", binDir) + t.Setenv("AGENTFIELD_HOME", t.TempDir()) + writeHarnessTestScript(t, binDir, "aforge", "[ \"$1\" = version ] && { printf 'aforge 9.9.9\\n'; exit 0; }; exit 1") + + report := buildDoctorReport(srv.URL) + status := report.HarnessProviders["aforge"] + if !status.Available { + t.Errorf("aforge should be available, got %+v", status) + } + if status.Version != "aforge 9.9.9" { + t.Errorf("aforge version = %q, want %q", status.Version, "aforge 9.9.9") + } + }) +} + // Contract: a provider with no ProbeArgs is skipped by --probe even when // detected, while a provider that declares them is still probed. The registry // is swapped for synthetic entries so no real coding-agent CLI is invoked.