webvoyager: validate --max-images and reject negative --limit

A 0/negative/non-numeric --max-images made lastShots' slice(-k) attach
every screenshot instead of the last k, risking judge token blowups and
spurious 0 rewards; parse it to a positive integer, falling back to the
15 default otherwise. A negative --limit likewise made tasks[:limit] drop
tasks off the end instead of taking the first N, so reject it.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Author: rgarcia

benchmarks/adapters/webvoyager/judge/src/judge.ts

@@ -32,6 +32,19 @@ export interface Args {
   detailsOut?: string;
 }
 
+const DEFAULT_MAX_IMAGES = 15;
+
+/**
+ * Parse `--max-images` into a positive integer last-k window. A 0, negative, or
+ * non-numeric value (e.g. a bad `WEBVOYAGER_MAX_IMAGES` env override) would make
+ * `lastShots`' `slice(-k)` attach *all* screenshots — blowing the judge's token
+ * budget — so anything that isn't a positive integer falls back to the default.
+ */
+export function parseMaxImages(raw: string | undefined): number {
+  const value = Number(raw ?? DEFAULT_MAX_IMAGES);
+  return Number.isInteger(value) && value > 0 ? value : DEFAULT_MAX_IMAGES;
+}
+
 function parseArgs(argv: string[]): Args {
   const flags = new Map<string, string>();
   for (let i = 0; i < argv.length; i += 1) {
@@ -51,7 +64,7 @@ function parseArgs(argv: string[]): Args {
     answer: flags.get("answer") ?? "/logs/agent/answer.txt",
     shots: flags.get("shots") ?? "/logs/agent/shots",
     judgeModel: flags.get("judge-model") ?? "claude-sonnet-4-5",
-    maxImages: Number(flags.get("max-images") ?? "15"),
+    maxImages: parseMaxImages(flags.get("max-images")),
     rewardOut: required("reward-out"),
     detailsOut: flags.get("details-out"),
   };

benchmarks/adapters/webvoyager/judge/test/judge.test.ts

@@ -3,7 +3,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { describe, expect, it } from "vitest";
 import type { Args } from "../src/judge.ts";
-import { run } from "../src/judge.ts";
+import { parseMaxImages, run } from "../src/judge.ts";
 import type { GradingDetails, JudgeContent, JudgeModel } from "../src/types.ts";
 
 /** A /logs/agent + /tests layout, plus the verifier output paths run() writes. */
@@ -108,3 +108,20 @@ describe("run", () => {
     expect(readDetails(args).error).toContain("ANTHROPIC_API_KEY");
   });
 });
+
+describe("parseMaxImages", () => {
+  it("keeps a valid positive integer", () => {
+    expect(parseMaxImages("3")).toBe(3);
+    expect(parseMaxImages("15")).toBe(15);
+  });
+
+  it("defaults to 15 when unset", () => {
+    expect(parseMaxImages(undefined)).toBe(15);
+  });
+
+  // A 0/negative/non-numeric last-k makes slice(-k) attach ALL screenshots, so
+  // anything that isn't a positive integer must fall back to the default.
+  it.each(["0", "-5", "abc", "", "2.5"])("falls back to 15 for invalid %o", (raw) => {
+    expect(parseMaxImages(raw)).toBe(15);
+  });
+});

benchmarks/adapters/webvoyager/src/webvoyager/adapter.py

@@ -99,6 +99,10 @@ def __init__(
         refresh: bool = False,
         **kwargs: object,
     ):
+        if limit is not None and limit < 0:
+            # tasks[:limit] with a negative limit drops tasks off the *end* instead
+            # of taking the first N, so reject it rather than silently mis-selecting.
+            raise ValueError(f"limit must be non-negative, got {limit}")
         self.output_dir = Path(output_dir)
         self.limit = limit
         self.overwrite = overwrite

benchmarks/adapters/webvoyager/tests/test_adapter.py

@@ -132,6 +132,12 @@ def test_limit_and_task_ids_select(tmp_path: Path) -> None:
     assert ids == {"Amazon--3", "Apple--1"}
 
 
+def test_negative_limit_rejected(tmp_path: Path) -> None:
+    # tasks[:limit] with a negative limit would drop tasks off the end, so it must error.
+    with pytest.raises(ValueError, match="non-negative"):
+        WebVoyagerAdapter(output_dir=tmp_path / "out", limit=-1)
+
+
 def test_overwrite_false_skips_existing(adapter: WebVoyagerAdapter) -> None:
     adapter.run()
     target = adapter.output_dir / "webvoyager-allrecipes--0" / "instruction.md"