feat(evals): core journeys green on real infra, environment healing behind its components, timeline observability

evals/packages/behaviors/package.json

@@ -33,6 +33,7 @@
   "dependencies": {
     "@openwork/cdp": "workspace:*",
     "@openwork/labs": "workspace:*",
-    "@openwork/matchers": "workspace:*"
+    "@openwork/matchers": "workspace:*",
+    "@openwork/timeline": "workspace:*"
   }
 }

evals/packages/behaviors/src/browser-handoff.ts

@@ -0,0 +1,151 @@
+import { chmod, mkdtemp, readFile, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { timed } from "@openwork/timeline";
+import { control, evalIn, waitFor } from "./desktop.ts";
+import type { Surface } from "@openwork/cdp";
+
+/**
+ * The desktop signs in by opening the system browser and finishing there. To
+ * observe that faithfully we capture what the app actually asks the OS to open
+ * (via a PATH shim for xdg-open, which is what Electron's shell.openExternal
+ * calls on Linux) rather than trusting that a browser appeared.
+ *
+ * Nothing here modifies the product: the app opens a URL exactly as it always
+ * does; we just record where it pointed.
+ */
+
+export interface UrlCapture {
+  /** Prepend to PATH when spawning the app. */
+  binDir: string;
+  /** Every URL the app asked the OS to open, oldest first. */
+  opened(): Promise<string[]>;
+  /** Wait for a URL matching a predicate, so a spec never races the handoff. */
+  waitForUrl(match: (url: string) => boolean, opts?: { timeoutMs?: number }): Promise<string>;
+}
+
+export async function captureOpenedUrls(): Promise<UrlCapture> {
+  const dir = await mkdtemp(join(tmpdir(), "openwork-open-external-"));
+  const logPath = join(dir, "opened-urls.log");
+  await writeFile(logPath, "", "utf8");
+  const shim = join(dir, "xdg-open");
+  await writeFile(shim, `#!/usr/bin/env bash\nprintf '%s\\n' "$1" >> ${JSON.stringify(logPath)}\nexit 0\n`, "utf8");
+  await chmod(shim, 0o755);
+
+  const opened = async (): Promise<string[]> => {
+    const contents = await readFile(logPath, "utf8").catch(() => "");
+    return contents.split("\n").map((line) => line.trim()).filter((line) => line.length > 0);
+  };
+
+  return {
+    binDir: dir,
+    opened,
+    async waitForUrl(match, { timeoutMs = 60_000 } = {}) {
+      const deadline = Date.now() + timeoutMs;
+      let seen: string[] = [];
+      while (Date.now() < deadline) {
+        seen = await opened();
+        const hit = seen.find((url) => match(url));
+        if (hit) return hit;
+        await new Promise((resolve) => setTimeout(resolve, 500));
+      }
+      throw new Error(`The app never asked to open a matching URL within ${timeoutMs}ms. Opened: ${seen.join(", ") || "<none>"}`);
+    },
+  };
+}
+
+/**
+ * Sign in on a real den web page in a real browser, the way a person does.
+ *
+ * Observed shape: den auth is two steps, not one form. First an email-only
+ * step ("Enter your email and we'll send you to the right sign-in step",
+ * button "Next"); only after that does the password step appear (button
+ * "Sign in"). There is no sign-up/sign-in toggle — the email step routes.
+ */
+export async function signInInBrowser(
+  browser: Surface,
+  url: string,
+  credentials: { email: string; password: string },
+): Promise<void> {
+  await timed("browser.signIn", async () => {
+    await evalIn(browser, `window.location.href = ${JSON.stringify(url)}`);
+    await waitFor(browser, `(() => {
+      if (document.querySelector('input[type="password"], input[name="password"]')) return true;
+      const email = document.querySelector('input[type="email"], input[name="email"]');
+      if (!email) return false;
+      const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, "value").set;
+      setter.call(email, ${JSON.stringify(credentials.email)});
+      email.dispatchEvent(new Event("input", { bubbles: true }));
+      const next = [...document.querySelectorAll("button")]
+        .find((candidate) => /next|continue|sign ?in/i.test(candidate.textContent ?? "") && !candidate.disabled);
+      if (!next) return false;
+      next.click();
+      return true;
+    })()`, { timeoutMs: 120_000, label: "den email step submitted" });
+    await waitFor(browser, `(() => {
+      if (/signed in/i.test(document.body?.innerText ?? "")) return true;
+      const password = document.querySelector('input[type="password"], input[name="password"]');
+      if (!password) return false;
+      const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, "value").set;
+      setter.call(password, ${JSON.stringify(credentials.password)});
+      password.dispatchEvent(new Event("input", { bubbles: true }));
+      const submit = [...document.querySelectorAll("button")]
+        .find((candidate) => /sign ?in|continue|log ?in/i.test(candidate.textContent ?? "") && !candidate.disabled);
+      if (!submit) return false;
+      submit.click();
+      return true;
+    })()`, { timeoutMs: 60_000, label: "den password step submitted" });
+    // Signed in means the outcome page, not a submit in flight ("Working...").
+    await waitFor(browser, `(() => {
+      const text = document.body?.innerText ?? "";
+      if (/signed in/i.test(text)) return true;
+      return [...document.querySelectorAll("input")]
+        .some((input) => (input.value ?? "").startsWith("openwork://"));
+    })()`, { timeoutMs: 60_000, label: "den sign-in outcome" });
+  }, credentials.email);
+}
+
+/**
+ * Read the deep link the browser is handed once sign-in completes.
+ *
+ * Observed shape: the app opens `<den>/?mode=sign-up&desktopAuth=1&desktopScheme=openwork`,
+ * the person signs in there, and Den shows "You're signed in" with an
+ * "Open OpenWork" button plus a readonly input holding the sign-in code —
+ * the full `openwork://den-auth?grant=…&denBaseUrl=…` URL a person would
+ * copy-paste into the app. Reading that input keeps the grant the real one
+ * Den issued for this session.
+ */
+export async function readHandoffDeepLink(browser: Surface, { timeoutMs = 120_000 } = {}): Promise<string> {
+  const found = await waitFor(browser, `(() => {
+    const fromInput = [...document.querySelectorAll("input")]
+      .map((input) => input.value)
+      .find((value) => typeof value === "string" && value.startsWith("openwork://") && value.includes("grant="));
+    if (fromInput) return fromInput;
+    const fromAnchor = [...document.querySelectorAll('a[href^="openwork://"]')]
+      .map((anchor) => anchor.getAttribute("href"))
+      .find((href) => typeof href === "string" && href.includes("grant="));
+    if (fromAnchor) return fromAnchor;
+    const inText = (document.body?.innerText ?? "").match(/openwork:\\/\\/[^\\s"']+grant=[^\\s"']+/);
+    return inText ? inText[0] : false;
+  })()`, { timeoutMs, label: "handoff deep link in the browser" });
+  if (typeof found !== "string") throw new Error("Could not read a handoff deep link from the browser page.");
+  return found;
+}
+
+/**
+ * Complete the hop back into the desktop.
+ *
+ * A real OS dispatches `openwork://den-auth?grant=…` to the app. A container has
+ * no protocol handler registered, so we hand the grant to the product's own
+ * documented entry point for exactly this situation (`auth.exchange-grant`,
+ * described in-product as signing in with a handoff grant). The grant itself is
+ * the real one the app generated and the browser session approved — only the OS
+ * dispatch is bridged, and that is stated wherever this is used.
+ */
+export async function completeDesktopHandoff(app: Surface, deepLinkOrGrantUrl: string, denBaseUrl: string): Promise<string> {
+  const url = new URL(deepLinkOrGrantUrl);
+  const grant = url.searchParams.get("grant") ?? "";
+  if (!grant) throw new Error(`No grant in the handoff URL: ${deepLinkOrGrantUrl}`);
+  await control(app, "auth.exchange-grant", { grant, baseUrl: denBaseUrl });
+  return grant;
+}

evals/packages/behaviors/src/cloud-plugins.ts

@@ -0,0 +1,155 @@
+import { timed } from "@openwork/timeline";
+import { denFetch } from "./den.ts";
+import type { DenSession } from "./den.ts";
+
+/**
+ * Authoring and sharing on the cloud side: a skill lives inside a plugin, and a
+ * plugin becomes shareable by being assigned to a marketplace that colleagues
+ * (or their teams) can use.
+ *
+ * These are plain API behaviours over the real den contract
+ * (ee/apps/den-api/src/routes/org/plugin-system), so the same calls work from a
+ * spec or a support script.
+ */
+
+export interface MarketplaceFacts {
+  id: string;
+  name: string;
+}
+
+export interface PluginFacts {
+  id: string;
+  name: string;
+  componentIds: string[];
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function requireId(value: unknown, what: string, status: number, body: unknown): string {
+  const item = isRecord(value) && isRecord(value.item) ? value.item : isRecord(value) ? value : null;
+  const id = item && typeof item.id === "string" ? item.id : "";
+  if (!id) throw new Error(`${what} did not return an id (status ${status}): ${JSON.stringify(body).slice(0, 300)}`);
+  return id;
+}
+
+export async function createMarketplace(admin: DenSession, input: { name: string; description?: string }): Promise<MarketplaceFacts> {
+  return timed("cloud.createMarketplace", async () => {
+    const { response, body } = await denFetch(admin, "/v1/marketplaces", {
+      method: "POST",
+      headers: { authorization: `Bearer ${admin.token}` },
+      body: JSON.stringify({ name: input.name, description: input.description ?? null }),
+    });
+    if (!response.ok) throw new Error(`Creating marketplace failed (${response.status}): ${JSON.stringify(body).slice(0, 300)}`);
+    return { id: requireId(body, "Creating a marketplace", response.status, body), name: input.name };
+  });
+}
+
+/**
+ * Create a plugin that CONTAINS a skill, and (optionally) publish it to a
+ * marketplace in the same call — the contract supports both, which mirrors what
+ * a person does in one sitting: author the skill, then share it.
+ */
+export async function createPluginWithSkill(
+  admin: DenSession,
+  input: { name: string; skillName: string; skillBody: string; marketplaceId?: string; orgWide?: boolean },
+): Promise<PluginFacts> {
+  return timed("cloud.createPluginWithSkill", async () => {
+    const skillMarkdown = `---\nname: ${input.skillName}\ndescription: Shared by an eval to prove skill sharing works.\n---\n\n${input.skillBody}\n`;
+    const { response, body } = await denFetch(admin, "/v1/plugins", {
+      method: "POST",
+      headers: { authorization: `Bearer ${admin.token}` },
+      body: JSON.stringify({
+        name: input.name,
+        description: "Created by the first-run cloud sharing eval.",
+        orgWide: input.orgWide ?? true,
+        marketplaceId: input.marketplaceId,
+        components: [{ type: "skill", input: { rawSourceText: skillMarkdown } }],
+      }),
+    });
+    if (!response.ok) throw new Error(`Creating plugin failed (${response.status}): ${JSON.stringify(body).slice(0, 300)}`);
+    const id = requireId(body, "Creating a plugin", response.status, body);
+    const item = isRecord(body) && isRecord(body.item) ? body.item : {};
+    const components = Array.isArray(item.components) ? item.components : [];
+    const componentIds = components
+      .map((component) => (isRecord(component) && typeof component.id === "string" ? component.id : ""))
+      .filter((value) => value.length > 0);
+    return { id, name: input.name, componentIds };
+  });
+}
+
+export async function assignPluginToMarketplace(admin: DenSession, marketplaceId: string, pluginId: string): Promise<void> {
+  await timed("cloud.assignPluginToMarketplace", async () => {
+    const { response, body } = await denFetch(admin, `/v1/marketplaces/${marketplaceId}/plugins`, {
+      method: "POST",
+      headers: { authorization: `Bearer ${admin.token}` },
+      body: JSON.stringify({ pluginId }),
+    });
+    if (!response.ok) throw new Error(`Assigning plugin to marketplace failed (${response.status}): ${JSON.stringify(body).slice(0, 300)}`);
+  });
+}
+
+/**
+ * Give a colleague (or their team, or the whole org) access to the marketplace.
+ * The API requires a role: viewer covers "can see and use what's shared".
+ */
+export async function grantMarketplaceAccess(
+  admin: DenSession,
+  marketplaceId: string,
+  grant: ({ orgWide: true } | { orgMembershipId: string } | { teamId: string }) & { role?: "viewer" | "editor" | "manager" },
+): Promise<void> {
+  await timed("cloud.grantMarketplaceAccess", async () => {
+    const { response, body } = await denFetch(admin, `/v1/marketplaces/${marketplaceId}/access`, {
+      method: "POST",
+      headers: { authorization: `Bearer ${admin.token}` },
+      body: JSON.stringify({ role: "viewer", ...grant }),
+    });
+    if (!response.ok) throw new Error(`Granting marketplace access failed (${response.status}): ${JSON.stringify(body).slice(0, 300)}`);
+  });
+}
+
+export interface ResolvedMarketplaceFacts {
+  pluginNames: string[];
+  skillNames: string[];
+  raw: unknown;
+}
+
+/**
+ * What a member actually sees in a marketplace — read as that member, not the
+ * admin. The resolved marketplace lists plugins (with component COUNTS only);
+ * the skill names live on each plugin's own resolved components
+ * (items[].configObject with objectType "skill" and the skill's title).
+ */
+export async function readResolvedMarketplace(member: DenSession, marketplaceId: string): Promise<ResolvedMarketplaceFacts> {
+  return timed("cloud.readResolvedMarketplace", async () => {
+    const { response, body } = await denFetch(member, `/v1/marketplaces/${marketplaceId}/resolved`, {
+      headers: { authorization: `Bearer ${member.token}` },
+    });
+    if (!response.ok) throw new Error(`Reading the resolved marketplace failed (${response.status}): ${JSON.stringify(body).slice(0, 300)}`);
+    const item = isRecord(body) && isRecord(body.item) ? body.item : isRecord(body) ? body : {};
+    const plugins = Array.isArray(item.plugins) ? item.plugins : [];
+    const pluginNames: string[] = [];
+    const pluginIds: string[] = [];
+    for (const plugin of plugins) {
+      if (!isRecord(plugin)) continue;
+      if (typeof plugin.name === "string") pluginNames.push(plugin.name);
+      if (typeof plugin.id === "string") pluginIds.push(plugin.id);
+    }
+    const skillNames: string[] = [];
+    for (const pluginId of pluginIds) {
+      const resolved = await denFetch(member, `/v1/plugins/${encodeURIComponent(pluginId)}/resolved`, {
+        headers: { authorization: `Bearer ${member.token}` },
+      });
+      if (!resolved.response.ok) continue; // Not readable by this member = not visible to them.
+      const items = isRecord(resolved.body) && Array.isArray(resolved.body.items) ? resolved.body.items : [];
+      for (const component of items) {
+        if (!isRecord(component) || !isRecord(component.configObject)) continue;
+        const configObject = component.configObject;
+        if (configObject.objectType !== "skill") continue;
+        if (typeof configObject.title === "string" && configObject.title) skillNames.push(configObject.title);
+      }
+    }
+    return { pluginNames, skillNames, raw: body };
+  });
+}

evals/packages/behaviors/src/desktop-boot.ts

@@ -51,9 +51,11 @@ export async function signInDesktopAs(app: Surface, den: DenRef, member: DenSess
     timeoutMs: 60_000,
     label: "active org resolved",
   });
-  await waitFor(app, "window.location.hash.includes('/onboarding')", {
+  // A first-time member lands on organization onboarding; a member whose app
+  // already has a workspace can come straight back to it.
+  await waitFor(app, `window.location.hash.includes("/onboarding") || /\\/(workspace|session)/.test(window.location.hash)`, {
     timeoutMs: 60_000,
-    label: "organization onboarding route",
+    label: "organization onboarding or workspace route",
   });
 }
 
@@ -101,6 +103,13 @@ async function resolveWorkspaceId(app: Surface): Promise<string> {
   return workspaceIdFromRoute(await currentHash(app));
 }
 
+/**
+ * THE arrangement path for a workspace: the product's own onboarding, driven
+ * the way a person drives it. A previous API seed (POST /workspaces/local +
+ * activate) produced a state the product itself never produces — a workspace
+ * with no engine and no model catalog — and specs failed on that arrangement,
+ * not on their subject. If a spec needs a workspace, it goes through here.
+ */
 export async function createAndSelectWorkspace(
   app: Surface,
   input: { path: string },
@@ -109,10 +118,20 @@ export async function createAndSelectWorkspace(
   const route = await currentHash(app);
   if (route.includes("/welcome")) {
     const workspace = await createLocalWorkspaceViaUi(app, input);
-    workspaceId = workspace.id || (await resolveWorkspaceId(app));
-    await clickButton(app, "Skip and use the free model", { timeoutMs: 30_000 });
-    await waitForText(app, "How did you hear about OpenWork?", { timeoutMs: 30_000 });
+    await clickButton(app, "Skip and use the free model", { timeoutMs: 90_000 });
+    await waitForText(app, "How did you hear about OpenWork?", { timeoutMs: 90_000 });
     await clickButton(app, "Skip", { timeoutMs: 15_000 });
+    // Only now is the workspace actually selected: resolving before the
+    // onboarding steps finish reads an id the app has not adopted yet.
+    workspaceId = workspace.id;
+    if (!workspaceId) {
+      await waitFor(app, `Boolean(localStorage.getItem("openwork.react.activeWorkspace"))
+        || /\\/workspace\\/[^/?#]+/.test(window.location.hash)`, {
+        timeoutMs: 180_000,
+        label: "workspace selected after onboarding",
+      });
+      workspaceId = await resolveWorkspaceId(app);
+    }
   } else {
     if (route.includes("/onboarding")) await completeOrganizationOnboarding(app);
     workspaceId = await resolveWorkspaceId(app);