fix: users never learned they were out of credits — 9 bugs, now covered on codex, claude code, cursor, hermes, pi and the CLI

esbuild.config.mjs

@@ -371,6 +371,7 @@ for (const h of hermesAll) {
 // bundle synchronously from session_start.
 const piWorker = [
   { entry: "dist/src/hooks/pi/wiki-worker.js", out: "wiki-worker" },
+  { entry: "dist/src/hooks/pi/notifications-worker.js", out: "notifications-worker" },
   { entry: "dist/src/skillify/skillify-worker.js", out: "skillify-worker" },
   { entry: "dist/src/skillify/autopull-worker.js", out: "autopull-worker" },
   // SkillOpt worker — pi spawns it on a user reaction (the extension can't import the

harnesses/pi/extension-source/hivemind.ts

@@ -527,6 +527,48 @@ const PI_SKILLIFY_WORKER_PATH = join(homedir(), ".pi", "agent", "hivemind", "ski
 // directly — pi can't import the TS module (raw .ts, zero deps), so it
 // routes through this child process. Keeps pi's pulled skills layout +
 // symlink fan-out in lockstep with the other agents automatically.
+const PI_NOTIFICATIONS_WORKER_PATH = join(homedir(), ".pi", "agent", "hivemind", "notifications-worker.js");
+
+/**
+ * Drain hivemind notifications and show them to the USER via pi's
+ * `ctx.ui.notify`.
+ *
+ * Pi is the only non-Claude-Code harness with a real user-visible channel
+ * (`notify(message, "info" | "warning" | "error")` — see
+ * `@mariozechner/pi-coding-agent` dist/core/extensions/types.d.ts). Cursor and
+ * Hermes have none, so their billing notices have to be relayed by the model;
+ * on pi we can just tell the user, which is strictly better.
+ *
+ * This extension is raw TS with no non-builtin imports, so the drain runs in
+ * the bundled worker and we read its stdout — the same pattern as autopull.
+ * 6s cap; any failure is swallowed, because a missed notification must never
+ * cost the user their session.
+ */
+function runNotificationsWorker(sessionId: string, reason: string): Array<{ text: string; severity: string }> {
+  if (!existsSync(PI_NOTIFICATIONS_WORKER_PATH)) {
+    logHm(`notifications: worker bundle missing at ${PI_NOTIFICATIONS_WORKER_PATH} — skipping`);
+    return [];
+  }
+  try {
+    const result = spawnSync(process.execPath, [PI_NOTIFICATIONS_WORKER_PATH, sessionId, reason], {
+      encoding: "utf-8",
+      timeout: 6_000,
+      env: process.env,
+    });
+    if (result.error) {
+      logHm(`notifications: spawn failed (swallowed): ${result.error.message}`);
+      return [];
+    }
+    const parsed = JSON.parse((result.stdout || "").trim() || "{}");
+    const list = Array.isArray(parsed?.notifications) ? parsed.notifications : [];
+    logHm(`notifications: worker returned ${list.length}`);
+    return list;
+  } catch (e: any) {
+    logHm(`notifications: swallowed: ${e?.message ?? e}`);
+    return [];
+  }
+}
+
 const PI_AUTOPULL_WORKER_PATH = join(homedir(), ".pi", "agent", "hivemind", "autopull-worker.js");
 
 /**
@@ -1427,6 +1469,21 @@ export default function hivemindExtension(pi: ExtensionAPI): void {
 
   pi.on("session_start", async (_event: any, ctx: any) => {
     logHm(`session_start: fired (capture=${captureEnabled}, embed=${process.env.HIVEMIND_EMBEDDINGS !== "false"}, table=${SESSIONS_TABLE})`);
+
+    // Tell the user about anything that needs their attention — most
+    // importantly that the org is out of Deeplake credits, in which case
+    // capture and recall silently return nothing. Before this, a pi user got
+    // no signal at all. notify() is a real user-visible toast, so unlike
+    // Cursor and Hermes the notice does not have to go through the model.
+    try {
+      const sid = ctx?.sessionManager?.getSessionId?.() ?? "";
+      for (const n of runNotificationsWorker(String(sid ?? ""), String(_event?.reason ?? ""))) {
+        if (n?.text) ctx.ui?.notify?.(n.text, n.severity === "error" ? "error" : n.severity === "warning" ? "warning" : "info");
+      }
+    } catch (e: any) {
+      logHm(`notifications: notify swallowed: ${e?.message ?? e}`);
+    }
+
     let creds = loadCreds();
     if (!creds) {
       logHm(`session_start: no credentials at ~/.deeplake/credentials.json — capture disabled this session`);

src/cli/install-pi.ts

@@ -58,6 +58,7 @@ const AUTOPULL_WORKER_PATH = join(WIKI_WORKER_DIR, "autopull-worker.js");
 // recently-used org skill and publish an improvement. Same shared module CC ships; pi
 // can't import the raw-.ts trigger so it shells this bundle. Sibling of the others.
 const SKILLOPT_WORKER_PATH = join(WIKI_WORKER_DIR, "skillopt-worker.js");
+const NOTIFICATIONS_WORKER_PATH = join(WIKI_WORKER_DIR, "notifications-worker.js");
 
 const HIVEMIND_BLOCK_BODY = `${HIVEMIND_BLOCK_START}
 ## Hivemind Memory
@@ -140,6 +141,13 @@ export function installPi(): void {
     ensureDir(WIKI_WORKER_DIR);
     copyFileSync(srcSkilloptWorker, SKILLOPT_WORKER_PATH);
   }
+  // Notification drain for pi's user-visible ctx.ui.notify channel.
+  const srcNotificationsWorker = join(pkgRoot(), "harnesses", "pi", "bundle", "notifications-worker.js");
+  if (existsSync(srcNotificationsWorker)) {
+    ensureDir(WIKI_WORKER_DIR);
+    copyFileSync(srcNotificationsWorker, NOTIFICATIONS_WORKER_PATH);
+  }
+
 
   ensureDir(VERSION_DIR);
   writeVersionStamp(VERSION_DIR, getVersion());

src/deeplake-api.ts

@@ -78,9 +78,42 @@ let _signalledBalanceExhausted = false;
  * DedupKey carries the UTC date so the banner re-fires daily until the
  * user tops up, rather than firing once-ever and then going quiet.
  */
+/**
+ * Turn a thrown fetch error into something a human can act on.
+ *
+ * `fetch` rejects with a bare `TypeError: fetch failed` for every transport
+ * failure — the real cause is buried in `.cause`. Surfacing the bare message
+ * is how `hivemind goal list` came to print `hivemind goal list: fetch failed`,
+ * which tells the user nothing about what to do.
+ *
+ * The common case is not a broken network: it is an agent sandbox with
+ * outbound access disabled. Verified 2026-08-13 — the same command, same org,
+ * same server, run under Codex's default `workspace-write` sandbox prints
+ * `fetch failed`, and under `danger-full-access` returns normally. Name that
+ * possibility rather than making the user discover it.
+ */
+export function describeNetworkFailure(e: unknown, apiUrl: string): Error {
+  const cause = (e as { cause?: { code?: string; message?: string } } | null)?.cause;
+  const detail = cause?.code ?? cause?.message
+    ?? (e instanceof Error ? e.message : String(e));
+  return new Error(
+    `Cannot reach the Deeplake API at ${apiUrl} (${detail}). `
+    + `If you are running inside an agent sandbox, outbound network access may be blocked — `
+    + `Codex's default sandbox blocks it, so run the command in your own terminal instead.`,
+  );
+}
+
+/**
+ * The server's "out of credits" response: HTTP 402 whose body carries
+ * `balance_cents`. Single source of truth for both the session-start banner
+ * and the human-readable error message thrown to CLI callers.
+ */
+export function isBalanceExhausted(status: number, bodyText: string): boolean {
+  return status === 402 && bodyText.includes("balance_cents");
+}
+
 function maybeSignalBalanceExhausted(status: number, bodyText: string): void {
-  if (status !== 402) return;
-  if (!bodyText.includes("balance_cents")) return;
+  if (!isBalanceExhausted(status, bodyText)) return;
   if (_signalledBalanceExhausted) return;
   _signalledBalanceExhausted = true;
   log(`balance exhausted — enqueuing session-start banner (body=${bodyText.slice(0, 120)})`);
@@ -95,7 +128,12 @@ function maybeSignalBalanceExhausted(status: number, bodyText: string): void {
     transient: true,
     title: "Hivemind credits exhausted — top up to keep capturing",
     body: `Sessions are not being saved and memory recall is returning empty. Top up at ${billingUrl()} to restore capture and recall.`,
-    dedupKey: { reason: "balance-zero" },
+    // Carries the org so a notice enqueued under one org is never rendered
+    // after switching to another (observed: switching to a funded org still
+    // showed the previous org's "credits exhausted" and linked to ITS billing
+    // page). drainSessionStart drops queued notices whose org no longer
+    // matches the credentials in force.
+    dedupKey: { reason: "balance-zero", orgId: loadCredentials()?.orgId ?? null },
     // User-facing billing notice → user channel only. Never the model's
     // additionalContext: a "top up at <url>" instruction in the agent prompt
     // is a prompt-injection pattern external agents flag.
@@ -106,20 +144,26 @@ function maybeSignalBalanceExhausted(status: number, bodyText: string): void {
 }
 
 /**
- * Construct the org-scoped billing URL from persisted credentials. The
- * canonical shape is `https://deeplake.ai/{orgName}/workspace/{workspaceId}/billing`
- * — the org and workspace come from `~/.deeplake/credentials.json`. Falls
- * back to the bare host when creds are missing or malformed (better to
- * point at *something* than at a URL with literal `undefined` segments).
+ * Construct the org-scoped billing URL from persisted credentials:
+ * `https://deeplake.ai/{orgId}/workspace/{workspaceId}/billing`.
+ *
+ * Keyed on the org ID, NOT `orgName`. `orgName` is a human display name, not a
+ * slug — the API returns e.g. `"mvincig11's Organization"`, which rendered as
+ * `deeplake.ai/mvincig11's%20Organization/workspace/default/billing`: an
+ * apostrophe and an escaped space in a path segment. A dead link at the exact
+ * moment the user needs to top up defeats the purpose of the notice. The API
+ * exposes no slug field (checked `/organizations/{id}` — it returns only `id`
+ * and the display `name`), so the UUID is the one unambiguous, URL-safe
+ * identifier available.
+ *
+ * Falls back to the bare host when creds are missing or malformed — better to
+ * point at *something* than at a URL with literal `undefined` segments.
  */
 function billingUrl(): string {
   try {
     const c = loadCredentials();
-    if (c?.orgName && c?.workspaceId) {
-      // URI-encode in case anyone has an org/workspace name with reserved chars.
-      // workspaceId is typically a UUID; orgName is typically a slug, but
-      // encodeURIComponent is a cheap guard against future weirdness.
-      return `https://deeplake.ai/${encodeURIComponent(c.orgName)}/workspace/${encodeURIComponent(c.workspaceId)}/billing`;
+    if (c?.orgId && c?.workspaceId) {
+      return `https://deeplake.ai/${encodeURIComponent(c.orgId)}/workspace/${encodeURIComponent(c.workspaceId)}/billing`;
     }
   } catch { /* fall through to default */ }
   return "https://deeplake.ai";
@@ -275,7 +319,7 @@ export class DeeplakeApi {
           lastError = new Error(`Query timeout after ${timeoutMs}ms`);
           throw lastError;
         }
-        lastError = e instanceof Error ? e : new Error(String(e));
+        lastError = describeNetworkFailure(e, this.apiUrl);
         if (attempt < MAX_RETRIES) {
           const delay = BASE_DELAY_MS * Math.pow(2, attempt) + Math.random() * 200;
           log(`query retry ${attempt + 1}/${MAX_RETRIES} (fetch error: ${lastError.message}) in ${delay.toFixed(0)}ms`);
@@ -309,6 +353,18 @@ export class DeeplakeApi {
       // Surface a session-start banner for the "out of credits" case before
       // throwing — see maybeSignalBalanceExhausted's docstring for why.
       maybeSignalBalanceExhausted(resp.status, text);
+      // The out-of-credits 402 is the one server error a user can act on, and
+      // it is the one they are most likely to read raw: `hivemind goal list`
+      // and friends print this message straight to the terminal. Emitting the
+      // API's JSON body verbatim ("Query failed: 402: {"balance_cents":0,...}")
+      // made it look like an internal fault rather than "your account is out
+      // of credits" — reported 2026-08-12 in #platform. Every other status
+      // keeps the raw shape, which is what the debugging paths expect.
+      if (isBalanceExhausted(resp.status, text)) {
+        throw new Error(
+          `Hivemind credits exhausted — sessions are not being saved and memory recall returns empty. Top up at ${billingUrl()} to restore capture and recall.`,
+        );
+      }
       throw new Error(`Query failed: ${resp.status}: ${text.slice(0, 200)}`);
     }
     throw lastError ?? new Error("Query failed: max retries exceeded");

src/hooks/codex/session-start.ts

@@ -21,8 +21,41 @@ import { log as _log } from "../../utils/debug.js";
 import { getInstalledVersion } from "../../utils/version-check.js";
 import { autoPullSkills } from "../../skillify/auto-pull.js";
 import { spawnGraphPullWorker } from "../../graph/spawn-pull-worker.js";
+import type { Notification } from "../../notifications/index.js";
+import { drainSessionStart, enqueueNotification, registerRule } from "../../notifications/index.js";
+import { bumpSessionCount } from "../../notifications/state.js";
+import { referralInviteRule } from "../../notifications/rules/referral-invite.js";
+import { renderCodexChannels } from "../../notifications/delivery/codex.js";
 const log = (msg: string) => _log("codex-session-start", msg);
 
+/** How long this hook waits on the notifications drain before emitting
+ *  without it. Codex kills the hook at 10s and this hook also carries the
+ *  memory/login context, so the drain never gets to be the reason the whole
+ *  output is lost. */
+const DRAIN_DEADLINE_MS = 4000;
+
+/** Hard ceiling for the whole hook. Codex kills it at 10s (see buildHooksJson
+ *  in src/cli/install-codex.ts) and discards EVERYTHING when it does — the
+ *  user sees `SessionStart hook (failed) error: hook timed out after 10s` and
+ *  loses the login context AND any billing CTA. Observed in real sessions.
+ *  Bounding only the drain was not enough: the auto-pull, the org-token heal
+ *  and module init all sit outside it. Emit whatever we have by this point. */
+const HOOK_BUDGET_MS = 7000;
+
+/** Resolves after `ms`. `unref` so a pending timer can't hold the process
+ *  open once the hook has written its output. */
+function deadline(ms: number): Promise<void> {
+  return new Promise<void>(resolve => {
+    const t = setTimeout(resolve, ms);
+    t.unref?.();
+  });
+}
+
+// Same rule registration as Claude Code's notifications hook
+// (src/hooks/session-notifications.ts). Rules are pure — registering them
+// costs nothing when none fire.
+registerRule(referralInviteRule);
+
 const __bundleDir = dirname(fileURLToPath(import.meta.url));
 // Codex DOES NOT have a model-only context channel for SessionStart hooks: any
 // `additionalContext` we emit is rendered as a `hook context: <text>` history
@@ -89,8 +122,53 @@ async function main(): Promise<void> {
   // disk; the only per-call cost is the SQL round-trip. autoPullSkills
   // never rejects — all errors are swallowed inside. Hard opt-out:
   // HIVEMIND_AUTOPULL_DISABLED=1.
-  const pullResult = await autoPullSkills();
+  // Notifications drain. Until this landed, Codex users never saw ANY
+  // notification the framework produced — most damagingly the
+  // `balance-exhausted` banner enqueued by deeplake-api's 402 handler. The
+  // result was hivemind failing completely silently on Codex: captures and
+  // recalls returned nothing and no CTA to top up ever surfaced (reported
+  // 2026-08-12 in #platform).
+  //
+  // The drain writes nothing itself — Codex accepts exactly ONE JSON object
+  // on a hook's stdout and this hook already owns it, so we collect the
+  // claimed notifications via the `deliver` override and merge them into the
+  // single output object below.
+  //
+  // Run in parallel with the auto-pull so the drain's fetches don't add to
+  // this blocking hook's wall time. drainSessionStart never throws (it
+  // catches internally); autoPullSkills never rejects.
+  //
+  // Deadline: unlike Claude Code — where the drain is its own hook command —
+  // this hook must ALSO deliver the memory/login context, and Codex kills the
+  // hook at 10s (see buildHooksJson in src/cli/install-codex.ts). A slow
+  // drain (goals SQL retries behind a stalled network) would take the whole
+  // hook down with it and drop everything, so we stop waiting well before
+  // that. If the drain lands late, its notifications go back on the queue
+  // for the next session rather than being marked shown but never rendered.
+  const rawSessionId = typeof input.session_id === "string" ? input.session_id.trim() : "";
+  const sessionId = rawSessionId.length > 0 ? rawSessionId : undefined;
+  const sessionCount = bumpSessionCount(sessionId);
+  let notified: Notification[] = [];
+  let emitted = false;
+  const drained = drainSessionStart({
+    agent: "codex",
+    creds,
+    sessionId,
+    source: input.source,
+    sessionCount,
+    deliver: (ns) => {
+      if (!emitted) { notified = ns; return; }
+      log(`notifications arrived after the deadline — re-queuing ${ns.length}`);
+      for (const n of ns) enqueueNotification(n).catch(() => undefined);
+    },
+  });
+  const [pullResult] = await Promise.all([
+    autoPullSkills(),
+    Promise.race([drained, deadline(DRAIN_DEADLINE_MS)]),
+  ]);
+  emitted = true;
   log(`autopull: pulled=${pullResult.pulled} skipped=${pullResult.skipped}`);
+  log(`notifications: ${notified.length} claimed`);
 
   let versionNotice = "";
   const current = getInstalledVersion(__bundleDir, ".codex-plugin");
@@ -150,14 +228,60 @@ async function main(): Promise<void> {
   const systemMessage = (!creds?.token && localMined > 0)
     ? `💡 ${localMined} ${skillNoun} mined from your local sessions live in ~/.claude/skills/. Run 'hivemind login' to share them with your team.`
     : undefined;
+
+  // Merge the drained notifications into the single JSON object Codex will
+  // accept. Notifications go FIRST in both channels — a "credits exhausted"
+  // warning must not be pushed below the routine login-state line.
+  const notifChannels = renderCodexChannels(notified);
+  const mergedSystemMessage = [notifChannels.systemMessage, systemMessage]
+    .filter(Boolean).join("\n\n");
+  const mergedContext = [notifChannels.additionalContext, additionalContext]
+    .filter(Boolean).join("\n\n");
+
   const output: Record<string, unknown> = {
     hookSpecificOutput: {
       hookEventName: "SessionStart",
-      additionalContext,
+      additionalContext: mergedContext,
     },
   };
-  if (systemMessage) output.systemMessage = systemMessage;
+  if (mergedSystemMessage) output.systemMessage = mergedSystemMessage;
   console.log(JSON.stringify(output));
 }
 
-main().catch((e) => { log(`fatal: ${e.message}`); process.exit(0); });
+/**
+ * Emit the minimal, always-correct output. Used when the hook blows its budget:
+ * a login line still beats codex discarding everything on a 10s timeout.
+ */
+function emitFallback(): void {
+  const creds = loadCredentials();
+  const additionalContext = creds?.token
+    ? `Hivemind: logged in as org ${creds.orgName ?? creds.orgId} (workspace: ${creds.workspaceId ?? "default"}).`
+    : "Hivemind: not logged in. Run `hivemind login` to enable shared memory + skill sharing.";
+  console.log(JSON.stringify({
+    hookSpecificOutput: { hookEventName: "SessionStart", additionalContext },
+  }));
+}
+
+// Watchdog: whatever happens inside main(), this process must produce its JSON
+// before Codex's 10s kill, because Codex discards the ENTIRE hook output on
+// timeout — login context and billing CTA alike.
+let wroteOutput = false;
+const originalLog = console.log.bind(console);
+console.log = (...args: unknown[]) => { wroteOutput = true; originalLog(...args); };
+
+const budget = setTimeout(() => {
+  if (wroteOutput) return;
+  log(`hook budget of ${HOOK_BUDGET_MS}ms exceeded — emitting fallback output`);
+  emitFallback();
+  process.exit(0);
+}, HOOK_BUDGET_MS);
+budget.unref?.();
+
+main()
+  .catch((e) => {
+    log(`fatal: ${e.message}`);
+    // Still give Codex something: a login line beats an empty hook cell.
+    if (!wroteOutput) emitFallback();
+    process.exit(0);
+  })
+  .finally(() => clearTimeout(budget));