synthetic-traffic: require evidence before paging on an all-failed tick

synthetic-traffic/README.md

@@ -46,7 +46,12 @@ bookkeeping, not identities.
 - **Reset-aware.** A ledger-sequence regression (testnet wipe, local recreate)
   archives state, alerts, and re-bootstraps from a fresh genesis.
 - **Dead-man alerting.** Silent in normal operation; the Discord webhook fires
-  on low funding runway, network resets, and all-actions-failed ticks.
+  on low funding runway, network resets, and all-actions-failed ticks. The
+  all-failed page needs evidence, not a ratio: a tick pages on its own only once
+  it ran `MIN_ALL_FAILED_ACTIONS` real actions, and thinner ticks page after
+  `ALL_FAILED_TICKS` consecutive all-failed ticks, then hourly while the outage
+  lasts (`alerts.ts`, unit-tested with `deno task test`). One transient on-chain
+  failure on a one-action off-peak tick is noise, not an outage.
 
 ## Local proving (before any testnet run)
 

synthetic-traffic/alerts.ts

@@ -0,0 +1,78 @@
+/**
+ * Alert policy — when an all-failed tick is evidence of an outage.
+ *
+ * Pure decision logic, no I/O, so the thresholds are unit-testable
+ * (`deno task test`). Delivery lives in funding.ts (`discordAlert`).
+ *
+ * The engine plans a Poisson batch per tick, so off-peak ticks routinely hold
+ * one or two real actions. "All actions failed" on such a tick is one transient
+ * failure (a bundle that FAILED on-chain, a flaky RPC) and looks identical to a
+ * total outage, which is how the 2026-08-03 19:44Z "1/1" page happened. A tick
+ * therefore only pages on its own once it is wide enough to be evidence; below
+ * that floor the engine waits for a run of consecutive all-failed ticks, which
+ * a real outage produces and an isolated transient does not.
+ */
+
+/** Real actions a single tick needs before "all of them failed" can page by
+ * itself. Four independent actions failing back to back is not plausible
+ * transient noise; one to three is the everyday off-peak batch size. */
+export const MIN_ALL_FAILED_ACTIONS = 4;
+
+/** Consecutive all-failed ticks that page regardless of how thin they are
+ * (3 ticks ~= 15 min at the deployed 5-min cadence). */
+export const ALL_FAILED_TICKS = 3;
+
+/** Further all-failed ticks between re-alerts (12 ~= hourly), so an outage
+ * that lasts pages once and then hourly instead of every tick. */
+export const ALL_FAILED_REALERT_TICKS = 12;
+
+export interface AllFailedState {
+  /** Consecutive all-failed ticks so far. */
+  streak: number;
+  /** Streak length at the last page; 0 while this streak has not paged. */
+  alertedAtStreak: number;
+}
+
+export function emptyAllFailedState(): AllFailedState {
+  return { streak: 0, alertedAtStreak: 0 };
+}
+
+/**
+ * Fold one tick's outcome into the all-failed state.
+ *
+ * `attempted` counts the real actions the tick actually ran (batch actions
+ * whose actor existed, plus aggregator payments) and `failures` counts how many
+ * of those threw. Intentional "seasoning" failures (`actFail`) submit a bundle
+ * that FAILS on-chain but return normally, so they land in `attempted` only —
+ * the alert has never fired on on-purpose errors and still does not.
+ */
+export function evaluateAllFailed(
+  state: AllFailedState,
+  attempted: number,
+  failures: number,
+): { state: AllFailedState; alert: string | null } {
+  if (attempted === 0 || failures < attempted) {
+    return { state: emptyAllFailedState(), alert: null };
+  }
+
+  const streak = state.streak + 1;
+  const broad = attempted >= MIN_ALL_FAILED_ACTIONS;
+  const sustained = streak >= ALL_FAILED_TICKS;
+  const carry = { streak, alertedAtStreak: state.alertedAtStreak };
+  if (!broad && !sustained) return { state: carry, alert: null };
+
+  // Already paged for this streak: hold until the re-alert interval.
+  if (
+    state.alertedAtStreak > 0 &&
+    streak - state.alertedAtStreak < ALL_FAILED_REALERT_TICKS
+  ) {
+    return { state: carry, alert: null };
+  }
+
+  const alert = streak === 1
+    ? `every action this tick failed (${failures}/${attempted}) — platform ` +
+      `down or config broken?`
+    : `every action failed for ${streak} consecutive ticks (latest ` +
+      `${failures}/${attempted}) — platform down or config broken?`;
+  return { state: { streak, alertedAtStreak: streak }, alert };
+}

synthetic-traffic/alerts_test.ts

@@ -0,0 +1,81 @@
+import { assertEquals, assertStringIncludes } from "@std/assert";
+import {
+  ALL_FAILED_REALERT_TICKS,
+  type AllFailedState,
+  emptyAllFailedState,
+  evaluateAllFailed,
+} from "./alerts.ts";
+
+/** Feed a sequence of [attempted, failures] ticks; collect what paged. */
+function run(ticks: Array<[number, number]>): {
+  alerts: string[];
+  state: AllFailedState;
+} {
+  let state = emptyAllFailedState();
+  const alerts: string[] = [];
+  for (const [attempted, failures] of ticks) {
+    const verdict = evaluateAllFailed(state, attempted, failures);
+    state = verdict.state;
+    if (verdict.alert) alerts.push(verdict.alert);
+  }
+  return { alerts, state };
+}
+
+Deno.test("thin tick with one real failure stays quiet (the 19:44Z page)", () => {
+  const { alerts, state } = run([[1, 1]]);
+  assertEquals(alerts, []);
+  assertEquals(state.streak, 1);
+});
+
+Deno.test("isolated thin failure between healthy ticks never pages", () => {
+  const { alerts, state } = run([[3, 0], [1, 1], [2, 0], [1, 1], [5, 0]]);
+  assertEquals(alerts, []);
+  assertEquals(state.streak, 0);
+});
+
+Deno.test("broad tick pages immediately", () => {
+  const { alerts } = run([[6, 6]]);
+  assertEquals(alerts.length, 1);
+  assertStringIncludes(alerts[0], "every action this tick failed (6/6)");
+});
+
+Deno.test("sustained thin failures page on the third consecutive tick", () => {
+  const { alerts } = run([[1, 1], [2, 2], [1, 1]]);
+  assertEquals(alerts.length, 1);
+  assertStringIncludes(alerts[0], "3 consecutive ticks");
+});
+
+Deno.test("a healthy tick resets the streak", () => {
+  const { alerts, state } = run([[1, 1], [2, 2], [2, 0], [1, 1], [1, 1]]);
+  assertEquals(alerts, []);
+  assertEquals(state.streak, 2);
+});
+
+Deno.test("a single success in a wide tick is not an outage", () => {
+  const { alerts } = run([[9, 8]]);
+  assertEquals(alerts, []);
+});
+
+Deno.test("an ongoing outage re-pages hourly, not every tick", () => {
+  const outage: Array<[number, number]> = Array.from(
+    { length: 30 },
+    () => [5, 5],
+  );
+  const { alerts } = run(outage);
+  assertEquals(alerts.length, 1 + Math.floor(29 / ALL_FAILED_REALERT_TICKS));
+});
+
+Deno.test("empty ticks are not failures", () => {
+  const { alerts, state } = run([[0, 0], [0, 0], [0, 0], [0, 0]]);
+  assertEquals(alerts, []);
+  assertEquals(state.streak, 0);
+});
+
+Deno.test("aggregator-only failures cannot make a healthy batch read as 100%", () => {
+  // Pre-fix arithmetic compared `failures` (batch + aggregator) against
+  // batch.length alone, so 2 aggregator failures next to 2 clean batch
+  // actions paged as "2/2". Counting both loops in the denominator makes it
+  // 2/4 — a bad half-tick, not an outage.
+  const { alerts } = run([[4, 2]]);
+  assertEquals(alerts, []);
+});

synthetic-traffic/deno.json

@@ -4,11 +4,13 @@
     "@colibri/core": "jsr:@colibri/core@^0.22.0",
     "@moonlight/moonlight-sdk": "jsr:@moonlight/moonlight-sdk@^0.12.1",
     "@opentelemetry/api": "npm:@opentelemetry/api@^1.9.0",
+    "@std/assert": "jsr:@std/assert@^1.0.0",
     "stellar-sdk": "npm:@stellar/stellar-sdk@^15.1.0",
     "stellar-sdk-14": "npm:@stellar/stellar-sdk@14.2.0"
   },
   "tasks": {
     "run": "deno run --allow-all main.ts",
-    "check": "deno check main.ts"
+    "check": "deno check main.ts",
+    "test": "deno test"
   }
 }

synthetic-traffic/main.ts

@@ -20,6 +20,7 @@ import {
   emptyState,
   type EngineState,
   entityKey,
+  type EntityState,
   loadState,
   saveState,
 } from "./state.ts";
@@ -43,6 +44,11 @@ import {
 } from "./actors.ts";
 import { describeDrop, firstDepositGateFactory, planTick } from "./traffic.ts";
 import { discordAlert, ensureRunway } from "./funding.ts";
+import {
+  type AllFailedState,
+  emptyAllFailedState,
+  evaluateAllFailed,
+} from "./alerts.ts";
 import {
   aggregatorPayment,
   ensurePayMirror,
@@ -171,14 +177,24 @@ async function reconcileRoster(
   }
 }
 
+/** What the tick actually ran: `attempted` counts the actions the engine got
+ * as far as executing (batch actions whose actor existed, plus aggregator
+ * payments), `failures` how many of those threw. The caller folds these into
+ * the all-failed alert policy — planned-but-skipped actions are deliberately
+ * absent from both, so the ratio describes real attempts only. */
+interface TrafficOutcome {
+  attempted: number;
+  failures: number;
+}
+
 async function runTraffic(
   env: ReturnType<typeof loadEngineEnv>,
   ring: KeyRing,
   state: EngineState,
   seed: string,
   nowDay: number,
   nowMs: number,
-): Promise<void> {
+): Promise<TrafficOutcome> {
   const slots = providerSlots(seed);
   const gate = firstDepositGateFactory(seed, slots, nowDay);
   const planned = planTick(
@@ -196,12 +212,22 @@ async function runTraffic(
   const dropNote = describeDrop(planned.length, batch.length);
   if (dropNote) console.log(`[traffic] ${dropNote}`);
 
+  let attempted = 0;
   let failures = 0;
   for (const a of batch) {
     const provider = state.providers[a.providerKey];
     const council = state.councils[provider.councilKey];
     const actor = state.entities[entityKey(a.providerKey, a.entityIdx)];
     if (!actor) continue;
+    // Resolve every skip condition before counting the attempt, so the
+    // all-failed ratio only ever describes actions the engine really ran.
+    let receiver: EntityState | undefined;
+    if (a.type === "send") {
+      receiver =
+        state.entities[entityKey(a.receiverProviderKey!, a.receiverIdx!)];
+      if (!receiver) continue;
+    }
+    attempted++;
     try {
       if (a.type === "deposit") {
         await actDeposit(
@@ -215,15 +241,12 @@ async function runTraffic(
           a.amount,
         );
       } else if (a.type === "send") {
-        const receiver =
-          state.entities[entityKey(a.receiverProviderKey!, a.receiverIdx!)];
-        if (!receiver) continue;
         await actSend(
           env,
           ring,
           state,
           actor,
-          receiver,
+          receiver!,
           council,
           provider.publicKey,
           a.assetCode,
@@ -274,6 +297,7 @@ async function runTraffic(
         const n = rng.poisson(lambda);
         for (let k = 0; k < n; k++) {
           const amount = Number(rng.lognormal(1.2, 0.6, 0.5, 3).toFixed(2));
+          attempted++;
           try {
             await aggregatorPayment(
               env,
@@ -301,12 +325,7 @@ async function runTraffic(
     }
   }
 
-  if (batch.length > 0 && failures === batch.length) {
-    await discordAlert(
-      env,
-      `every action this tick failed (${failures}/${batch.length}) — platform down or config broken?`,
-    );
-  }
+  return { attempted, failures };
 }
 
 async function runwayCheck(
@@ -330,7 +349,7 @@ async function runwayCheck(
 async function tick(
   env: ReturnType<typeof loadEngineEnv>,
   ring: KeyRing,
-): Promise<void> {
+): Promise<TrafficOutcome> {
   let state = loadState(env.stateFile);
 
   if (state && state.networkPassphrase !== env.networkPassphrase) {
@@ -366,10 +385,18 @@ async function tick(
 
   await reconcileRoster(env, ring, state, ring.rngSeed, nowDay);
   await runwayCheck(env, ring, state);
-  await runTraffic(env, ring, state, ring.rngSeed, nowDay, nowMs);
+  const outcome = await runTraffic(
+    env,
+    ring,
+    state,
+    ring.rngSeed,
+    nowDay,
+    nowMs,
+  );
 
   state.lastLedgerSeq = Math.max(state.lastLedgerSeq, 0);
   saveState(env.stateFile, state);
+  return outcome;
 }
 
 async function main() {
@@ -393,10 +420,21 @@ async function main() {
   // none of the batch/runway/reset alerts ever fire), so the streak itself
   // must page: first at STUCK_ALERT_TICKS, then every REALERT_TICKS while
   // the condition holds, and once more on recovery.
+  // The two paging paths are exclusive per tick: a tick that throws never
+  // reaches the all-failed evaluation, and a tick that completes never counts
+  // toward the stuck streak.
   let failStreak = 0;
+  let allFailed: AllFailedState = emptyAllFailedState();
   while (true) {
     try {
-      await tick(env, ring);
+      const outcome = await tick(env, ring);
+      const verdict = evaluateAllFailed(
+        allFailed,
+        outcome.attempted,
+        outcome.failures,
+      );
+      allFailed = verdict.state;
+      if (verdict.alert) await discordAlert(env, verdict.alert);
       if (failStreak >= STUCK_ALERT_TICKS) {
         await discordAlert(
           env,