From 69d38b8fbd1a7a1e2c304ef67c4cabf064b1ddfc Mon Sep 17 00:00:00 2001 From: Gorka Date: Mon, 3 Aug 2026 17:55:51 -0300 Subject: [PATCH] synthetic-traffic: require evidence before paging on an all-failed tick. A tick with one real action and one transient on-chain failure read as 100% and paged as a total outage. The all-failed alert now needs a tick wide enough to stand on its own or a run of consecutive all-failed ticks, and counts aggregator payments in the denominator they already failed into. --- synthetic-traffic/README.md | 7 ++- synthetic-traffic/alerts.ts | 78 ++++++++++++++++++++++++++++++ synthetic-traffic/alerts_test.ts | 81 ++++++++++++++++++++++++++++++++ synthetic-traffic/deno.json | 4 +- synthetic-traffic/deno.lock | 12 +++++ synthetic-traffic/main.ts | 66 ++++++++++++++++++++------ 6 files changed, 232 insertions(+), 16 deletions(-) create mode 100644 synthetic-traffic/alerts.ts create mode 100644 synthetic-traffic/alerts_test.ts diff --git a/synthetic-traffic/README.md b/synthetic-traffic/README.md index 8b96e10..25e07a2 100644 --- a/synthetic-traffic/README.md +++ b/synthetic-traffic/README.md @@ -46,7 +46,12 @@ bookkeeping, not identities. - **Reset-aware.** A ledger-sequence regression (testnet wipe, local recreate) archives state, alerts, and re-bootstraps from a fresh genesis. - **Dead-man alerting.** Silent in normal operation; the Discord webhook fires - on low funding runway, network resets, and all-actions-failed ticks. + on low funding runway, network resets, and all-actions-failed ticks. The + all-failed page needs evidence, not a ratio: a tick pages on its own only once + it ran `MIN_ALL_FAILED_ACTIONS` real actions, and thinner ticks page after + `ALL_FAILED_TICKS` consecutive all-failed ticks, then hourly while the outage + lasts (`alerts.ts`, unit-tested with `deno task test`). One transient on-chain + failure on a one-action off-peak tick is noise, not an outage. ## Local proving (before any testnet run) diff --git a/synthetic-traffic/alerts.ts b/synthetic-traffic/alerts.ts new file mode 100644 index 0000000..7c26208 --- /dev/null +++ b/synthetic-traffic/alerts.ts @@ -0,0 +1,78 @@ +/** + * Alert policy — when an all-failed tick is evidence of an outage. + * + * Pure decision logic, no I/O, so the thresholds are unit-testable + * (`deno task test`). Delivery lives in funding.ts (`discordAlert`). + * + * The engine plans a Poisson batch per tick, so off-peak ticks routinely hold + * one or two real actions. "All actions failed" on such a tick is one transient + * failure (a bundle that FAILED on-chain, a flaky RPC) and looks identical to a + * total outage, which is how the 2026-08-03 19:44Z "1/1" page happened. A tick + * therefore only pages on its own once it is wide enough to be evidence; below + * that floor the engine waits for a run of consecutive all-failed ticks, which + * a real outage produces and an isolated transient does not. + */ + +/** Real actions a single tick needs before "all of them failed" can page by + * itself. Four independent actions failing back to back is not plausible + * transient noise; one to three is the everyday off-peak batch size. */ +export const MIN_ALL_FAILED_ACTIONS = 4; + +/** Consecutive all-failed ticks that page regardless of how thin they are + * (3 ticks ~= 15 min at the deployed 5-min cadence). */ +export const ALL_FAILED_TICKS = 3; + +/** Further all-failed ticks between re-alerts (12 ~= hourly), so an outage + * that lasts pages once and then hourly instead of every tick. */ +export const ALL_FAILED_REALERT_TICKS = 12; + +export interface AllFailedState { + /** Consecutive all-failed ticks so far. */ + streak: number; + /** Streak length at the last page; 0 while this streak has not paged. */ + alertedAtStreak: number; +} + +export function emptyAllFailedState(): AllFailedState { + return { streak: 0, alertedAtStreak: 0 }; +} + +/** + * Fold one tick's outcome into the all-failed state. + * + * `attempted` counts the real actions the tick actually ran (batch actions + * whose actor existed, plus aggregator payments) and `failures` counts how many + * of those threw. Intentional "seasoning" failures (`actFail`) submit a bundle + * that FAILS on-chain but return normally, so they land in `attempted` only — + * the alert has never fired on on-purpose errors and still does not. + */ +export function evaluateAllFailed( + state: AllFailedState, + attempted: number, + failures: number, +): { state: AllFailedState; alert: string | null } { + if (attempted === 0 || failures < attempted) { + return { state: emptyAllFailedState(), alert: null }; + } + + const streak = state.streak + 1; + const broad = attempted >= MIN_ALL_FAILED_ACTIONS; + const sustained = streak >= ALL_FAILED_TICKS; + const carry = { streak, alertedAtStreak: state.alertedAtStreak }; + if (!broad && !sustained) return { state: carry, alert: null }; + + // Already paged for this streak: hold until the re-alert interval. + if ( + state.alertedAtStreak > 0 && + streak - state.alertedAtStreak < ALL_FAILED_REALERT_TICKS + ) { + return { state: carry, alert: null }; + } + + const alert = streak === 1 + ? `every action this tick failed (${failures}/${attempted}) — platform ` + + `down or config broken?` + : `every action failed for ${streak} consecutive ticks (latest ` + + `${failures}/${attempted}) — platform down or config broken?`; + return { state: { streak, alertedAtStreak: streak }, alert }; +} diff --git a/synthetic-traffic/alerts_test.ts b/synthetic-traffic/alerts_test.ts new file mode 100644 index 0000000..36ca62a --- /dev/null +++ b/synthetic-traffic/alerts_test.ts @@ -0,0 +1,81 @@ +import { assertEquals, assertStringIncludes } from "@std/assert"; +import { + ALL_FAILED_REALERT_TICKS, + type AllFailedState, + emptyAllFailedState, + evaluateAllFailed, +} from "./alerts.ts"; + +/** Feed a sequence of [attempted, failures] ticks; collect what paged. */ +function run(ticks: Array<[number, number]>): { + alerts: string[]; + state: AllFailedState; +} { + let state = emptyAllFailedState(); + const alerts: string[] = []; + for (const [attempted, failures] of ticks) { + const verdict = evaluateAllFailed(state, attempted, failures); + state = verdict.state; + if (verdict.alert) alerts.push(verdict.alert); + } + return { alerts, state }; +} + +Deno.test("thin tick with one real failure stays quiet (the 19:44Z page)", () => { + const { alerts, state } = run([[1, 1]]); + assertEquals(alerts, []); + assertEquals(state.streak, 1); +}); + +Deno.test("isolated thin failure between healthy ticks never pages", () => { + const { alerts, state } = run([[3, 0], [1, 1], [2, 0], [1, 1], [5, 0]]); + assertEquals(alerts, []); + assertEquals(state.streak, 0); +}); + +Deno.test("broad tick pages immediately", () => { + const { alerts } = run([[6, 6]]); + assertEquals(alerts.length, 1); + assertStringIncludes(alerts[0], "every action this tick failed (6/6)"); +}); + +Deno.test("sustained thin failures page on the third consecutive tick", () => { + const { alerts } = run([[1, 1], [2, 2], [1, 1]]); + assertEquals(alerts.length, 1); + assertStringIncludes(alerts[0], "3 consecutive ticks"); +}); + +Deno.test("a healthy tick resets the streak", () => { + const { alerts, state } = run([[1, 1], [2, 2], [2, 0], [1, 1], [1, 1]]); + assertEquals(alerts, []); + assertEquals(state.streak, 2); +}); + +Deno.test("a single success in a wide tick is not an outage", () => { + const { alerts } = run([[9, 8]]); + assertEquals(alerts, []); +}); + +Deno.test("an ongoing outage re-pages hourly, not every tick", () => { + const outage: Array<[number, number]> = Array.from( + { length: 30 }, + () => [5, 5], + ); + const { alerts } = run(outage); + assertEquals(alerts.length, 1 + Math.floor(29 / ALL_FAILED_REALERT_TICKS)); +}); + +Deno.test("empty ticks are not failures", () => { + const { alerts, state } = run([[0, 0], [0, 0], [0, 0], [0, 0]]); + assertEquals(alerts, []); + assertEquals(state.streak, 0); +}); + +Deno.test("aggregator-only failures cannot make a healthy batch read as 100%", () => { + // Pre-fix arithmetic compared `failures` (batch + aggregator) against + // batch.length alone, so 2 aggregator failures next to 2 clean batch + // actions paged as "2/2". Counting both loops in the denominator makes it + // 2/4 — a bad half-tick, not an outage. + const { alerts } = run([[4, 2]]); + assertEquals(alerts, []); +}); diff --git a/synthetic-traffic/deno.json b/synthetic-traffic/deno.json index 4aa7e0d..66b8c3d 100644 --- a/synthetic-traffic/deno.json +++ b/synthetic-traffic/deno.json @@ -4,11 +4,13 @@ "@colibri/core": "jsr:@colibri/core@^0.22.0", "@moonlight/moonlight-sdk": "jsr:@moonlight/moonlight-sdk@^0.12.1", "@opentelemetry/api": "npm:@opentelemetry/api@^1.9.0", + "@std/assert": "jsr:@std/assert@^1.0.0", "stellar-sdk": "npm:@stellar/stellar-sdk@^15.1.0", "stellar-sdk-14": "npm:@stellar/stellar-sdk@14.2.0" }, "tasks": { "run": "deno run --allow-all main.ts", - "check": "deno check main.ts" + "check": "deno check main.ts", + "test": "deno test" } } diff --git a/synthetic-traffic/deno.lock b/synthetic-traffic/deno.lock index f196120..2aa5888 100644 --- a/synthetic-traffic/deno.lock +++ b/synthetic-traffic/deno.lock @@ -7,7 +7,9 @@ "jsr:@noble/curves@^1.8.0": "1.9.0", "jsr:@noble/hashes@1.8.0": "1.8.0", "jsr:@noble/hashes@^1.6.1": "1.8.0", + "jsr:@std/assert@1": "1.0.19", "jsr:@std/collections@^1.1.3": "1.3.0", + "jsr:@std/internal@^1.0.12": "1.0.14", "jsr:@std/toml@^1.0.11": "1.0.11", "npm:@opentelemetry/api@^1.9.0": "1.9.1", "npm:@stellar/stellar-sdk@14.2.0": "14.2.0", @@ -50,9 +52,18 @@ "@noble/hashes@1.8.0": { "integrity": "b52a2fcb4d02f8d8137871564a31f1ee9e2b0d15eedabbf32d2f7333f0abc939" }, + "@std/assert@1.0.19": { + "integrity": "eaada96ee120cb980bc47e040f82814d786fe8162ecc53c91d8df60b8755991e", + "dependencies": [ + "jsr:@std/internal" + ] + }, "@std/collections@1.3.0": { "integrity": "eb36b43d784477ea0b476483ac034a14bdd182aff921c812ecf662a1fcef9498" }, + "@std/internal@1.0.14": { + "integrity": "291516b3d4c35024d6ffbc0a9df5bf4c64116e05b50012cf846710152d2ffdf7" + }, "@std/toml@1.0.11": { "integrity": "e084988b872ca4bad6aedfb7350f6eeed0e8ba88e9ee5e1590621c5b5bb8f715", "dependencies": [ @@ -441,6 +452,7 @@ "dependencies": [ "jsr:@colibri/core@0.22", "jsr:@moonlight/moonlight-sdk@~0.12.1", + "jsr:@std/assert@1", "npm:@opentelemetry/api@^1.9.0", "npm:@stellar/stellar-sdk@14.2.0", "npm:@stellar/stellar-sdk@^15.1.0" diff --git a/synthetic-traffic/main.ts b/synthetic-traffic/main.ts index fa10fe0..1e8e74e 100644 --- a/synthetic-traffic/main.ts +++ b/synthetic-traffic/main.ts @@ -20,6 +20,7 @@ import { emptyState, type EngineState, entityKey, + type EntityState, loadState, saveState, } from "./state.ts"; @@ -43,6 +44,11 @@ import { } from "./actors.ts"; import { describeDrop, firstDepositGateFactory, planTick } from "./traffic.ts"; import { discordAlert, ensureRunway } from "./funding.ts"; +import { + type AllFailedState, + emptyAllFailedState, + evaluateAllFailed, +} from "./alerts.ts"; import { aggregatorPayment, ensurePayMirror, @@ -171,6 +177,16 @@ async function reconcileRoster( } } +/** What the tick actually ran: `attempted` counts the actions the engine got + * as far as executing (batch actions whose actor existed, plus aggregator + * payments), `failures` how many of those threw. The caller folds these into + * the all-failed alert policy — planned-but-skipped actions are deliberately + * absent from both, so the ratio describes real attempts only. */ +interface TrafficOutcome { + attempted: number; + failures: number; +} + async function runTraffic( env: ReturnType, ring: KeyRing, @@ -178,7 +194,7 @@ async function runTraffic( seed: string, nowDay: number, nowMs: number, -): Promise { +): Promise { const slots = providerSlots(seed); const gate = firstDepositGateFactory(seed, slots, nowDay); const planned = planTick( @@ -196,12 +212,22 @@ async function runTraffic( const dropNote = describeDrop(planned.length, batch.length); if (dropNote) console.log(`[traffic] ${dropNote}`); + let attempted = 0; let failures = 0; for (const a of batch) { const provider = state.providers[a.providerKey]; const council = state.councils[provider.councilKey]; const actor = state.entities[entityKey(a.providerKey, a.entityIdx)]; if (!actor) continue; + // Resolve every skip condition before counting the attempt, so the + // all-failed ratio only ever describes actions the engine really ran. + let receiver: EntityState | undefined; + if (a.type === "send") { + receiver = + state.entities[entityKey(a.receiverProviderKey!, a.receiverIdx!)]; + if (!receiver) continue; + } + attempted++; try { if (a.type === "deposit") { await actDeposit( @@ -215,15 +241,12 @@ async function runTraffic( a.amount, ); } else if (a.type === "send") { - const receiver = - state.entities[entityKey(a.receiverProviderKey!, a.receiverIdx!)]; - if (!receiver) continue; await actSend( env, ring, state, actor, - receiver, + receiver!, council, provider.publicKey, a.assetCode, @@ -274,6 +297,7 @@ async function runTraffic( const n = rng.poisson(lambda); for (let k = 0; k < n; k++) { const amount = Number(rng.lognormal(1.2, 0.6, 0.5, 3).toFixed(2)); + attempted++; try { await aggregatorPayment( env, @@ -301,12 +325,7 @@ async function runTraffic( } } - if (batch.length > 0 && failures === batch.length) { - await discordAlert( - env, - `every action this tick failed (${failures}/${batch.length}) — platform down or config broken?`, - ); - } + return { attempted, failures }; } async function runwayCheck( @@ -330,7 +349,7 @@ async function runwayCheck( async function tick( env: ReturnType, ring: KeyRing, -): Promise { +): Promise { let state = loadState(env.stateFile); if (state && state.networkPassphrase !== env.networkPassphrase) { @@ -366,10 +385,18 @@ async function tick( await reconcileRoster(env, ring, state, ring.rngSeed, nowDay); await runwayCheck(env, ring, state); - await runTraffic(env, ring, state, ring.rngSeed, nowDay, nowMs); + const outcome = await runTraffic( + env, + ring, + state, + ring.rngSeed, + nowDay, + nowMs, + ); state.lastLedgerSeq = Math.max(state.lastLedgerSeq, 0); saveState(env.stateFile, state); + return outcome; } async function main() { @@ -393,10 +420,21 @@ async function main() { // none of the batch/runway/reset alerts ever fire), so the streak itself // must page: first at STUCK_ALERT_TICKS, then every REALERT_TICKS while // the condition holds, and once more on recovery. + // The two paging paths are exclusive per tick: a tick that throws never + // reaches the all-failed evaluation, and a tick that completes never counts + // toward the stuck streak. let failStreak = 0; + let allFailed: AllFailedState = emptyAllFailedState(); while (true) { try { - await tick(env, ring); + const outcome = await tick(env, ring); + const verdict = evaluateAllFailed( + allFailed, + outcome.attempted, + outcome.failures, + ); + allFailed = verdict.state; + if (verdict.alert) await discordAlert(env, verdict.alert); if (failStreak >= STUCK_ALERT_TICKS) { await discordAlert( env,