diff --git a/docs/now/2026-09-10-inngest-probe-suite-as-a-t27-spec.md b/docs/now/2026-09-10-inngest-probe-suite-as-a-t27-spec.md new file mode 100644 index 0000000000..c9f7ce4d26 --- /dev/null +++ b/docs/now/2026-09-10-inngest-probe-suite-as-a-t27-spec.md @@ -0,0 +1,9 @@ +# NOW -- Inngest probe suite as a .t27 spec; training-stuck-check retires 404 trainings (2026-09-10) + +## Inngest probe suite as a .t27 spec; training-stuck-check retires 404 trainings (Closes #3559) + +- `specs/automation/inngest-probe-suite.t27` states the safe probe suite behind the bot's `/inngest_probe` (host gHashTag/999-multibots-telegraf #2325, #2326, #2331): payload = manifest `safe_probe` + `e2e_test: true` forced; transport = Inngest GQL `invokeFunction`; expectations `FAILED-at-guard | COMPLETED | skip`; verdicts `match | mismatch | timeout | invoke-error | skipped`; forbidden side effects (`charges-balance`, `paid-api`, `messages-user`, `messages-owners`); one suite at a time; admin only. +- Judge rule as measured on production 2026-09-09/10: a guard that is a step name fails that step; a body-level guard leaves only the `function error` span; the server may leave the span of a step whose `NonRetriableError` ended the run as RUNNING, so on a terminal run the culprit is the first step that is not COMPLETED. Mirror run 2026-09-10 02:16Z with the bot's own suite code: 21 of 28 probes were never judged because a fast guard was already FAILED when the run was discovered; fixed in host #2331, re-judged offline 28/28. +- `specs/functions/training-stuck-check.t27` NOTE: a training Replicate answers 404 for is retired -- `model_trainings.status` set to `failed` with a fixed error text, one warn to the admin, no event, no user message; skipped in safe mode (host #2328). Two rows from 2025-12-02 had been re-asked every 30 minutes since; the first production tick after the deploy (2026-09-10 01:30Z) retired both. +- Published-figures pin: test blocks 12462 -> 12465, the three `test` blocks of the new spec (#3557 had moved it 12456 -> 12462 the same day); the movement is written in the pin's own note, as the check demands. +- Not claimed: `LIVE_RUN_STATUS` in the spec stays `not-yet-run` for the bot's own report -- the two suites verified so far were the admin's run of 2026-09-09 19:11Z (judged offline) and the mirror run of 2026-09-10 (judged offline after #2331); the 404 branch of training-stuck-check is stated in NOTE, not yet as a step; `gate-topology` and `untrusted-input` were failing on this PR against the pre-rebase base and are re-measured after the rebase, not fixed here. diff --git a/specs/automation/inngest-probe-suite.t27 b/specs/automation/inngest-probe-suite.t27 new file mode 100644 index 0000000000..0b2f5a5c58 --- /dev/null +++ b/specs/automation/inngest-probe-suite.t27 @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: Apache-2.0 +// specs/automation/inngest-probe-suite.t27 -- safe probe of every served Inngest function +// of 999-multibots-telegraf, started by the admin command /inngest_probe in the bot. +// Host: 999-multibots-telegraf src/inngest_app/probe/probeSuite.ts (the logic), +// src/inngest_app/probe/inngestProbeClient.ts (the one GraphQL mutation), +// src/commands/inngestProbeCommand.ts (the Telegram command). +// The per-function expectation lives in the manifest (probe_expect) and on each +// function card in specs/functions/.t27 (GUARD, SAFE_PROBE); this spec states the +// suite: what is sent, how a run is found, how a verdict is reached, and what may not happen. +// phi^2 + 1/phi^2 = 3 | TRINITY + +module automation::inngest_probe_suite { + + pub const KIND : str = "automation"; + pub const ID : str = "inngest-probe-suite"; + pub const REPO : str = "999-multibots-telegraf"; + pub const VERSION : u8 = 1; + + // How the suite is started and by whom. + pub const COMMAND : str = "/inngest_probe"; + pub const COMMAND_STATUS : str = "/inngest_probe status"; + pub const ACTOR : str = "admin"; // ADMIN_IDS only; a stranger is refused + pub const CONFIRM : str = "inline-button"; // nothing is invoked until the admin presses it + pub const CONCURRENCY : u8 = 1; // one suite at a time, in-process lock + + // What is sent. The payload is the manifest's safe_probe object with the safe-mode + // flag forced on; a safe_probe that tries to set the flag to false is overridden. + pub const SAFE_FLAG : str = "e2e_test"; + pub const TRANSPORT : str = "inngest-gql:invokeFunction"; + pub const SLUG_FORMAT : str = "-"; + pub const APP_ID : str = "telegram-bot-client"; + pub const ORDER : str = "manifest order, sequential"; + + // How the run of an invoke is found: runs of that functionID queued after the invoke + // (minus a small skew) whose eventName starts with the invoke prefix. A cron tick of the + // same function in the window is not the probe's run and is ignored. + pub const RUN_LOOKBACK_MS : u32 = 5000; + pub const INVOKED_EVENT_PREFIX : str = "inngest/function.invoked"; + pub const POLL_MS : u32 = 3000; + pub const BUDGET_MS : u32 = 120000; // per probe; after it the verdict is timeout + + // The verdict, per probe. match is the only good one. + // FAILED-at-guard : run FAILED and the first FAILED step is the card's GUARD + // (GUARD "none"/"unknown": any FAILED run matches) + // COMPLETED : run COMPLETED + // skip : not invoked (no safe payload or paid path), reported as skipped + pub const EXPECTATIONS : [3]str = ["FAILED-at-guard", "COMPLETED", "skip"]; + pub const VERDICTS : [5]str = ["match", "mismatch", "timeout", "invoke-error", "skipped"]; + pub const SUITE_OK_IFF : str = "no mismatch, no timeout, no invoke-error"; + + // What may not happen while the suite runs. These are properties of the functions' + // safe-mode branches (safeMode.ts), which the suite relies on and the report checks by + // expecting FAILED at the guard or a COMPLETED that reached only ADMIN_CHAT_ID. + pub const FORBIDDEN_SIDE_EFFECTS : [4]str = ["charges-balance", "paid-api", "messages-user", "messages-owners"]; + pub const ALLOWED_RECIPIENT : str = "ADMIN_CHAT_ID"; + + // Where the count of probes comes from: every manifest entry with control "spec+code". + // At main @251571c that is 28 = 17 FAILED-at-guard + 11 COMPLETED + 0 skip. + pub const PLANNED_AT_251571C : u8 = 28; + pub const EXPECT_FAILED_AT_GUARD_AT_251571C : u8 = 17; + pub const EXPECT_COMPLETED_AT_251571C : u8 = 11; + + // Honesty. The suite has not been run against production from this spec's commit: the + // GraphQL endpoint is private to the Railway network, so the first real run happens + // after deploy, from the admin chat. The unit tests drive the orchestrator with a fake + // Inngest (src/__tests__/inngest/probeSuite.test.ts, src/__tests__/bot/inngestProbeCommand.test.ts). + pub const LIVE_RUN_STATUS : str = "not-yet-run"; + + struct ProbePlan { + id: str, + slug: str, + expect: str, // one of EXPECTATIONS + guard: str, + payload_json: str, // always contains "e2e_test": true + } + + struct ProbeResult { + id: str, + verdict: str, // one of VERDICTS + run_id: str, + status: str, // Inngest run status, "" when no run + failed_step: str, // "" when none + } + + struct ProbeSuiteReport { + version: u8, + planned: u32, + match: u32, + mismatch: u32, + timeout: u32, + invoke_error: u32, + skipped: u32, + ok: bool, + } + + test "guard failure is the good outcome for a guarded function" + const plan = ProbePlan { id: "neuro-image-generate", slug: "telegram-bot-client-neuro-image-generate", expect: "FAILED-at-guard", guard: "validate-input", payload_json: "{\"e2e_test\":true}" }; + const result = ProbeResult { id: "neuro-image-generate", verdict: "match", run_id: "r1", status: "FAILED", failed_step: "validate-input" }; + assert(plan.expect == "FAILED-at-guard"); + assert(result.failed_step == plan.guard); + assert(result.verdict == "match"); + + test "a guarded function that completed on a probe is a mismatch" + const result = ProbeResult { id: "neuro-image-generate", verdict: "mismatch", run_id: "r2", status: "COMPLETED", failed_step: "" }; + assert(result.status == "COMPLETED"); + assert(result.verdict == "mismatch"); + + test "the suite is ok only when every probe matched or was skipped" + const good = ProbeSuiteReport { version: 1, planned: 28, match: 28, mismatch: 0, timeout: 0, invoke_error: 0, skipped: 0, ok: true }; + const bad = ProbeSuiteReport { version: 1, planned: 28, match: 27, mismatch: 1, timeout: 0, invoke_error: 0, skipped: 0, ok: false }; + assert(good.ok); + assert(!bad.ok); + assert(good.match + good.mismatch + good.timeout + good.invoke_error + good.skipped == good.planned); +} diff --git a/specs/functions/training-stuck-check.t27 b/specs/functions/training-stuck-check.t27 index 9f403df629..02fba83a54 100644 --- a/specs/functions/training-stuck-check.t27 +++ b/specs/functions/training-stuck-check.t27 @@ -39,4 +39,4 @@ pub const GUARD : str = "none"; pub const SAFE_PROBE : str = ""; pub const PROBE_RESULT : str = "not-deployed"; pub const CONTROL : str = "spec+code"; -pub const NOTE : str = "Not on the production Railway build as of 2026-09-09 (deployed_2026_09_09 false in the manifest). RETRIES is not declared in the code, so the Inngest JS SDK v3 default of 4 applies. The same schedule is stated as a cron card, specs/crons ID inngest/999-multibots-telegraf/check-stuck-trainings."; +pub const NOTE : str = "Not on the production Railway build as of 2026-09-09 (deployed_2026_09_09 false in the manifest). RETRIES is not declared in the code, so the Inngest JS SDK v3 default of 4 applies. The same schedule is stated as a cron card, specs/crons ID inngest/999-multibots-telegraf/check-stuck-trainings. Since host PR #2328 (2026-09-10) a training that Replicate answers 404 for is retired: model_trainings.status set to failed with the fixed error text, one warn to the admin, no event and no user message; two rows from 2025-12-02 had been re-asked every 30 minutes for nine months. Skipped in safe mode. The 404 branch is not yet stated as a step; the code is its only witness."; diff --git a/tools/published_figures.py b/tools/published_figures.py index 92b2f4ee5c..163d07b464 100755 --- a/tools/published_figures.py +++ b/tools/published_figures.py @@ -65,9 +65,11 @@ # byte-identical to their twin. 12644 - 188 = 12456, which is what a # re-derivation gives -- the first thing this file caught, on its first run. # #3557 added specs/ui/viewport.t27 with six test blocks: 12456 + 6 = 12462. + # #3556 (Closes #3559) added specs/automation/inngest-probe-suite.t27 with + # three test blocks: 12462 + 3 = 12465. ("test blocks", "blocks", - r"^\s*test\s+(?:\"[^\"]*\"|[A-Za-z_][\w\-]*)\s*\{?\s*$", 12462, - "#3479 pinned 12644; #3482 removed 188; #3557 added 6"), + r"^\s*test\s+(?:\"[^\"]*\"|[A-Za-z_][\w\-]*)\s*\{?\s*$", 12465, + "#3479 pinned 12644; #3482 removed 188; #3557 added 6; #3556 added 3"), ]