diff --git a/.github/workflows/untrusted-input-gate.yml b/.github/workflows/untrusted-input-gate.yml index 1e158d1025..622ae63f49 100644 --- a/.github/workflows/untrusted-input-gate.yml +++ b/.github/workflows/untrusted-input-gate.yml @@ -43,6 +43,14 @@ jobs: # it needs no Coq toolchain: the test builds its own two-file tree. run: python3 scripts/ci/test_admitted_gate_reads_both_files.py + - name: An empty rings matrix refuses instead of skipping the build + # #3069. The build job is guarded by `count != '0'`, a skipped job is + # green, and a crate rename matches rings-rust.yml's own paths filter -- + # so the commit that empties the matcher runs the workflow and collects + # a green tick. Runs here: needs no Rust, and the test builds its own + # ring tree and executes the workflow step under GitHub's shell flags. + run: python3 scripts/ci/test_rings_matrix_refuses_an_empty_population.py + - name: No untrusted event data interpolated into shell run: python3 scripts/ci/check_untrusted_shell_interp.py diff --git a/docs/now/2026-09-04-an-empty-matrix-is-not-a-clean-build.md b/docs/now/2026-09-04-an-empty-matrix-is-not-a-clean-build.md new file mode 100644 index 0000000000..d3d8661e32 --- /dev/null +++ b/docs/now/2026-09-04-an-empty-matrix-is-not-a-clean-build.md @@ -0,0 +1,27 @@ +# NOW -- An empty matrix is not a clean build (2026-09-04) + +## A crate rename made rings-rust compile nothing and report success + +- `rings_matrix.py` drops a directory unless the name starts `ring-`, ends + `-rust`, and holds a `Cargo.toml`. Any of the three is one rename away. +- With the matrix empty the build job is skipped -- `if: needs.discover.outputs + .count != '0'` -- and **a skipped job is green**. `discover` printed + `Discovered 0 ring-*-rust crate(s).` and succeeded. +- The trigger is the same commit: `paths: rings/ring-*-rust/**` means the rename + matches the filter, runs the workflow, and collects a green tick for it. +- The script now refuses an empty population and exits **2**, naming what it + looked for and where. Verified with GitHub's own shell flags that + `MATRIX="$(python3 …)"` aborts the step, so `discover` goes red instead. +- Today the population is 17, all carrying `Cargo.toml`; the control asserts a + real tree still emits its matrix and reports its count. +- The workflow's own header records the previous visit: seven master runs with + all 17 crate jobs failing, every one concluding `success`. That was fixed per + job. This is the same door one step earlier -- with zero jobs there are no + verdict rows to write. +- The control caught a false pass. Emptying `$GITHUB_OUTPUT` in the harness made + all three defect arms fail on line 2's redirect rather than on the script's + exit 2, and every one of them "passed". Fixed, and each arm now also asserts + that the refusal is what stopped it. +- Prior art, checked: pytest reserves exit code **5** for "No tests were + collected" as a public-API outcome. This repository's 2-for-everything is + coarser than the field's by one distinction. diff --git a/scripts/ci/rings_matrix.py b/scripts/ci/rings_matrix.py index 967c2172e7..f92a51fe75 100755 --- a/scripts/ci/rings_matrix.py +++ b/scripts/ci/rings_matrix.py @@ -41,6 +41,43 @@ def discover(repo_root: Path) -> list[dict[str, str]]: def main() -> int: repo_root = Path(__file__).resolve().parents[2] include = discover(repo_root) + + # AN EMPTY MATRIX IS NOT A CLEAN BUILD (#3069). + # + # `discover` drops a directory on three conditions -- the name must start + # `ring-`, end `-rust`, and hold a `Cargo.toml` -- and every one of them is a + # rename away. With the matrix empty the workflow does not fail: the build + # job is guarded by `if: needs.discover.outputs.count != '0'`, a SKIPPED job + # is green, and the whole run concludes success having compiled nothing. + # + # The trigger is not hypothetical. `.github/workflows/rings-rust.yml` filters + # on `rings/ring-*-rust/**`, so the very commit that renames the crates + # matches the filter, RUNS this workflow, and gets a green tick for it. + # + # This file has been here before. Its workflow's own header records seven + # master runs between 2026-05-23 and 2026-08-20 in which all 17 crate jobs + # failed and every run concluded `success`, because the summary printed a + # COUNT and never read what the matrix had measured. That door was closed + # per job; this is the same door one step earlier, at the matrix itself. + # + # Exit 2, not 1: nothing failed to compile, the population was never built. + # `t27c corpus` refuses a spec tree with no specs the same way, `scripts/tri` + # uses 2 for an unbuilt compiler, and pytest reserves a code of its own -- + # 5, "No tests were collected" -- for exactly this outcome. + if not include: + rings_dir = repo_root / "rings" + print( + f"rings_matrix: REFUSED -- no ring-*-rust crate with a Cargo.toml under " + f"{rings_dir}.\n" + " Nothing was compiled and nothing failed to compile: the matrix is empty,\n" + " the build job would be SKIPPED, and a skipped job reads as green.\n" + " If the crates were renamed or removed on purpose, update this script and\n" + " the `paths:` filter in .github/workflows/rings-rust.yml in the same commit.\n" + " Exit code 2 = could not take a reading, not a failed build.", + file=sys.stderr, + ) + return 2 + matrix = {"include": include} out = json.dumps(matrix, separators=(",", ":")) # GitHub Actions consumes `matrix=...` on a single line. diff --git a/scripts/ci/test_rings_matrix_refuses_an_empty_population.py b/scripts/ci/test_rings_matrix_refuses_an_empty_population.py new file mode 100644 index 0000000000..c13740c42e --- /dev/null +++ b/scripts/ci/test_rings_matrix_refuses_an_empty_population.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""#3069: an empty matrix is not a clean build. + +`discover` drops a directory on three conditions -- the name must start `ring-`, +end `-rust`, and hold a `Cargo.toml` -- and every one of them is a rename away. +With the matrix empty the workflow did not fail: the build job is guarded by +`if: needs.discover.outputs.count != '0'`, a SKIPPED job is green, and the run +concluded success having compiled nothing. + +The trigger is not hypothetical. rings-rust.yml filters on `rings/ring-*-rust/**`, +so the very commit that renames the crates matches the filter, RUNS the workflow, +and collects a green tick for it. + +Every assertion has a control: a tree that DOES carry ring crates must still emit +its matrix and exit 0, or this file cannot fail. + +The step body is extracted from the workflow rather than restated, so a change to +the YAML that stops the failure propagating is caught -- GitHub runs steps under +`bash --noprofile --norc -eo pipefail`, and this runs them the same way. +""" + +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +WF = REPO / ".github/workflows/rings-rust.yml" +SCRIPT = REPO / "scripts/ci/rings_matrix.py" +FAILURES = [] + + +def check(name, ok, detail=""): + print(f" {'ok ' if ok else 'FAILED '}{name}") + if not ok: + FAILURES.append(f"{name}: {detail}") + + +def tree(dirs): + """A scratch repo holding the script and the named ring directories.""" + d = tempfile.mkdtemp(prefix=f"rings-matrix-{os.getpid()}-") + os.makedirs(os.path.join(d, "scripts/ci")) + shutil.copy(SCRIPT, os.path.join(d, "scripts/ci/rings_matrix.py")) + for name, with_cargo in dirs: + p = os.path.join(d, "rings", name) + os.makedirs(p) + if with_cargo: + with open(os.path.join(p, "Cargo.toml"), "w") as f: + f.write("[package]\nname = \"x\"\n") + return d + + +def gen_step(): + """The `Generate matrix` step body, as the workflow ships it.""" + y = WF.read_text() + m = re.search(r"- name: Generate matrix\n.*?\n run: \|\n(.*?)(?=\n [a-z]|\n - |\n\Z)", y, re.S) + if not m: + return None + return "\n".join(l[10:] if l.startswith(" " * 10) else l for l in m.group(1).split("\n")) + + +def run_step(cwd, body): + """Run the step the way GitHub does, with a real $GITHUB_OUTPUT. + + Emptying that variable instead of pointing it somewhere made every defect + arm pass for the WRONG reason: the step died on line 2's redirect to "", + not on line 1's exit 2. The control caught it, which is the whole reason a + control that asserts SUCCESS sits beside three that assert failure. + """ + env = dict(os.environ) + fd, path = tempfile.mkstemp(prefix="gh-output-") + os.close(fd) + env["GITHUB_OUTPUT"] = path + try: + return subprocess.run(["bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", body], + capture_output=True, text=True, cwd=cwd, env=env) + finally: + os.unlink(path) + + +def main(): + body = gen_step() + check("the Generate matrix step body was found in the workflow", body is not None, + "the extractor no longer matches -- this test would assert nothing") + + # THE DEFECT, three ways the population empties. + for label, dirs in ( + ("crates renamed away from -rust", [("ring-088-rs", True)]), + ("a ring directory with no Cargo.toml", [("ring-088-rust", False)]), + ("no rings/ directory at all", []), + ): + d = tree(dirs) + try: + r = subprocess.run([sys.executable, "scripts/ci/rings_matrix.py"], + capture_output=True, text=True, cwd=d) + check(f"an empty population exits 2 ({label})", r.returncode == 2, + f"rc={r.returncode} out={(r.stdout + r.stderr)[:200]!r}") + check(f"and prints no matrix to stdout ({label})", "include" not in r.stdout, + f"stdout={r.stdout!r}") + if body is not None: + s = run_step(d, body) + out = s.stdout + s.stderr + check(f"and the workflow step fails rather than reporting 0 ({label})", + s.returncode != 0, f"rc={s.returncode} out={out[:200]!r}") + # For the RIGHT reason: the script's own refusal, not a broken + # harness. Without this the arm passes on any failure at all. + check(f"and it is the script's refusal that stopped it ({label})", + "rings_matrix: REFUSED" in out, f"out={out[:300]!r}") + finally: + shutil.rmtree(d, ignore_errors=True) + + # CONTROLS. Without these, a script that always exits 2 passes everything above. + d = tree([("ring-088-rust", True), ("ring-089-rust", True), ("not-a-ring", True)]) + try: + r = subprocess.run([sys.executable, "scripts/ci/rings_matrix.py"], + capture_output=True, text=True, cwd=d) + check("control: a tree with ring crates still exits 0", r.returncode == 0, + f"rc={r.returncode} err={r.stderr[:200]!r}") + ok = False + try: + ok = [e["crate"] for e in json.loads(r.stdout)["include"]] == ["ring-088-rust", "ring-089-rust"] + except Exception as exc: # noqa: BLE001 + check("control: and emits a parseable matrix", False, f"{exc}: {r.stdout!r}") + check("control: naming exactly the two ring crates, sorted", ok, f"stdout={r.stdout!r}") + if body is not None: + s = run_step(d, body) + check("control: and the workflow step succeeds, reporting 2", + s.returncode == 0 and "Discovered 2 " in s.stdout, + f"rc={s.returncode} out={(s.stdout + s.stderr)[:200]!r}") + finally: + shutil.rmtree(d, ignore_errors=True) + + print() + if FAILURES: + print("FAILED:") + for f in FAILURES: + print(f" - {f}") + return 1 + print("ok: an empty matrix refuses; a real one still builds.") + return 0 + + +if __name__ == "__main__": + sys.exit(main())