diff --git a/.github/workflows/spec-guards.yml b/.github/workflows/spec-guards.yml index 9ea91f134..845d2159f 100644 --- a/.github/workflows/spec-guards.yml +++ b/.github/workflows/spec-guards.yml @@ -15,6 +15,12 @@ # ring_spec_differential.py a matching signature is not matching behaviour: # ring-090 read 16 of 16 identical and disagreed on # 126 of 1190 cases (#3420, #3424). +# diagnostic_capacity.py clang's default -ferror-limit=20 had been +# censoring every C error total published for this +# repository: 15133 errors, reported as 3849. A +# ratchet on the MEASURING APPARATUS -- it fails if +# an instrument that reported completely starts +# truncating (#3450). # check_duplicate_declarations.py # three specs declare 25 top-level names twice and # `t27c check` exited 0 on all of them. Every @@ -30,8 +36,8 @@ # runs zero times on the default branch and its history is empty (#2925 family). # # Cost, measured rather than estimated: 12s for the seal scan, 0s for the drift -# scan, ~1s per differential pair, 7s for the duplicate-declaration ratchet, on -# top of the compiler build. +# scan, ~1s per differential pair, 7s for the duplicate-declaration ratchet, +# under 1s for the diagnostic-capacity ratchet, on top of the compiler build. name: Spec Guards on: @@ -46,6 +52,7 @@ on: - 'tools/ring_spec_differential.py' - 'tools/check_duplicate_declarations.py' - 'tools/duplicate_declarations_baseline.txt' + - 'tools/diagnostic_capacity.py' - '.github/workflows/spec-guards.yml' push: branches: [master] @@ -59,6 +66,7 @@ on: - 'tools/ring_spec_differential.py' - 'tools/check_duplicate_declarations.py' - 'tools/duplicate_declarations_baseline.txt' + - 'tools/diagnostic_capacity.py' - '.github/workflows/spec-guards.yml' workflow_dispatch: @@ -98,6 +106,11 @@ jobs: - name: Self-check -- duplicate declarations run: python3 tools/check_duplicate_declarations.py --self-check + # The fixtures, before the counts. Two of the first five produced the + # wrong KIND of diagnostic and both read as findings about the compiler. + - name: Self-check -- diagnostic capacity fixtures + run: python3 tools/diagnostic_capacity.py --self-check + - name: Does every seal still describe what the compiler emits run: python3 tools/check_seal_currency.py @@ -110,6 +123,12 @@ jobs: - name: Does any spec declare one name twice run: python3 tools/check_duplicate_declarations.py + # A ratchet on the instruments, not on the code: if a compiler that + # reported completely starts truncating, every total measured with it + # silently becomes a floor. + - name: Does every compiler still report what it finds + run: python3 tools/diagnostic_capacity.py + - name: Run each CONVERGED pair on the same inputs shell: bash run: | diff --git a/docs/now/2026-09-08-i-asked-the-other-five-instruments.md b/docs/now/2026-09-08-i-asked-the-other-five-instruments.md new file mode 100644 index 000000000..aec2292ef --- /dev/null +++ b/docs/now/2026-09-08-i-asked-the-other-five-instruments.md @@ -0,0 +1,8 @@ +# NOW -- I asked the other five instruments (2026-09-08) + +## I asked the other five instruments (Closes #3450, Refs #3448) + +- One instrument had been censoring its own totals, so the obvious question was whether the others do. Nobody had asked. Planting **50 known errors per language**: `clang` default reports **20** and truncates; `clang -ferror-limit=0`, `rustc`, `zig` and `iverilog` all report **50** and are complete; **`yosys` reports 1 and aborts on the first error** -- not capped but stopped, so an error COUNT from it is 0 or 1 and means nothing. Only its pass/fail is a measurement. +- **The fixtures were the hard part and two of five were wrong**, both in ways that read as findings about the compiler rather than about me. The Zig fixture named its functions `f16` and `f32`, which **shadow Zig primitives**, so it failed with 2 errors of a different kind before reaching the undefined names -- indistinguishable from a cap. The Verilog fixture used implicitly declared identifiers, which **yosys treats as a warning**, so it reported zero -- indistinguishable from silence. And `iverilog` prints **two** diagnostic lines per error, so a naive line count doubles it. Three of five instruments were misread on the first pass, none of them at fault. +- The damage is bounded, and the bound was measured rather than hoped. **Acceptance counts are safe**: clang's exit code is 1 with and without the limit on a failing file and 0 on a clean one, so a count of FILES that compile cannot be censored by a per-file cap -- every `cc accepts N` figure stands. **The repository's own gates are clean**: the only tool invoking `cc` reads `returncode`, never a diagnostic count. **The Rust totals stand too**, since `rustc` reports completely. What was wrong is exactly the C error totals in my own reports: 3849 was **15188**, 3825 was **15133**, and after the last change **15126**. +- `tools/diagnostic_capacity.py` records the table and fails if an instrument that reported completely starts truncating -- a ratchet on the MEASURING APPARATUS rather than on the code, wired into `Spec Guards` at under a second. It **degrades rather than lying**: a runner without zig or yosys checks what it has and NAMES what it skipped, and only a run with nothing available exits 2. Four controls: a planted truncation reddens it, a one-compiler PATH passes while naming four skips, an empty PATH exits 2, and a fixture that stops producing its error fails the self-check. diff --git a/tools/diagnostic_capacity.py b/tools/diagnostic_capacity.py new file mode 100755 index 000000000..d856cdf63 --- /dev/null +++ b/tools/diagnostic_capacity.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Does each compiler REPORT every error it finds, or stop early? + +Every error total published for the generated C in this repository was a floor. +`cc -fsyntax-only | grep -c 'error:'` looks like a count and is not: clang's +default is `-ferror-limit=20`, 141 corpus files reach it, and the corpus emits +15133 errors rather than the 3849 reported (#3448). A repair that certainly +removed errors showed a delta of ZERO -- three files sitting at exactly 20 +before and after -- which is how it was found. + +So this asks the same question of every instrument, by planting a KNOWN number +of errors and counting what comes back. + +Measured 2026-09-08, 50 planted: + + clang, default 20 CAPPED at -ferror-limit=20 + clang, -ferror-limit=0 50 complete + rustc 50 complete + zig 50 complete + iverilog 50 complete (two diagnostic LINES per error) + yosys 1 ABORTS on the first error + +yosys is not capped, it stops: any error COUNT from it is 0 or 1 and means +nothing. Only its pass/fail is a measurement. + +Acceptance columns are safe, and that was checked rather than assumed: clang's +exit code is 1 with and without the limit on a failing file, and 0 on a clean +one. A count of FILES that compile is unaffected by a cap on diagnostics per +file. + +THE FIXTURES ARE THE HARD PART, and two of the first five were wrong in ways +that looked like findings: + + * the Zig fixture named its functions `f16`, `f32` -- which shadow Zig + primitives -- so it reported 2 errors of a different kind and read as a cap; + * the Verilog fixture used implicitly declared identifiers, which yosys + treats as a WARNING, so it reported 0 errors and read as silence. + +Hence `--self-check`: every fixture must produce the expected KIND of +diagnostic before its count is believed. + +Usage: + tools/diagnostic_capacity.py report, and fail if a cap appeared + tools/diagnostic_capacity.py --self-check fixtures only + +Exit codes: + 0 every instrument reports as recorded below + 1 an instrument that reported completely now truncates + 2 COULD NOT RUN (an instrument is missing, or a fixture stopped working) +""" + +import os +import re +import subprocess +import sys +import tempfile + +PLANTED = 50 + +# name -> (fixture builder, argv builder, diagnostic regex, expected count, +# note). `expected` is what was MEASURED, not what the manual claims. +EXPECT = { + "clang (default)": 20, + "clang (-ferror-limit=0)": PLANTED, + "rustc": PLANTED, + "zig": PLANTED, + "iverilog": PLANTED, + "yosys": 1, +} + + +def fixtures(d: str) -> dict: + c = os.path.join(d, "c50.c") + with open(c, "w") as fh: + fh.write("".join(f"int f{i}(void) {{ return undefined_{i}; }}\n" for i in range(PLANTED))) + r = os.path.join(d, "r50.rs") + with open(r, "w") as fh: + fh.write("".join(f"pub fn f{i}() -> i32 {{ undefined_{i} }}\n" for i in range(PLANTED))) + # NOT `f16`/`f32`: those shadow Zig primitives and the file fails for a + # different reason before the undefined names are ever reached. + z = os.path.join(d, "z50.zig") + with open(z, "w") as fh: + fh.write("".join(f"pub fn probe_{i}() i32 {{ return undefined_{i}; }}\n" for i in range(PLANTED))) + v = os.path.join(d, "v50.v") + with open(v, "w") as fh: + fh.write("module m;\n" + "".join(f" assign w{i} = undefined_{i};\n" for i in range(PLANTED)) + "endmodule\n") + # yosys treats an implicit declaration as a WARNING, so the iverilog + # fixture measures nothing there. A missing module is a real yosys error. + y = os.path.join(d, "y50.v") + with open(y, "w") as fh: + fh.write("".join(f"module m{i}; nosuchmod_{i} u{i} (); endmodule\n" for i in range(PLANTED))) + return {"c": c, "rs": r, "zig": z, "v": v, "y": y} + + +def run(argv, cwd=None) -> str: + try: + p = subprocess.run(argv, capture_output=True, text=True, cwd=cwd) + except FileNotFoundError: + return None + return p.stdout + p.stderr + + +def counts(d: str, f: dict) -> dict: + out = os.path.join(d, "out") + os.makedirs(out, exist_ok=True) + got = {} + + t = run(["cc", "-std=c11", "-fsyntax-only", f["c"]]) + got["clang (default)"] = None if t is None else len(re.findall(r"error:", t)) + t = run(["cc", "-std=c11", "-ferror-limit=0", "-fsyntax-only", f["c"]]) + got["clang (-ferror-limit=0)"] = None if t is None else len(re.findall(r"error:", t)) + + t = run(["rustc", "--edition", "2021", "--crate-type", "lib", "--crate-name", "m", + "-A", "warnings", "--emit=metadata", "-o", os.path.join(out, "m.rmeta"), f["rs"]]) + # `error: aborting due to N previous errors` is a summary, not an error. + got["rustc"] = None if t is None else len( + [l for l in t.splitlines() if l.startswith("error") and "aborting due to" not in l]) + + t = run(["zig", "build-obj", f["zig"], "-femit-bin=" + os.path.join(out, "z.o")]) + got["zig"] = None if t is None else len(re.findall(r"error:", t)) + + t = run(["iverilog", "-o", os.path.join(out, "v.vvp"), f["v"]]) + # TWO diagnostic lines per planted error ("Unable to bind" and "Unable to + # elaborate"), so the unit is the SOURCE LINE cited, not the line printed. + got["iverilog"] = None if t is None else len( + set(re.findall(r"^[^\s:]+\.v:(\d+):\s*error:", t, re.M))) + + t = run(["yosys", "-q", "-p", f"read_verilog {f['y']}; hierarchy -check"]) + got["yosys"] = None if t is None else len(re.findall(r"(?m)^ERROR:", t)) + return got + + +def self_check(d: str, f: dict) -> int: + """Each fixture must produce the KIND of diagnostic it was written for. + + Two of the first five did not, and both read as findings about the + instrument: the Zig fixture tripped over primitive shadowing, and the + Verilog one produced warnings where yosys needed errors. + """ + ok = True + present = 0 + checks = [ + ("clang", ["cc", "-std=c11", "-ferror-limit=0", "-fsyntax-only", f["c"]], r"use of undeclared identifier"), + ("rustc", ["rustc", "--edition", "2021", "--crate-type", "lib", "--crate-name", "m", + "-A", "warnings", "--emit=metadata", "-o", os.path.join(d, "m.rmeta"), f["rs"]], + r"cannot find value"), + ("zig", ["zig", "build-obj", f["zig"], "-femit-bin=" + os.path.join(d, "z.o")], r"use of undeclared identifier"), + ("iverilog", ["iverilog", "-o", os.path.join(d, "v.vvp"), f["v"]], r"Unable to bind"), + ("yosys", ["yosys", "-q", "-p", f"read_verilog {f['y']}; hierarchy -check"], r"is not part of the design"), + ] + for name, argv, want in checks: + t = run(argv) + if t is None: + # NOT a failure: a runner without zig or yosys can still check the + # fixtures for the compilers it has, and the one that matters -- + # clang, the capped one -- is everywhere. Named, never dropped. + print(f" {name:10} NOT ON PATH, not checked") + continue + present += 1 + hit = re.search(want, t) is not None + print(f" {name:10} expects /{want}/ -> {'PASS' if hit else 'FAIL'}") + ok &= hit + if present == 0: + # A check that could not run has not passed. + print(" no instrument on PATH. Exit 2 = COULD NOT RUN.", file=sys.stderr) + return 2 + print(f" checked {present} of {len(checks)} fixtures") + return 0 if ok else 1 + + +def main() -> int: + with tempfile.TemporaryDirectory() as d: + f = fixtures(d) + if "--self-check" in sys.argv: + return self_check(d, f) + got = counts(d, f) + print(f"planted {PLANTED} errors per language\n") + bad = False + absent = [] + checked = 0 + for name, want in EXPECT.items(): + n = got.get(name) + if n is None: + print(f" {name:24} - NOT ON PATH, not checked") + absent.append(name) + continue + checked += 1 + verdict = "complete" if n >= PLANTED else ("ABORTS on the first" if n <= 1 else "TRUNCATES") + flag = "" if n == want else " <-- CHANGED, recorded " + str(want) + print(f" {name:24} {n:4} {verdict}{flag}") + if n < want: + bad = True + print() + print("Acceptance counts are unaffected: clang's exit code is 1 with and") + print("without the limit on a failing file, and 0 on a clean one -- so a") + print("count of FILES that compile cannot be censored by a per-file cap.") + if absent: + # NAMED, not silently dropped: a run that checked two instruments + # and a run that checked six print different things. + print(f"\nNOT CHECKED ({len(absent)} of {len(EXPECT)}): {', '.join(absent)}") + print("Install them to widen this, or read the result as covering the rest.") + if checked == 0: + # A check that could not run has not passed. + print("\nNo instrument was available. Exit 2 = COULD NOT RUN.", file=sys.stderr) + return 2 + print(f"\nchecked {checked} of {len(EXPECT)}") + return 1 if bad else 0 + + +if __name__ == "__main__": + sys.exit(main())