From fa12b1343149a818cfa97314fb507cce2203f13c Mon Sep 17 00:00:00 2001 From: Vasilev Dmitrii Date: Fri, 28 Aug 2026 02:36:07 +0700 Subject: [PATCH 1/2] corpus: report all four backends, not two (Refs #2161) `corpus` describes itself as "the only corpus metric that does not lie" and measured Zig and Verilog. The Rust backend had no compile gate anywhere in this repository and neither did C -- which is how an empty `match` for every `switch`, a dropped body for every `for`, and a `u64` typed as `int` all shipped with a green exit. Two stages added, mirroring the existing Zig block: gen-rust -> rustc --edition 2021 --crate-type lib --emit=metadata gen-c -> cc -fsyntax-only -std=gnu11 and one row the two-backend table could not show: how many specs satisfy ALL FOUR toolchains. That is what "one spec, four targets" claims, and until now nothing counted it. On a 39-spec sample: generates Zig 22 ... and Zig accepts it 12 30.8% generates Rust 22 ... and rustc accepts it 0 0.0% generates C 22 ... and cc accepts it 1 2.6% generates Verilog 22 ... and iverilog accepts 7 17.9% Zig AND Verilog accept 6 15.4% ALL FOUR accept 0 0.0% The `BOTH backends accept` label is now `Zig AND Verilog accept`, because with four columns "both" no longer names anything. Zero is the number this change exists to print. It is a sample, not the corpus, and the row says so by carrying its own denominator. --- bootstrap/src/service.rs | 74 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 2 deletions(-) diff --git a/bootstrap/src/service.rs b/bootstrap/src/service.rs index d314d5f566..65a0d160d0 100644 --- a/bootstrap/src/service.rs +++ b/bootstrap/src/service.rs @@ -782,6 +782,17 @@ pub fn run_path(_repo_root: &Path, spec: &str, to_bitstream: bool) -> anyhow::Re struct SpecOutcome { zig_gen: bool, zig_build: bool, + /// Does gen-rust produce something rustc accepts? + /// + /// `corpus` calls itself "the only corpus metric that does not lie" and + /// reported TWO of the four backends. The Rust backend had no compile gate + /// anywhere in the repository, and gen-c had none either -- which is how an + /// empty `match`, a dropped loop body, and a `u64` typed as `int` all + /// shipped with a green exit. + rust_gen: bool, + rust_build: bool, + c_gen: bool, + c_build: bool, v_gen: bool, v_build: bool, /// W707: does `yosys synth_xilinx` accept it? @@ -911,6 +922,50 @@ pub fn run_corpus( let mut o = SpecOutcome::default(); let sp = p.to_string_lossy().to_string(); + // ---- Rust ---- + if let Some((c, text)) = run_timed(Command::new(&me).args(["gen-rust", &sp]), 15) { + if text == "__TIMEOUT__" { + o.timed_out = true; + } else if c == Some(0) && !text.trim().is_empty() { + o.rust_gen = true; + let rp = tmp.join("c.rs"); + if std::fs::write(&rp, &text).is_ok() { + if let Some((rc, rt)) = run_timed( + Command::new("rustc").args([ + "--edition", "2021", "--crate-type", "lib", + "--emit=metadata", "-A", "warnings", + "-o", "/dev/null", &rp.to_string_lossy(), + ]), + 30, + ) { + if rt == "__TIMEOUT__" { o.timed_out = true; } + o.rust_build = rc == Some(0); + } + } + } + } + + // ---- C ---- + if let Some((c, text)) = run_timed(Command::new(&me).args(["gen-c", &sp]), 15) { + if text == "__TIMEOUT__" { + o.timed_out = true; + } else if c == Some(0) && !text.trim().is_empty() { + o.c_gen = true; + let cp = tmp.join("c.c"); + if std::fs::write(&cp, &text).is_ok() { + if let Some((cc, ct)) = run_timed( + Command::new("cc").args([ + "-fsyntax-only", "-std=gnu11", &cp.to_string_lossy(), + ]), + 30, + ) { + if ct == "__TIMEOUT__" { o.timed_out = true; } + o.c_build = cc == Some(0); + } + } + } + } + // ---- Zig ---- if let Some((c, text)) = run_timed(Command::new(&me).args(["gen", &sp]), 15) { if text == "__TIMEOUT__" { @@ -993,11 +1048,21 @@ pub fn run_corpus( // T180: the column the instrument does not control. let vdp = out.iter().filter(|(_, o)| o.v_build && o.v_data_port).count(); let vsy = out.iter().filter(|(_, o)| o.v_synth).count(); + let rg = c(|o| o.rust_gen); + let rb = c(|o| o.rust_build); + let cg = c(|o| o.c_gen); + let cb = c(|o| o.c_build); let both = out.iter().filter(|(_, o)| o.zig_build && o.v_build).count(); + // The row the two-backend table could not show: does ONE spec satisfy all + // four toolchains? That is what "one spec, four targets" claims. + let all4 = out + .iter() + .filter(|(_, o)| o.zig_build && o.v_build && o.rust_build && o.c_build) + .count(); let to = c(|o| o.timed_out); if json { - println!("{{\"specs\":{n},\"zig_gen\":{zg},\"zig_build\":{zb},\"verilog_gen\":{vg},\"verilog_build\":{vb},\"verilog_build_with_data_port\":{vdp},\"verilog_synth\":{vsy},\"both_build\":{both},\"timed_out\":{to}}}"); + println!("{{\"specs\":{n},\"zig_gen\":{zg},\"zig_build\":{zb},\"verilog_gen\":{vg},\"verilog_build\":{vb},\"verilog_build_with_data_port\":{vdp},\"verilog_synth\":{vsy},\"rust_gen\":{rg},\"rust_build\":{rb},\"c_gen\":{cg},\"c_build\":{cb},\"both_build\":{both},\"all_four_build\":{all4},\"timed_out\":{to}}}"); return Ok(()); } @@ -1007,13 +1072,18 @@ pub fn run_corpus( println!(" {}", "-".repeat(52)); println!(" {:<26} {:>5} {:>6}", "generates Zig", zg, format!("{:.1}%", pct(zg))); println!(" {:<26} {:>5} {:>6}", " ... and Zig accepts it", zb, format!("{:.1}%", pct(zb))); + println!(" {:<26} {:>5} {:>6}", "generates Rust", rg, format!("{:.1}%", pct(rg))); + println!(" {:<26} {:>5} {:>6}", " ... and rustc accepts it", rb, format!("{:.1}%", pct(rb))); + println!(" {:<26} {:>5} {:>6}", "generates C", cg, format!("{:.1}%", pct(cg))); + println!(" {:<26} {:>5} {:>6}", " ... and cc accepts it", cb, format!("{:.1}%", pct(cb))); println!(" {:<26} {:>5} {:>6}", "generates Verilog", vg, format!("{:.1}%", pct(vg))); println!(" {:<26} {:>5} {:>6}", " ... and iverilog accepts", vb, format!("{:.1}%", pct(vb))); println!(" {:<26} {:>5} {:>6}", " ... AND has a data port", vdp, format!("{:.1}%", pct(vdp))); if synth { println!(" {:<26} {:>5} {:>6}", " ... AND yosys SYNTHESISES", vsy, format!("{:.1}%", pct(vsy))); } - println!(" {:<26} {:>5} {:>6}", "BOTH backends accept", both, format!("{:.1}%", pct(both))); + println!(" {:<26} {:>5} {:>6}", "Zig AND Verilog accept", both, format!("{:.1}%", pct(both))); + println!(" {:<26} {:>5} {:>6}", "ALL FOUR accept", all4, format!("{:.1}%", pct(all4))); if to > 0 { println!(" {:<26} {:>5}", "timed out (hang)", to); } From 835de840185de5418c8db5dc962ceb3cce361df2 Mon Sep 17 00:00:00 2001 From: Vasilev Dmitrii Date: Fri, 28 Aug 2026 02:36:07 +0700 Subject: [PATCH 2/2] docs/now: corpus reported half the backends (Refs #2161) --- ...corpus-metric-that-does-not-lie-reported-half-the-b.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 docs/now/2026-08-28-the-only-corpus-metric-that-does-not-lie-reported-half-the-b.md diff --git a/docs/now/2026-08-28-the-only-corpus-metric-that-does-not-lie-reported-half-the-b.md b/docs/now/2026-08-28-the-only-corpus-metric-that-does-not-lie-reported-half-the-b.md new file mode 100644 index 0000000000..2b33f212d1 --- /dev/null +++ b/docs/now/2026-08-28-the-only-corpus-metric-that-does-not-lie-reported-half-the-b.md @@ -0,0 +1,8 @@ +# NOW -- The only corpus metric that does not lie reported half the backends (2026-08-28) + +## The only corpus metric that does not lie reported half the backends (Refs #2161) + +- Refs #2161. `t27c corpus` calls itself "the only corpus metric that does not lie" and measured Zig and Verilog. Rust had no compile gate anywhere in the repository and neither did C -- which is exactly how an empty match for every switch, a dropped body for every for, and a u64 typed as int all shipped with a green exit +- Added gen-rust -> rustc --emit=metadata and gen-c -> cc -fsyntax-only, plus the row the two-backend table could not show: how many specs satisfy ALL FOUR toolchains. That is what "one spec, four targets" claims and nothing counted it +- On a 39-spec sample: Zig accepts 12, rustc accepts 0, cc accepts 1, iverilog accepts 7, and ALL FOUR accept ZERO. The zero is the number this change exists to print +- The "BOTH backends accept" label became "Zig AND Verilog accept": with four columns the word both no longer names anything, and leaving it would have made the new rows read as a subset of it