diff --git a/.claude/workflows/README.md b/.claude/workflows/README.md new file mode 100644 index 0000000000..d38f45daf5 --- /dev/null +++ b/.claude/workflows/README.md @@ -0,0 +1,48 @@ +# .claude/workflows/ — reusable multi-agent runs + +Each file here is a Workflow script: a deterministic orchestration of subagents, +invoked by name rather than re-authored each time. + +``` +Workflow({ name: "loop-recon", args: { repo: "/path/to/checkout" } }) +``` + +## What is here + +| workflow | question it answers | +|---|---| +| `loop-recon.js` | What is weakest in this repository right now, what is the field already doing that we are not, and which `tri` command is missing — each finding adversarially refuted before it reaches a plan | + +## The shape these follow, and why + +**Pipeline, not barrier.** Findings from one dimension go to verification while +another dimension is still searching. A barrier between the phases would idle +the fast dimensions behind the slowest one, and there is no cross-dimension +dependency to justify it. + +**Every finding is refuted before it is believed.** A separate agent is asked to +*kill* each claim, with `stands=false` as the default when it cannot be +confirmed from evidence the verifier gathered itself. This exists because a +plausible-but-wrong finding costs more than a missed one: it sends the next hour +somewhere real work is not. + +**Structured output, not parsed prose.** `schema:` forces the subagent through a +validating tool call, so the script never regex-scrapes an answer out of English. + +**The prompt states what is already known.** Each recon prompt lists the findings +already in hand and says "do NOT re-report these". Without it, every run +rediscovers the same top three and reads like progress. + +**Effort is part of the finding.** `minutes | hours | days`, and the verifier is +asked to correct it. A true finding with a wrong cost estimate still plans the +next iteration badly. + +## What they deliberately do not do + +They do not edit anything. Recon returns findings; the decision about what to +act on stays with the main loop, where the context to weigh it lives. + +They do not hard-code a checkout path. `REPO` comes from `args.repo` or the +working directory — baking one machine's absolute path into a shared file is +what `secret-scan` rejects, and a workflow is not exempt from a rule the +repository applies to everything else. diff --git a/.claude/workflows/loop-recon.js b/.claude/workflows/loop-recon.js new file mode 100644 index 0000000000..e3bc4079e1 --- /dev/null +++ b/.claude/workflows/loop-recon.js @@ -0,0 +1,143 @@ +export const meta = { + name: 'loop-recon', + description: 'Weak points of t27 and who else is in this space, adversarially verified', + whenToUse: 'Each /loop iteration: find what is weakest and what the field already solved. Pass {repo: ""} to point it at a checkout.', + phases: [ + { title: 'Recon', detail: 'weak points, competitors, unmeasured claims, quick wins' }, + { title: 'Verify', detail: 'refute each finding before it reaches the plan' }, + ], +} + +// The repository this is run from. Hard-coding one worktree's absolute path +// is the same defect `secret-scan` rejects, so it comes from args or the cwd. +const REPO = (args && args.repo) || process.env.PWD || '.' + +const FINDINGS = { + type: 'object', + required: ['findings'], + properties: { + findings: { + type: 'array', + items: { + type: 'object', + required: ['title', 'evidence', 'why_it_matters', 'effort'], + properties: { + title: { type: 'string' }, + evidence: { type: 'string', description: 'file:line, command output, or URL' }, + why_it_matters: { type: 'string' }, + effort: { type: 'string', enum: ['minutes', 'hours', 'days'] }, + remedy: { type: 'string' }, + }, + }, + }, + }, +} + +const VERDICT = { + type: 'object', + required: ['stands', 'reasoning'], + properties: { + stands: { type: 'boolean' }, + reasoning: { type: 'string' }, + }, +} + +const DIMENSIONS = [ + { + key: 'weakest', + prompt: `In ${REPO} (the t27 spec-first ternary language: .t27 specs -> Verilog/C/Rust/Zig via t27c). + +Current state, already known — do NOT re-report these: + - suite 2424 passed / 0 failed + - seal gate, specs-generate gate, gate-preconditions all exit 0 + - t27c 0.2.0 published to crates.io today + - #2747 nothing builds the Lean proofs (250 theorems, 45 workflows, no lake build) + - #2754 five gates have no master baseline; five tracked .py files do not parse + +Find what is WEAKEST that is NOT on that list. Concretely: + - run \`./target/release/t27c corpus\` and \`./target/release/t27c suite\` and read what they say is worst + - the gap between "generates" and "accepts" per backend -- which backend is furthest behind and what is the single largest cause + - any tool in tools/ that reports a number nobody acts on + - anything where a count is going the WRONG way + +Rank by (impact / effort). Quote real command output. Effort must be honest: "minutes" means a single edit.`, + }, + { + key: 'rustc', + prompt: `In ${REPO}, the corpus report says rustc accepts 0 of 559 generated Rust files -- the only backend at zero, while Zig accepts 217 and cc accepts 157. + +Measure WHY, precisely: + - generate Rust for a sample of specs (\`./target/release/t27c gen-rust \`), compile each with rustc, and classify the errors + - report the top error classes by COUNT and by NUMBER OF SPECS BLOCKED (these rank differently and the second is what matters) + - for the largest class, say whether it is one emitter defect or many + - state how many specs would compile if the single largest cause were fixed -- measure this by actually removing/working around it on a sample, not by assuming + +A previous measurement said 499 distinct error classes with the largest being 688 occurrences of a missing \`serde\`. Verify or refute that, and say which measurement is right.`, + }, + { + key: 'competitors', + prompt: `Research who else works on this and what they have that t27 does not. Use WebSearch and WebFetch (load them with ToolSearch first). + +t27 is: a spec-first language where a .t27 spec is the single source of truth and t27c emits Verilog, C, Rust and Zig from it, with seals (hashes pinning each spec to its four outputs), a corpus ratchet, and a formal Lean model of the Verilog-lowerable subset. + +Search for the actual field: + - spec-first / single-source-of-truth HDL generation (Chisel, SpinalHDL, Amaranth/nMigen, Veryl, PyMTL3, Bluespec, Clash, Filament, Calyx) + - multi-target codegen from one spec (Kaitai Struct, Protobuf/Cap'n Proto as prior art for the "one spec, N backends" pattern) + - formally verified compiler backends (CompCert, Vellvm, Lean/Coq-modelled lowering) + - ternary / BitNet / low-precision hardware toolchains + +For each of the 5-8 most relevant: what it does, what it has that t27 lacks, what t27 has that it lacks, and one concrete idea worth stealing. Give URLs. Be blunt about where t27 is behind — the point is to find work, not to reassure.`, + }, + { + key: 'tri-cli', + prompt: `In ${REPO}, read cli/tri/src/ and list every subcommand \`tri\` has today (\`./target/release/tri --help\`, and each subcommand's --help). + +Then propose NEW commands that would speed up the recurring work in this repository. The work that actually recurs, from the last day: + - "did my change move the corpus?" -- currently requires building two binaries and diffing per spec by hand + - "which gates have no baseline on master?" -- currently requires reading 45 workflow files and gh run list per file + - "is this number still true?" -- claims in docs drifting from what the tools print + - "what did this loop iteration change?" -- assembling the report by hand each time + +For each proposal: the exact command line, what it prints, which manual sequence it replaces, and roughly how it would be implemented (which existing code it can reuse). Prefer 2-3 commands that are genuinely load-bearing over a long list. Say explicitly if an existing command already covers it.`, + }, +] + +phase('Recon') +const recon = await pipeline( + DIMENSIONS, + d => agent(d.prompt, { label: `recon:${d.key}`, phase: 'Recon', schema: FINDINGS }), + (res, d) => { + if (!res || !res.findings || res.findings.length === 0) return [] + return parallel( + res.findings.slice(0, 6).map(f => () => + agent( + `Refute this, using the repository at ${REPO} and the network where the claim is about the outside world. + +CLAIM: ${f.title} +EVIDENCE: ${f.evidence} +WHY IT MATTERS: ${f.why_it_matters} +EFFORT CLAIMED: ${f.effort} + +Check it yourself -- run the command, read the file, fetch the URL. Default to stands=false if you cannot confirm it with evidence you gathered. If the claim is true but the EFFORT is wrong, say stands=true and correct the effort in your reasoning.`, + { label: `verify:${(f.title || '').slice(0, 34)}`, phase: 'Verify', schema: VERDICT } + ).then(v => ({ ...f, dimension: d.key, verdict: v })) + ) + ) + } +) + +const all = recon.flat().filter(Boolean) +const confirmed = all.filter(f => f.verdict && f.verdict.stands) +log(`${all.length} findings, ${confirmed.length} survived`) + +return { + confirmed: confirmed.map(f => ({ + dimension: f.dimension, + title: f.title, + effort: f.effort, + evidence: f.evidence, + remedy: f.remedy, + note: f.verdict.reasoning.slice(0, 400), + })), + refuted: all.filter(f => f.verdict && !f.verdict.stands).map(f => ({ title: f.title, why: f.verdict.reasoning.slice(0, 200) })), +} diff --git a/cli/tri/src/gates.rs b/cli/tri/src/gates.rs index 90e493f574..6b73f1fdd5 100644 --- a/cli/tri/src/gates.rs +++ b/cli/tri/src/gates.rs @@ -165,6 +165,26 @@ pub enum GatesCmd { #[arg(long, default_value_t = 50)] min_runs: u64, }, + /// Workflows with no recent run on the default branch: their green is + /// about frequency, not health. + /// + /// `dead` asks "ran a lot and never passed". This asks the opposite and + /// harder question: "never ran, so nobody knows". Three gates in this + /// repository were in that state at once -- rings-rust, secret-scan and + /// cli-tri, all `paths:`-filtered on the root Cargo.toml, which nothing + /// had edited in months. Editing it woke all three: seventeen ring crates + /// had never compiled, 233 files carried a developer's home directory, and + /// `tri rtl check` had been dying on a submodule that was declared but not + /// registered. Every one of them had been reading as passing. + Unmeasured { + /// owner/repo, repeatable. Defaults to the repository you are in. + #[arg(long = "repo")] + repos: Vec, + /// Call a workflow unmeasured when its last default-branch run is + /// older than this many days, or when it has none at all. + #[arg(long, default_value_t = 30)] + stale_days: u64, + }, } /// Gate scripts whose control lives in a SEPARATE file, and the file that @@ -2216,6 +2236,14 @@ pub fn run(cmd: &GatesCmd) -> Result<()> { dir.as_deref(), ), GatesCmd::Prs { repo } => prs(repo.as_deref()), + GatesCmd::Unmeasured { repos, stale_days } => { + let list: Vec = if repos.is_empty() { + vec![current_repo()?] + } else { + repos.clone() + }; + unmeasured(&list, *stale_days) + } GatesCmd::Dead { repos, min_runs } => { let list: Vec = if repos.is_empty() { ["gHashTag/trinity", "gHashTag/trinity-fpga", "gHashTag/t27"] @@ -2263,6 +2291,168 @@ fn too_few_runs_to_judge(total: u64, min_runs: u64) -> bool { total < min_runs } +/// The repository this working tree belongs to, as `owner/name`. +fn current_repo() -> Result { + let s = gh(&["repo", "view", "--json", "nameWithOwner", "--jq", ".nameWithOwner"])?; + let s = s.trim().to_string(); + if s.is_empty() { + anyhow::bail!("`gh repo view` named no repository -- run this inside a checkout, or pass --repo"); + } + Ok(s) +} + +/// Does this workflow restrict itself with `paths:`? A path filter is what +/// turns "has not failed" into "has not run", so it is reported beside the +/// staleness rather than left for the reader to go and look up. +fn has_path_filter(root: &std::path::Path, rel: &str) -> bool { + let p = root.join(rel); + match std::fs::read_to_string(&p) { + Ok(t) => t.lines().any(|l| { + let t = l.trim_start(); + t.starts_with("paths:") || t.starts_with("paths-ignore:") + }), + Err(_) => false, + } +} + +/// Can a human get the missing reading at all? Without `workflow_dispatch:` +/// there is no way to fire it against the default branch on purpose, so the +/// gap cannot be closed even by someone who wants to. +fn has_dispatch(root: &std::path::Path, rel: &str) -> bool { + match std::fs::read_to_string(root.join(rel)) { + Ok(t) => t.lines().any(|l| l.trim_start().starts_with("workflow_dispatch:")), + Err(_) => false, + } +} + +fn unmeasured(repos: &[String], stale_days: u64) -> Result<()> { + let root = repo_root()?; + let mut rows: Vec<(String, String, String, bool, bool)> = Vec::new(); + let mut checked = 0usize; + let mut unreadable = 0usize; + + for repo in repos { + let default_branch = gh(&[ + "api", + &format!("repos/{repo}"), + "--jq", + ".default_branch", + ])? + .trim() + .to_string(); + + let listing = gh(&[ + "api", + &format!("repos/{repo}/actions/workflows?per_page=100"), + "--jq", + r#".workflows[]|select(.state=="active")|"\(.id)\t\(.name)\t\(.path)""#, + ])?; + + for line in listing.lines() { + let mut it = line.splitn(3, '\t'); + let (id, name, path) = match (it.next(), it.next(), it.next()) { + (Some(a), Some(b), Some(c)) => (a, b, c), + _ => continue, + }; + checked += 1; + + // The most recent run ON THE DEFAULT BRANCH. A run on a pull + // request says nothing about the branch everything merges into. + // The jq here was once written as a raw string with the closing + // `""` trimmed off at runtime, which produced `... // ` -- invalid + // jq. Every query then failed, `unwrap_or_default` turned each + // failure into an empty string, and the command reported that all + // 58 workflows had never run. A confident wrong answer, produced by + // the exact mechanism this command exists to find. + // + // So: no default on error. A query that did not run is not a + // workflow that did not run, and the two are now said differently. + let query = format!( + "repos/{repo}/actions/workflows/{id}/runs?branch={default_branch}&per_page=1" + ); + let last = match gh(&["api", &query, "--jq", ".workflow_runs[0].created_at // \"\""]) { + Ok(v) => v.trim().to_string(), + Err(e) => { + eprintln!(" ? could not ask about {name}: {e}"); + unreadable += 1; + continue; + } + }; + + let stale = if last.is_empty() { + true + } else { + match days_since(&last) { + Some(d) => d > stale_days, + // An unparseable date is not a fresh one. Saying "fine" + // here is the same defect this command exists to find. + None => true, + } + }; + if stale { + rows.push(( + repo.clone(), + name.to_string(), + if last.is_empty() { "never".into() } else { last[..10].to_string() }, + has_path_filter(&root, path), + has_dispatch(&root, path), + )); + } + } + } + + if unreadable > 0 { + println!( + " {unreadable} workflow(s) could not be asked about; they are NOT counted as \ + either fresh or stale." + ); + } + if rows.is_empty() { + println!( + "Every active workflow has run on the default branch within {stale_days} days \ + ({checked} checked)." + ); + return Ok(()); + } + + rows.sort_by(|a, b| a.2.cmp(&b.2).then(a.1.cmp(&b.1))); + println!( + "{} of {} active workflow(s) have no default-branch run within {} days.\n", + rows.len(), + checked, + stale_days + ); + println!(" {:<10} {:<7} {:<9} {}", "LAST", "paths:", "dispatch", "WORKFLOW"); + for (repo, name, last, filtered, dispatch) in &rows { + println!( + " {:<10} {:<7} {:<9} {} ({})", + last, + if *filtered { "yes" } else { "-" }, + if *dispatch { "yes" } else { "NO" }, + name, + repo + ); + } + println!( + "\n A gate that has not run on the default branch is not passing there; it is\n\ + unmeasured. `paths: yes` is usually the reason. `dispatch: NO` means the\n\ + reading cannot be taken on purpose -- add `workflow_dispatch:` first." + ); + Ok(()) +} + +/// Whole days between an ISO-8601 timestamp and now, or None if it will not +/// parse. Kept separate so the staleness rule can be tested without a network. +fn days_since(iso: &str) -> Option { + let ts = chrono::DateTime::parse_from_rfc3339(iso).ok()?; + let now = chrono::Utc::now(); + let secs = now.signed_duration_since(ts.with_timezone(&chrono::Utc)).num_seconds(); + if secs < 0 { + return Some(0); + } + Some(secs as u64 / 86_400) +} + fn dead(repos: &[String], min_runs: u64) -> Result<()> { let mut rows: Vec<(String, String, u64)> = Vec::new(); for repo in repos { @@ -2406,6 +2596,45 @@ mod tests { /// The floor `tri gates dead` actually ships with, read back out of clap /// rather than repeated as a literal here. + /// `days_since` is the whole staleness rule, so it is exercised without a + /// network. The case that matters is the LAST one: an unparseable date must + /// not read as fresh, because "I could not tell" and "it is fine" are the + /// two answers this command exists to keep apart. + #[test] + fn days_since_counts_whole_days_and_refuses_to_guess() { + let now = chrono::Utc::now(); + let mk = |d: i64| (now - chrono::Duration::days(d)).to_rfc3339(); + + assert_eq!(super::days_since(&mk(0)), Some(0)); + assert_eq!(super::days_since(&mk(1)), Some(1)); + assert_eq!(super::days_since(&mk(45)), Some(45)); + + // A clock skewed into the future is 0 days old, not a negative number + // that would underflow the comparison. + let future = (now + chrono::Duration::days(3)).to_rfc3339(); + assert_eq!(super::days_since(&future), Some(0)); + + // Not a date. None, so the caller treats it as stale. + assert_eq!(super::days_since("never"), None); + assert_eq!(super::days_since(""), None); + assert_eq!(super::days_since("2026-08-28"), None); + } + + /// The staleness decision itself, spelled out: None must mean stale. + #[test] + fn an_unreadable_date_is_stale_not_fresh() { + let stale_days = 30u64; + let decide = |iso: &str| match super::days_since(iso) { + Some(d) => d > stale_days, + None => true, + }; + let fresh = (chrono::Utc::now() - chrono::Duration::days(2)).to_rfc3339(); + let old = (chrono::Utc::now() - chrono::Duration::days(200)).to_rfc3339(); + assert!(!decide(&fresh), "a two-day-old run is not stale"); + assert!(decide(&old), "a 200-day-old run is stale"); + assert!(decide("garbage"), "an unreadable date must not read as fresh"); + } + fn shipped_floor() -> u64 { match Root::parse_from(["tri-gates", "dead"]).action { GatesCmd::Dead { min_runs, .. } => min_runs, diff --git a/docs/now/2026-08-29-a-command-that-reports-what-nobody-measured-and-got-it-wrong.md b/docs/now/2026-08-29-a-command-that-reports-what-nobody-measured-and-got-it-wrong.md new file mode 100644 index 0000000000..cec216b194 --- /dev/null +++ b/docs/now/2026-08-29-a-command-that-reports-what-nobody-measured-and-got-it-wrong.md @@ -0,0 +1,8 @@ +# NOW -- A command that reports what nobody measured, and got it wrong first (2026-08-29) + +## A command that reports what nobody measured, and got it wrong first (Refs #2754) + +- tri gates unmeasured finds workflows with no default-branch run: 28 of 58 here, against tri gates dead which asks the opposite question +- its first version said 58 of 58 -- broken jq, and unwrap_or_default turned every failed query into never ran +- no default on error now: could not ask is counted separately and never rendered as did not run +- loop-recon saved as a reusable workflow with its shape written down beside it