diff --git a/Cargo.lock b/Cargo.lock index 0f6368e4497cb..4d40476b0e69b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -872,6 +872,7 @@ dependencies = [ "semver", "serde", "serde_json", + "tempfile", "tracing", "tracing-subscriber", "unified-diff", diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index 363802093a13a..2989b1247afb9 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -7,7 +7,7 @@ default-run = "bootstrap" [features] build-metrics = ["dep:sysinfo", "build_helper/metrics"] -tracing = ["dep:tracing", "dep:tracing-chrome", "dep:tracing-subscriber", "dep:chrono", "dep:tempfile"] +tracing = ["dep:tracing", "dep:tracing-chrome", "dep:tracing-subscriber", "dep:chrono"] [lib] path = "src/lib.rs" @@ -55,6 +55,7 @@ termcolor = "1.4" toml = "0.5" walkdir = "2.4" xz2 = "0.1" +tempfile = "3.15.0" # Dependencies needed by the build-metrics feature sysinfo = { version = "0.39.0", default-features = false, optional = true, features = ["system"] } @@ -64,7 +65,6 @@ chrono = { version = "0.4", default-features = false, optional = true, features tracing = { version = "0.1", optional = true, features = ["attributes"] } tracing-chrome = { version = "0.7", optional = true } tracing-subscriber = { version = "0.3", optional = true, features = ["env-filter", "fmt", "registry", "std"] } -tempfile = { version = "3.15.0", optional = true } [target.'cfg(windows)'.dependencies.junction] version = "1.3.0" @@ -83,7 +83,6 @@ features = [ [dev-dependencies] pretty_assertions = "1.4" -tempfile = "3.15.0" insta = "1.43" # We care a lot about bootstrap's compile times, so don't include debuginfo for diff --git a/src/bootstrap/src/bin/rustc.rs b/src/bootstrap/src/bin/rustc.rs index bb974e4645ea6..72c4b713c3ac9 100644 --- a/src/bootstrap/src/bin/rustc.rs +++ b/src/bootstrap/src/bin/rustc.rs @@ -21,8 +21,8 @@ use std::process::{Child, Command}; use std::time::Instant; use shared_helpers::{ - dylib_path, dylib_path_var, exe, maybe_dump, parse_rustc_stage, parse_rustc_verbose, - parse_value_from_args, + ArgFileCommand, dylib_path, dylib_path_var, exe, maybe_dump, parse_rustc_stage, + parse_rustc_verbose, parse_value_from_args, }; #[path = "../utils/shared_helpers.rs"] @@ -112,11 +112,11 @@ fn main() { let mut cmd = match env::var_os("RUSTC_WRAPPER_REAL") { Some(wrapper) if !wrapper.is_empty() => { - let mut cmd = Command::new(wrapper); + let mut cmd = ArgFileCommand::new(wrapper); cmd.arg(rustc_driver); cmd } - _ => Command::new(rustc_driver), + _ => ArgFileCommand::new(rustc_driver), }; cmd.args(&args).env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); @@ -271,6 +271,7 @@ fn main() { eprintln!("{prefix} libdir: {libdir:?}"); } + let (mut cmd, _arg_file) = cmd.build().unwrap(); maybe_dump(format!("stage{}-rustc", stage + 1), &cmd); let start = Instant::now(); diff --git a/src/bootstrap/src/bin/rustdoc.rs b/src/bootstrap/src/bin/rustdoc.rs index 2925892fed283..ea3c77e7051ce 100644 --- a/src/bootstrap/src/bin/rustdoc.rs +++ b/src/bootstrap/src/bin/rustdoc.rs @@ -4,10 +4,9 @@ use std::env; use std::path::PathBuf; -use std::process::Command; use shared_helpers::{ - dylib_path, dylib_path_var, maybe_dump, parse_rustc_stage, parse_rustc_verbose, + ArgFileCommand, dylib_path, dylib_path_var, maybe_dump, parse_rustc_stage, parse_rustc_verbose, parse_value_from_args, }; @@ -31,7 +30,7 @@ fn main() { let mut dylib_path = dylib_path(); dylib_path.insert(0, PathBuf::from(libdir.clone())); - let mut cmd = Command::new(rustdoc); + let mut cmd = ArgFileCommand::new(rustdoc); if target.is_some() { // The stage0 compiler has a special sysroot distinct from what we @@ -81,6 +80,7 @@ fn main() { } } + let (mut cmd, _arg_file) = cmd.build().unwrap(); maybe_dump(format!("stage{}-rustdoc", stage + 1), &cmd); if verbose > 1 { diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 68a4f928464f1..3dbf4a576203a 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -2662,8 +2662,8 @@ pub fn run_cargo( ) -> Vec { // `target_root_dir` looks like $dir/$target/release let target_root_dir = stamp.path().parent().unwrap(); - // `target_deps_dir` looks like $dir/$target/release/deps - let target_deps_dir = target_root_dir.join("deps"); + // `target_build_dir` looks like $dir/$target/release/build + let target_build_dir = target_root_dir.join("build"); // `host_root_dir` looks like $dir/release let host_root_dir = target_root_dir .parent() @@ -2734,7 +2734,7 @@ pub fn run_cargo( // If this was output in the `deps` dir then this is a precise file // name (hash included) so we start tracking it. - if filename.starts_with(&target_deps_dir) { + if filename.starts_with(&target_build_dir) { deps.push((filename.to_path_buf(), DependencyType::Target)); continue; } @@ -2769,11 +2769,18 @@ pub fn run_cargo( // Ok now we need to actually find all the files listed in `toplevel`. We've // got a list of prefix/extensions and we basically just need to find the - // most recent file in the `deps` folder corresponding to each one. - let contents = target_deps_dir + // most recent file in the `build` folder corresponding to each one. + // + // Cargo's build folder is structured as `build///out/` so + // we need to traverse multiple directory layers to get to actual files. + let read_dir = |path: &Path| path.read_dir().ok().into_iter().flatten().filter_map(Result::ok); + let contents = target_build_dir .read_dir() - .unwrap_or_else(|e| panic!("Couldn't read {}: {}", target_deps_dir.display(), e)) - .map(|e| t!(e)) + .unwrap_or_else(|e| panic!("Couldn't read {}: {}", target_build_dir.display(), e)) + .map(|e| e.unwrap()) + .flat_map(|e| read_dir(&e.path())) + .flat_map(|e| read_dir(&e.path())) + .flat_map(|e| read_dir(&e.path())) .map(|e| (e.path(), e.file_name().into_string().unwrap(), t!(e.metadata()))) .collect::>(); for (prefix, extension, expected_len) in toplevel { diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index ed5c2586a5ed6..bc0ee1efa1250 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -10,7 +10,7 @@ //! return `ToolBuildResult` and should never prepare `cargo` invocations manually. use std::ffi::OsStr; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::{env, fs}; use crate::core::build_steps::compile::is_lto_stage; @@ -1622,8 +1622,9 @@ impl Builder<'_> { // // Notably this munges the dynamic library lookup path to point to the // right location to run `compiler`. - let mut lib_paths: Vec = - vec![self.cargo_out(compiler, Mode::ToolBootstrap, *host).join("deps")]; + let mut lib_paths: Vec = discover_out_dirs_with_dylibs( + self.cargo_out(compiler, Mode::ToolBootstrap, *host).join("build"), + ); // On MSVC a tool may invoke a C compiler (e.g., compiletest in run-make // mode) and that C compiler may need some extra PATH modification. Do @@ -1651,3 +1652,23 @@ impl Builder<'_> { cmd } } + +/// Gets all of the `out` dirs in a given Cargo `build-dir//build` dir. +fn discover_out_dirs_with_dylibs(dir: PathBuf) -> Vec { + if !dir.exists() { + return Vec::new(); + } + let read_dir = |path: &Path| path.read_dir().ok().into_iter().flatten().filter_map(Result::ok); + let has_dylib = |path: &Path| { + read_dir(path) + .any(|e| e.path().extension().is_some_and(|ext| ext == std::env::consts::DLL_EXTENSION)) + }; + dir.read_dir() + .unwrap_or_else(|e| panic!("Couldn't read {}: {}", dir.display(), e)) + .map(|e| e.unwrap()) + .flat_map(|e| read_dir(&e.path())) + .flat_map(|e| read_dir(&e.path())) + .map(|e| e.path()) + .filter(|path| path.ends_with("out") && has_dylib(path)) + .collect::>() +} diff --git a/src/bootstrap/src/core/builder/cargo.rs b/src/bootstrap/src/core/builder/cargo.rs index 8247e8739f35c..328802d47a393 100644 --- a/src/bootstrap/src/core/builder/cargo.rs +++ b/src/bootstrap/src/core/builder/cargo.rs @@ -600,6 +600,8 @@ impl Builder<'_> { let mut hostflags = HostFlags::default(); + cargo.env("CARGO_UNSTABLE_BUILD_DIR_NEW_LAYOUT", "true"); + // Codegen backends are not yet tracked by -Zbinary-dep-depinfo, // so we need to explicitly clear out if they've been updated. for backend in self.codegen_backends(compiler) { diff --git a/src/bootstrap/src/utils/shared_helpers.rs b/src/bootstrap/src/utils/shared_helpers.rs index d620cc4bbb6b4..ced215665672d 100644 --- a/src/bootstrap/src/utils/shared_helpers.rs +++ b/src/bootstrap/src/utils/shared_helpers.rs @@ -14,12 +14,15 @@ #![allow(dead_code)] use std::env; -use std::ffi::OsString; +use std::ffi::{OsStr, OsString}; use std::fs::OpenOptions; use std::io::Write; -use std::process::Command; +use std::path::Path; +use std::process::{Command, CommandEnvs}; use std::str::FromStr; +use tempfile::NamedTempFile; + /// Returns the environment variable which the dynamic library lookup path /// resides in for this platform. pub fn dylib_path_var() -> &'static str { @@ -126,3 +129,111 @@ pub fn parse_value_from_args<'a>(args: &'a [OsString], key: &str) -> Option<&'a None } + +/// A wrapper around [`Command`] that adds support for arg files. +/// This is useful as we have some commands that can get very long and at times +/// hit the OS limit (usually Windows) +/// +/// This implementation is based off the of `ProcessBuilder` implementation in Cargo +/// but simplified. +/// +/// NOTE: In most scenarios we want to avoid arg files as it makes debugging more complicated +/// so we try to avoid it if the command is not close the the OS limit. +#[derive(Debug)] +pub struct ArgFileCommand { + command: Command, + args: Vec, +} + +impl ArgFileCommand { + #[track_caller] + pub fn new>(program: S) -> Self { + let command = Command::new(program); + Self { command, args: Vec::new() } + } + pub fn arg>(&mut self, arg: S) -> &mut Self { + self.args.push(arg.as_ref().to_os_string()); + self + } + + pub fn args(&mut self, args: I) -> &mut Self + where + I: IntoIterator, + S: AsRef, + { + self.args.extend(args.into_iter().map(|s| s.as_ref().to_os_string())); + self + } + + pub fn env(&mut self, key: K, val: V) -> &mut Self + where + K: AsRef, + V: AsRef, + { + self.command.env(key, val); + self + } + + pub fn get_envs(&self) -> CommandEnvs<'_> { + self.command.get_envs() + } + + pub fn env_remove>(&mut self, key: K) -> &mut Self { + self.command.env_remove(key); + self + } + + pub fn current_dir>(&mut self, dir: P) -> &mut Self { + self.command.current_dir(dir); + self + } + + pub fn stdin(&mut self, stdin: std::process::Stdio) -> &mut Self { + self.command.stdin(stdin); + self + } + + pub fn build(mut self) -> std::io::Result<(Command, NamedTempFile)> { + let mut tmp = tempfile::Builder::new().prefix("bootstrap-argfile.").tempfile()?; + + // On Windows there is a hard limit of ~32KB, so we cut off at 30KB to + // give some buffer just incase. + #[cfg(windows)] + let threshold: usize = 30 * 1024; + // On unix the limit is defined by ARG_MAX. If its not explicitly set we set it to 1MB + // which is fairly large but lower than the ~2MB that it defaults to on most systems. + #[cfg(unix)] + let threshold: usize = + std::env::var("ARG_MAX").ok().and_then(|v| v.parse().ok()).unwrap_or(1024 * 1024); + + let total_arg_len: usize = self.args.iter().map(|a| a.len() + 1).sum(); + if total_arg_len <= threshold { + self.command.args(self.args); + return Ok((self.command, tmp)); + } + + let mut arg = OsString::from("@"); + arg.push(tmp.path()); + self.command.arg(arg); + + let cap = self.args.iter().map(|arg| arg.len() + 1).sum::(); + let mut buf = Vec::with_capacity(cap); + for arg in &self.args { + let arg = arg.to_str().ok_or_else(|| { + std::io::Error::other(format!( + "argument for argfile contains invalid UTF-8 characters: `{}`", + arg.to_string_lossy() + )) + })?; + if arg.contains('\n') { + return Err(std::io::Error::other(format!( + "argument for argfile contains newlines: `{arg}`" + ))); + } + writeln!(buf, "{arg}")?; + } + tmp.write_all(&buf)?; + + Ok((self.command, tmp)) + } +} diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index 97ddde799a757..51af86472ff99 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -2,7 +2,12 @@ # dynamically in CI from ci.yml. runners: - &base-job - env: { } + env: + # This force enables the new Cargo build-dir layout for all CI jobs. + # The new layout will become the default soon so we enable it on everything + # to catch issues as soon as possible. + # See: https://github.com/rust-lang/cargo/issues/15010 + CARGO_UNSTABLE_BUILD_DIR_NEW_LAYOUT: true - &job-linux-4c os: ubuntu-24.04 diff --git a/src/tools/compiletest/Cargo.toml b/src/tools/compiletest/Cargo.toml index f9cbea591862a..0d8cbdf789b3f 100644 --- a/src/tools/compiletest/Cargo.toml +++ b/src/tools/compiletest/Cargo.toml @@ -31,6 +31,7 @@ rustfix = "0.8.1" semver = { version = "1.0.23", features = ["serde"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +tempfile = "3.23.0" tracing = "0.1" tracing-subscriber = { version = "0.3.3", default-features = false, features = ["ansi", "env-filter", "fmt", "parking_lot", "smallvec"] } unified-diff = "0.2.1" diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index a2bf6c0839df4..c527b527997c7 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -23,7 +23,7 @@ use crate::errors::{Error, ErrorKind, load_errors}; use crate::output_capture::ConsoleOut; use crate::read2::{Truncated, read2_abbreviated}; use crate::runtest::compute_diff::{DiffLine, diff_by_lines, make_diff, write_diff}; -use crate::util::{Utf8PathBufExt, add_dylib_path, static_regex}; +use crate::util::{ArgFileCommand, Utf8PathBufExt, add_dylib_path, static_regex}; use crate::{json, stamp_file_path}; // Helper modules that implement test running logic for each test suite. @@ -383,7 +383,7 @@ impl<'test> TestCx<'test> { } } - /// Runs a [`Command`] and waits for it to finish, then converts its exit + /// Runs a [`ArgFileCommand`] and waits for it to finish, then converts its exit /// status and output streams into a [`ProcRes`]. /// /// The command might have succeeded or failed; it is the caller's @@ -393,7 +393,8 @@ impl<'test> TestCx<'test> { /// Panics if the command couldn't be executed at all /// (e.g. because the executable could not be found). #[must_use = "caller should check whether the command succeeded"] - fn run_command_to_procres(&self, cmd: &mut Command) -> ProcRes { + fn run_command_to_procres(&self, cmd: ArgFileCommand) -> ProcRes { + let (mut cmd, _arg_file) = cmd.build().unwrap(); let output = cmd .output() .unwrap_or_else(|e| self.fatal(&format!("failed to exec `{cmd:?}` because: {e}"))); diff --git a/src/tools/compiletest/src/runtest/coverage.rs b/src/tools/compiletest/src/runtest/coverage.rs index 5b94d9567d8fc..437f788567dbf 100644 --- a/src/tools/compiletest/src/runtest/coverage.rs +++ b/src/tools/compiletest/src/runtest/coverage.rs @@ -8,7 +8,7 @@ use glob::glob; use crate::common::{TestSuite, UI_COVERAGE, UI_COVERAGE_MAP}; use crate::runtest::{Emit, ProcRes, TestCx, WillExecute}; -use crate::util::static_regex; +use crate::util::{ArgFileCommand, static_regex}; impl<'test> TestCx<'test> { fn coverage_dump_path(&self) -> &Utf8Path { @@ -27,9 +27,9 @@ impl<'test> TestCx<'test> { } drop(proc_res); - let mut dump_command = Command::new(coverage_dump_path); + let mut dump_command = ArgFileCommand::new(coverage_dump_path); dump_command.arg(llvm_ir_path); - let proc_res = self.run_command_to_procres(&mut dump_command); + let proc_res = self.run_command_to_procres(dump_command); if !proc_res.status.success() { self.fatal_proc_rec("coverage-dump failed!", &proc_res); } @@ -227,7 +227,11 @@ impl<'test> TestCx<'test> { } } - fn run_llvm_tool(&self, name: &str, configure_cmd_fn: impl FnOnce(&mut Command)) -> ProcRes { + fn run_llvm_tool( + &self, + name: &str, + configure_cmd_fn: impl FnOnce(&mut ArgFileCommand), + ) -> ProcRes { let tool_path = self .config .llvm_bin_dir @@ -235,10 +239,10 @@ impl<'test> TestCx<'test> { .expect("this test expects the LLVM bin dir to be available") .join(name); - let mut cmd = Command::new(tool_path); + let mut cmd = ArgFileCommand::new(tool_path); configure_cmd_fn(&mut cmd); - self.run_command_to_procres(&mut cmd) + self.run_command_to_procres(cmd) } fn normalize_coverage_output(&self, coverage: &str) -> Result { diff --git a/src/tools/compiletest/src/runtest/debuginfo.rs b/src/tools/compiletest/src/runtest/debuginfo.rs index f2f13bc882647..6db74750e89d9 100644 --- a/src/tools/compiletest/src/runtest/debuginfo.rs +++ b/src/tools/compiletest/src/runtest/debuginfo.rs @@ -9,6 +9,7 @@ use tracing::debug; use super::debugger::DebuggerCommands; use super::{Debugger, Emit, ProcRes, TestCx, Truncated, WillExecute}; use crate::debuggers::extract_gdb_version; +use crate::util::ArgFileCommand; impl TestCx<'_> { pub(super) fn run_debuginfo_test(&self) { @@ -458,7 +459,7 @@ impl TestCx<'_> { // make sure `PATH` points to all the dlls necessary to run the debugee let path = prepend_to_path(&self.config.target_run_lib_path); - let mut cmd = Command::new(lldb); + let mut cmd = ArgFileCommand::new(lldb); cmd.arg("--one-line") .arg("script --language python -- import lldb_batchmode; lldb_batchmode.main()") .env("LLDB_BATCHMODE_TARGET_PATH", test_executable) @@ -467,7 +468,7 @@ impl TestCx<'_> { .env("PYTHONPATH", pythonpath) .env("PATH", path); - self.run_command_to_procres(&mut cmd) + self.run_command_to_procres(cmd) } } diff --git a/src/tools/compiletest/src/runtest/js_doc.rs b/src/tools/compiletest/src/runtest/js_doc.rs index 53c40515af74f..1b5d4608f2728 100644 --- a/src/tools/compiletest/src/runtest/js_doc.rs +++ b/src/tools/compiletest/src/runtest/js_doc.rs @@ -1,6 +1,5 @@ -use std::process::Command; - use super::{DocKind, TestCx}; +use crate::util::ArgFileCommand; impl TestCx<'_> { pub(super) fn run_rustdoc_js_test(&self) { @@ -10,18 +9,18 @@ impl TestCx<'_> { self.document(&out_dir, DocKind::Html); let file_stem = self.testpaths.file.file_stem().expect("no file stem"); - let res = self.run_command_to_procres( - Command::new(&nodejs) - .arg(self.config.src_root.join("src/tools/rustdoc-js/tester.js")) - .arg("--doc-folder") - .arg(out_dir) - .arg("--crate-name") - .arg(file_stem.replace("-", "_")) - .arg("--test-file") - .arg(self.testpaths.file.with_extension("js")) - .arg("--revision") - .arg(self.revision.unwrap_or_default()), - ); + + let mut cmd = ArgFileCommand::new(&nodejs); + cmd.arg(self.config.src_root.join("src/tools/rustdoc-js/tester.js")) + .arg("--doc-folder") + .arg(out_dir) + .arg("--crate-name") + .arg(file_stem.replace("-", "_")) + .arg("--test-file") + .arg(self.testpaths.file.with_extension("js")) + .arg("--revision") + .arg(self.revision.unwrap_or_default()); + let res = self.run_command_to_procres(cmd); if !res.status.success() { self.fatal_proc_rec("rustdoc-js test failed!", &res); } diff --git a/src/tools/compiletest/src/runtest/run_make.rs b/src/tools/compiletest/src/runtest/run_make.rs index 1044683ae6426..e1aaa2a03880f 100644 --- a/src/tools/compiletest/src/runtest/run_make.rs +++ b/src/tools/compiletest/src/runtest/run_make.rs @@ -1,3 +1,4 @@ +use std::path::{Path, PathBuf}; use std::process::{Command, Output, Stdio}; use std::{env, fs}; @@ -6,7 +7,7 @@ use camino::{Utf8Path, Utf8PathBuf}; use super::{ProcRes, TestCx, disable_error_reporting}; use crate::common::TestSuite; -use crate::util::{copy_dir_all, dylib_env_var}; +use crate::util::{ArgFileCommand, copy_dir_all, dylib_env_var}; impl TestCx<'_> { pub(super) fn run_rmake_test(&self) { @@ -66,8 +67,8 @@ impl TestCx<'_> { // build// // ├── bootstrap-tools/ // │ ├── /release/librun_make_support.rlib // <- support rlib itself - // │ ├── /release/deps/ // <- deps - // │ └── release/deps/ // <- deps of deps + // │ ├── /release/build///out // <- deps + // │ └── release/build///out // <- deps of deps // ``` // // FIXME(jieyouxu): there almost certainly is a better way to do this (specifically how the @@ -77,8 +78,8 @@ impl TestCx<'_> { let support_host_path = tools_bin.join(&self.config.host).join("release"); let support_lib_path = support_host_path.join("librun_make_support.rlib"); - let support_lib_deps = support_host_path.join("deps"); - let support_lib_deps_deps = tools_bin.join("release").join("deps"); + let support_lib_deps = discover_out_dirs(support_host_path.join("build")); + let support_lib_deps_deps = discover_out_dirs(tools_bin.join("release").join("build")); // To compile the recipe with rustc, we need to provide suitable dynamic library search // paths to rustc. This includes both: @@ -100,6 +101,10 @@ impl TestCx<'_> { p }; + let out_dirs_to_args = |paths: Vec| { + paths.into_iter().map(|p| format!("-Ldependency={}", p.display())).collect::>() + }; + // run-make-support and run-make tests are compiled using the stage0 compiler // If the stage is 0, then the compiler that we test (either bootstrap or an explicitly // set compiler) is the one that actually compiled run-make-support. @@ -108,7 +113,7 @@ impl TestCx<'_> { .stage0_rustc_path .as_ref() .expect("stage0 rustc is required to run run-make tests"); - let mut rustc = Command::new(&stage0_rustc); + let mut rustc = ArgFileCommand::new(&stage0_rustc); rustc // `rmake.rs` **must** be buildable by a stable compiler, it may not use *any* unstable // library or compiler features. Here, we force the stage 0 rustc to consider itself as @@ -119,8 +124,8 @@ impl TestCx<'_> { .arg(&recipe_bin) // Specify library search paths for `run_make_support`. .arg(format!("-Ldependency={}", &support_lib_path.parent().unwrap())) - .arg(format!("-Ldependency={}", &support_lib_deps)) - .arg(format!("-Ldependency={}", &support_lib_deps_deps)) + .args(out_dirs_to_args(support_lib_deps)) + .args(out_dirs_to_args(support_lib_deps_deps)) // Provide `run_make_support` as extern prelude, so test writers don't need to write // `extern run_make_support;`. .arg("--extern") @@ -134,7 +139,7 @@ impl TestCx<'_> { rustc.arg("-Dunused_must_use"); // Now run rustc to build the recipe. - let res = self.run_command_to_procres(&mut rustc); + let res = self.run_command_to_procres(rustc); if !res.status.success() { self.fatal_proc_rec("run-make test failed: could not build `rmake.rs` recipe", &res); } @@ -337,3 +342,19 @@ impl TestCx<'_> { } } } + +/// Gets all of the `out` dirs in a given Cargo `build-dir//build` dir. +fn discover_out_dirs(dir: Utf8PathBuf) -> Vec { + let read_dir = |path: &Path| path.read_dir().ok().into_iter().flatten().filter_map(Result::ok); + let contents = dir + .read_dir() + .unwrap_or_else(|e| panic!("Couldn't read {}: {}", dir, e)) + .map(|e| e.unwrap()) + .flat_map(|e| read_dir(&e.path())) + .flat_map(|e| read_dir(&e.path())) + .map(|e| e.path()) + .filter(|path| path.ends_with("out")) + .collect::>(); + + return contents; +} diff --git a/src/tools/compiletest/src/runtest/rustdoc.rs b/src/tools/compiletest/src/runtest/rustdoc.rs index 8907848e0ca5d..a6a826a377301 100644 --- a/src/tools/compiletest/src/runtest/rustdoc.rs +++ b/src/tools/compiletest/src/runtest/rustdoc.rs @@ -1,6 +1,5 @@ -use std::process::Command; - use super::{DocKind, TestCx, remove_and_create_dir_all}; +use crate::util::ArgFileCommand; impl TestCx<'_> { pub(super) fn run_rustdoc_html_test(&self) { @@ -19,14 +18,14 @@ impl TestCx<'_> { if self.props.check_test_line_numbers_match { self.check_rustdoc_test_option(proc_res); } else { - let mut cmd = Command::new(&self.config.python); + let mut cmd = ArgFileCommand::new(&self.config.python); cmd.arg(self.config.src_root.join("src/etc/htmldocck.py")) .arg(&out_dir) .arg(&self.testpaths.file); if self.config.bless { cmd.arg("--bless"); } - let res = self.run_command_to_procres(&mut cmd); + let res = self.run_command_to_procres(cmd); if !res.status.success() { self.fatal_proc_rec("htmldocck failed!", &res); } diff --git a/src/tools/compiletest/src/runtest/rustdoc_json.rs b/src/tools/compiletest/src/runtest/rustdoc_json.rs index d6afb77c2e8df..29626b1f40c04 100644 --- a/src/tools/compiletest/src/runtest/rustdoc_json.rs +++ b/src/tools/compiletest/src/runtest/rustdoc_json.rs @@ -1,6 +1,5 @@ -use std::process::Command; - use super::{DocKind, TestCx, remove_and_create_dir_all}; +use crate::util::ArgFileCommand; impl TestCx<'_> { pub(super) fn run_rustdoc_json_test(&self) { @@ -23,13 +22,9 @@ impl TestCx<'_> { self.fatal_proc_rec("rustdoc failed!", &proc_res); } - let res = self.run_command_to_procres( - Command::new(self.config.jsondocck_path.as_ref().unwrap()) - .arg("--doc-dir") - .arg(&out_dir) - .arg("--template") - .arg(&self.testpaths.file), - ); + let mut cmd = ArgFileCommand::new(self.config.jsondocck_path.as_ref().unwrap()); + cmd.arg("--doc-dir").arg(&out_dir).arg("--template").arg(&self.testpaths.file); + let res = self.run_command_to_procres(cmd); if !res.status.success() { self.fatal_proc_rec_general("jsondocck failed!", None, &res, || { @@ -41,9 +36,9 @@ impl TestCx<'_> { let mut json_out = out_dir.join(self.testpaths.file.file_stem().unwrap()); json_out.set_extension("json"); - let res = self.run_command_to_procres( - Command::new(self.config.jsondoclint_path.as_ref().unwrap()).arg(&json_out), - ); + let mut cmd = ArgFileCommand::new(self.config.jsondoclint_path.as_ref().unwrap()); + cmd.arg(&json_out); + let res = self.run_command_to_procres(cmd); if !res.status.success() { self.fatal_proc_rec("jsondoclint failed!", &res); diff --git a/src/tools/compiletest/src/util.rs b/src/tools/compiletest/src/util.rs index 2bfdd8889399d..db3b08801880e 100644 --- a/src/tools/compiletest/src/util.rs +++ b/src/tools/compiletest/src/util.rs @@ -1,7 +1,11 @@ use std::env; -use std::process::Command; +use std::ffi::{OsStr, OsString}; +use std::io::Write; +use std::path::Path; +use std::process::{Command, CommandEnvs}; use camino::{Utf8Path, Utf8PathBuf}; +use tempfile::NamedTempFile; #[cfg(test)] mod tests; @@ -149,3 +153,112 @@ macro_rules! string_enum { } pub(crate) use string_enum; + +/// A wrapper around [`Command`] that adds support for arg files. +/// This is useful as we have some commands that can get very long and at times +/// hit the OS limit (usually Windows) +/// +/// This implementation is based off the of `ProcessBuilder` implementation in Cargo +/// but simplified. +/// +/// NOTE: In most scenarios we want to avoid arg files as it makes debugging more complicated +/// so we try to avoid it if the command is not close the the OS limit. +#[derive(Debug)] +pub(crate) struct ArgFileCommand { + command: Command, + args: Vec, +} + +#[allow(dead_code)] // Roughly match the `std::process::Command` API +impl ArgFileCommand { + #[track_caller] + pub(crate) fn new>(program: S) -> Self { + let command = Command::new(program); + Self { command, args: Vec::new() } + } + pub(crate) fn arg>(&mut self, arg: S) -> &mut Self { + self.args.push(arg.as_ref().to_os_string()); + self + } + + pub(crate) fn args(&mut self, args: I) -> &mut Self + where + I: IntoIterator, + S: AsRef, + { + self.args.extend(args.into_iter().map(|s| s.as_ref().to_os_string())); + self + } + + pub(crate) fn env(&mut self, key: K, val: V) -> &mut Self + where + K: AsRef, + V: AsRef, + { + self.command.env(key, val); + self + } + + pub(crate) fn get_envs(&self) -> CommandEnvs<'_> { + self.command.get_envs() + } + + pub(crate) fn env_remove>(&mut self, key: K) -> &mut Self { + self.command.env_remove(key); + self + } + + pub(crate) fn current_dir>(&mut self, dir: P) -> &mut Self { + self.command.current_dir(dir); + self + } + + pub(crate) fn stdin(&mut self, stdin: std::process::Stdio) -> &mut Self { + self.command.stdin(stdin); + self + } + + pub(crate) fn build(mut self) -> std::io::Result<(Command, NamedTempFile)> { + let mut tmp = tempfile::Builder::new().prefix("compiletest-argfile.").tempfile()?; + + // On Windows there is a hard limit of ~32KB, so we cut off at 30KB to + // give some buffer just incase. + #[cfg(windows)] + let threshold: usize = 30 * 1024; + // On unix the limit is defined by ARG_MAX. If its not explicitly set we set it to 1MB + // which is fairly large but lower than the ~2MB that it defaults to on most systems. + #[cfg(unix)] + let threshold: usize = + std::env::var("ARG_MAX").ok().and_then(|v| v.parse().ok()).unwrap_or(1024 * 1024); + + let total_arg_len: usize = self.args.iter().map(|a| a.len() + 1).sum(); + if total_arg_len <= threshold { + self.command.args(self.args); + return Ok((self.command, tmp)); + } + + let mut arg = OsString::from("@"); + arg.push(tmp.path()); + self.command.arg(arg); + + let cap = self.args.iter().map(|arg| arg.len() + 1).sum::(); + let mut buf = Vec::with_capacity(cap); + for arg in &self.args { + let arg = arg.to_str().ok_or_else(|| { + std::io::Error::other(format!( + "argument for argfile contains invalid UTF-8 characters: `{}`", + arg.to_string_lossy() + )) + })?; + if arg.contains('\n') { + return Err(std::io::Error::other(format!( + "argument for argfile contains newlines: `{arg}`" + ))); + } + writeln!(buf, "{arg}")?; + } + tmp.write_all(&buf)?; + + Ok((self.command, tmp)) + } +} diff --git a/src/tools/miri/test-cargo-miri/run-test.py b/src/tools/miri/test-cargo-miri/run-test.py index 6b3b6343b8258..bf1272f5bca0b 100755 --- a/src/tools/miri/test-cargo-miri/run-test.py +++ b/src/tools/miri/test-cargo-miri/run-test.py @@ -215,7 +215,7 @@ def test_cargo_miri_multi_target(): if os.listdir(target_dir) != ["miri"]: fail(f"`{target_dir}` contains unexpected files") # Ensure something exists inside that target dir. - os.access(os.path.join(target_dir, "miri", "debug", "deps"), os.F_OK) + os.access(os.path.join(target_dir, "miri", "debug"), os.F_OK) print("\nTEST SUCCESSFUL!") sys.exit(0) diff --git a/src/tools/rustfmt/.github/workflows/linux.yml b/src/tools/rustfmt/.github/workflows/linux.yml index 369b048f12fec..c8cbf00804bc3 100644 --- a/src/tools/rustfmt/.github/workflows/linux.yml +++ b/src/tools/rustfmt/.github/workflows/linux.yml @@ -36,4 +36,7 @@ jobs: rustup target add ${{ matrix.target }} - name: Build and Test + env: + RUSTFLAGS: -D warnings + CARGO_UNSTABLE_BUILD_DIR_NEW_LAYOUT: true run: ./ci/build_and_test.sh diff --git a/src/tools/rustfmt/.github/workflows/mac.yml b/src/tools/rustfmt/.github/workflows/mac.yml index b2e8401131b73..0838800dff06e 100644 --- a/src/tools/rustfmt/.github/workflows/mac.yml +++ b/src/tools/rustfmt/.github/workflows/mac.yml @@ -32,4 +32,7 @@ jobs: rustup target add ${{ matrix.target }} - name: Build and Test + env: + RUSTFLAGS: -D warnings + CARGO_UNSTABLE_BUILD_DIR_NEW_LAYOUT: true run: ./ci/build_and_test.sh diff --git a/src/tools/rustfmt/.github/workflows/windows.yml b/src/tools/rustfmt/.github/workflows/windows.yml index ef8d54fb93f57..af47fddcf59e1 100644 --- a/src/tools/rustfmt/.github/workflows/windows.yml +++ b/src/tools/rustfmt/.github/workflows/windows.yml @@ -59,4 +59,7 @@ jobs: - name: Build and Test shell: cmd + env: + RUSTFLAGS: -D warnings + CARGO_UNSTABLE_BUILD_DIR_NEW_LAYOUT: true run: ci\build_and_test.bat diff --git a/src/tools/rustfmt/src/test/mod.rs b/src/tools/rustfmt/src/test/mod.rs index 36e6aa84fc2d5..4eded7c49eb50 100644 --- a/src/tools/rustfmt/src/test/mod.rs +++ b/src/tools/rustfmt/src/test/mod.rs @@ -1077,8 +1077,23 @@ fn rustfmt() -> PathBuf { let mut me = env::current_exe().expect("failed to get current executable"); // Chop of the test name. me.pop(); - // Chop off `deps`. - me.pop(); + + // Handle Cargo's old and new filesystem layouts + // * v1: `target//deps/test-bin-[HASH][EXE]` + // * v2: `target//build//[HASH]/out/test-bin-[HASH][EXE]` + if me.ends_with("deps") { + // Chop off `deps`. + me.pop(); + } else if me.ends_with("out") { + // Chop off `out`. + me.pop(); + // Chop off ``. + me.pop(); + // Chop off ``. + me.pop(); + // Chop off `build`. + me.pop(); + } me.push("rustfmt"); assert!( diff --git a/src/tools/rustfmt/tests/cargo-fmt/main.rs b/src/tools/rustfmt/tests/cargo-fmt/main.rs index dcdbca07a77c9..63cc12521a8f7 100644 --- a/src/tools/rustfmt/tests/cargo-fmt/main.rs +++ b/src/tools/rustfmt/tests/cargo-fmt/main.rs @@ -10,8 +10,17 @@ use rustfmt_config_proc_macro::rustfmt_only_ci_test; fn cargo_fmt(args: &[&str]) -> (String, String) { let mut bin_dir = env::current_exe().unwrap(); bin_dir.pop(); // chop off test exe name + + // Handle Cargo's old and new filesystem layouts + // * v1: `target//deps/test-bin-[HASH][EXE]` + // * v2: `target//build//[HASH]/out/test-bin-[HASH][EXE]` if bin_dir.ends_with("deps") { bin_dir.pop(); + } else if bin_dir.ends_with("out") { + bin_dir.pop(); // chop off `out` + bin_dir.pop(); // chop off `` + bin_dir.pop(); // chop off `` + bin_dir.pop(); // chop off `build` } let cmd = bin_dir.join(format!("cargo-fmt{}", env::consts::EXE_SUFFIX));