From d7fb70b5430fe09c2c98fced46e5222c46c853b9 Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Tue, 25 Jul 2023 13:52:53 -0700 Subject: [PATCH 1/8] Add support for chunking; use clap for arg parsing --- Cargo.lock | 299 ++++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 1 + src/lib.rs | 76 ++++++++++-- src/test_runner.rs | 5 +- 4 files changed, 371 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5e54526..7f251c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,21 +2,174 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "anstream" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is-terminal", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" + +[[package]] +name = "anstyle-parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +dependencies = [ + "anstyle", + "windows-sys", +] + +[[package]] +name = "bitflags" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" + [[package]] name = "cargo-all-features" -version = "1.8.0" +version = "1.9.0" dependencies = [ + "clap", "itertools", "json", "termcolor", ] +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + +[[package]] +name = "clap" +version = "4.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd304a20bff958a57f04c4e96a2e7594cc4490a0e809cbd48bb6437edaa452d" +dependencies = [ + "clap_builder", + "clap_derive", + "once_cell", +] + +[[package]] +name = "clap_builder" +version = "4.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01c6a3f08f1fe5662a35cfe393aec09c4df95f60ee93b7556505260f75eee9e1" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + [[package]] name = "either" version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +[[package]] +name = "errno" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" + +[[package]] +name = "is-terminal" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +dependencies = [ + "hermit-abi", + "rustix", + "windows-sys", +] + [[package]] name = "itertools" version = "0.10.1" @@ -32,6 +185,72 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd" +[[package]] +name = "libc" +version = "0.2.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" + +[[package]] +name = "linux-raw-sys" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0" + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "proc-macro2" +version = "1.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fe8a65d69dd0808184ebb5f836ab526bb259db23c657efa38711b1072ee47f0" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustix" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "2.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b60f673f44a8255b9c8c657daf66a596d435f2da81a555b06dc644d080ba45e0" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "termcolor" version = "1.1.2" @@ -41,6 +260,18 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "unicode-ident" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" + +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + [[package]] name = "winapi" version = "0.3.9" @@ -71,3 +302,69 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" diff --git a/Cargo.toml b/Cargo.toml index 0b4d34a..155c10b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,3 +20,4 @@ path = "src/bin/cargo-test-all-features.rs" json = "0.12" itertools = "0.10" termcolor = "1" +clap = { version = "4.3.19", features = ["derive"] } diff --git a/src/lib.rs b/src/lib.rs index 82c909f..1a153e2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ +use clap::{error::ErrorKind, Command, Parser}; use std::{env, error, ffi, process}; pub mod cargo_metadata; @@ -5,18 +6,77 @@ pub mod features_finder; pub mod test_runner; mod types; +#[derive(Parser)] +#[command(author, version, about = "See https://crates.io/crates/cargo-all-features", long_about = None)] +struct Cli { + #[arg( + long, + default_value_t = 1, + help = "Split the workspace into n chunks, each chunk containing a roughly equal number of crates" + )] + n_chunks: usize, + #[arg( + long, + default_value_t = 0, + requires = "n_chunks", + help = "Which chunk to test, indexed at 0" + )] + chunk: usize, + + #[arg( + help = "arguments to pass down to cargo", + allow_hyphen_values = true, + trailing_var_arg = true + )] + cargo_args: Vec, +} + pub fn run(cargo_command: test_runner::CargoCommand) -> Result<(), Box> { - if let Some(arg) = env::args().nth(1) { - if arg == "--help" { - println!("See https://crates.io/crates/cargo-all-features"); - return Ok(()); - } + let cli = Cli::parse(); + let mut cmd = Command::new("cargo-all-features"); + + if cli.chunk >= cli.n_chunks { + cmd.error( + ErrorKind::InvalidValue, + "Must not ask for chunks out of bounds", + ) + .print()?; + process::exit(1); + } + + if cli.n_chunks == 0 { + cmd.error(ErrorKind::InvalidValue, "--n-chunks must be at least 1") + .print()?; + process::exit(1) } let packages = determine_packages_to_test()?; - for package in packages { - let outcome = test_all_features_for_package(&package, cargo_command)?; + // chunks() takes a chunk size, not a number of chunks + // we must adjust to deal with the fact that if things are not a perfect multiple, + // len / n_chunks will end up with an uncounted remainder chunk + let mut chunk_size = packages.len() / cli.n_chunks; + if packages.len() % cli.n_chunks != 0 { + chunk_size += 1; + } + + let chunk = if let Some(chunk) = packages.chunks(chunk_size).nth(cli.chunk) { + chunk + } else { + println!("Chunk is empty (did you ask for more chunks than there are packages?"); + return Ok(()); + }; + if cli.n_chunks != 1 { + let packages: String = chunk.iter().map(|p| [&p.name, ","]).flatten().collect(); + let packages = packages.trim_end_matches(','); + println!( + "Running on chunk {} out of {} ({chunk_size} packages: {packages})", + cli.chunk, cli.n_chunks + ); + } + + for package in chunk { + let outcome = test_all_features_for_package(&package, cargo_command, &cli.cargo_args)?; if let TestOutcome::Fail(exit_status) = outcome { process::exit(exit_status.code().unwrap()); @@ -29,6 +89,7 @@ pub fn run(cargo_command: test_runner::CargoCommand) -> Result<(), Box Result> { let feature_sets = crate::features_finder::fetch_feature_sets(package); @@ -37,6 +98,7 @@ fn test_all_features_for_package( command, package.name.clone(), feature_set.clone(), + cargo_args, package .manifest_path .parent() diff --git a/src/test_runner.rs b/src/test_runner.rs index 1fa253e..44d8933 100644 --- a/src/test_runner.rs +++ b/src/test_runner.rs @@ -1,5 +1,5 @@ use crate::types::FeatureList; -use std::{env, error, path, process}; +use std::{error, path, process}; use termcolor::WriteColor; pub struct TestRunner { @@ -16,6 +16,7 @@ impl TestRunner { cargo_command: CargoCommand, crate_name: String, feature_set: FeatureList, + cargo_args: &[String], working_dir: path::PathBuf, ) -> Self { let mut command = process::Command::new(&crate::cargo_cmd()); @@ -39,7 +40,7 @@ impl TestRunner { } // Pass through cargo args - for arg in env::args().skip(2) { + for arg in cargo_args { command.arg(&arg); } From 1f56d079d96913650f68280e212973b31b240509 Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Wed, 26 Jul 2023 14:51:39 -0700 Subject: [PATCH 2/8] Requires both --- src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib.rs b/src/lib.rs index 1a153e2..768e93c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,6 +12,7 @@ struct Cli { #[arg( long, default_value_t = 1, + requires = "chunk", help = "Split the workspace into n chunks, each chunk containing a roughly equal number of crates" )] n_chunks: usize, From 3785ae35ff5b32909977af18252b8001fcaeb8ec Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Wed, 26 Jul 2023 14:55:27 -0700 Subject: [PATCH 3/8] 1-indexed --- src/lib.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 768e93c..0117fc6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,9 +18,9 @@ struct Cli { n_chunks: usize, #[arg( long, - default_value_t = 0, + default_value_t = 1, requires = "n_chunks", - help = "Which chunk to test, indexed at 0" + help = "Which chunk to test, indexed at 1" )] chunk: usize, @@ -36,7 +36,7 @@ pub fn run(cargo_command: test_runner::CargoCommand) -> Result<(), Box= cli.n_chunks { + if cli.chunk > cli.n_chunks || cli.chunk < 1 { cmd.error( ErrorKind::InvalidValue, "Must not ask for chunks out of bounds", @@ -61,7 +61,8 @@ pub fn run(cargo_command: test_runner::CargoCommand) -> Result<(), Box Date: Wed, 26 Jul 2023 15:28:19 -0700 Subject: [PATCH 4/8] Fix cargo subcommand behavior --- src/lib.rs | 19 ++++++++++++++++--- src/test_runner.rs | 23 ++++++++++++++++++----- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 0117fc6..0eb6cf0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,7 +6,19 @@ pub mod features_finder; pub mod test_runner; mod types; -#[derive(Parser)] +#[derive(Parser, Clone)] +#[command(author, version, about = "See https://crates.io/crates/cargo-all-features", long_about = None)] +#[command(bin_name = "cargo")] +/// The cargo wrapper so that `cargo check-all-features ...` will work, since it internally invokes `check-all-features` with itself +/// as the first argument +enum CargoCli { + #[command(name = "check-all-features")] + #[command(alias = "build-all-features")] + #[command(alias = "test-all-features")] + Subcommand(Cli), +} + +#[derive(Parser, Clone)] #[command(author, version, about = "See https://crates.io/crates/cargo-all-features", long_about = None)] struct Cli { #[arg( @@ -33,9 +45,10 @@ struct Cli { } pub fn run(cargo_command: test_runner::CargoCommand) -> Result<(), Box> { - let cli = Cli::parse(); + let cli = match CargoCli::parse() { + CargoCli::Subcommand(cli) => cli, + }; let mut cmd = Command::new("cargo-all-features"); - if cli.chunk > cli.n_chunks || cli.chunk < 1 { cmd.error( ErrorKind::InvalidValue, diff --git a/src/test_runner.rs b/src/test_runner.rs index 44d8933..12ad293 100644 --- a/src/test_runner.rs +++ b/src/test_runner.rs @@ -21,11 +21,7 @@ impl TestRunner { ) -> Self { let mut command = process::Command::new(&crate::cargo_cmd()); - command.arg(match cargo_command { - CargoCommand::Build => "build", - CargoCommand::Check => "check", - CargoCommand::Test => "test", - }); + command.arg(cargo_command.get_name()); command.arg("--no-default-features"); let mut features = feature_set @@ -91,3 +87,20 @@ pub enum CargoCommand { Check, Test, } + +impl CargoCommand { + pub fn get_name(self) -> &'static str { + match self { + CargoCommand::Build => "build", + CargoCommand::Check => "check", + CargoCommand::Test => "test", + } + } + pub fn get_cli_name(self) -> &'static str { + match self { + CargoCommand::Build => "build-all-features", + CargoCommand::Check => "check-all-features", + CargoCommand::Test => "test-all-features", + } + } +} From 591e98d05e2f3b51e3015ef294e63f45a6b6001b Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Wed, 26 Jul 2023 15:44:20 -0700 Subject: [PATCH 5/8] clippy --- src/lib.rs | 8 +++----- src/test_runner.rs | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 0eb6cf0..1c237cd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -45,9 +45,7 @@ struct Cli { } pub fn run(cargo_command: test_runner::CargoCommand) -> Result<(), Box> { - let cli = match CargoCli::parse() { - CargoCli::Subcommand(cli) => cli, - }; + let CargoCli::Subcommand(cli) = CargoCli::parse(); let mut cmd = Command::new("cargo-all-features"); if cli.chunk > cli.n_chunks || cli.chunk < 1 { cmd.error( @@ -82,7 +80,7 @@ pub fn run(cargo_command: test_runner::CargoCommand) -> Result<(), Box Result<(), Box Date: Wed, 26 Jul 2023 15:46:13 -0700 Subject: [PATCH 6/8] more clippy --- src/types.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/types.rs b/src/types.rs index 6d7a2b8..3bc8044 100644 --- a/src/types.rs +++ b/src/types.rs @@ -22,13 +22,13 @@ impl FromIterator for FeatureList { impl AsMut<::Target> for &mut FeatureList { fn as_mut(&mut self) -> &mut ::Target { - self.deref_mut() + self } } impl AsRef<::Target> for &FeatureList { fn as_ref(&self) -> &::Target { - self.deref() + self } } From 26066c626d810e767017332851273dabc2d92ff3 Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Wed, 26 Jul 2023 16:43:27 -0700 Subject: [PATCH 7/8] docs --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index d2b4222..febb193 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,8 @@ max_combination_size = 4 allowlist = ["foo", "bar"] ``` +The project also supports chunking: `--n-chunks 3 --chunks 1` will split the crates being tested into three sets (alphabetically, currently), and run the requested command for the first set of crates only. This is useful for splitting up CI jobs or performing disk cleanups since for large workspaces `check-all-features` and friends can take a very long time and produce a ton of artifacts. + ## License Licensed under either of From 98524e04d4f782b832b5ddb56d12590e2c3e0fdf Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Mon, 31 Jul 2023 12:00:38 -0700 Subject: [PATCH 8/8] clippy --- src/types.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types.rs b/src/types.rs index 3bc8044..a5c3926 100644 --- a/src/types.rs +++ b/src/types.rs @@ -48,7 +48,7 @@ impl Deref for FeatureList { impl AsRef for &Feature { fn as_ref(&self) -> &str { - self.deref() + self } }