diff --git a/benches/benchsuite/Cargo.toml b/benches/benchsuite/Cargo.toml index c16f4f86646..59776da2461 100644 --- a/benches/benchsuite/Cargo.toml +++ b/benches/benchsuite/Cargo.toml @@ -10,12 +10,20 @@ description = "Benchmarking suite for Cargo." [dependencies] cargo = { path = "../.." } +cargo-test-support = { path = "../../crates/cargo-test-support" } # Consider removing html_reports in 0.4 and switching to `cargo criterion`. criterion = { version = "0.3.5", features = ["html_reports"] } flate2 = { version = "1.0.3", default-features = false, features = ["zlib"] } tar = { version = "0.4.38", default-features = false } url = "2.2.2" +[lib] +bench = false + [[bench]] name = "resolve" harness = false + +[[bench]] +name = "workspace_initialization" +harness = false diff --git a/benches/benchsuite/benches/resolve.rs b/benches/benchsuite/benches/resolve.rs index 84455232eb9..d03cd620e27 100644 --- a/benches/benchsuite/benches/resolve.rs +++ b/benches/benchsuite/benches/resolve.rs @@ -1,145 +1,12 @@ +use benchsuite::fixtures; use cargo::core::compiler::{CompileKind, RustcTargetData}; -use cargo::core::resolver::features::{CliFeatures, FeatureOpts, FeatureResolver, ForceAllTargets}; -use cargo::core::resolver::{HasDevUnits, ResolveBehavior}; +use cargo::core::resolver::features::{FeatureOpts, FeatureResolver}; +use cargo::core::resolver::{CliFeatures, ForceAllTargets, HasDevUnits, ResolveBehavior}; use cargo::core::{PackageIdSpec, Workspace}; use cargo::ops::WorkspaceResolve; use cargo::Config; use criterion::{criterion_group, criterion_main, Criterion}; -use std::fs; -use std::path::{Path, PathBuf}; -use std::process::Command; -use url::Url; - -// This is an arbitrary commit that existed when I started. This helps -// ensure consistent results. It can be updated if needed, but that can -// make it harder to compare results with older versions of cargo. -const CRATES_IO_COMMIT: &str = "85f7bfd61ea4fee08ec68c468762e886b2aebec6"; - -fn setup() { - create_home(); - create_target_dir(); - clone_index(); - unpack_workspaces(); -} - -fn root() -> PathBuf { - let mut p = PathBuf::from(env!("CARGO_TARGET_TMPDIR")); - p.push("bench"); - p -} - -fn target_dir() -> PathBuf { - let mut p = root(); - p.push("target"); - p -} - -fn cargo_home() -> PathBuf { - let mut p = root(); - p.push("chome"); - p -} - -fn index() -> PathBuf { - let mut p = root(); - p.push("index"); - p -} - -fn workspaces_path() -> PathBuf { - let mut p = root(); - p.push("workspaces"); - p -} - -fn registry_url() -> Url { - Url::from_file_path(index()).unwrap() -} - -fn create_home() { - let home = cargo_home(); - if !home.exists() { - fs::create_dir_all(&home).unwrap(); - } - fs::write( - home.join("config.toml"), - format!( - r#" - [source.crates-io] - replace-with = 'local-snapshot' - - [source.local-snapshot] - registry = '{}' - "#, - registry_url() - ), - ) - .unwrap(); -} - -fn create_target_dir() { - // This is necessary to ensure the .rustc_info.json file is written. - // Otherwise it won't be written, and it is very expensive to create. - if !target_dir().exists() { - std::fs::create_dir_all(target_dir()).unwrap(); - } -} - -/// This clones crates.io at a specific point in time into tmp/index. -fn clone_index() { - let index = index(); - let maybe_git = |command: &str| { - let status = Command::new("git") - .current_dir(&index) - .args(command.split_whitespace().collect::>()) - .status() - .expect("git should be installed"); - status.success() - }; - let git = |command: &str| { - if !maybe_git(command) { - panic!("failed to run git command: {}", command); - } - }; - if index.exists() { - if maybe_git(&format!( - "rev-parse -q --verify {}^{{commit}}", - CRATES_IO_COMMIT - )) { - // Already fetched. - return; - } - } else { - fs::create_dir_all(&index).unwrap(); - git("init --bare"); - git("remote add origin https://github.com/rust-lang/crates.io-index"); - } - git(&format!("fetch origin {}", CRATES_IO_COMMIT)); - git("branch -f master FETCH_HEAD"); -} - -/// This unpacks the compressed workspace skeletons into tmp/workspaces. -fn unpack_workspaces() { - let ws_dir = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("workspaces"); - let archives = fs::read_dir(ws_dir) - .unwrap() - .map(|e| e.unwrap().path()) - .filter(|p| p.extension() == Some(std::ffi::OsStr::new("tgz"))); - for archive in archives { - let name = archive.file_stem().unwrap(); - let f = fs::File::open(&archive).unwrap(); - let f = flate2::read::GzDecoder::new(f); - let dest = workspaces_path().join(&name); - if dest.exists() { - fs::remove_dir_all(&dest).unwrap(); - } - let mut archive = tar::Archive::new(f); - archive.unpack(workspaces_path()).unwrap(); - } -} +use std::path::Path; struct ResolveInfo<'cfg> { ws: Workspace<'cfg>, @@ -152,36 +19,12 @@ struct ResolveInfo<'cfg> { ws_resolve: WorkspaceResolve<'cfg>, } -/// Vec of `(ws_name, ws_root)`. -fn workspaces() -> Vec<(String, PathBuf)> { - // CARGO_BENCH_WORKSPACES can be used to override, otherwise it just uses - // the workspaces in the workspaces directory. - let mut ps: Vec<_> = match std::env::var_os("CARGO_BENCH_WORKSPACES") { - Some(s) => std::env::split_paths(&s).collect(), - None => fs::read_dir(workspaces_path()) - .unwrap() - .map(|e| e.unwrap().path()) - // These currently fail in most cases on Windows due to long - // filenames in the git checkouts. - .filter(|p| { - !(cfg!(windows) - && matches!(p.file_name().unwrap().to_str().unwrap(), "servo" | "tikv")) - }) - .collect(), - }; - // Sort so it is consistent. - ps.sort(); - ps.into_iter() - .map(|p| (p.file_name().unwrap().to_str().unwrap().to_owned(), p)) - .collect() -} - /// Helper for resolving a workspace. This will run the resolver once to /// download everything, and returns all the data structures that are used /// during resolution. fn do_resolve<'cfg>(config: &'cfg Config, ws_root: &Path) -> ResolveInfo<'cfg> { let requested_kinds = [CompileKind::Host]; - let ws = cargo::core::Workspace::new(&ws_root.join("Cargo.toml"), config).unwrap(); + let ws = Workspace::new(&ws_root.join("Cargo.toml"), config).unwrap(); let target_data = RustcTargetData::new(&ws, &requested_kinds).unwrap(); let cli_features = CliFeatures::from_command_line(&[], false, true).unwrap(); let pkgs = cargo::ops::Packages::Default; @@ -212,38 +55,14 @@ fn do_resolve<'cfg>(config: &'cfg Config, ws_root: &Path) -> ResolveInfo<'cfg> { } } -/// Creates a new Config. -/// -/// This is separate from `do_resolve` to deal with the ownership and lifetime. -fn make_config(ws_root: &Path) -> Config { - let shell = cargo::core::Shell::new(); - let mut config = cargo::util::Config::new(shell, ws_root.to_path_buf(), cargo_home()); - // Configure is needed to set the target_dir which is needed to write - // the .rustc_info.json file which is very expensive. - config - .configure( - 0, - false, - None, - false, - false, - false, - &Some(target_dir()), - &[], - &[], - ) - .unwrap(); - config -} - /// Benchmark of the full `resolve_ws_with_opts` which runs the resolver /// twice, the feature resolver, and more. This is a major component of a /// regular cargo build. fn resolve_ws(c: &mut Criterion) { - setup(); + let fixtures = fixtures!(); let mut group = c.benchmark_group("resolve_ws"); - for (ws_name, ws_root) in workspaces() { - let config = make_config(&ws_root); + for (ws_name, ws_root) in fixtures.workspaces() { + let config = fixtures.make_config(&ws_root); // The resolver info is initialized only once in a lazy fashion. This // allows criterion to skip this workspace if the user passes a filter // on the command-line (like `cargo bench -- resolve_ws/tikv`). @@ -282,10 +101,10 @@ fn resolve_ws(c: &mut Criterion) { /// Benchmark of the feature resolver. fn feature_resolver(c: &mut Criterion) { - setup(); + let fixtures = fixtures!(); let mut group = c.benchmark_group("feature_resolver"); - for (ws_name, ws_root) in workspaces() { - let config = make_config(&ws_root); + for (ws_name, ws_root) in fixtures.workspaces() { + let config = fixtures.make_config(&ws_root); let mut lazy_info = None; group.bench_function(&ws_name, |b| { let ResolveInfo { diff --git a/benches/benchsuite/benches/workspace_initialization.rs b/benches/benchsuite/benches/workspace_initialization.rs new file mode 100644 index 00000000000..af68efe76ce --- /dev/null +++ b/benches/benchsuite/benches/workspace_initialization.rs @@ -0,0 +1,27 @@ +use benchsuite::fixtures; +use cargo::core::Workspace; +use criterion::{criterion_group, criterion_main, Criterion}; + +fn workspace_initialization(c: &mut Criterion) { + let fixtures = fixtures!(); + let mut group = c.benchmark_group("workspace_initialization"); + for (ws_name, ws_root) in fixtures.workspaces() { + let config = fixtures.make_config(&ws_root); + // The resolver info is initialized only once in a lazy fashion. This + // allows criterion to skip this workspace if the user passes a filter + // on the command-line (like `cargo bench -- workspace_initialization/tikv`). + group.bench_function(ws_name, |b| { + b.iter(|| Workspace::new(&ws_root.join("Cargo.toml"), &config).unwrap()) + }); + } + group.finish(); +} + +// Criterion complains about the measurement time being too small, but the +// measurement time doesn't seem important to me, what is more important is +// the number of iterations which defaults to 100, which seems like a +// reasonable default. Otherwise, the measurement time would need to be +// changed per workspace. We wouldn't want to spend 60s on every workspace, +// that would take too long and isn't necessary for the smaller workspaces. +criterion_group!(benches, workspace_initialization); +criterion_main!(benches); diff --git a/benches/benchsuite/src/lib.rs b/benches/benchsuite/src/lib.rs new file mode 100644 index 00000000000..42e0f513c26 --- /dev/null +++ b/benches/benchsuite/src/lib.rs @@ -0,0 +1,197 @@ +use cargo::Config; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use url::Url; + +#[macro_export] +macro_rules! fixtures { + () => { + $crate::Fixtures::new(env!("CARGO_TARGET_TMPDIR")) + }; +} + +// This is an arbitrary commit that existed when I started. This helps +// ensure consistent results. It can be updated if needed, but that can +// make it harder to compare results with older versions of cargo. +const CRATES_IO_COMMIT: &str = "85f7bfd61ea4fee08ec68c468762e886b2aebec6"; + +pub struct Fixtures { + cargo_target_tmpdir: PathBuf, +} + +impl Fixtures { + pub fn new(cargo_target_tmpdir: &str) -> Self { + let bench = Self { + cargo_target_tmpdir: PathBuf::from(cargo_target_tmpdir), + }; + bench.create_home(); + bench.create_target_dir(); + bench.clone_index(); + bench.unpack_workspaces(); + bench + } + + fn root(&self) -> PathBuf { + self.cargo_target_tmpdir.join("bench") + } + + fn target_dir(&self) -> PathBuf { + let mut p = self.root(); + p.push("target"); + p + } + + fn cargo_home(&self) -> PathBuf { + let mut p = self.root(); + p.push("chome"); + p + } + + fn index(&self) -> PathBuf { + let mut p = self.root(); + p.push("index"); + p + } + + fn workspaces_path(&self) -> PathBuf { + let mut p = self.root(); + p.push("workspaces"); + p + } + + fn registry_url(&self) -> Url { + Url::from_file_path(self.index()).unwrap() + } + + fn create_home(&self) { + let home = self.cargo_home(); + if !home.exists() { + fs::create_dir_all(&home).unwrap(); + } + fs::write( + home.join("config.toml"), + format!( + r#" + [source.crates-io] + replace-with = 'local-snapshot' + + [source.local-snapshot] + registry = '{}' + "#, + self.registry_url() + ), + ) + .unwrap(); + } + + fn create_target_dir(&self) { + // This is necessary to ensure the .rustc_info.json file is written. + // Otherwise it won't be written, and it is very expensive to create. + if !self.target_dir().exists() { + fs::create_dir_all(self.target_dir()).unwrap(); + } + } + + /// This clones crates.io at a specific point in time into tmp/index. + fn clone_index(&self) { + let index = self.index(); + let maybe_git = |command: &str| { + let status = Command::new("git") + .current_dir(&index) + .args(command.split_whitespace().collect::>()) + .status() + .expect("git should be installed"); + status.success() + }; + let git = |command: &str| { + if !maybe_git(command) { + panic!("failed to run git command: {}", command); + } + }; + if index.exists() { + if maybe_git(&format!( + "rev-parse -q --verify {}^{{commit}}", + CRATES_IO_COMMIT + )) { + // Already fetched. + return; + } + } else { + fs::create_dir_all(&index).unwrap(); + git("init --bare"); + git("remote add origin https://github.com/rust-lang/crates.io-index"); + } + git(&format!("fetch origin {}", CRATES_IO_COMMIT)); + git("branch -f master FETCH_HEAD"); + } + + /// This unpacks the compressed workspace skeletons into tmp/workspaces. + fn unpack_workspaces(&self) { + let ws_dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .join("workspaces"); + let archives = fs::read_dir(ws_dir) + .unwrap() + .map(|e| e.unwrap().path()) + .filter(|p| p.extension() == Some(std::ffi::OsStr::new("tgz"))); + for archive in archives { + let name = archive.file_stem().unwrap(); + let f = fs::File::open(&archive).unwrap(); + let f = flate2::read::GzDecoder::new(f); + let dest = self.workspaces_path().join(&name); + if dest.exists() { + fs::remove_dir_all(&dest).unwrap(); + } + let mut archive = tar::Archive::new(f); + archive.unpack(self.workspaces_path()).unwrap(); + } + } + + /// Vec of `(ws_name, ws_root)`. + pub fn workspaces(&self) -> Vec<(String, PathBuf)> { + // CARGO_BENCH_WORKSPACES can be used to override, otherwise it just uses + // the workspaces in the workspaces directory. + let mut ps: Vec<_> = match std::env::var_os("CARGO_BENCH_WORKSPACES") { + Some(s) => std::env::split_paths(&s).collect(), + None => fs::read_dir(self.workspaces_path()) + .unwrap() + .map(|e| e.unwrap().path()) + // These currently fail in most cases on Windows due to long + // filenames in the git checkouts. + .filter(|p| { + !(cfg!(windows) + && matches!(p.file_name().unwrap().to_str().unwrap(), "servo" | "tikv")) + }) + .collect(), + }; + // Sort so it is consistent. + ps.sort(); + ps.into_iter() + .map(|p| (p.file_name().unwrap().to_str().unwrap().to_owned(), p)) + .collect() + } + + /// Creates a new Config. + pub fn make_config(&self, ws_root: &Path) -> Config { + let shell = cargo::core::Shell::new(); + let mut config = Config::new(shell, ws_root.to_path_buf(), self.cargo_home()); + // Configure is needed to set the target_dir which is needed to write + // the .rustc_info.json file which is very expensive. + config + .configure( + 0, + false, + None, + false, + false, + false, + &Some(self.target_dir()), + &[], + &[], + ) + .unwrap(); + config + } +} diff --git a/benches/workspaces/rust-ws-inherit.tgz b/benches/workspaces/rust-ws-inherit.tgz new file mode 100644 index 00000000000..6e7b6691ff7 Binary files /dev/null and b/benches/workspaces/rust-ws-inherit.tgz differ