feat(cargo-codspeed): throw an error when building a bench target with default harness

Author: GuillaumeLagrange

Cargo.lock

@@ -27,6 +27,7 @@ itertools = { workspace = true }
 anstyle = "1.0.8"
 serde = { workspace = true }
 serde_json = { workspace = true }
+toml = "0.8"
 codspeed = { path = "../codspeed", version = "=4.0.5" }
 
 [dev-dependencies]

crates/cargo-codspeed/Cargo.toml

@@ -4,7 +4,9 @@ use crate::{
     measurement_mode::MeasurementMode,
     prelude::*,
 };
+use anyhow::Context;
 use cargo_metadata::{camino::Utf8PathBuf, Message, Metadata, TargetKind};
+use std::collections::HashMap;
 use std::process::{exit, Command, Stdio};
 
 struct BuildOptions<'a> {
@@ -31,6 +33,50 @@ pub struct BuildConfig {
     pub passthrough_flags: Vec<String>,
 }
 
+fn get_bench_harness_value(
+    manifest_path: &Utf8PathBuf,
+    bench_name: &str,
+    cache: &mut HashMap<Utf8PathBuf, toml::Table>,
+) -> Result<bool> {
+    let manifest_table = if let Some(table) = cache.get(manifest_path) {
+        table
+    } else {
+        // Read and parse the Cargo.toml file
+        let manifest_content = std::fs::read_to_string(manifest_path)
+            .with_context(|| format!("Failed to read manifest at {manifest_path}"))?;
+        let table: toml::Table = toml::from_str(&manifest_content)
+            .with_context(|| format!("Failed to parse TOML in {manifest_path}"))?;
+        cache.insert(manifest_path.clone(), table);
+        cache.get(manifest_path).unwrap()
+    };
+
+    // Look for [[bench]] sections
+    let Some(benches) = manifest_table.get("bench").and_then(|v| v.as_array()) else {
+        // If no [[bench]] sections, it's not an error, benches present in <root>/benches/<name>.rs
+        // are still collected with harness = true
+        return Ok(true);
+    };
+
+    // Find the bench entry with matching name
+    let matching_bench = benches
+        .iter()
+        .filter_map(|bench| bench.as_table())
+        .find(|bench_table| {
+            bench_table
+                .get("name")
+                .and_then(|v| v.as_str())
+                .is_some_and(|name| name == bench_name)
+        });
+
+    // Check if harness is enabled (defaults to true)
+    let harness_enabled = matching_bench
+        .and_then(|bench_table| bench_table.get("harness"))
+        .and_then(|v| v.as_bool())
+        .unwrap_or(true);
+
+    Ok(harness_enabled)
+}
+
 impl BuildOptions<'_> {
     /// Builds the benchmarks by invoking cargo
     /// Returns a list of built benchmarks, with path to associated executables
@@ -60,6 +106,8 @@ impl BuildOptions<'_> {
         );
 
         let mut built_benches = Vec::new();
+        let mut bench_targets_with_default_harness = Vec::new();
+        let mut manifest_cache = HashMap::new();
 
         let package_names = self
             .package_filters
@@ -95,6 +143,15 @@ impl BuildOptions<'_> {
                     let add_bench_to_codspeed_dir = package_names.iter().contains(&package.name);
 
                     if add_bench_to_codspeed_dir {
+                        if get_bench_harness_value(
+                            &package.manifest_path,
+                            &bench_target_name,
+                            &mut manifest_cache,
+                        )? {
+                            bench_targets_with_default_harness
+                                .push((package.name.to_string(), bench_target_name.clone()));
+                        }
+
                         built_benches.push(BuiltBench {
                             package: package.name.to_string(),
                             bench: bench_target_name,
@@ -114,6 +171,25 @@ impl BuildOptions<'_> {
             exit(status.code().expect("Could not get exit code"));
         }
 
+        if !bench_targets_with_default_harness.is_empty() {
+            let targets_list = bench_targets_with_default_harness
+                .into_iter()
+                .map(|(package, bench)| format!("  - `{bench}` in package `{package}`"))
+                .join("\n");
+
+            bail!("\
+CodSpeed will not work with the following benchmark targets:
+{targets_list}
+
+CodSpeed requires benchmark targets to disable the default test harness because benchmark frameworks handle harnessing themselves.
+
+Either disable the default harness by adding `harness = false` to the corresponding \
+`[[bench]]` section in the Cargo.toml, or specify which targets to build by using \
+`cargo codspeed build -p package_name --bench first_target --bench second_target`.
+
+See `cargo codspeed build --help` for more information.");
+        }
+
         for built_bench in &built_benches {
             eprintln!(
                 "Built benchmark `{}` in package `{}`",

crates/cargo-codspeed/src/build.rs

@@ -134,7 +134,7 @@ pub fn run_benches(
 
         command
             .status()
-            .map_err(|e| anyhow!("failed to execute the benchmark process: {}", e))
+            .map_err(|e| anyhow!("failed to execute the benchmark process: {e}"))
             .and_then(|status| {
                 if status.success() {
                     Ok(())

crates/cargo-codspeed/src/run.rs

@@ -11,9 +11,9 @@ repository = "https://github.com/CodSpeedHQ/codspeed-rust"
 homepage = "https://codspeed.io"
 license = "MIT OR Apache-2.0"
 categories = [
-  "development-tools",
-  "development-tools::profiling",
-  "development-tools::testing",
+    "development-tools",
+    "development-tools::profiling",
+    "development-tools::testing",
 ]
 keywords = ["codspeed", "benchmark", "criterion"]
 [dependencies]
@@ -26,7 +26,7 @@ regex = { version = "1.5", default-features = false, features = ["std"] }
 futures = { version = "0.3", default-features = false, optional = true }
 smol = { version = "2.0", default-features = false, optional = true }
 tokio = { version = "1.39", default-features = false, features = [
-  "rt",
+    "rt",
 ], optional = true }
 async-std = { version = "1.12", optional = true }
 

crates/criterion_compat/Cargo.toml

@@ -11,9 +11,9 @@ repository = "https://github.com/CodSpeedHQ/codspeed-rust"
 homepage = "https://codspeed.io"
 license = "MIT OR Apache-2.0"
 categories = [
-  "development-tools",
-  "development-tools::profiling",
-  "development-tools::testing",
+    "development-tools",
+    "development-tools::profiling",
+    "development-tools::testing",
 ]
 keywords = ["codspeed", "benchmark", "divan"]