diff --git a/Cargo.lock b/Cargo.lock index d70880f9f..fc6b5156c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -135,130 +135,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" -[[package]] -name = "async-attributes" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3203e79f4dd9bdda415ed03cf14dae5a2bf775c683a00f94e9cd1faf0f596e5" -dependencies = [ - "quote", - "syn 1.0.109", -] - -[[package]] -name = "async-channel" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" -dependencies = [ - "concurrent-queue", - "event-listener 2.5.3", - "futures-core", -] - -[[package]] -name = "async-channel" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" -dependencies = [ - "concurrent-queue", - "event-listener-strategy", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-executor" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30ca9a001c1e8ba5149f91a74362376cc6bc5b919d92d988668657bd570bdcec" -dependencies = [ - "async-task", - "concurrent-queue", - "fastrand", - "futures-lite", - "slab", -] - -[[package]] -name = "async-global-executor" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" -dependencies = [ - "async-channel 2.3.1", - "async-executor", - "async-io", - "async-lock", - "blocking", - "futures-lite", - "once_cell", -] - -[[package]] -name = "async-io" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a2b323ccce0a1d90b449fd71f2a06ca7faa7c54c2751f06c9bd851fc061059" -dependencies = [ - "async-lock", - "cfg-if", - "concurrent-queue", - "futures-io", - "futures-lite", - "parking", - "polling", - "rustix 0.38.44", - "slab", - "tracing", - "windows-sys", -] - -[[package]] -name = "async-lock" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" -dependencies = [ - "event-listener 5.4.0", - "event-listener-strategy", - "pin-project-lite", -] - -[[package]] -name = "async-std" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "730294c1c08c2e0f85759590518f6333f0d5a0a766a27d519c1b244c3dfd8a24" -dependencies = [ - "async-attributes", - "async-channel 1.9.0", - "async-global-executor", - "async-io", - "async-lock", - "crossbeam-utils", - "futures-channel", - "futures-core", - "futures-io", - "futures-lite", - "gloo-timers", - "kv-log-macro", - "log", - "memchr", - "once_cell", - "pin-project-lite", - "pin-utils", - "slab", - "wasm-bindgen-futures", -] - -[[package]] -name = "async-task" -version = "4.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" - [[package]] name = "async-trait" version = "0.1.88" @@ -270,12 +146,6 @@ dependencies = [ "syn 2.0.100", ] -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - [[package]] name = "autocfg" version = "1.4.0" @@ -457,19 +327,6 @@ dependencies = [ "generic-array 0.14.7", ] -[[package]] -name = "blocking" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703f41c54fc768e63e091340b424302bb1c29ef4aa0c7f10fe849dfb114d29ea" -dependencies = [ - "async-channel 2.3.1", - "async-task", - "futures-io", - "futures-lite", - "piper", -] - [[package]] name = "bls-signatures" version = "0.15.0" @@ -699,15 +556,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "concurrent-queue" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "config" version = "0.12.0" @@ -909,12 +757,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" dependencies = [ "anes", - "async-std", "cast", "ciborium", "clap", "criterion-plot", - "futures", "is-terminal", "itertools 0.10.5", "num-traits", @@ -1318,33 +1164,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "event-listener" -version = "2.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" - -[[package]] -name = "event-listener" -version = "5.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener-strategy" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" -dependencies = [ - "event-listener 5.4.0", - "pin-project-lite", -] - [[package]] name = "execute" version = "0.2.13" @@ -1588,108 +1407,6 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" -[[package]] -name = "futures" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" - -[[package]] -name = "futures-executor" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" - -[[package]] -name = "futures-lite" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5edaec856126859abb19ed65f39e90fea3a9574b9707f13539acf4abf7eb532" -dependencies = [ - "fastrand", - "futures-core", - "futures-io", - "parking", - "pin-project-lite", -] - -[[package]] -name = "futures-macro" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "futures-sink" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" - -[[package]] -name = "futures-task" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" - -[[package]] -name = "futures-util" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - [[package]] name = "fvm" version = "3.13.1" @@ -1746,7 +1463,6 @@ name = "fvm_conformance_tests" version = "0.1.0" dependencies = [ "anyhow", - "async-std", "base64", "cid", "colored", @@ -1754,7 +1470,6 @@ dependencies = [ "either", "env_logger 0.11.8", "flate2", - "futures", "fvm", "fvm_ipld_blockstore", "fvm_ipld_car", @@ -1767,6 +1482,7 @@ dependencies = [ "log", "num-traits", "num_cpus", + "rayon", "regex", "serde", "serde_json", @@ -1987,18 +1703,6 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" -[[package]] -name = "gloo-timers" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" -dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", -] - [[package]] name = "group" version = "0.13.0" @@ -2044,12 +1748,6 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" -[[package]] -name = "hermit-abi" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" - [[package]] name = "hermit-abi" version = "0.5.0" @@ -2423,15 +2121,6 @@ dependencies = [ "cpufeatures", ] -[[package]] -name = "kv-log-macro" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" -dependencies = [ - "log", -] - [[package]] name = "lazy_static" version = "1.5.0" @@ -2509,9 +2198,6 @@ name = "log" version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" -dependencies = [ - "value-bag", -] [[package]] name = "mach2" @@ -2792,12 +2478,6 @@ dependencies = [ "group", ] -[[package]] -name = "parking" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" - [[package]] name = "parking_lot" version = "0.12.3" @@ -2857,29 +2537,6 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" -[[package]] -name = "pin-project-lite" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "piper" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c8c490f422ef9a4efd2cb5b42b76c8613d7e7dfc1caf667b8a3350a5acc066" -dependencies = [ - "atomic-waker", - "fastrand", - "futures-io", -] - [[package]] name = "pkg-config" version = "0.3.32" @@ -2914,21 +2571,6 @@ dependencies = [ "plotters-backend", ] -[[package]] -name = "polling" -version = "3.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a604568c3202727d1507653cb121dbd627a58684eb09a820fd746bee38b4442f" -dependencies = [ - "cfg-if", - "concurrent-queue", - "hermit-abi 0.4.0", - "pin-project-lite", - "rustix 0.38.44", - "tracing", - "windows-sys", -] - [[package]] name = "portable-atomic" version = "1.11.0" @@ -3468,15 +3110,6 @@ dependencies = [ "rand_core", ] -[[package]] -name = "slab" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] - [[package]] name = "smallvec" version = "1.15.0" @@ -3842,22 +3475,6 @@ dependencies = [ "winnow", ] -[[package]] -name = "tracing" -version = "0.1.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" -dependencies = [ - "pin-project-lite", - "tracing-core", -] - -[[package]] -name = "tracing-core" -version = "0.1.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" - [[package]] name = "trait-set" version = "0.3.0" @@ -3934,12 +3551,6 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" -[[package]] -name = "value-bag" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943ce29a8a743eb10d6082545d861b24f9d1b160b7d741e0f2cdf726bec909c5" - [[package]] name = "version_check" version = "0.9.5" @@ -3997,19 +3608,6 @@ dependencies = [ "wasm-bindgen-shared", ] -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.50" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" -dependencies = [ - "cfg-if", - "js-sys", - "once_cell", - "wasm-bindgen", - "web-sys", -] - [[package]] name = "wasm-bindgen-macro" version = "0.2.100" diff --git a/testing/conformance/Cargo.toml b/testing/conformance/Cargo.toml index 8a8768e1a..f17d9814a 100644 --- a/testing/conformance/Cargo.toml +++ b/testing/conformance/Cargo.toml @@ -21,8 +21,6 @@ cid = { workspace = true } serde = { version = "1.0", features = ["derive"] } lazy_static = "1.5.0" log = "0.4.27" -futures = "0.3.31" -async-std = { version = "1.13", features = ["attributes"] } wasmtime = { workspace = true } base64 = "0.22.1" flate2 = { version = "1.1" } @@ -44,7 +42,8 @@ m2-native = [] [dev-dependencies] env_logger = "0.11.8" -criterion = { version = "0.5", features = ["async_std"] } +criterion = { version = "0.5" } +rayon = "1.10.0" [[bin]] name = "perf-conformance" diff --git a/testing/conformance/benches/bench_drivers.rs b/testing/conformance/benches/bench_drivers.rs index b25281f45..04420967e 100644 --- a/testing/conformance/benches/bench_drivers.rs +++ b/testing/conformance/benches/bench_drivers.rs @@ -99,7 +99,7 @@ pub fn bench_vector_file( name: &str, engines: &MultiEngine, ) -> anyhow::Result<()> { - let (bs, _) = async_std::task::block_on(vector.seed_blockstore()).unwrap(); + let (bs, _) = vector.seed_blockstore().unwrap(); for variant in vector.preconditions.variants.iter() { let name = format!("{} | {}", name, variant.id); diff --git a/testing/conformance/src/actors.rs b/testing/conformance/src/actors.rs index 275ad08d2..7356a2138 100644 --- a/testing/conformance/src/actors.rs +++ b/testing/conformance/src/actors.rs @@ -1,6 +1,5 @@ // Copyright 2021-2023 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT -use std::io::Read; use std::sync::Mutex; use fvm_ipld_blockstore::{Blockstore, MemoryBlockstore}; @@ -20,10 +19,7 @@ fn load_bundles(bundles: &[&[u8]]) -> anyhow::Result { for bundle in bundles { let mut reader = tar::Archive::new(zstd::Decoder::with_buffer(*bundle)?); for entry in reader.entries()? { - // We need to read it to a vec first as we can't send it between threads (async issues). - let mut car = Vec::new(); - entry?.read_to_end(&mut car)?; - load_car(&bs, &*car)?; + load_car(&bs, entry?)?; } } Ok(bs) diff --git a/testing/conformance/src/bin/perf-conformance.rs b/testing/conformance/src/bin/perf-conformance.rs index 2400a2d34..f4651e778 100644 --- a/testing/conformance/src/bin/perf-conformance.rs +++ b/testing/conformance/src/bin/perf-conformance.rs @@ -47,7 +47,7 @@ fn main() { ) .expect("failed to construct engine"); - let (bs, _) = async_std::task::block_on(vector.seed_blockstore()).unwrap(); + let (bs, _) = vector.seed_blockstore().unwrap(); for variant in vector.preconditions.variants.iter() { run_variant_for_perf(bs.clone(), &vector, variant, &engine, itt_info) } diff --git a/testing/conformance/src/lib.rs b/testing/conformance/src/lib.rs index b6e9f7ef6..afc7b1f9a 100644 --- a/testing/conformance/src/lib.rs +++ b/testing/conformance/src/lib.rs @@ -18,4 +18,10 @@ macro_rules! report { ($status:expr, $path:expr, $id:expr) => { println!("[{}] vector: {} | variant: {}", $status, $path, $id); }; + ($status:expr, $path:expr, $id:expr, $reason:expr) => { + println!( + "[{}] vector: {} | variant: {}\n\t|> reason: {:#}", + $status, $path, $id, $reason + ); + }; } diff --git a/testing/conformance/src/vector.rs b/testing/conformance/src/vector.rs index fdb2d2280..04edc77a1 100644 --- a/testing/conformance/src/vector.rs +++ b/testing/conformance/src/vector.rs @@ -181,7 +181,7 @@ impl MessageVector { impl MessageVector { /// Seeds a new blockstore with the CAR encoded in the test vector and all available bundled /// actors. Returns the blockstore and the root CID. - pub async fn seed_blockstore(&self) -> anyhow::Result<(MemoryBlockstore, Vec)> { + pub fn seed_blockstore(&self) -> anyhow::Result<(MemoryBlockstore, Vec)> { let blockstore = MemoryBlockstore::new(); load_actors(&blockstore)?; diff --git a/testing/conformance/tests/runner.rs b/testing/conformance/tests/runner.rs index a9debdca1..22ff5f62e 100644 --- a/testing/conformance/tests/runner.rs +++ b/testing/conformance/tests/runner.rs @@ -1,69 +1,30 @@ // Copyright 2021-2023 Protocol Labs // Copyright 2019-2022 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT -use std::collections::HashMap; +use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}; use std::env::var; -use std::fs::File; -use std::io::BufReader; -use std::iter; use std::path::{Path, PathBuf}; -use std::str::FromStr; -use anyhow::{Context as _, anyhow}; -use async_std::{stream, sync, task}; +use anyhow::anyhow; use colored::*; -use futures::{Future, StreamExt, TryFutureExt, TryStreamExt}; use fvm::engine::MultiEngine; use fvm_conformance_tests::driver::*; use fvm_conformance_tests::report; -use fvm_conformance_tests::tracing::{TestTraceExporter, TestTraceExporterRef}; -use fvm_conformance_tests::vector::{MessageVector, Selector}; -use fvm_conformance_tests::vm::{TestStatsGlobal, TestStatsRef}; +use fvm_conformance_tests::tracing::TestTraceExporter; +use fvm_conformance_tests::vector::MessageVector; +use fvm_conformance_tests::vm::TestStatsGlobal; +use fvm_ipld_blockstore::MemoryBlockstore; use itertools::Itertools; -use lazy_static::lazy_static; use walkdir::WalkDir; -enum ErrorAction { - Error, - Warn, - Ignore, -} - -impl FromStr for ErrorAction { - type Err = String; - fn from_str(s: &str) -> Result { - match s.to_lowercase().as_str() { - "error" => Ok(Self::Error), - "warn" => Ok(Self::Warn), - "ignore" => Ok(Self::Ignore), - _ => Err("must be one of error|warn|ignore".into()), - } - } -} - -lazy_static! { - /// The maximum parallelism when processing test vectors. Capped at 48. - static ref TEST_VECTOR_PARALLELISM: usize = std::env::var_os("TEST_VECTOR_PARALLELISM") - .map(|s| { - let s = s.to_str().unwrap(); - s.parse().expect("parallelism must be an integer") - }).unwrap_or_else(num_cpus::get).min(48); - - /// By default a post-condition error is fatal and stops all testing. We can use this env var to relax that - /// and let the test carry on (optionally with a warning); there's a correctness check against the post condition anyway. - static ref TEST_VECTOR_POSTCONDITION_MISSING_ACTION: ErrorAction = std::env::var_os("TEST_VECTOR_POSTCONDITION_MISSING_ACTION") - .map(|s| { - let s = s.to_str().unwrap(); - s.parse().expect("unexpected post condition error action") - }).unwrap_or(ErrorAction::Warn); - - static ref ENGINES: MultiEngine = MultiEngine::new(*TEST_VECTOR_PARALLELISM as u32); -} - -#[async_std::test] -async fn conformance_test_runner() -> anyhow::Result<()> { +#[test] +fn conformance_test_runner() -> anyhow::Result<()> { env_logger::init(); + let parallelism = rayon::current_num_threads().try_into().unwrap(); + let engine: MultiEngine = MultiEngine::new(parallelism); + println!("running with {parallelism} threads"); + let path = var("VECTOR").unwrap_or_else(|_| "test-vectors/corpus".to_owned()); let path = Path::new(path.as_str()).to_path_buf(); let stats = TestStatsGlobal::new_ref(); @@ -73,87 +34,129 @@ async fn conformance_test_runner() -> anyhow::Result<()> { .ok() .map(|path| TestTraceExporter::new(Path::new(path.as_str()).to_path_buf())); - let vector_results = if path.is_file() { - let stats = stats.clone(); - let tracer = tracer.clone(); - either::Either::Left( - iter::once(async move { - let res = run_vector(path.clone(), stats, tracer) - .await - .with_context(|| format!("failed to run vector: {}", path.display()))?; - anyhow::Ok((path, res)) - }) - .map(futures::future::Either::Left), - ) + // Collect the test vector files. + let vector_paths = if path.is_file() { + vec![path] } else { - either::Either::Right( - WalkDir::new(path) - .into_iter() - .filter_ok(is_runnable) - .map(|e| { - let stats = stats.clone(); - let tracer = tracer.clone(); - async move { - let path = e?.path().to_path_buf(); - let res = run_vector(path.clone(), stats, tracer) - .await - .with_context(|| format!("failed to run vector: {}", path.display()))?; - Ok((path, res)) - } - }) - .map(futures::future::Either::Right), - ) + WalkDir::new(path) + .into_iter() + .filter_ok(is_runnable) + .map_ok(|de| de.into_path()) + .collect::, _>>()? }; - let mut results = Box::pin( - stream::from_iter(vector_results) - // Will _load_ up to 100 vectors at once in any order. We won't actually run the vectors in - // parallel (yet), but that shouldn't be too hard. - .map(|task| { - async move { - let (path, jobs) = task.await?; - Ok(stream::from_iter(jobs).map(move |job| { - let path = path.clone(); - Ok(async move { anyhow::Ok((path, job.await?)) }) - })) - } - .try_flatten_stream() - }) - .flatten() - .try_buffer_unordered(*TEST_VECTOR_PARALLELISM), - ); + // Collect the test vectors. + let vectors: Vec<(PathBuf, MessageVector)> = vector_paths + .into_par_iter() + .map(|p| { + let v = MessageVector::from_file(&p)?; + Ok((p, v)) + }) + .collect::>()?; + + #[derive(Default)] + struct Counters { + succeeded: u32, + failed: u32, + skipped: u32, + } - let mut succeeded = 0; - let mut failed = 0; - let mut skipped = 0; + let counters: Counters = vectors + .par_iter() + .flat_map_iter({ + let stats = &stats; + let tracer = &tracer; + let engine = &engine; + move |(path, vector)| { + use rayon::iter::Either; + + // Skip unsupported vectors. + if !vector.is_supported() { + return Either::Left(vector.preconditions.variants.iter().map( + move |variant| { + ( + path, + VariantResult::Skipped { + reason: "unsupported".into(), + id: variant.id.clone(), + }, + ) + }, + )); + } - while let Some((path, res)) = results.next().await.transpose()? { - match res { + // Load the vector's blockstore. + let bs = load_vector_bs(vector); + + Either::Right(vector.preconditions.variants.iter().map(move |variant| { + let bs = match &bs { + Ok(bs) => bs.clone(), + Err(e) => { + return ( + path, + VariantResult::Failed { + reason: anyhow!("failed to load vector state: {e}"), + id: variant.id.clone(), + }, + ); + } + }; + let res = run_variant( + bs.clone(), + vector, + variant, + engine, + true, + stats.clone(), + tracer + .clone() + .map(|t| t.export_fun(path.clone(), variant.id.clone())), + ) + .unwrap_or_else(|e| VariantResult::Failed { + reason: e, + id: variant.id.clone(), + }); + (path, res) + })) + } + }) + .map(|(path, res)| match res { VariantResult::Ok { id } => { report!("OK".on_green(), path.display(), id); - succeeded += 1; + Counters { + succeeded: 1, + ..Default::default() + } } VariantResult::Failed { reason, id } => { - report!("FAIL".white().on_red(), path.display(), id); - println!("\t|> reason: {:#}", reason); - failed += 1; + report!("FAIL".white().on_red(), path.display(), id, reason); + Counters { + failed: 1, + ..Default::default() + } } VariantResult::Skipped { reason, id } => { - report!("SKIP".on_yellow(), path.display(), id); - println!("\t|> reason: {}", reason); - skipped += 1; + report!("SKIP".on_yellow(), path.display(), id, reason); + Counters { + skipped: 1, + ..Default::default() + } } - } - } + }) + .reduce(Counters::default, |a, b| Counters { + succeeded: a.succeeded + b.succeeded, + failed: a.failed + b.failed, + skipped: a.skipped + b.skipped, + }); println!(); println!( "{}", format!( "conformance tests result: {}/{} tests passed ({} skipped)", - succeeded, - failed + succeeded, - skipped, + counters.succeeded, + counters.failed + counters.succeeded, + counters.skipped, ) .bold() ); @@ -177,125 +180,30 @@ async fn conformance_test_runner() -> anyhow::Result<()> { tracer.export_tombstones()?; } - if failed > 0 { + if counters.failed > 0 { Err(anyhow!("some vectors failed")) } else { Ok(()) } } -/// Runs a single test vector and returns a list of VectorResults, -/// one per variant. -async fn run_vector( - path: PathBuf, - stats: TestStatsRef, - tracer: TestTraceExporterRef, -) -> anyhow::Result>>> { - let file = File::open(&path)?; - let reader = BufReader::new(file); - - // Test vectors have the form: - // - // { "class": ..., rest... } - // - // Unfortunately: - // 1. That means we need to use serde's "flatten" and/or "tag" feature to decode them. - // 2. Serde's JSON library doesn't support arbitrary precision numbers when doing this. - // 3. The circulating supply doesn't fit in a u64, and f64 isn't precise enough. - // - // So we manually: - // 1. Decode into a map of `String` -> `raw data`. - // 2. Pull off the class. - // 3. Re-serialize. - // 4. Decode into the correct type. - // - // Upstream bug is https://github.com/serde-rs/serde/issues/1183 (or at least that looks like - // the most appropriate one out of all the related issues). - let mut vector: HashMap> = - serde_json::from_reader(reader).context("failed to parse vector")?; - let class_json = vector - .remove("class") - .context("expected test vector to have a class")?; - - let class: &str = - serde_json::from_str(class_json.get()).context("failed to parse test vector class")?; - let vector_json = serde_json::to_string(&vector)?; - - match class { - "message" => { - let v: MessageVector = - serde_json::from_str(&vector_json).context("failed to parse message vector")?; - let skip = !v.selector.as_ref().is_none_or(Selector::supported); - if skip { - Ok(either::Either::Left( - v.preconditions.variants.into_iter().map(|variant| { - futures::future::Either::Left(async move { - Ok(VariantResult::Skipped { - id: variant.id, - reason: "selector not supported".to_owned(), - }) - }) - }), - )) - } else { - // First import the blockstore and do some sanity checks. - let (bs, imported_root) = v.seed_blockstore().await?; - if !imported_root.contains(&v.preconditions.state_tree.root_cid) { - return Err(anyhow!( - "imported roots ({}) do not contain precondition CID {}", - imported_root.iter().join(", "), - v.preconditions.state_tree.root_cid - )); - } - if !imported_root.contains(&v.postconditions.state_tree.root_cid) { - let msg = format!( - "imported roots ({}) do not contain postcondition CID {}", - imported_root.iter().join(", "), - v.postconditions.state_tree.root_cid - ); - - match *TEST_VECTOR_POSTCONDITION_MISSING_ACTION { - ErrorAction::Error => { - return Err(anyhow!(msg)); - } - ErrorAction::Warn => { - eprintln!("WARN: {msg} in {}", path.display()) - } - ErrorAction::Ignore => (), - } - } +fn load_vector_bs(v: &MessageVector) -> anyhow::Result { + let (bs, imported_root) = v.seed_blockstore()?; + if !imported_root.contains(&v.preconditions.state_tree.root_cid) { + return Err(anyhow!( + "imported roots ({}) do not contain precondition CID {}", + imported_root.iter().join(", "), + v.preconditions.state_tree.root_cid + )); + } + if !imported_root.contains(&v.postconditions.state_tree.root_cid) { + let msg = format!( + "imported roots ({}) do not contain postcondition CID {}", + imported_root.iter().join(", "), + v.postconditions.state_tree.root_cid + ); - let v = sync::Arc::new(v); - Ok(either::Either::Right( - (0..v.preconditions.variants.len()).map(move |i| { - let v = v.clone(); - let bs = bs.clone(); - let path = path.clone(); - let variant_id = v.preconditions.variants[i].id.clone(); - let name = format!("{} | {}", path.display(), variant_id); - let stats = stats.clone(); - let tracer = tracer.clone(); - futures::future::Either::Right( - task::Builder::new() - .name(name.clone()) - .spawn(async move { - run_variant( - bs, - &v, - &v.preconditions.variants[i], - &ENGINES, - true, - stats, - tracer.map(|t| t.export_fun(path, variant_id)), - ) - .with_context(|| format!("failed to run {name}")) - }) - .unwrap(), - ) - }), - )) - } - } - other => Err(anyhow!("unknown test vector class: {}", other)), + return Err(anyhow!(msg)); } + Ok(bs) }