From a9d51f678cbaf6b21e4ad5bba895be4479501c00 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 10 Mar 2025 16:37:52 +0100 Subject: [PATCH 01/33] feat(e2e): update e2e test command line arguments --- .../mithril-end-to-end/src/main.rs | 59 ++++++++++++++----- .../src/mithril/infrastructure.rs | 9 ++- 2 files changed, 52 insertions(+), 16 deletions(-) diff --git a/mithril-test-lab/mithril-end-to-end/src/main.rs b/mithril-test-lab/mithril-end-to-end/src/main.rs index 87898022b3e..24d6363d5f6 100644 --- a/mithril-test-lab/mithril-end-to-end/src/main.rs +++ b/mithril-test-lab/mithril-end-to-end/src/main.rs @@ -51,9 +51,13 @@ pub struct Args { #[clap(long, default_value = ".")] bin_directory: PathBuf, - /// Number of Pool nodes in the devnet - #[clap(long, default_value_t = 3, value_parser = has_at_least_two_pool_nodes)] - number_of_pool_nodes: u8, + /// Number of aggregators + #[clap(long, default_value_t = 1)] + number_of_aggregators: u8, + + /// Number of signers + #[clap(long, default_value_t = 2)] + number_of_signers: u8, /// Length of a Cardano slot in the devnet (in s) #[clap(long, default_value_t = 0.10)] @@ -135,17 +139,21 @@ impl Args { _ => Level::Trace, } } -} -fn has_at_least_two_pool_nodes(s: &str) -> Result { - let number_of_pool_nodes: u8 = s.parse().map_err(|_| format!("`{}` isn't a number", s))?; - if number_of_pool_nodes >= 2 { - Ok(number_of_pool_nodes) - } else { - Err(format!( - "At least two pool nodes are required (one for the aggregator, one for at least one \ - signer), number given: {s}", - )) + fn validate(&self) -> StdResult<()> { + if self.number_of_aggregators == 0 { + return Err(anyhow!("At least one aggregator is required")); + } + if self.number_of_signers == 0 { + return Err(anyhow!("At least one signer is required")); + } + if !self.use_p2p_network && self.number_of_aggregators >= 2 { + return Err(anyhow!( + "The P2P mode must be activated to run more than one aggregator" + )); + } + + Ok(()) } } @@ -308,6 +316,7 @@ impl App { artifacts_dir: PathBuf, ) -> StdResult<()> { let server_port = 8080; + args.validate()?; let run_only_mode = args.run_only; let use_p2p_network_mode = args.use_p2p_network; let use_p2p_passive_relays = args.use_p2p_passive_relays; @@ -315,7 +324,7 @@ impl App { let devnet = Devnet::bootstrap(&DevnetBootstrapArgs { devnet_scripts_dir: args.devnet_scripts_directory, artifacts_target_dir: work_dir.join("devnet"), - number_of_pool_nodes: args.number_of_pool_nodes, + number_of_pool_nodes: args.number_of_aggregators + args.number_of_signers, cardano_slot_length: args.cardano_slot_length, cardano_epoch_length: args.cardano_epoch_length, cardano_node_version: args.cardano_node_version.to_owned(), @@ -326,6 +335,8 @@ impl App { *self.devnet.lock().await = Some(devnet.clone()); let mut infrastructure = MithrilInfrastructure::start(&MithrilInfrastructureConfig { + number_of_aggregators: args.number_of_aggregators, + number_of_signers: args.number_of_signers, server_port, devnet: devnet.clone(), artifacts_dir, @@ -487,4 +498,24 @@ mod tests { AppResult::Cancelled(_) )); } + + #[test] + fn args_fails_validation() { + let args = Args::parse_from(["", "--number-of-signers", "0"]); + args.validate() + .expect_err("validate should fail without signers"); + + let args = Args::parse_from(["", "--number-of-aggregators", "0"]); + args.validate() + .expect_err("validate should fail without aggregators"); + + let args = Args::parse_from(["", "--number-of-aggregators", "2"]); + args.validate().expect_err( + "validate should fail with more than one aggregator if p2p network is not used", + ); + + let args = Args::parse_from(["", "--use-p2p-network", "--number-of-aggregators", "2"]); + args.validate() + .expect("validate should succeed with more than one aggregator if p2p network is used"); + } } diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs index b2f6a468f5c..ae46197e432 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs @@ -16,6 +16,8 @@ use crate::{ use super::signer::SignerConfig; pub struct MithrilInfrastructureConfig { + pub number_of_aggregators: u8, + pub number_of_signers: u8, pub server_port: u64, pub devnet: Devnet, pub work_dir: PathBuf, @@ -54,8 +56,11 @@ impl MithrilInfrastructure { config.devnet.run().await?; let devnet_topology = config.devnet.topology(); // Clap check that we always have at least 2 pools, no need to be defensive here - let aggregator_cardano_node = &devnet_topology.pool_nodes[0]; - let signer_cardano_nodes = &devnet_topology.pool_nodes[1..]; + let number_of_aggregators = config.number_of_aggregators as usize; + let number_of_signers = config.number_of_signers as usize; + let aggregator_cardano_nodes = &devnet_topology.pool_nodes[0..number_of_aggregators]; + let signer_cardano_nodes = &devnet_topology.pool_nodes + [number_of_aggregators..number_of_aggregators + number_of_signers]; let signer_party_ids = signer_cardano_nodes .iter() .map(|s| s.party_id()) From 6eddc27ec8c67dda2a648ac205be87bf5557646d Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 11 Mar 2025 15:52:00 +0100 Subject: [PATCH 02/33] feat(e2e): 'MithrilInfrastructure' supports multiple aggregators First aggregator is 'master', and others (if any) are 'slave' to the 'master'. --- .../mithril-end-to-end/src/end_to_end_spec.rs | 35 +-- .../mithril-end-to-end/src/main.rs | 11 +- .../src/mithril/aggregator.rs | 21 +- .../src/mithril/infrastructure.rs | 199 +++++++++++------- .../mithril-end-to-end/src/mithril/signer.rs | 7 +- .../mithril-end-to-end/src/run_only.rs | 15 +- .../src/stress_test/aggregator_helpers.rs | 3 + 7 files changed, 183 insertions(+), 108 deletions(-) diff --git a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs index 4dc5e8a309a..46c0ae00e47 100644 --- a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs +++ b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs @@ -44,12 +44,14 @@ impl<'a> Spec<'a> { } pub async fn run(&mut self) -> StdResult<()> { - let aggregator_endpoint = self.infrastructure.aggregator().endpoint(); - assertions::wait_for_enough_immutable(self.infrastructure.aggregator().db_directory()) - .await?; + let aggregator_endpoint = self.infrastructure.master_aggregator().endpoint(); + assertions::wait_for_enough_immutable( + self.infrastructure.master_aggregator().db_directory(), + ) + .await?; let start_epoch = self .infrastructure - .chain_observer() + .master_chain_observer() .get_current_epoch() .await? .unwrap_or_default(); @@ -63,19 +65,20 @@ impl<'a> Spec<'a> { // Wait 4 epochs after start epoch for the aggregator to be able to bootstrap a genesis certificate let mut target_epoch = start_epoch + 4; assertions::wait_for_target_epoch( - self.infrastructure.chain_observer(), + self.infrastructure.master_chain_observer(), target_epoch, "minimal epoch for the aggregator to be able to bootstrap genesis certificate" .to_string(), ) .await?; - assertions::bootstrap_genesis_certificate(self.infrastructure.aggregator_mut()).await?; + assertions::bootstrap_genesis_certificate(self.infrastructure.master_aggregator_mut()) + .await?; assertions::wait_for_epoch_settings(&aggregator_endpoint).await?; // Wait 2 epochs before changing stake distribution, so that we use at least one original stake distribution target_epoch += 2; assertions::wait_for_target_epoch( - self.infrastructure.chain_observer(), + self.infrastructure.master_chain_observer(), target_epoch, "epoch after which the stake distribution will change".to_string(), ) @@ -87,17 +90,17 @@ impl<'a> Spec<'a> { // Wait 2 epochs before changing protocol parameters target_epoch += 2; assertions::wait_for_target_epoch( - self.infrastructure.chain_observer(), + self.infrastructure.master_chain_observer(), target_epoch, "epoch after which the protocol parameters will change".to_string(), ) .await?; - assertions::update_protocol_parameters(self.infrastructure.aggregator_mut()).await?; + assertions::update_protocol_parameters(self.infrastructure.master_aggregator_mut()).await?; // Wait 6 epochs after protocol parameters update, so that we make sure that we use new protocol parameters as well as new stake distribution a few times target_epoch += 6; assertions::wait_for_target_epoch( - self.infrastructure.chain_observer(), + self.infrastructure.master_chain_observer(), target_epoch, "epoch after which the certificate chain will be long enough to catch most common troubles with stake distribution and protocol parameters".to_string(), ) @@ -114,7 +117,7 @@ impl<'a> Spec<'a> { .await?; target_epoch += 5; assertions::wait_for_target_epoch( - self.infrastructure.chain_observer(), + self.infrastructure.master_chain_observer(), target_epoch, "epoch after which the era switch will have triggered".to_string(), ) @@ -122,11 +125,13 @@ impl<'a> Spec<'a> { // Proceed to a re-genesis of the certificate chain if self.regenesis_on_era_switch { - assertions::bootstrap_genesis_certificate(self.infrastructure.aggregator_mut()) - .await?; + assertions::bootstrap_genesis_certificate( + self.infrastructure.master_aggregator_mut(), + ) + .await?; target_epoch += 5; assertions::wait_for_target_epoch( - self.infrastructure.chain_observer(), + self.infrastructure.master_chain_observer(), target_epoch, "epoch after which the re-genesis on era switch will be completed".to_string(), ) @@ -141,7 +146,7 @@ impl<'a> Spec<'a> { } async fn verify_artifacts_production(&self, target_epoch: Epoch) -> StdResult { - let aggregator_endpoint = self.infrastructure.aggregator().endpoint(); + let aggregator_endpoint = self.infrastructure.master_aggregator().endpoint(); let expected_epoch_min = target_epoch - 3; // Verify that mithril stake distribution artifacts are produced and signed correctly { diff --git a/mithril-test-lab/mithril-end-to-end/src/main.rs b/mithril-test-lab/mithril-end-to-end/src/main.rs index 24d6363d5f6..0460ab9ef36 100644 --- a/mithril-test-lab/mithril-end-to-end/src/main.rs +++ b/mithril-test-lab/mithril-end-to-end/src/main.rs @@ -201,13 +201,18 @@ async fn main_exec() -> StdResult<()> { fs::create_dir(&path).expect("Artifacts dir creation failure"); path }; + let store_dir = { + let path = work_dir.join("stores"); + fs::create_dir(&path).expect("Stores dir creation failure"); + path + }; let mut app = App::new(); let mut app_stopper = AppStopper::new(&app); let mut join_set = JoinSet::new(); with_gracefull_shutdown(&mut join_set); - join_set.spawn(async move { app.run(args, work_dir, artifacts_dir).await }); + join_set.spawn(async move { app.run(args, work_dir, store_dir, artifacts_dir).await }); let res = match join_set.join_next().await { Some(Ok(tasks_result)) => tasks_result, @@ -313,6 +318,7 @@ impl App { &mut self, args: Args, work_dir: PathBuf, + store_dir: PathBuf, artifacts_dir: PathBuf, ) -> StdResult<()> { let server_port = 8080; @@ -339,8 +345,9 @@ impl App { number_of_signers: args.number_of_signers, server_port, devnet: devnet.clone(), - artifacts_dir, work_dir, + store_dir, + artifacts_dir, bin_dir: args.bin_directory, cardano_node_version: args.cardano_node_version, mithril_run_interval: args.mithril_run_interval, diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs index 2d8ddcfcd01..30679b072f3 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs @@ -15,10 +15,12 @@ use tokio::process::Child; #[derive(Debug)] pub struct AggregatorConfig<'a> { + pub index: usize, pub server_port: u64, pub pool_node: &'a PoolNode, pub cardano_cli_path: &'a Path, pub work_dir: &'a Path, + pub store_dir: &'a Path, pub artifacts_dir: &'a Path, pub bin_dir: &'a Path, pub cardano_node_version: &'a str, @@ -28,6 +30,7 @@ pub struct AggregatorConfig<'a> { pub mithril_era_marker_address: &'a str, pub signed_entity_types: &'a [String], pub chain_observer_type: &'a str, + pub master_aggregator_endpoint: &'a Option, } #[derive(Debug)] @@ -57,7 +60,7 @@ impl Aggregator { let signed_entity_types = aggregator_config.signed_entity_types.join(","); let mithril_run_interval = format!("{}", aggregator_config.mithril_run_interval); let public_server_url = format!("http://localhost:{server_port_parameter}/aggregator"); - let env = HashMap::from([ + let mut env = HashMap::from([ ("NETWORK", "devnet"), ("RUN_INTERVAL", &mithril_run_interval), ("SERVER_IP", "0.0.0.0"), @@ -71,7 +74,10 @@ impl Aggregator { aggregator_config.artifacts_dir.to_str().unwrap(), ), ("NETWORK_MAGIC", &magic_id), - ("DATA_STORES_DIRECTORY", "./stores/aggregator"), + ( + "DATA_STORES_DIRECTORY", + aggregator_config.store_dir.to_str().unwrap(), + ), ( "CARDANO_NODE_SOCKET_PATH", aggregator_config.pool_node.socket_path.to_str().unwrap(), @@ -103,19 +109,28 @@ impl Aggregator { ("CARDANO_TRANSACTIONS_SIGNING_CONFIG__STEP", "15"), ("PERSIST_USAGE_REPORT_INTERVAL_IN_SECONDS", "3"), ]); + if let Some(master_aggregator_endpoint) = aggregator_config.master_aggregator_endpoint { + env.insert("MASTER_AGGREGATOR_ENDPOINT", master_aggregator_endpoint); + } let args = vec![ "--db-directory", aggregator_config.pool_node.db_path.to_str().unwrap(), "-vvv", ]; - let command = MithrilCommand::new( + let mut command = MithrilCommand::new( "mithril-aggregator", aggregator_config.work_dir, aggregator_config.bin_dir, env, &args, )?; + let name = if aggregator_config.index == 0 { + "mithril-aggregator".to_string() + } else { + format!("mithril-aggregator-slave-{}", aggregator_config.index) + }; + command.set_log_name(&name); Ok(Self { server_port: aggregator_config.server_port, diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs index ae46197e432..e0d1e30044d 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs @@ -1,4 +1,3 @@ -use anyhow::Context; use mithril_common::chain_observer::{ChainObserver, PallasChainObserver}; use mithril_common::entities::{Epoch, PartyId, ProtocolParameters}; use mithril_common::{CardanoNetwork, StdResult}; @@ -21,6 +20,7 @@ pub struct MithrilInfrastructureConfig { pub server_port: u64, pub devnet: Devnet, pub work_dir: PathBuf, + pub store_dir: PathBuf, pub artifacts_dir: PathBuf, pub bin_dir: PathBuf, pub cardano_node_version: String, @@ -38,12 +38,12 @@ pub struct MithrilInfrastructure { artifacts_dir: PathBuf, bin_dir: PathBuf, devnet: Devnet, - aggregator: Aggregator, + aggregators: Vec, signers: Vec, relay_aggregators: Vec, relay_signers: Vec, relay_passives: Vec, - cardano_chain_observer: Arc, + cardano_chain_observers: Vec>, run_only_mode: bool, current_era: String, era_reader_adapter: String, @@ -55,7 +55,6 @@ impl MithrilInfrastructure { let chain_observer_type = "pallas"; config.devnet.run().await?; let devnet_topology = config.devnet.topology(); - // Clap check that we always have at least 2 pools, no need to be defensive here let number_of_aggregators = config.number_of_aggregators as usize; let number_of_signers = config.number_of_signers as usize; let aggregator_cardano_nodes = &devnet_topology.pool_nodes[0..number_of_aggregators]; @@ -66,34 +65,47 @@ impl MithrilInfrastructure { .map(|s| s.party_id()) .collect::>>()?; - let aggregator = - Self::start_aggregator(config, aggregator_cardano_node, chain_observer_type).await?; + let aggregators = + Self::start_aggregators(config, aggregator_cardano_nodes, chain_observer_type).await?; + let aggregator_endpoints = aggregators + .iter() + .map(|aggregator| aggregator.endpoint()) + .collect::>(); + let master_aggregator_endpoint = aggregator_endpoints[0].to_owned(); let (relay_aggregators, relay_signers, relay_passives) = - Self::start_relays(config, aggregator.endpoint(), &signer_party_ids)?; + Self::start_relays(config, &aggregator_endpoints, &signer_party_ids)?; let signers = Self::start_signers( config, - aggregator.endpoint(), + master_aggregator_endpoint, signer_cardano_nodes, &relay_signers, )?; - let cardano_chain_observer = Arc::new(PallasChainObserver::new( - &aggregator_cardano_node.socket_path, - CardanoNetwork::DevNet(DEVNET_MAGIC_ID), - )); + fn build_chain_observer( + cardano_node: &PoolNode, + network: CardanoNetwork, + ) -> Arc { + Arc::new(PallasChainObserver::new(&cardano_node.socket_path, network)) + } + let cardano_chain_observers: Vec> = aggregator_cardano_nodes + .iter() + .map(|cardano_node| { + build_chain_observer(cardano_node, CardanoNetwork::DevNet(DEVNET_MAGIC_ID)) + }) + .collect(); Ok(Self { bin_dir: config.bin_dir.to_path_buf(), artifacts_dir: config.artifacts_dir.to_path_buf(), devnet: config.devnet.clone(), - aggregator, + aggregators, signers, relay_aggregators, relay_signers, relay_passives, - cardano_chain_observer, + cardano_chain_observers, run_only_mode: config.run_only_mode, current_era: config.mithril_era.clone(), era_reader_adapter: config.mithril_era_reader_adapter.clone(), @@ -121,15 +133,16 @@ impl MithrilInfrastructure { pub async fn register_switch_to_next_era(&mut self, next_era: &str) -> StdResult<()> { let next_era_epoch = self - .chain_observer() + .master_chain_observer() .get_current_epoch() .await? .unwrap_or_default() + 1; if self.era_reader_adapter == "cardano-chain" { + let devnet = self.devnet.clone(); assertions::register_era_marker( - &mut self.aggregator, - &self.devnet, + self.master_aggregator_mut(), + &devnet, next_era, next_era_epoch, ) @@ -140,53 +153,68 @@ impl MithrilInfrastructure { Ok(()) } - async fn start_aggregator( + async fn start_aggregators( config: &MithrilInfrastructureConfig, - pool_node: &PoolNode, + pool_nodes: &[PoolNode], chain_observer_type: &str, - ) -> StdResult { - let aggregator_artifacts_directory = config.artifacts_dir.join("mithril-aggregator"); - if !aggregator_artifacts_directory.exists() { - fs::create_dir_all(&aggregator_artifacts_directory).with_context(|| { - format!( - "Could not create artifacts directory '{}'", - aggregator_artifacts_directory.display() - ) + ) -> StdResult> { + let mut aggregators = vec![]; + let mut master_aggregator_endpoint: Option = None; + for (index, pool_node) in pool_nodes.iter().enumerate() { + let aggregator_artifacts_dir = if index == 0 { + config.artifacts_dir.join("mithril-aggregator") + } else { + config + .artifacts_dir + .join(format!("mithril-aggregator-slave-{index}")) + }; + let aggregator_store_dir = if index == 0 { + config.store_dir.join("aggregator") + } else { + config.store_dir.join(format!("aggregator-slave-{index}")) + }; + + let mut aggregator = Aggregator::new(&AggregatorConfig { + index, + server_port: config.server_port + index as u64, + pool_node, + cardano_cli_path: &config.devnet.cardano_cli_path(), + work_dir: &config.work_dir, + store_dir: &aggregator_store_dir, + artifacts_dir: &aggregator_artifacts_dir, + bin_dir: &config.bin_dir, + cardano_node_version: &config.cardano_node_version, + mithril_run_interval: config.mithril_run_interval, + mithril_era: &config.mithril_era, + mithril_era_reader_adapter: &config.mithril_era_reader_adapter, + mithril_era_marker_address: &config.devnet.mithril_era_marker_address()?, + signed_entity_types: &config.signed_entity_types, + chain_observer_type, + master_aggregator_endpoint: &master_aggregator_endpoint.clone(), })?; + + aggregator.set_protocol_parameters(&ProtocolParameters { + k: 75, + m: 105, + phi_f: 0.95, + }); + + Self::register_startup_era(&mut aggregator, config).await?; + + aggregator.serve()?; + + if master_aggregator_endpoint.is_none() { + master_aggregator_endpoint = Some(aggregator.endpoint()); + } + aggregators.push(aggregator); } - let mut aggregator = Aggregator::new(&AggregatorConfig { - server_port: config.server_port, - pool_node, - cardano_cli_path: &config.devnet.cardano_cli_path(), - work_dir: &config.work_dir, - artifacts_dir: &aggregator_artifacts_directory, - bin_dir: &config.bin_dir, - cardano_node_version: &config.cardano_node_version, - mithril_run_interval: config.mithril_run_interval, - mithril_era: &config.mithril_era, - mithril_era_reader_adapter: &config.mithril_era_reader_adapter, - mithril_era_marker_address: &config.devnet.mithril_era_marker_address()?, - signed_entity_types: &config.signed_entity_types, - chain_observer_type, - })?; - - aggregator.set_protocol_parameters(&ProtocolParameters { - k: 75, - m: 105, - phi_f: 0.95, - }); - - Self::register_startup_era(&mut aggregator, config).await?; - - aggregator.serve()?; - - Ok(aggregator) + Ok(aggregators) } fn start_relays( config: &MithrilInfrastructureConfig, - aggregator_endpoint: String, + aggregator_endpoints: &[String], signers_party_ids: &[PartyId], ) -> StdResult<(Vec, Vec, Vec)> { if !config.use_p2p_network_mode { @@ -199,19 +227,23 @@ impl MithrilInfrastructure { info!("Starting the Mithril infrastructure in P2P mode (experimental)"); - let mut relay_aggregator = RelayAggregator::new( - config.server_port + 100, - &aggregator_endpoint, - &config.work_dir, - &config.bin_dir, - )?; - relay_aggregator.start()?; + for (index, aggregator_endpoint) in aggregator_endpoints.iter().enumerate() { + let mut relay_aggregator = RelayAggregator::new( + config.server_port + index as u64 + 100, + aggregator_endpoint, + &config.work_dir, + &config.bin_dir, + )?; + relay_aggregator.start()?; + relay_aggregators.push(relay_aggregator); + } + let master_aggregator = &relay_aggregators[0]; let mut relay_passive_id = 1; if config.use_p2p_passive_relays { let mut relay_passive_aggregator = RelayPassive::new( config.server_port + 200, - relay_aggregator.peer_addr().to_owned(), + master_aggregator.peer_addr().to_owned(), format!("{relay_passive_id}"), &config.work_dir, &config.bin_dir, @@ -224,8 +256,8 @@ impl MithrilInfrastructure { let mut relay_signer = RelaySigner::new( config.server_port + index as u64 + 300, config.server_port + index as u64 + 400, - relay_aggregator.peer_addr().to_owned(), - &aggregator_endpoint, + master_aggregator.peer_addr().to_owned(), + &aggregator_endpoints[0], party_id.clone(), &config.work_dir, &config.bin_dir, @@ -248,14 +280,12 @@ impl MithrilInfrastructure { relay_signers.push(relay_signer); } - relay_aggregators.push(relay_aggregator); - Ok((relay_aggregators, relay_signers, relay_passives)) } fn start_signers( config: &MithrilInfrastructureConfig, - aggregator_endpoint: String, + master_aggregator_endpoint: String, pool_nodes: &[PoolNode], relay_signers: &[RelaySigner], ) -> StdResult> { @@ -269,7 +299,7 @@ impl MithrilInfrastructure { let aggregator_endpoint = if config.use_p2p_network_mode { relay_signers[index].endpoint() } else { - aggregator_endpoint.clone() + master_aggregator_endpoint.clone() }; let mut signer = Signer::new(&SignerConfig { @@ -278,6 +308,9 @@ impl MithrilInfrastructure { pool_node, cardano_cli_path: &config.devnet.cardano_cli_path(), work_dir: &config.work_dir, + store_dir: &config + .store_dir + .join(format!("signer-{}", pool_node.party_id()?)), bin_dir: &config.bin_dir, mithril_run_interval: config.mithril_run_interval, mithril_era: &config.mithril_era, @@ -294,13 +327,15 @@ impl MithrilInfrastructure { } pub async fn stop_nodes(&mut self) -> StdResult<()> { - // Note: The aggregator should be stopped *last* since signers depends on it + // Note: The aggregators should be stopped *last* since signers depends on it info!("Stopping Mithril infrastructure"); for signer in self.signers.as_mut_slice() { signer.stop().await?; } - self.aggregator.stop().await?; + for aggregator in self.aggregators.as_mut_slice() { + aggregator.stop().await?; + } Ok(()) } @@ -309,12 +344,12 @@ impl MithrilInfrastructure { &self.devnet } - pub fn aggregator(&self) -> &Aggregator { - &self.aggregator + pub fn master_aggregator(&self) -> &Aggregator { + &self.aggregators[0] } - pub fn aggregator_mut(&mut self) -> &mut Aggregator { - self.aggregator.borrow_mut() + pub fn master_aggregator_mut(&mut self) -> &mut Aggregator { + self.aggregators[0].borrow_mut() } pub fn signers(&self) -> &[Signer] { @@ -337,8 +372,8 @@ impl MithrilInfrastructure { &self.relay_passives } - pub fn chain_observer(&self) -> Arc { - self.cardano_chain_observer.clone() + pub fn master_chain_observer(&self) -> Arc { + self.cardano_chain_observers[0].clone() } pub fn build_client(&self) -> StdResult { @@ -354,7 +389,11 @@ impl MithrilInfrastructure { artifacts_dir }; - Client::new(self.aggregator.endpoint(), &work_dir, &self.bin_dir) + Client::new( + self.master_aggregator().endpoint(), + &work_dir, + &self.bin_dir, + ) } pub fn run_only_mode(&self) -> bool { @@ -362,7 +401,7 @@ impl MithrilInfrastructure { } pub async fn tail_logs(&self, number_of_line: u64) -> StdResult<()> { - self.aggregator().tail_logs(number_of_line).await?; + self.master_aggregator().tail_logs(number_of_line).await?; for signer in self.signers() { signer.tail_logs(number_of_line).await?; } @@ -380,7 +419,7 @@ impl MithrilInfrastructure { } pub async fn last_error_in_logs(&self, number_of_error: u64) -> StdResult<()> { - self.aggregator() + self.master_aggregator() .last_error_in_logs(number_of_error) .await?; diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/signer.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/signer.rs index b91dee9ed66..d92633c1e76 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/signer.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/signer.rs @@ -16,6 +16,7 @@ pub struct SignerConfig<'a> { pub pool_node: &'a PoolNode, pub cardano_cli_path: &'a Path, pub work_dir: &'a Path, + pub store_dir: &'a Path, pub bin_dir: &'a Path, pub mithril_run_interval: u32, pub mithril_era: &'a str, @@ -36,7 +37,6 @@ impl Signer { pub fn new(signer_config: &SignerConfig) -> StdResult { let party_id = signer_config.pool_node.party_id()?; let magic_id = DEVNET_MAGIC_ID.to_string(); - let data_stores_path = format!("./stores/signer-{party_id}"); let era_reader_adapter_params = if signer_config.mithril_era_reader_adapter == "cardano-chain" { format!( @@ -58,7 +58,10 @@ impl Signer { "DB_DIRECTORY", signer_config.pool_node.db_path.to_str().unwrap(), ), - ("DATA_STORES_DIRECTORY", &data_stores_path), + ( + "DATA_STORES_DIRECTORY", + signer_config.store_dir.to_str().unwrap(), + ), ("STORE_RETENTION_LIMIT", "10"), ("NETWORK_MAGIC", &magic_id), ( diff --git a/mithril-test-lab/mithril-end-to-end/src/run_only.rs b/mithril-test-lab/mithril-end-to-end/src/run_only.rs index 3fbd20b19d1..dda8042795b 100644 --- a/mithril-test-lab/mithril-end-to-end/src/run_only.rs +++ b/mithril-test-lab/mithril-end-to-end/src/run_only.rs @@ -12,12 +12,14 @@ impl<'a> RunOnly<'a> { } pub async fn start(&mut self) -> StdResult<()> { - let aggregator_endpoint = self.infrastructure.aggregator().endpoint(); - assertions::wait_for_enough_immutable(self.infrastructure.aggregator().db_directory()) - .await?; + let aggregator_endpoint = self.infrastructure.master_aggregator().endpoint(); + assertions::wait_for_enough_immutable( + self.infrastructure.master_aggregator().db_directory(), + ) + .await?; let start_epoch = self .infrastructure - .chain_observer() + .master_chain_observer() .get_current_epoch() .await? .unwrap_or_default(); @@ -25,13 +27,14 @@ impl<'a> RunOnly<'a> { // Wait 3 epochs after start epoch for the aggregator to be able to bootstrap a genesis certificate let target_epoch = start_epoch + 3; assertions::wait_for_target_epoch( - self.infrastructure.chain_observer(), + self.infrastructure.master_chain_observer(), target_epoch, "minimal epoch for the aggregator to be able to bootstrap genesis certificate" .to_string(), ) .await?; - assertions::bootstrap_genesis_certificate(self.infrastructure.aggregator_mut()).await?; + assertions::bootstrap_genesis_certificate(self.infrastructure.master_aggregator_mut()) + .await?; assertions::wait_for_epoch_settings(&aggregator_endpoint).await?; // Transfer some funds on the devnet to have some Cardano transactions to sign diff --git a/mithril-test-lab/mithril-end-to-end/src/stress_test/aggregator_helpers.rs b/mithril-test-lab/mithril-end-to-end/src/stress_test/aggregator_helpers.rs index cf25ac1c9dd..059b242108e 100644 --- a/mithril-test-lab/mithril-end-to-end/src/stress_test/aggregator_helpers.rs +++ b/mithril-test-lab/mithril-end-to-end/src/stress_test/aggregator_helpers.rs @@ -19,10 +19,12 @@ pub async fn bootstrap_aggregator( let chain_observer_type = "cardano-cli"; let mut aggregator = Aggregator::new(&AggregatorConfig { + index: 0, server_port: args.server_port as u64, pool_node: &args.pool_node, cardano_cli_path: &args.cardano_cli_path, work_dir: &args.work_dir, + store_dir: &args.work_dir, artifacts_dir: &args.work_dir, bin_dir: &args.bin_dir, cardano_node_version: "1.2.3", @@ -32,6 +34,7 @@ pub async fn bootstrap_aggregator( mithril_era_reader_adapter: "dummy", signed_entity_types: &signed_entity_types, chain_observer_type, + master_aggregator_endpoint: &None, }) .unwrap(); From b917665e3df2b032bde3c37e40c0238232360041 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 12 Mar 2025 14:35:24 +0100 Subject: [PATCH 03/33] refactor(e2e): implement interior mutability for 'Aggregator', 'RunOnly' and 'Spec' --- .../mithril-end-to-end/src/assertions/exec.rs | 14 ++-- .../mithril-end-to-end/src/end_to_end_spec.rs | 27 +++---- .../mithril-end-to-end/src/main.rs | 8 +- .../src/mithril/aggregator.rs | 73 +++++++++++-------- .../src/mithril/infrastructure.rs | 42 +++++------ .../mithril-end-to-end/src/run_only.rs | 9 +-- .../src/stress_test/aggregator_helpers.rs | 28 ++++--- 7 files changed, 108 insertions(+), 93 deletions(-) diff --git a/mithril-test-lab/mithril-end-to-end/src/assertions/exec.rs b/mithril-test-lab/mithril-end-to-end/src/assertions/exec.rs index 4bab328576e..2dcc0247d25 100644 --- a/mithril-test-lab/mithril-end-to-end/src/assertions/exec.rs +++ b/mithril-test-lab/mithril-end-to-end/src/assertions/exec.rs @@ -5,7 +5,7 @@ use mithril_common::entities::{Epoch, ProtocolParameters}; use mithril_common::StdResult; use slog_scope::info; -pub async fn bootstrap_genesis_certificate(aggregator: &mut Aggregator) -> StdResult<()> { +pub async fn bootstrap_genesis_certificate(aggregator: &Aggregator) -> StdResult<()> { info!("Bootstrap genesis certificate"); info!("> stopping aggregator"); @@ -13,13 +13,13 @@ pub async fn bootstrap_genesis_certificate(aggregator: &mut Aggregator) -> StdRe info!("> bootstrapping genesis using signers registered two epochs ago..."); aggregator.bootstrap_genesis().await?; info!("> done, restarting aggregator"); - aggregator.serve()?; + aggregator.serve().await?; Ok(()) } pub async fn register_era_marker( - aggregator: &mut Aggregator, + aggregator: &Aggregator, devnet: &Devnet, mithril_era: &str, era_epoch: Epoch, @@ -56,7 +56,7 @@ pub async fn transfer_funds(devnet: &Devnet) -> StdResult<()> { Ok(()) } -pub async fn update_protocol_parameters(aggregator: &mut Aggregator) -> StdResult<()> { +pub async fn update_protocol_parameters(aggregator: &Aggregator) -> StdResult<()> { info!("Update protocol parameters"); info!("> stopping aggregator"); @@ -70,9 +70,11 @@ pub async fn update_protocol_parameters(aggregator: &mut Aggregator) -> StdResul "> updating protocol parameters to {:?}...", protocol_parameters_new ); - aggregator.set_protocol_parameters(&protocol_parameters_new); + aggregator + .set_protocol_parameters(&protocol_parameters_new) + .await; info!("> done, restarting aggregator"); - aggregator.serve()?; + aggregator.serve().await?; Ok(()) } diff --git a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs index 46c0ae00e47..9401c20c953 100644 --- a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs +++ b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs @@ -6,7 +6,7 @@ use mithril_common::{ }; pub struct Spec<'a> { - pub infrastructure: &'a mut MithrilInfrastructure, + pub infrastructure: &'a MithrilInfrastructure, is_signing_cardano_transactions: bool, is_signing_cardano_stake_distribution: bool, is_signing_cardano_database: bool, @@ -16,7 +16,7 @@ pub struct Spec<'a> { impl<'a> Spec<'a> { pub fn new( - infrastructure: &'a mut MithrilInfrastructure, + infrastructure: &'a MithrilInfrastructure, signed_entity_types: Vec, next_era: Option, regenesis_on_era_switch: bool, @@ -43,7 +43,7 @@ impl<'a> Spec<'a> { } } - pub async fn run(&mut self) -> StdResult<()> { + pub async fn run(&self) -> StdResult<()> { let aggregator_endpoint = self.infrastructure.master_aggregator().endpoint(); assertions::wait_for_enough_immutable( self.infrastructure.master_aggregator().db_directory(), @@ -71,8 +71,7 @@ impl<'a> Spec<'a> { .to_string(), ) .await?; - assertions::bootstrap_genesis_certificate(self.infrastructure.master_aggregator_mut()) - .await?; + assertions::bootstrap_genesis_certificate(self.infrastructure.master_aggregator()).await?; assertions::wait_for_epoch_settings(&aggregator_endpoint).await?; // Wait 2 epochs before changing stake distribution, so that we use at least one original stake distribution @@ -95,7 +94,7 @@ impl<'a> Spec<'a> { "epoch after which the protocol parameters will change".to_string(), ) .await?; - assertions::update_protocol_parameters(self.infrastructure.master_aggregator_mut()).await?; + assertions::update_protocol_parameters(self.infrastructure.master_aggregator()).await?; // Wait 6 epochs after protocol parameters update, so that we make sure that we use new protocol parameters as well as new stake distribution a few times target_epoch += 6; @@ -125,10 +124,8 @@ impl<'a> Spec<'a> { // Proceed to a re-genesis of the certificate chain if self.regenesis_on_era_switch { - assertions::bootstrap_genesis_certificate( - self.infrastructure.master_aggregator_mut(), - ) - .await?; + assertions::bootstrap_genesis_certificate(self.infrastructure.master_aggregator()) + .await?; target_epoch += 5; assertions::wait_for_target_epoch( self.infrastructure.master_chain_observer(), @@ -165,7 +162,7 @@ impl<'a> Spec<'a> { self.infrastructure.signers().len(), ) .await?; - let mut client = self.infrastructure.build_client()?; + let mut client = self.infrastructure.build_client().await?; assertions::assert_client_can_verify_mithril_stake_distribution(&mut client, &hash) .await?; } @@ -187,7 +184,7 @@ impl<'a> Spec<'a> { ) .await?; - let mut client = self.infrastructure.build_client()?; + let mut client = self.infrastructure.build_client().await?; assertions::assert_client_can_verify_snapshot(&mut client, &digest).await?; } @@ -213,7 +210,7 @@ impl<'a> Spec<'a> { assertions::assert_node_producing_cardano_database_digests_map(&aggregator_endpoint) .await?; - let mut client = self.infrastructure.build_client()?; + let mut client = self.infrastructure.build_client().await?; assertions::assert_client_can_verify_cardano_database(&mut client, &hash).await?; } @@ -239,7 +236,7 @@ impl<'a> Spec<'a> { .infrastructure .devnet() .mithril_payments_transaction_hashes()?; - let mut client = self.infrastructure.build_client()?; + let mut client = self.infrastructure.build_client().await?; assertions::assert_client_can_verify_transactions(&mut client, transaction_hashes) .await?; } @@ -265,7 +262,7 @@ impl<'a> Spec<'a> { ) .await?; - let mut client = self.infrastructure.build_client()?; + let mut client = self.infrastructure.build_client().await?; assertions::assert_client_can_verify_cardano_stake_distribution( &mut client, &hash, diff --git a/mithril-test-lab/mithril-end-to-end/src/main.rs b/mithril-test-lab/mithril-end-to-end/src/main.rs index 0460ab9ef36..3b7da24c783 100644 --- a/mithril-test-lab/mithril-end-to-end/src/main.rs +++ b/mithril-test-lab/mithril-end-to-end/src/main.rs @@ -340,7 +340,7 @@ impl App { .await?; *self.devnet.lock().await = Some(devnet.clone()); - let mut infrastructure = MithrilInfrastructure::start(&MithrilInfrastructureConfig { + let infrastructure = MithrilInfrastructure::start(&MithrilInfrastructureConfig { number_of_aggregators: args.number_of_aggregators, number_of_signers: args.number_of_signers, server_port, @@ -363,12 +363,12 @@ impl App { let runner: StdResult<()> = match run_only_mode { true => { - let mut run_only = RunOnly::new(&mut infrastructure); + let run_only = RunOnly::new(&infrastructure); run_only.start().await } false => { - let mut spec = Spec::new( - &mut infrastructure, + let spec = Spec::new( + &infrastructure, args.signed_entity_types, args.mithril_next_era, args.mithril_era_regenesis_on_switch, diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs index 30679b072f3..2531213d397 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs @@ -10,8 +10,10 @@ use slog_scope::info; use std::cmp; use std::collections::HashMap; use std::path::{Path, PathBuf}; +use std::sync::Arc; use std::time::Duration; use tokio::process::Child; +use tokio::sync::RwLock; #[derive(Debug)] pub struct AggregatorConfig<'a> { @@ -37,8 +39,8 @@ pub struct AggregatorConfig<'a> { pub struct Aggregator { server_port: u64, db_directory: PathBuf, - command: MithrilCommand, - process: Option, + command: Arc>, + process: RwLock>, } impl Aggregator { @@ -135,8 +137,8 @@ impl Aggregator { Ok(Self { server_port: aggregator_config.server_port, db_directory: aggregator_config.pool_node.db_path.clone(), - command, - process: None, + command: Arc::new(RwLock::new(command)), + process: RwLock::new(None), }) } @@ -145,7 +147,7 @@ impl Aggregator { server_port: other.server_port, db_directory: other.db_directory.clone(), command: other.command.clone(), - process: None, + process: RwLock::new(None), } } @@ -157,14 +159,16 @@ impl Aggregator { &self.db_directory } - pub fn serve(&mut self) -> StdResult<()> { - self.process = Some(self.command.start(&["serve".to_string()])?); + pub async fn serve(&self) -> StdResult<()> { + let mut command = self.command.write().await; + let mut process = self.process.write().await; + *process = Some(command.start(&["serve".to_string()])?); Ok(()) } - pub async fn bootstrap_genesis(&mut self) -> StdResult<()> { + pub async fn bootstrap_genesis(&self) -> StdResult<()> { // Clone the command so we can alter it without affecting the original - let mut command = self.command.clone(); + let mut command = self.command.write().await; let process_name = "mithril-aggregator-genesis-bootstrap"; command.set_log_name(process_name); @@ -177,7 +181,7 @@ impl Aggregator { if exit_status.success() { Ok(()) } else { - self.command.tail_logs(Some(process_name), 40).await?; + command.tail_logs(Some(process_name), 40).await?; Err(match exit_status.code() { Some(c) => { @@ -190,20 +194,21 @@ impl Aggregator { } } - pub async fn stop(&mut self) -> StdResult<()> { - if let Some(process) = self.process.as_mut() { + pub async fn stop(&self) -> StdResult<()> { + let mut process = self.process.write().await; + if let Some(mut process_running) = process.take() { info!("Stopping aggregator"); - process + process_running .kill() .await .with_context(|| "Could not kill aggregator")?; - self.process = None; + *process = None; } Ok(()) } pub async fn era_generate_tx_datum( - &mut self, + &self, target_path: &Path, mithril_era: &str, next_era_activation_epoch: entities::Epoch, @@ -234,8 +239,8 @@ impl Aggregator { args.push(next_era_epoch.to_string()); } - let exit_status = self - .command + let mut command = self.command.write().await; + let exit_status = command .start(&args)? .wait() .await @@ -257,45 +262,51 @@ impl Aggregator { } } - pub fn set_protocol_parameters(&mut self, protocol_parameters: &entities::ProtocolParameters) { - self.command.set_env_var( + pub async fn set_protocol_parameters( + &self, + protocol_parameters: &entities::ProtocolParameters, + ) { + let mut command = self.command.write().await; + command.set_env_var( "PROTOCOL_PARAMETERS__K", &format!("{}", protocol_parameters.k), ); - self.command.set_env_var( + command.set_env_var( "PROTOCOL_PARAMETERS__M", &format!("{}", protocol_parameters.m), ); - self.command.set_env_var( + command.set_env_var( "PROTOCOL_PARAMETERS__PHI_F", &format!("{}", protocol_parameters.phi_f), ); } - pub fn set_mock_cardano_cli_file_path( - &mut self, + pub async fn set_mock_cardano_cli_file_path( + &self, stake_distribution_file: &Path, epoch_file_path: &Path, ) { - self.command.set_env_var( + let mut command = self.command.write().await; + command.set_env_var( "MOCK_STAKE_DISTRIBUTION_FILE", stake_distribution_file.to_str().unwrap(), ); - self.command - .set_env_var("MOCK_EPOCH_FILE", epoch_file_path.to_str().unwrap()); + command.set_env_var("MOCK_EPOCH_FILE", epoch_file_path.to_str().unwrap()); } /// Change the run interval of the aggregator state machine (default: 400ms) - pub fn change_run_interval(&mut self, interval: Duration) { - self.command - .set_env_var("RUN_INTERVAL", &format!("{}", interval.as_millis())) + pub async fn change_run_interval(&self, interval: Duration) { + let mut command = self.command.write().await; + command.set_env_var("RUN_INTERVAL", &format!("{}", interval.as_millis())) } pub async fn tail_logs(&self, number_of_line: u64) -> StdResult<()> { - self.command.tail_logs(None, number_of_line).await + let command = self.command.write().await; + command.tail_logs(None, number_of_line).await } pub async fn last_error_in_logs(&self, number_of_error: u64) -> StdResult<()> { - self.command.last_error_in_logs(None, number_of_error).await + let command = self.command.write().await; + command.last_error_in_logs(None, number_of_error).await } } diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs index e0d1e30044d..d5d99958791 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs @@ -2,10 +2,10 @@ use mithril_common::chain_observer::{ChainObserver, PallasChainObserver}; use mithril_common::entities::{Epoch, PartyId, ProtocolParameters}; use mithril_common::{CardanoNetwork, StdResult}; use slog_scope::info; -use std::borrow::BorrowMut; use std::fs; use std::path::PathBuf; use std::sync::Arc; +use tokio::sync::RwLock; use crate::{ assertions, Aggregator, AggregatorConfig, Client, Devnet, PoolNode, RelayAggregator, @@ -45,7 +45,7 @@ pub struct MithrilInfrastructure { relay_passives: Vec, cardano_chain_observers: Vec>, run_only_mode: bool, - current_era: String, + current_era: RwLock, era_reader_adapter: String, use_era_specific_work_dir: bool, } @@ -107,14 +107,14 @@ impl MithrilInfrastructure { relay_passives, cardano_chain_observers, run_only_mode: config.run_only_mode, - current_era: config.mithril_era.clone(), + current_era: RwLock::new(config.mithril_era.clone()), era_reader_adapter: config.mithril_era_reader_adapter.clone(), use_era_specific_work_dir: config.use_era_specific_work_dir, }) } async fn register_startup_era( - aggregator: &mut Aggregator, + aggregator: &Aggregator, config: &MithrilInfrastructureConfig, ) -> StdResult<()> { let era_epoch = Epoch(0); @@ -131,7 +131,7 @@ impl MithrilInfrastructure { Ok(()) } - pub async fn register_switch_to_next_era(&mut self, next_era: &str) -> StdResult<()> { + pub async fn register_switch_to_next_era(&self, next_era: &str) -> StdResult<()> { let next_era_epoch = self .master_chain_observer() .get_current_epoch() @@ -141,14 +141,15 @@ impl MithrilInfrastructure { if self.era_reader_adapter == "cardano-chain" { let devnet = self.devnet.clone(); assertions::register_era_marker( - self.master_aggregator_mut(), + self.master_aggregator(), &devnet, next_era, next_era_epoch, ) .await?; } - self.current_era = next_era.to_owned(); + let mut current_era = self.current_era.write().await; + *current_era = next_era.to_owned(); Ok(()) } @@ -174,7 +175,7 @@ impl MithrilInfrastructure { config.store_dir.join(format!("aggregator-slave-{index}")) }; - let mut aggregator = Aggregator::new(&AggregatorConfig { + let aggregator = Aggregator::new(&AggregatorConfig { index, server_port: config.server_port + index as u64, pool_node, @@ -193,15 +194,17 @@ impl MithrilInfrastructure { master_aggregator_endpoint: &master_aggregator_endpoint.clone(), })?; - aggregator.set_protocol_parameters(&ProtocolParameters { - k: 75, - m: 105, - phi_f: 0.95, - }); + aggregator + .set_protocol_parameters(&ProtocolParameters { + k: 75, + m: 105, + phi_f: 0.95, + }) + .await; - Self::register_startup_era(&mut aggregator, config).await?; + Self::register_startup_era(&aggregator, config).await?; - aggregator.serve()?; + aggregator.serve().await?; if master_aggregator_endpoint.is_none() { master_aggregator_endpoint = Some(aggregator.endpoint()); @@ -348,10 +351,6 @@ impl MithrilInfrastructure { &self.aggregators[0] } - pub fn master_aggregator_mut(&mut self) -> &mut Aggregator { - self.aggregators[0].borrow_mut() - } - pub fn signers(&self) -> &[Signer] { &self.signers } @@ -376,11 +375,12 @@ impl MithrilInfrastructure { self.cardano_chain_observers[0].clone() } - pub fn build_client(&self) -> StdResult { + pub async fn build_client(&self) -> StdResult { let work_dir = { let mut artifacts_dir = self.artifacts_dir.join("mithril-client"); if self.use_era_specific_work_dir { - artifacts_dir = artifacts_dir.join(format!("era.{}", self.current_era)); + let current_era = self.current_era.read().await; + artifacts_dir = artifacts_dir.join(format!("era.{}", current_era)); } if !artifacts_dir.exists() { fs::create_dir_all(&artifacts_dir)?; diff --git a/mithril-test-lab/mithril-end-to-end/src/run_only.rs b/mithril-test-lab/mithril-end-to-end/src/run_only.rs index dda8042795b..146c996e4e6 100644 --- a/mithril-test-lab/mithril-end-to-end/src/run_only.rs +++ b/mithril-test-lab/mithril-end-to-end/src/run_only.rs @@ -3,15 +3,15 @@ use crate::MithrilInfrastructure; use mithril_common::StdResult; pub struct RunOnly<'a> { - pub infrastructure: &'a mut MithrilInfrastructure, + pub infrastructure: &'a MithrilInfrastructure, } impl<'a> RunOnly<'a> { - pub fn new(infrastructure: &'a mut MithrilInfrastructure) -> Self { + pub fn new(infrastructure: &'a MithrilInfrastructure) -> Self { Self { infrastructure } } - pub async fn start(&mut self) -> StdResult<()> { + pub async fn start(&self) -> StdResult<()> { let aggregator_endpoint = self.infrastructure.master_aggregator().endpoint(); assertions::wait_for_enough_immutable( self.infrastructure.master_aggregator().db_directory(), @@ -33,8 +33,7 @@ impl<'a> RunOnly<'a> { .to_string(), ) .await?; - assertions::bootstrap_genesis_certificate(self.infrastructure.master_aggregator_mut()) - .await?; + assertions::bootstrap_genesis_certificate(self.infrastructure.master_aggregator()).await?; assertions::wait_for_epoch_settings(&aggregator_endpoint).await?; // Transfer some funds on the devnet to have some Cardano transactions to sign diff --git a/mithril-test-lab/mithril-end-to-end/src/stress_test/aggregator_helpers.rs b/mithril-test-lab/mithril-end-to-end/src/stress_test/aggregator_helpers.rs index 059b242108e..ee1d6e69c1a 100644 --- a/mithril-test-lab/mithril-end-to-end/src/stress_test/aggregator_helpers.rs +++ b/mithril-test-lab/mithril-end-to-end/src/stress_test/aggregator_helpers.rs @@ -43,18 +43,24 @@ pub async fn bootstrap_aggregator( // Extremely large interval since, for the two following starts, only the http_server part // of the aggregator is relevant as we need to send signer registrations. - aggregator.change_run_interval(Duration::from_secs(20000)); - aggregator.set_mock_cardano_cli_file_path( - &args.mock_stake_distribution_file_path(), - &args.mock_epoch_file_path(), - ); - aggregator.set_protocol_parameters(&signers_fixture.protocol_parameters()); + aggregator + .change_run_interval(Duration::from_secs(20000)) + .await; + aggregator + .set_mock_cardano_cli_file_path( + &args.mock_stake_distribution_file_path(), + &args.mock_epoch_file_path(), + ) + .await; + aggregator + .set_protocol_parameters(&signers_fixture.protocol_parameters()) + .await; info!( ">> Starting the aggregator with a large run interval to call the http_server\ without being bothered by the state machine cycles" ); - aggregator.serve().unwrap(); + aggregator.serve().await.unwrap(); wait::for_aggregator_http_server_to_start(&aggregator, Duration::from_secs(10)).await?; restart_aggregator_and_move_one_epoch_forward(&mut aggregator, current_epoch, args).await?; @@ -98,7 +104,7 @@ pub async fn bootstrap_aggregator( { info!(">> Compute genesis certificate"); - let mut genesis_aggregator = Aggregator::copy_configuration(&aggregator); + let genesis_aggregator = Aggregator::copy_configuration(&aggregator); genesis_aggregator .bootstrap_genesis() .await @@ -106,8 +112,8 @@ pub async fn bootstrap_aggregator( } info!(">> Restart aggregator with a normal run interval"); - aggregator.change_run_interval(Duration::from_secs(3)); - aggregator.serve().unwrap(); + aggregator.change_run_interval(Duration::from_secs(3)).await; + aggregator.serve().await.unwrap(); wait::for_aggregator_http_server_to_start(&aggregator, Duration::from_secs(10)).await?; @@ -128,7 +134,7 @@ async fn restart_aggregator_and_move_one_epoch_forward( fake_chain::set_epoch(&args.mock_epoch_file_path(), *current_epoch); info!(">> Restarting the aggregator with a large run interval"); - aggregator.serve().unwrap(); + aggregator.serve().await.unwrap(); wait::for_aggregator_http_server_to_start(aggregator, Duration::from_secs(10)).await?; wait::for_epoch_settings_at_epoch(aggregator, Duration::from_secs(10), *current_epoch).await?; From f92420c0903b643a86002fbbedf6520dce4d32e3 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Thu, 13 Mar 2025 16:38:13 +0100 Subject: [PATCH 04/33] refactor(relay): support for dialing to peer in relays of e2e test --- .../src/mithril/relay_aggregator.rs | 21 ++++++++++++++++--- .../src/mithril/relay_passive.rs | 10 ++++----- .../src/mithril/relay_signer.rs | 8 ++++--- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/relay_aggregator.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/relay_aggregator.rs index dcea189e2ba..ab028505199 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/relay_aggregator.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/relay_aggregator.rs @@ -6,6 +6,7 @@ use tokio::process::Child; #[derive(Debug)] pub struct RelayAggregator { + index: usize, listen_port: u64, command: MithrilCommand, process: Option, @@ -13,22 +14,28 @@ pub struct RelayAggregator { impl RelayAggregator { pub fn new( + index: usize, listen_port: u64, + dial_to: Option, aggregator_endpoint: &str, work_dir: &Path, bin_dir: &Path, ) -> StdResult { let listen_port_str = format!("{listen_port}"); - let env = HashMap::from([ + let mut env = HashMap::from([ ("LISTEN_PORT", listen_port_str.as_str()), ("AGGREGATOR_ENDPOINT", aggregator_endpoint), ]); + if let Some(dial_to) = &dial_to { + env.insert("DIAL_TO", dial_to); + } let args = vec!["-vvv", "aggregator"]; let mut command = MithrilCommand::new("mithril-relay", work_dir, bin_dir, env, &args)?; - command.set_log_name("mithril-relay-aggregator"); + command.set_log_name(&Self::command_name(index)); Ok(Self { + index, listen_port, command, process: None, @@ -39,6 +46,14 @@ impl RelayAggregator { format!("/ip4/127.0.0.1/tcp/{}", self.listen_port) } + fn command_name(index: usize) -> String { + if index == 0 { + "mithril-relay-aggregator".to_string() + } else { + format!("mithril-relay-aggregator-slave-{}", index) + } + } + pub fn start(&mut self) -> StdResult<()> { self.process = Some(self.command.start(&[])?); Ok(()) @@ -46,7 +61,7 @@ impl RelayAggregator { pub async fn tail_logs(&self, number_of_line: u64) -> StdResult<()> { self.command - .tail_logs(Some("mithril-relay-aggregator"), number_of_line) + .tail_logs(Some(&Self::command_name(self.index)), number_of_line) .await } } diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/relay_passive.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/relay_passive.rs index e5f096cf29a..bd019d50a76 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/relay_passive.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/relay_passive.rs @@ -15,16 +15,16 @@ pub struct RelayPassive { impl RelayPassive { pub fn new( listen_port: u64, - dial_to: String, + dial_to: Option, relay_id: String, work_dir: &Path, bin_dir: &Path, ) -> StdResult { let listen_port_str = format!("{listen_port}"); - let env = HashMap::from([ - ("LISTEN_PORT", listen_port_str.as_str()), - ("DIAL_TO", &dial_to), - ]); + let mut env = HashMap::from([("LISTEN_PORT", listen_port_str.as_str())]); + if let Some(dial_to) = &dial_to { + env.insert("DIAL_TO", dial_to); + } let args = vec!["-vvv", "passive"]; let mut command = MithrilCommand::new("mithril-relay", work_dir, bin_dir, env, &args)?; diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/relay_signer.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/relay_signer.rs index c4c22f77373..70c433abe5f 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/relay_signer.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/relay_signer.rs @@ -18,7 +18,7 @@ impl RelaySigner { pub fn new( listen_port: u64, server_port: u64, - dial_to: String, + dial_to: Option, aggregator_endpoint: &str, party_id: PartyId, work_dir: &Path, @@ -26,13 +26,15 @@ impl RelaySigner { ) -> StdResult { let listen_port_str = format!("{listen_port}"); let server_port_str = format!("{server_port}"); - let env = HashMap::from([ + let mut env = HashMap::from([ ("LISTEN_PORT", listen_port_str.as_str()), ("SERVER_PORT", server_port_str.as_str()), ("AGGREGATOR_ENDPOINT", aggregator_endpoint), - ("DIAL_TO", &dial_to), ("SIGNER_REPEATER_DELAY", "100"), ]); + if let Some(dial_to) = &dial_to { + env.insert("DIAL_TO", dial_to); + } let args = vec!["-vvv", "signer"]; let mut command = MithrilCommand::new("mithril-relay", work_dir, bin_dir, env, &args)?; From 2d948a02f597d7ea7c9d83cc7c0f53b43102bdfd Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Thu, 13 Mar 2025 16:41:06 +0100 Subject: [PATCH 05/33] refactor(e2e): enhance 'MithrilInfrastructure' support for multiple aggregators Better P2P relays topology and fix log files collisions. --- .../src/mithril/aggregator.rs | 13 ++- .../src/mithril/infrastructure.rs | 86 +++++++++++++------ 2 files changed, 71 insertions(+), 28 deletions(-) diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs index 2531213d397..9800cb5e6df 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs @@ -37,6 +37,7 @@ pub struct AggregatorConfig<'a> { #[derive(Debug)] pub struct Aggregator { + index: usize, server_port: u64, db_directory: PathBuf, command: Arc>, @@ -135,6 +136,7 @@ impl Aggregator { command.set_log_name(&name); Ok(Self { + index: aggregator_config.index, server_port: aggregator_config.server_port, db_directory: aggregator_config.pool_node.db_path.clone(), command: Arc::new(RwLock::new(command)), @@ -144,6 +146,7 @@ impl Aggregator { pub fn copy_configuration(other: &Aggregator) -> Self { Self { + index: other.index, server_port: other.server_port, db_directory: other.db_directory.clone(), command: other.command.clone(), @@ -169,8 +172,12 @@ impl Aggregator { pub async fn bootstrap_genesis(&self) -> StdResult<()> { // Clone the command so we can alter it without affecting the original let mut command = self.command.write().await; - let process_name = "mithril-aggregator-genesis-bootstrap"; - command.set_log_name(process_name); + let command_name = if self.index == 0 { + "mithril-aggregator-genesis-bootstrap" + } else { + &format!("mithril-aggregator-genesis-bootstrap-slave-{}", self.index) + }; + command.set_log_name(command_name); let exit_status = command .start(&["genesis".to_string(), "bootstrap".to_string()])? @@ -181,7 +188,7 @@ impl Aggregator { if exit_status.success() { Ok(()) } else { - command.tail_logs(Some(process_name), 40).await?; + command.tail_logs(Some(command_name), 40).await?; Err(match exit_status.code() { Some(c) => { diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs index d5d99958791..d20ba4ef73c 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs @@ -202,13 +202,13 @@ impl MithrilInfrastructure { }) .await; - Self::register_startup_era(&aggregator, config).await?; - - aggregator.serve().await?; - if master_aggregator_endpoint.is_none() { master_aggregator_endpoint = Some(aggregator.endpoint()); + Self::register_startup_era(&aggregator, config).await?; } + + aggregator.serve().await?; + aggregators.push(aggregator); } @@ -227,60 +227,69 @@ impl MithrilInfrastructure { let mut relay_aggregators: Vec = vec![]; let mut relay_signers: Vec = vec![]; let mut relay_passives: Vec = vec![]; + let master_aggregator_endpoint = &aggregator_endpoints[0]; info!("Starting the Mithril infrastructure in P2P mode (experimental)"); + let mut bootstrap_peer_addr = None; for (index, aggregator_endpoint) in aggregator_endpoints.iter().enumerate() { let mut relay_aggregator = RelayAggregator::new( + index, config.server_port + index as u64 + 100, + bootstrap_peer_addr.clone(), aggregator_endpoint, &config.work_dir, &config.bin_dir, )?; + if bootstrap_peer_addr.is_none() { + bootstrap_peer_addr = Some(relay_aggregator.peer_addr().to_owned()); + } relay_aggregator.start()?; relay_aggregators.push(relay_aggregator); } - let master_aggregator = &relay_aggregators[0]; - - let mut relay_passive_id = 1; - if config.use_p2p_passive_relays { - let mut relay_passive_aggregator = RelayPassive::new( - config.server_port + 200, - master_aggregator.peer_addr().to_owned(), - format!("{relay_passive_id}"), - &config.work_dir, - &config.bin_dir, - )?; - relay_passive_aggregator.start()?; - relay_passives.push(relay_passive_aggregator); - } for (index, party_id) in signers_party_ids.iter().enumerate() { let mut relay_signer = RelaySigner::new( + config.server_port + index as u64 + 200, config.server_port + index as u64 + 300, - config.server_port + index as u64 + 400, - master_aggregator.peer_addr().to_owned(), - &aggregator_endpoints[0], + bootstrap_peer_addr.clone(), + master_aggregator_endpoint, party_id.clone(), &config.work_dir, &config.bin_dir, )?; relay_signer.start()?; - if config.use_p2p_passive_relays { + relay_signers.push(relay_signer); + } + + if config.use_p2p_passive_relays { + let mut relay_passive_id = 1; + for (index, _aggregator_endpoint) in aggregator_endpoints.iter().enumerate() { + let mut relay_passive_aggregator = RelayPassive::new( + config.server_port + index as u64 + 400, + bootstrap_peer_addr.clone(), + format!("{relay_passive_id}"), + &config.work_dir, + &config.bin_dir, + )?; + relay_passive_aggregator.start()?; + relay_passives.push(relay_passive_aggregator); relay_passive_id += 1; + } + + for (index, _party_id) in signers_party_ids.iter().enumerate() { let mut relay_passive_signer = RelayPassive::new( config.server_port + index as u64 + 500, - relay_signer.peer_addr().to_owned(), + bootstrap_peer_addr.clone(), format!("{relay_passive_id}"), &config.work_dir, &config.bin_dir, )?; relay_passive_signer.start()?; relay_passives.push(relay_passive_signer); + relay_passive_id += 1; } - - relay_signers.push(relay_signer); } Ok((relay_aggregators, relay_signers, relay_passives)) @@ -348,9 +357,21 @@ impl MithrilInfrastructure { } pub fn master_aggregator(&self) -> &Aggregator { + assert!( + !self.aggregators.is_empty(), + "No master aggregator available for this infrastructure" + ); &self.aggregators[0] } + pub fn slave_aggregators(&self) -> &[Aggregator] { + &self.aggregators[1..] + } + + pub fn slave_aggregator(&self, index: usize) -> &Aggregator { + &self.aggregators[index + 1] + } + pub fn signers(&self) -> &[Signer] { &self.signers } @@ -372,9 +393,21 @@ impl MithrilInfrastructure { } pub fn master_chain_observer(&self) -> Arc { + assert!( + !self.aggregators.is_empty(), + "No master chain observer available for this infrastructure" + ); self.cardano_chain_observers[0].clone() } + pub fn slave_chain_observers(&self) -> &[Arc] { + &self.cardano_chain_observers[1..] + } + + pub fn slave_chain_observer(&self, index: usize) -> Arc { + self.cardano_chain_observers[index + 1].clone() + } + pub async fn build_client(&self) -> StdResult { let work_dir = { let mut artifacts_dir = self.artifacts_dir.join("mithril-client"); @@ -402,6 +435,9 @@ impl MithrilInfrastructure { pub async fn tail_logs(&self, number_of_line: u64) -> StdResult<()> { self.master_aggregator().tail_logs(number_of_line).await?; + for aggregator in self.slave_aggregators() { + aggregator.tail_logs(number_of_line).await?; + } for signer in self.signers() { signer.tail_logs(number_of_line).await?; } From a6ffd0fb464439d8ef6f6c772997bbdeadf45dc8 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Thu, 13 Mar 2025 16:42:45 +0100 Subject: [PATCH 06/33] feat(e2e): 'RunOnly' supports multiple aggregators --- .../src/mithril/aggregator.rs | 6 +- .../mithril-end-to-end/src/run_only.rs | 98 +++++++++++++++---- 2 files changed, 84 insertions(+), 20 deletions(-) diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs index 9800cb5e6df..b66ba9cd653 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs @@ -154,6 +154,10 @@ impl Aggregator { } } + pub fn is_master(&self) -> bool { + self.index == 0 + } + pub fn endpoint(&self) -> String { format!("http://localhost:{}/aggregator", &self.server_port) } @@ -172,7 +176,7 @@ impl Aggregator { pub async fn bootstrap_genesis(&self) -> StdResult<()> { // Clone the command so we can alter it without affecting the original let mut command = self.command.write().await; - let command_name = if self.index == 0 { + let command_name = if self.is_master() { "mithril-aggregator-genesis-bootstrap" } else { &format!("mithril-aggregator-genesis-bootstrap-slave-{}", self.index) diff --git a/mithril-test-lab/mithril-end-to-end/src/run_only.rs b/mithril-test-lab/mithril-end-to-end/src/run_only.rs index 146c996e4e6..c624f1ff04f 100644 --- a/mithril-test-lab/mithril-end-to-end/src/run_only.rs +++ b/mithril-test-lab/mithril-end-to-end/src/run_only.rs @@ -1,25 +1,83 @@ -use crate::assertions; -use crate::MithrilInfrastructure; +use std::sync::Arc; + +use anyhow::anyhow; +use mithril_common::chain_observer::ChainObserver; +use tokio::sync::RwLock; +use tokio::task::JoinSet; + use mithril_common::StdResult; -pub struct RunOnly<'a> { - pub infrastructure: &'a MithrilInfrastructure, +use crate::{assertions, Aggregator, MithrilInfrastructure}; + +pub struct RunOnly { + pub infrastructure: Arc>>, } -impl<'a> RunOnly<'a> { - pub fn new(infrastructure: &'a MithrilInfrastructure) -> Self { +impl RunOnly { + pub fn new(infrastructure: Arc>>) -> Self { Self { infrastructure } } - pub async fn start(&self) -> StdResult<()> { - let aggregator_endpoint = self.infrastructure.master_aggregator().endpoint(); - assertions::wait_for_enough_immutable( - self.infrastructure.master_aggregator().db_directory(), + pub async fn run(self) -> StdResult<()> { + let run_only = Arc::new(self); + let mut join_set = JoinSet::new(); + + let run_only_clone = run_only.clone(); + join_set.spawn(async move { run_only_clone.start_master().await }); + + let infrastructure_guard = run_only.infrastructure.read().await; + let slave_aggregators = infrastructure_guard + .as_ref() + .ok_or(anyhow!("No infrastructure found"))? + .slave_aggregators(); + for index in 0..slave_aggregators.len() { + let run_only_clone = run_only.clone(); + join_set.spawn(async move { run_only_clone.start_slave(index).await }); + } + + while let Some(res) = join_set.join_next().await { + res??; + } + + Ok(()) + } + + pub async fn start_master(&self) -> StdResult<()> { + let infrastructure_guard = self.infrastructure.read().await; + let infrastructure = infrastructure_guard + .as_ref() + .ok_or(anyhow!("No infrastructure found"))?; + + self.start_aggregator( + infrastructure.master_aggregator(), + infrastructure.master_chain_observer(), + infrastructure, ) - .await?; - let start_epoch = self - .infrastructure - .master_chain_observer() + .await + } + + pub async fn start_slave(&self, slave_index: usize) -> StdResult<()> { + let infrastructure_guard = self.infrastructure.read().await; + let infrastructure = infrastructure_guard + .as_ref() + .ok_or(anyhow!("No infrastructure found"))?; + + self.start_aggregator( + infrastructure.slave_aggregator(slave_index), + infrastructure.slave_chain_observer(slave_index), + infrastructure, + ) + .await + } + + pub async fn start_aggregator( + &self, + aggregator: &Aggregator, + chain_observer: Arc, + infrastructure: &MithrilInfrastructure, + ) -> StdResult<()> { + assertions::wait_for_enough_immutable(aggregator.db_directory()).await?; + let start_epoch = chain_observer .get_current_epoch() .await? .unwrap_or_default(); @@ -27,17 +85,19 @@ impl<'a> RunOnly<'a> { // Wait 3 epochs after start epoch for the aggregator to be able to bootstrap a genesis certificate let target_epoch = start_epoch + 3; assertions::wait_for_target_epoch( - self.infrastructure.master_chain_observer(), + chain_observer, target_epoch, "minimal epoch for the aggregator to be able to bootstrap genesis certificate" .to_string(), ) .await?; - assertions::bootstrap_genesis_certificate(self.infrastructure.master_aggregator()).await?; - assertions::wait_for_epoch_settings(&aggregator_endpoint).await?; + assertions::bootstrap_genesis_certificate(aggregator).await?; + assertions::wait_for_epoch_settings(&aggregator.endpoint()).await?; - // Transfer some funds on the devnet to have some Cardano transactions to sign - assertions::transfer_funds(self.infrastructure.devnet()).await?; + if aggregator.is_master() { + // Transfer some funds on the devnet to have some Cardano transactions to sign + assertions::transfer_funds(infrastructure.devnet()).await?; + } Ok(()) } From 5e1bde8bc0b6d009d4cbd6537bc7718186800eab Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Thu, 13 Mar 2025 17:40:30 +0100 Subject: [PATCH 07/33] feat(e2e): 'Spec' supports multiple aggregators --- .../mithril-end-to-end/src/end_to_end_spec.rs | 208 ++++++++++++------ 1 file changed, 137 insertions(+), 71 deletions(-) diff --git a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs index 9401c20c953..91defbab31f 100644 --- a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs +++ b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs @@ -1,12 +1,19 @@ -use crate::assertions; -use crate::MithrilInfrastructure; +use std::sync::Arc; + +use anyhow::anyhow; +use tokio::sync::RwLock; +use tokio::task::JoinSet; + use mithril_common::{ + chain_observer::ChainObserver, entities::{Epoch, SignedEntityTypeDiscriminants}, StdResult, }; -pub struct Spec<'a> { - pub infrastructure: &'a MithrilInfrastructure, +use crate::{assertions, Aggregator, MithrilInfrastructure}; + +pub struct Spec { + pub infrastructure: Arc>>, is_signing_cardano_transactions: bool, is_signing_cardano_stake_distribution: bool, is_signing_cardano_database: bool, @@ -14,9 +21,9 @@ pub struct Spec<'a> { regenesis_on_era_switch: bool, } -impl<'a> Spec<'a> { +impl Spec { pub fn new( - infrastructure: &'a MithrilInfrastructure, + infrastructure: Arc>>, signed_entity_types: Vec, next_era: Option, regenesis_on_era_switch: bool, @@ -43,80 +50,134 @@ impl<'a> Spec<'a> { } } - pub async fn run(&self) -> StdResult<()> { - let aggregator_endpoint = self.infrastructure.master_aggregator().endpoint(); - assertions::wait_for_enough_immutable( - self.infrastructure.master_aggregator().db_directory(), - ) - .await?; - let start_epoch = self - .infrastructure - .master_chain_observer() - .get_current_epoch() - .await? - .unwrap_or_default(); + pub async fn run(self) -> StdResult<()> { + let spec = Arc::new(self); + let infrastructure_guard = spec.infrastructure.read().await; + let infrastructure = infrastructure_guard + .as_ref() + .ok_or(anyhow!("No infrastructure found"))?; // Transfer some funds on the devnet to have some Cardano transactions to sign. // This step needs to be executed early in the process so that the transactions are available // for signing in the penultimate immutable chunk before the end of the test. // As we get closer to the tip of the chain when signing, we'll be able to relax this constraint. - assertions::transfer_funds(self.infrastructure.devnet()).await?; + assertions::transfer_funds(infrastructure.devnet()).await?; + + let mut join_set = JoinSet::new(); + let spec_clone = spec.clone(); + join_set.spawn(async move { spec_clone.start_master().await }); + for index in 0..infrastructure.slave_aggregators().len() { + let spec_clone = spec.clone(); + join_set.spawn(async move { spec_clone.start_slave(index).await }); + } + while let Some(res) = join_set.join_next().await { + res??; + } + + Ok(()) + } + + pub async fn start_master(&self) -> StdResult<()> { + let infrastructure_guard = self.infrastructure.read().await; + let infrastructure = infrastructure_guard + .as_ref() + .ok_or(anyhow!("No infrastructure found"))?; + + self.start_aggregator( + infrastructure.master_aggregator(), + infrastructure.master_chain_observer(), + infrastructure, + ) + .await + } + + pub async fn start_slave(&self, slave_index: usize) -> StdResult<()> { + let infrastructure_guard = self.infrastructure.read().await; + let infrastructure = infrastructure_guard + .as_ref() + .ok_or(anyhow!("No infrastructure found"))?; + + self.start_aggregator( + infrastructure.slave_aggregator(slave_index), + infrastructure.slave_chain_observer(slave_index), + infrastructure, + ) + .await + } + + pub async fn start_aggregator( + &self, + aggregator: &Aggregator, + chain_observer: Arc, + infrastructure: &MithrilInfrastructure, + ) -> StdResult<()> { + assertions::wait_for_enough_immutable(aggregator.db_directory()).await?; + let start_epoch = chain_observer + .get_current_epoch() + .await? + .unwrap_or_default(); // Wait 4 epochs after start epoch for the aggregator to be able to bootstrap a genesis certificate let mut target_epoch = start_epoch + 4; + /* if !aggregator.is_master() { + target_epoch += 2; + } */ assertions::wait_for_target_epoch( - self.infrastructure.master_chain_observer(), + chain_observer.clone(), target_epoch, "minimal epoch for the aggregator to be able to bootstrap genesis certificate" .to_string(), ) .await?; - assertions::bootstrap_genesis_certificate(self.infrastructure.master_aggregator()).await?; - assertions::wait_for_epoch_settings(&aggregator_endpoint).await?; + assertions::bootstrap_genesis_certificate(aggregator).await?; + assertions::wait_for_epoch_settings(&aggregator.endpoint()).await?; // Wait 2 epochs before changing stake distribution, so that we use at least one original stake distribution target_epoch += 2; assertions::wait_for_target_epoch( - self.infrastructure.master_chain_observer(), + chain_observer.clone(), target_epoch, "epoch after which the stake distribution will change".to_string(), ) .await?; - let delegation_round = 1; - assertions::delegate_stakes_to_pools(self.infrastructure.devnet(), delegation_round) - .await?; + + if aggregator.is_master() { + // Delegate some stakes to pools + let delegation_round = 1; + assertions::delegate_stakes_to_pools(infrastructure.devnet(), delegation_round).await?; + } // Wait 2 epochs before changing protocol parameters target_epoch += 2; assertions::wait_for_target_epoch( - self.infrastructure.master_chain_observer(), + chain_observer.clone(), target_epoch, "epoch after which the protocol parameters will change".to_string(), ) .await?; - assertions::update_protocol_parameters(self.infrastructure.master_aggregator()).await?; + assertions::update_protocol_parameters(aggregator).await?; // Wait 6 epochs after protocol parameters update, so that we make sure that we use new protocol parameters as well as new stake distribution a few times target_epoch += 6; assertions::wait_for_target_epoch( - self.infrastructure.master_chain_observer(), + chain_observer.clone(), target_epoch, "epoch after which the certificate chain will be long enough to catch most common troubles with stake distribution and protocol parameters".to_string(), ) .await?; // Verify that artifacts are produced and signed correctly - let mut target_epoch = self.verify_artifacts_production(target_epoch).await?; + let mut target_epoch = self + .verify_artifacts_production(target_epoch, aggregator, infrastructure) + .await?; // Verify that artifacts are produced and signed correctly after era switch if let Some(next_era) = &self.next_era { // Switch to next era - self.infrastructure - .register_switch_to_next_era(next_era) - .await?; + infrastructure.register_switch_to_next_era(next_era).await?; target_epoch += 5; assertions::wait_for_target_epoch( - self.infrastructure.master_chain_observer(), + chain_observer.clone(), target_epoch, "epoch after which the era switch will have triggered".to_string(), ) @@ -124,11 +185,10 @@ impl<'a> Spec<'a> { // Proceed to a re-genesis of the certificate chain if self.regenesis_on_era_switch { - assertions::bootstrap_genesis_certificate(self.infrastructure.master_aggregator()) - .await?; + assertions::bootstrap_genesis_certificate(aggregator).await?; target_epoch += 5; assertions::wait_for_target_epoch( - self.infrastructure.master_chain_observer(), + chain_observer.clone(), target_epoch, "epoch after which the re-genesis on era switch will be completed".to_string(), ) @@ -136,107 +196,113 @@ impl<'a> Spec<'a> { } // Verify that artifacts are produced and signed correctly - self.verify_artifacts_production(target_epoch).await?; + self.verify_artifacts_production(target_epoch, aggregator, infrastructure) + .await?; } Ok(()) } - async fn verify_artifacts_production(&self, target_epoch: Epoch) -> StdResult { - let aggregator_endpoint = self.infrastructure.master_aggregator().endpoint(); + async fn verify_artifacts_production( + &self, + target_epoch: Epoch, + aggregator: &Aggregator, + infrastructure: &MithrilInfrastructure, + ) -> StdResult { let expected_epoch_min = target_epoch - 3; // Verify that mithril stake distribution artifacts are produced and signed correctly { - let hash = - assertions::assert_node_producing_mithril_stake_distribution(&aggregator_endpoint) - .await?; + let hash = assertions::assert_node_producing_mithril_stake_distribution( + &aggregator.endpoint(), + ) + .await?; let certificate_hash = assertions::assert_signer_is_signing_mithril_stake_distribution( - &aggregator_endpoint, + &aggregator.endpoint(), &hash, expected_epoch_min, ) .await?; assertions::assert_is_creating_certificate_with_enough_signers( - &aggregator_endpoint, + &aggregator.endpoint(), &certificate_hash, - self.infrastructure.signers().len(), + infrastructure.signers().len(), ) .await?; - let mut client = self.infrastructure.build_client().await?; + let mut client = infrastructure.build_client().await?; assertions::assert_client_can_verify_mithril_stake_distribution(&mut client, &hash) .await?; } // Verify that snapshot artifacts are produced and signed correctly { - let digest = assertions::assert_node_producing_snapshot(&aggregator_endpoint).await?; + let digest = assertions::assert_node_producing_snapshot(&aggregator.endpoint()).await?; let certificate_hash = assertions::assert_signer_is_signing_snapshot( - &aggregator_endpoint, + &aggregator.endpoint(), &digest, expected_epoch_min, ) .await?; assertions::assert_is_creating_certificate_with_enough_signers( - &aggregator_endpoint, + &aggregator.endpoint(), &certificate_hash, - self.infrastructure.signers().len(), + infrastructure.signers().len(), ) .await?; - let mut client = self.infrastructure.build_client().await?; + let mut client = infrastructure.build_client().await?; assertions::assert_client_can_verify_snapshot(&mut client, &digest).await?; } // Verify that Cardano database snapshot artifacts are produced and signed correctly if self.is_signing_cardano_database { let hash = - assertions::assert_node_producing_cardano_database_snapshot(&aggregator_endpoint) + assertions::assert_node_producing_cardano_database_snapshot(&aggregator.endpoint()) .await?; let certificate_hash = assertions::assert_signer_is_signing_cardano_database_snapshot( - &aggregator_endpoint, + &aggregator.endpoint(), &hash, expected_epoch_min, ) .await?; assertions::assert_is_creating_certificate_with_enough_signers( - &aggregator_endpoint, + &aggregator.endpoint(), &certificate_hash, - self.infrastructure.signers().len(), + infrastructure.signers().len(), ) .await?; - assertions::assert_node_producing_cardano_database_digests_map(&aggregator_endpoint) + assertions::assert_node_producing_cardano_database_digests_map(&aggregator.endpoint()) .await?; - let mut client = self.infrastructure.build_client().await?; + let mut client = infrastructure.build_client().await?; assertions::assert_client_can_verify_cardano_database(&mut client, &hash).await?; } // Verify that Cardano transactions artifacts are produced and signed correctly if self.is_signing_cardano_transactions { - let hash = assertions::assert_node_producing_cardano_transactions(&aggregator_endpoint) - .await?; + let hash = + assertions::assert_node_producing_cardano_transactions(&aggregator.endpoint()) + .await?; let certificate_hash = assertions::assert_signer_is_signing_cardano_transactions( - &aggregator_endpoint, + &aggregator.endpoint(), &hash, expected_epoch_min, ) .await?; assertions::assert_is_creating_certificate_with_enough_signers( - &aggregator_endpoint, + &aggregator.endpoint(), &certificate_hash, - self.infrastructure.signers().len(), + infrastructure.signers().len(), ) .await?; - let transaction_hashes = self - .infrastructure + let transaction_hashes = infrastructure .devnet() .mithril_payments_transaction_hashes()?; - let mut client = self.infrastructure.build_client().await?; + let mut client = infrastructure.build_client().await?; assertions::assert_client_can_verify_transactions(&mut client, transaction_hashes) .await?; } @@ -245,24 +311,24 @@ impl<'a> Spec<'a> { if self.is_signing_cardano_stake_distribution { { let (hash, epoch) = assertions::assert_node_producing_cardano_stake_distribution( - &aggregator_endpoint, + &aggregator.endpoint(), ) .await?; let certificate_hash = assertions::assert_signer_is_signing_cardano_stake_distribution( - &aggregator_endpoint, + &aggregator.endpoint(), &hash, expected_epoch_min, ) .await?; assertions::assert_is_creating_certificate_with_enough_signers( - &aggregator_endpoint, + &aggregator.endpoint(), &certificate_hash, - self.infrastructure.signers().len(), + infrastructure.signers().len(), ) .await?; - let mut client = self.infrastructure.build_client().await?; + let mut client = infrastructure.build_client().await?; assertions::assert_client_can_verify_cardano_stake_distribution( &mut client, &hash, From 62f183cd603ac14586daa7ed82dd91ccdcd33a14 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Thu, 13 Mar 2025 17:41:12 +0100 Subject: [PATCH 08/33] feat(e2e): runner supports multiple aggregators --- .../mithril-end-to-end/src/main.rs | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/mithril-test-lab/mithril-end-to-end/src/main.rs b/mithril-test-lab/mithril-end-to-end/src/main.rs index 3b7da24c783..e4d478eec27 100644 --- a/mithril-test-lab/mithril-end-to-end/src/main.rs +++ b/mithril-test-lab/mithril-end-to-end/src/main.rs @@ -12,7 +12,7 @@ use std::{ use thiserror::Error; use tokio::{ signal::unix::{signal, SignalKind}, - sync::Mutex, + sync::{Mutex, RwLock}, task::JoinSet, }; @@ -287,19 +287,19 @@ impl From> for AppResult { struct App { devnet: Arc>>, - infrastructure: Arc>>, + infrastructure: Arc>>, } impl App { fn new() -> Self { Self { devnet: Arc::new(Mutex::new(None)), - infrastructure: Arc::new(Mutex::new(None)), + infrastructure: Arc::new(RwLock::new(None)), } } async fn tail_logs(&self) { - if let Some(infrastructure) = self.infrastructure.lock().await.as_ref() { + if let Some(infrastructure) = self.infrastructure.read().await.as_ref() { let _ = infrastructure.tail_logs(40).await.inspect_err(|e| { error!("Failed to tail logs: {}", e); }); @@ -307,7 +307,7 @@ impl App { } async fn last_error_in_logs(&self) { - if let Some(infrastructure) = self.infrastructure.lock().await.as_ref() { + if let Some(infrastructure) = self.infrastructure.read().await.as_ref() { let _ = infrastructure.last_error_in_logs(1).await.inspect_err(|e| { error!("Failed to grep error in logs: {}", e); }); @@ -360,23 +360,21 @@ impl App { use_era_specific_work_dir: args.mithril_next_era.is_some(), }) .await?; + *self.infrastructure.write().await = Some(infrastructure); let runner: StdResult<()> = match run_only_mode { - true => { - let run_only = RunOnly::new(&infrastructure); - run_only.start().await - } + true => RunOnly::new(self.infrastructure.clone()).run().await, false => { - let spec = Spec::new( - &infrastructure, + Spec::new( + self.infrastructure.clone(), args.signed_entity_types, args.mithril_next_era, args.mithril_era_regenesis_on_switch, - ); - spec.run().await + ) + .run() + .await } }; - *self.infrastructure.lock().await = Some(infrastructure); match runner.with_context(|| "Mithril End to End test failed") { Ok(()) if run_only_mode => loop { @@ -395,7 +393,7 @@ impl App { struct AppStopper { devnet: Arc>>, - infrastructure: Arc>>, + infrastructure: Arc>>, } impl AppStopper { @@ -407,7 +405,7 @@ impl AppStopper { } pub async fn stop(&mut self) { - if let Some(infrastructure) = self.infrastructure.lock().await.as_mut() { + if let Some(infrastructure) = self.infrastructure.write().await.as_mut() { let _ = infrastructure.stop_nodes().await.inspect_err(|e| { error!("Failed to stop nodes: {}", e); }); From c3eda6eca80501d81d46946cc3469180b193b5ee Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 14 Mar 2025 11:07:26 +0100 Subject: [PATCH 09/33] refactor(e2e): enhance naming of aggregators and associated relays --- .../mithril-end-to-end/src/end_to_end_spec.rs | 10 ++--- .../src/mithril/aggregator.rs | 43 +++++++++++++------ .../src/mithril/infrastructure.rs | 7 ++- .../src/mithril/relay_aggregator.rs | 19 +++++--- 4 files changed, 53 insertions(+), 26 deletions(-) diff --git a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs index 91defbab31f..27e8c0e9d43 100644 --- a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs +++ b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs @@ -228,7 +228,7 @@ impl Spec { infrastructure.signers().len(), ) .await?; - let mut client = infrastructure.build_client().await?; + let mut client = infrastructure.build_client(aggregator).await?; assertions::assert_client_can_verify_mithril_stake_distribution(&mut client, &hash) .await?; } @@ -250,7 +250,7 @@ impl Spec { ) .await?; - let mut client = infrastructure.build_client().await?; + let mut client = infrastructure.build_client(aggregator).await?; assertions::assert_client_can_verify_snapshot(&mut client, &digest).await?; } @@ -276,7 +276,7 @@ impl Spec { assertions::assert_node_producing_cardano_database_digests_map(&aggregator.endpoint()) .await?; - let mut client = infrastructure.build_client().await?; + let mut client = infrastructure.build_client(aggregator).await?; assertions::assert_client_can_verify_cardano_database(&mut client, &hash).await?; } @@ -302,7 +302,7 @@ impl Spec { let transaction_hashes = infrastructure .devnet() .mithril_payments_transaction_hashes()?; - let mut client = infrastructure.build_client().await?; + let mut client = infrastructure.build_client(aggregator).await?; assertions::assert_client_can_verify_transactions(&mut client, transaction_hashes) .await?; } @@ -328,7 +328,7 @@ impl Spec { ) .await?; - let mut client = infrastructure.build_client().await?; + let mut client = infrastructure.build_client(aggregator).await?; assertions::assert_client_can_verify_cardano_stake_distribution( &mut client, &hash, diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs index b66ba9cd653..5bac69843b0 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs @@ -128,12 +128,10 @@ impl Aggregator { env, &args, )?; - let name = if aggregator_config.index == 0 { - "mithril-aggregator".to_string() - } else { - format!("mithril-aggregator-slave-{}", aggregator_config.index) - }; - command.set_log_name(&name); + command.set_log_name(&format!( + "mithril-aggregator-{}", + Self::name_suffix(aggregator_config.index), + )); Ok(Self { index: aggregator_config.index, @@ -154,10 +152,26 @@ impl Aggregator { } } + pub fn index(&self) -> usize { + self.index + } + pub fn is_master(&self) -> bool { self.index == 0 } + pub fn name(&self) -> String { + format!("mithril-aggregator-{}", Self::name_suffix(self.index)) + } + + pub fn name_suffix(index: usize) -> String { + if index == 0 { + "master".to_string() + } else { + format!("slave-{}", index) + } + } + pub fn endpoint(&self) -> String { format!("http://localhost:{}/aggregator", &self.server_port) } @@ -176,11 +190,10 @@ impl Aggregator { pub async fn bootstrap_genesis(&self) -> StdResult<()> { // Clone the command so we can alter it without affecting the original let mut command = self.command.write().await; - let command_name = if self.is_master() { - "mithril-aggregator-genesis-bootstrap" - } else { - &format!("mithril-aggregator-genesis-bootstrap-slave-{}", self.index) - }; + let command_name = &format!( + "mithril-aggregator-genesis-bootstrap-{}", + Self::name_suffix(self.index), + ); command.set_log_name(command_name); let exit_status = command @@ -208,7 +221,7 @@ impl Aggregator { pub async fn stop(&self) -> StdResult<()> { let mut process = self.process.write().await; if let Some(mut process_running) = process.take() { - info!("Stopping aggregator"); + info!("Stopping {}", self.name()); process_running .kill() .await @@ -313,11 +326,13 @@ impl Aggregator { pub async fn tail_logs(&self, number_of_line: u64) -> StdResult<()> { let command = self.command.write().await; - command.tail_logs(None, number_of_line).await + command.tail_logs(Some(&self.name()), number_of_line).await } pub async fn last_error_in_logs(&self, number_of_error: u64) -> StdResult<()> { let command = self.command.write().await; - command.last_error_in_logs(None, number_of_error).await + command + .last_error_in_logs(Some(&self.name()), number_of_error) + .await } } diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs index d20ba4ef73c..a3905d35786 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs @@ -408,9 +408,12 @@ impl MithrilInfrastructure { self.cardano_chain_observers[index + 1].clone() } - pub async fn build_client(&self) -> StdResult { + pub async fn build_client(&self, aggregator: &Aggregator) -> StdResult { let work_dir = { - let mut artifacts_dir = self.artifacts_dir.join("mithril-client"); + let mut artifacts_dir = self.artifacts_dir.join(format!( + "mithril-client-aggregator-{}", + Aggregator::name_suffix(aggregator.index()) + )); if self.use_era_specific_work_dir { let current_era = self.current_era.read().await; artifacts_dir = artifacts_dir.join(format!("era.{}", current_era)); diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/relay_aggregator.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/relay_aggregator.rs index ab028505199..80228d597dc 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/relay_aggregator.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/relay_aggregator.rs @@ -32,7 +32,10 @@ impl RelayAggregator { let args = vec!["-vvv", "aggregator"]; let mut command = MithrilCommand::new("mithril-relay", work_dir, bin_dir, env, &args)?; - command.set_log_name(&Self::command_name(index)); + command.set_log_name(&format!( + "mithril-relay-aggregator-{}", + Self::name_suffix(index), + )); Ok(Self { index, @@ -46,11 +49,11 @@ impl RelayAggregator { format!("/ip4/127.0.0.1/tcp/{}", self.listen_port) } - fn command_name(index: usize) -> String { + pub fn name_suffix(index: usize) -> String { if index == 0 { - "mithril-relay-aggregator".to_string() + "master".to_string() } else { - format!("mithril-relay-aggregator-slave-{}", index) + format!("slave-{}", index) } } @@ -61,7 +64,13 @@ impl RelayAggregator { pub async fn tail_logs(&self, number_of_line: u64) -> StdResult<()> { self.command - .tail_logs(Some(&Self::command_name(self.index)), number_of_line) + .tail_logs( + Some(&format!( + "mithril-relay-aggregator-{}", + Self::name_suffix(self.index), + )), + number_of_line, + ) .await } } From 596387f0a5c05fce86f895d48ce966b386d01831 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 14 Mar 2025 12:57:24 +0100 Subject: [PATCH 10/33] refactor(e2e): enhance assertions checks By providing information about the targeted aggregator in logs and errors. --- .../src/assertions/check.rs | 176 ++++++++++++------ .../mithril-end-to-end/src/end_to_end_spec.rs | 48 ++--- 2 files changed, 137 insertions(+), 87 deletions(-) diff --git a/mithril-test-lab/mithril-end-to-end/src/assertions/check.rs b/mithril-test-lab/mithril-end-to-end/src/assertions/check.rs index c09bf95c1dd..85d4365fa78 100644 --- a/mithril-test-lab/mithril-end-to-end/src/assertions/check.rs +++ b/mithril-test-lab/mithril-end-to-end/src/assertions/check.rs @@ -18,7 +18,7 @@ use mithril_common::{ }; use crate::{ - attempt, utils::AttemptResult, CardanoDbCommand, CardanoDbV2Command, + attempt, utils::AttemptResult, Aggregator, CardanoDbCommand, CardanoDbV2Command, CardanoStakeDistributionCommand, CardanoTransactionCommand, Client, ClientCommand, MithrilStakeDistributionCommand, }; @@ -37,10 +37,13 @@ async fn get_json_response(url: String) -> StdResult StdResult { - let url = format!("{aggregator_endpoint}/artifact/mithril-stake-distributions"); - info!("Waiting for the aggregator to produce a mithril stake distribution"); + let url = format!( + "{}/artifact/mithril-stake-distributions", + aggregator.endpoint() + ); + info!("Waiting for the aggregator to produce a mithril stake distribution"; "aggregator" => &aggregator.name()); async fn fetch_last_mithril_stake_distribution_hash(url: String) -> StdResult> { match get_json_response::(url) @@ -49,7 +52,7 @@ pub async fn assert_node_producing_mithril_stake_distribution( { Ok([stake_distribution, ..]) => Ok(Some(stake_distribution.hash.clone())), Ok(&[]) => Ok(None), - Err(err) => Err(anyhow!("Invalid mithril stake distribution body : {err}",)), + Err(err) => Err(anyhow!("Invalid mithril stake distribution body: {err}",)), } } @@ -64,35 +67,44 @@ pub async fn assert_node_producing_mithril_stake_distribution( AttemptResult::Timeout() => Err(anyhow!( "Timeout exhausted assert_node_producing_mithril_stake_distribution, no response from `{url}`" )), - } + }.with_context(|| { + format!( + "Requesting aggregator `{}`", + aggregator.name() + ) + }) } pub async fn assert_signer_is_signing_mithril_stake_distribution( - aggregator_endpoint: &str, + aggregator: &Aggregator, hash: &str, expected_epoch_min: Epoch, ) -> StdResult { - let url = format!("{aggregator_endpoint}/artifact/mithril-stake-distribution/{hash}"); + let url = format!( + "{}/artifact/mithril-stake-distribution/{hash}", + aggregator.endpoint() + ); info!( "Asserting the aggregator is signing the mithril stake distribution message `{}` with an expected min epoch of `{}`", hash, - expected_epoch_min + expected_epoch_min; + "aggregator" => &aggregator.name() ); async fn fetch_mithril_stake_distribution_message( url: String, expected_epoch_min: Epoch, ) -> StdResult> { - match get_json_response::(url) + match get_json_response::(url.clone()) .await? { Ok(stake_distribution) => match stake_distribution.epoch { epoch if epoch >= expected_epoch_min => Ok(Some(stake_distribution)), epoch => Err(anyhow!( - "Minimum expected mithril stake distribution epoch not reached : {epoch} < {expected_epoch_min}" + "Minimum expected mithril stake distribution epoch not reached: {epoch} < {expected_epoch_min}" )), }, - Err(err) => Err(anyhow!("Invalid mithril stake distribution body : {err}",)), + Err(err) => Err(anyhow!("Invalid mithril stake distribution body: {err}",)), } } @@ -107,12 +119,17 @@ pub async fn assert_signer_is_signing_mithril_stake_distribution( AttemptResult::Timeout() => Err(anyhow!( "Timeout exhausted assert_signer_is_signing_mithril_stake_distribution, no response from `{url}`" )), - } + }.with_context(|| { + format!( + "Requesting aggregator `{}`", + aggregator.name() + ) + }) } -pub async fn assert_node_producing_snapshot(aggregator_endpoint: &str) -> StdResult { - let url = format!("{aggregator_endpoint}/artifact/snapshots"); - info!("Waiting for the aggregator to produce a snapshot"); +pub async fn assert_node_producing_snapshot(aggregator: &Aggregator) -> StdResult { + let url = format!("{}/artifact/snapshots", aggregator.endpoint()); + info!("Waiting for the aggregator to produce a snapshot"; "aggregator" => &aggregator.name()); async fn fetch_last_snapshot_digest(url: String) -> StdResult> { match get_json_response::>(url) @@ -121,7 +138,7 @@ pub async fn assert_node_producing_snapshot(aggregator_endpoint: &str) -> StdRes { Ok([snapshot, ..]) => Ok(Some(snapshot.digest.clone())), Ok(&[]) => Ok(None), - Err(err) => Err(anyhow!("Invalid snapshot body : {err}",)), + Err(err) => Err(anyhow!("Invalid snapshot body: {err}",)), } } @@ -137,18 +154,20 @@ pub async fn assert_node_producing_snapshot(aggregator_endpoint: &str) -> StdRes "Timeout exhausted assert_node_producing_snapshot, no response from `{url}`" )), } + .with_context(|| format!("Requesting aggregator `{}`", aggregator.name())) } pub async fn assert_signer_is_signing_snapshot( - aggregator_endpoint: &str, + aggregator: &Aggregator, digest: &str, expected_epoch_min: Epoch, ) -> StdResult { - let url = format!("{aggregator_endpoint}/artifact/snapshot/{digest}"); + let url = format!("{}/artifact/snapshot/{digest}", aggregator.endpoint()); info!( "Asserting the aggregator is signing the snapshot message `{}` with an expected min epoch of `{}`", digest, - expected_epoch_min + expected_epoch_min; + "aggregator" => &aggregator.name() ); async fn fetch_snapshot_message( @@ -159,7 +178,7 @@ pub async fn assert_signer_is_signing_snapshot( Ok(snapshot) => match snapshot.beacon.epoch { epoch if epoch >= expected_epoch_min => Ok(Some(snapshot)), epoch => Err(anyhow!( - "Minimum expected snapshot epoch not reached : {epoch} < {expected_epoch_min}" + "Minimum expected snapshot epoch not reached: {epoch} < {expected_epoch_min}" )), }, Err(err) => Err(anyhow!(err).context("Invalid snapshot body")), @@ -173,18 +192,21 @@ pub async fn assert_signer_is_signing_snapshot( info!("Signer signed a snapshot"; "certificate_hash" => &snapshot.certificate_hash); Ok(snapshot.certificate_hash) } - AttemptResult::Err(error) => Err(error), + AttemptResult::Err(error) => { + Err(error).with_context(|| format!("Requesting aggregator `{}`", aggregator.name())) + } AttemptResult::Timeout() => Err(anyhow!( "Timeout exhausted assert_signer_is_signing_snapshot, no response from `{url}`" )), } + .with_context(|| format!("Requesting aggregator `{}`", aggregator.name())) } pub async fn assert_node_producing_cardano_database_snapshot( - aggregator_endpoint: &str, + aggregator: &Aggregator, ) -> StdResult { - let url = format!("{aggregator_endpoint}/artifact/cardano-database"); - info!("Waiting for the aggregator to produce a Cardano database snapshot"); + let url = format!("{}/artifact/cardano-database", aggregator.endpoint()); + info!("Waiting for the aggregator to produce a Cardano database snapshot"; "aggregator" => &aggregator.name()); async fn fetch_last_cardano_database_snapshot_hash(url: String) -> StdResult> { match get_json_response::(url) @@ -193,7 +215,7 @@ pub async fn assert_node_producing_cardano_database_snapshot( { Ok([cardano_database_snapshot, ..]) => Ok(Some(cardano_database_snapshot.hash.clone())), Ok(&[]) => Ok(None), - Err(err) => Err(anyhow!("Invalid Cardano database snapshot body : {err}",)), + Err(err) => Err(anyhow!("Invalid Cardano database snapshot body: {err}",)), } } @@ -209,18 +231,20 @@ pub async fn assert_node_producing_cardano_database_snapshot( "Timeout exhausted assert_node_producing_snapshot, no response from `{url}`" )), } + .with_context(|| format!("Requesting aggregator `{}`", aggregator.name())) } pub async fn assert_signer_is_signing_cardano_database_snapshot( - aggregator_endpoint: &str, + aggregator: &Aggregator, hash: &str, expected_epoch_min: Epoch, ) -> StdResult { - let url = format!("{aggregator_endpoint}/artifact/cardano-database/{hash}"); + let url = format!("{}/artifact/cardano-database/{hash}", aggregator.endpoint()); info!( "Asserting the aggregator is signing the Cardano database snapshot message `{}` with an expected min epoch of `{}`", hash, - expected_epoch_min + expected_epoch_min; + "aggregator" => &aggregator.name() ); async fn fetch_cardano_database_snapshot_message( @@ -233,7 +257,7 @@ pub async fn assert_signer_is_signing_cardano_database_snapshot( Ok(cardano_database_snapshot) => match cardano_database_snapshot.beacon.epoch { epoch if epoch >= expected_epoch_min => Ok(Some(cardano_database_snapshot)), epoch => Err(anyhow!( - "Minimum expected Cardano database snapshot epoch not reached : {epoch} < {expected_epoch_min}" + "Minimum expected Cardano database snapshot epoch not reached: {epoch} < {expected_epoch_min}" )), }, Err(err) => Err(anyhow!(err).context("Invalid Cardano database snapshot body")), @@ -252,13 +276,17 @@ pub async fn assert_signer_is_signing_cardano_database_snapshot( "Timeout exhausted assert_signer_is_signing_snapshot, no response from `{url}`" )), } + .with_context(|| format!("Requesting aggregator `{}`", aggregator.name())) } pub async fn assert_node_producing_cardano_database_digests_map( - aggregator_endpoint: &str, + aggregator: &Aggregator, ) -> StdResult> { - let url = format!("{aggregator_endpoint}/artifact/cardano-database/digests"); - info!("Waiting for the aggregator to produce a Cardano database digests map"); + let url = format!( + "{}/artifact/cardano-database/digests", + aggregator.endpoint() + ); + info!("Waiting for the aggregator to produce a Cardano database digests map"; "aggregator" => &aggregator.name()); async fn fetch_cardano_database_digests_map( url: String, @@ -274,7 +302,7 @@ pub async fn assert_node_producing_cardano_database_digests_map( .map(|item| (item.immutable_file_name.clone(), item.digest.clone())) .collect(), )), - Err(err) => Err(anyhow!("Invalid Cardano database digests map body : {err}",)), + Err(err) => Err(anyhow!("Invalid Cardano database digests map body: {err}",)), } } @@ -289,14 +317,19 @@ pub async fn assert_node_producing_cardano_database_digests_map( AttemptResult::Timeout() => Err(anyhow!( "Timeout exhausted assert_node_producing_cardano_database_digests_map, no response from `{url}`" )), - } + }.with_context(|| { + format!( + "Requesting aggregator `{}`", + aggregator.name() + ) + }) } pub async fn assert_node_producing_cardano_transactions( - aggregator_endpoint: &str, + aggregator: &Aggregator, ) -> StdResult { - let url = format!("{aggregator_endpoint}/artifact/cardano-transactions"); - info!("Waiting for the aggregator to produce a Cardano transactions artifact"); + let url = format!("{}/artifact/cardano-transactions", aggregator.endpoint()); + info!("Waiting for the aggregator to produce a Cardano transactions artifact"; "aggregator" => &aggregator.name()); async fn fetch_last_cardano_transaction_snapshot_hash( url: String, @@ -307,9 +340,7 @@ pub async fn assert_node_producing_cardano_transactions( { Ok([artifact, ..]) => Ok(Some(artifact.hash.clone())), Ok(&[]) => Ok(None), - Err(err) => Err(anyhow!( - "Invalid Cardano transactions artifact body : {err}", - )), + Err(err) => Err(anyhow!("Invalid Cardano transactions artifact body: {err}",)), } } @@ -325,18 +356,23 @@ pub async fn assert_node_producing_cardano_transactions( "Timeout exhausted assert_node_producing_cardano_transactions, no response from `{url}`" )), } + .with_context(|| format!("Requesting aggregator `{}`", aggregator.name())) } pub async fn assert_signer_is_signing_cardano_transactions( - aggregator_endpoint: &str, + aggregator: &Aggregator, hash: &str, expected_epoch_min: Epoch, ) -> StdResult { - let url = format!("{aggregator_endpoint}/artifact/cardano-transaction/{hash}"); + let url = format!( + "{}/artifact/cardano-transaction/{hash}", + aggregator.endpoint() + ); info!( "Asserting the aggregator is signing the Cardano transactions artifact `{}` with an expected min epoch of `{}`", hash, - expected_epoch_min + expected_epoch_min; + "aggregator" => &aggregator.name() ); async fn fetch_cardano_transaction_snapshot_message( @@ -347,7 +383,7 @@ pub async fn assert_signer_is_signing_cardano_transactions( Ok(artifact) => match artifact.epoch { epoch if epoch >= expected_epoch_min => Ok(Some(artifact)), epoch => Err(anyhow!( - "Minimum expected artifact epoch not reached : {epoch} < {expected_epoch_min}" + "Minimum expected artifact epoch not reached: {epoch} < {expected_epoch_min}" )), }, Err(err) => Err(anyhow!(err).context("Invalid Cardano transactions artifact body")), @@ -365,14 +401,22 @@ pub async fn assert_signer_is_signing_cardano_transactions( AttemptResult::Timeout() => Err(anyhow!( "Timeout exhausted assert_signer_is_signing_cardano_transactions, no response from `{url}`" )), - } + }.with_context(|| { + format!( + "Requesting aggregator `{}`", + aggregator.name() + ) + }) } pub async fn assert_node_producing_cardano_stake_distribution( - aggregator_endpoint: &str, + aggregator: &Aggregator, ) -> StdResult<(String, Epoch)> { - let url = format!("{aggregator_endpoint}/artifact/cardano-stake-distributions"); - info!("Waiting for the aggregator to produce a Cardano stake distribution"); + let url = format!( + "{}/artifact/cardano-stake-distributions", + aggregator.endpoint() + ); + info!("Waiting for the aggregator to produce a Cardano stake distribution"; "aggregator" => &aggregator.name()); async fn fetch_last_cardano_stake_distribution_message( url: String, @@ -386,7 +430,7 @@ pub async fn assert_node_producing_cardano_stake_distribution( stake_distribution.epoch, ))), Ok(&[]) => Ok(None), - Err(err) => Err(anyhow!("Invalid Cardano stake distribution body : {err}",)), + Err(err) => Err(anyhow!("Invalid Cardano stake distribution body: {err}",)), } } @@ -401,19 +445,28 @@ pub async fn assert_node_producing_cardano_stake_distribution( AttemptResult::Timeout() => Err(anyhow!( "Timeout exhausted assert_node_producing_cardano_stake_distribution, no response from `{url}`" )), - } + }.with_context(|| { + format!( + "Requesting aggregator `{}`", + aggregator.name() + ) + }) } pub async fn assert_signer_is_signing_cardano_stake_distribution( - aggregator_endpoint: &str, + aggregator: &Aggregator, hash: &str, expected_epoch_min: Epoch, ) -> StdResult { - let url = format!("{aggregator_endpoint}/artifact/cardano-stake-distribution/{hash}"); + let url = format!( + "{}/artifact/cardano-stake-distribution/{hash}", + aggregator.endpoint() + ); info!( "Asserting the aggregator is signing the Cardano stake distribution message `{}` with an expected min epoch of `{}`", hash, - expected_epoch_min + expected_epoch_min; + "aggregator" => &aggregator.name() ); async fn fetch_cardano_stake_distribution_message( @@ -426,7 +479,7 @@ pub async fn assert_signer_is_signing_cardano_stake_distribution( Ok(stake_distribution) => match stake_distribution.epoch { epoch if epoch >= expected_epoch_min => Ok(Some(stake_distribution)), epoch => Err(anyhow!( - "Minimum expected Cardano stake distribution epoch not reached : {epoch} < {expected_epoch_min}" + "Minimum expected Cardano stake distribution epoch not reached: {epoch} < {expected_epoch_min}" )), }, Err(err) => Err(anyhow!(err).context("Invalid Cardano stake distribution body",)), @@ -444,15 +497,21 @@ pub async fn assert_signer_is_signing_cardano_stake_distribution( AttemptResult::Timeout() => Err(anyhow!( "Timeout exhausted assert_signer_is_signing_cardano_stake_distribution, no response from `{url}`" )), - } + }.with_context(|| { + format!( + "Requesting aggregator `{}`", + aggregator.name() + ) + }) } pub async fn assert_is_creating_certificate_with_enough_signers( - aggregator_endpoint: &str, + aggregator: &Aggregator, certificate_hash: &str, total_signers_expected: usize, ) -> StdResult<()> { - let url = format!("{aggregator_endpoint}/certificate/{certificate_hash}"); + let url = format!("{}/certificate/{certificate_hash}", aggregator.endpoint()); + info!("Waiting for the aggregator to create a certificate with enough signers"; "aggregator" => &aggregator.name()); async fn fetch_certificate_message(url: String) -> StdResult> { match get_json_response::(url).await? { @@ -486,6 +545,7 @@ pub async fn assert_is_creating_certificate_with_enough_signers( "Timeout exhausted assert_is_creating_certificate, no response from `{url}`" )), } + .with_context(|| format!("Requesting aggregator `{}`", aggregator.name())) } pub async fn assert_client_can_verify_snapshot(client: &mut Client, digest: &str) -> StdResult<()> { diff --git a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs index 27e8c0e9d43..4134471f1eb 100644 --- a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs +++ b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs @@ -119,9 +119,6 @@ impl Spec { // Wait 4 epochs after start epoch for the aggregator to be able to bootstrap a genesis certificate let mut target_epoch = start_epoch + 4; - /* if !aggregator.is_master() { - target_epoch += 2; - } */ assertions::wait_for_target_epoch( chain_observer.clone(), target_epoch, @@ -212,18 +209,16 @@ impl Spec { let expected_epoch_min = target_epoch - 3; // Verify that mithril stake distribution artifacts are produced and signed correctly { - let hash = assertions::assert_node_producing_mithril_stake_distribution( - &aggregator.endpoint(), - ) - .await?; + let hash = + assertions::assert_node_producing_mithril_stake_distribution(aggregator).await?; let certificate_hash = assertions::assert_signer_is_signing_mithril_stake_distribution( - &aggregator.endpoint(), + aggregator, &hash, expected_epoch_min, ) .await?; assertions::assert_is_creating_certificate_with_enough_signers( - &aggregator.endpoint(), + aggregator, &certificate_hash, infrastructure.signers().len(), ) @@ -235,16 +230,16 @@ impl Spec { // Verify that snapshot artifacts are produced and signed correctly { - let digest = assertions::assert_node_producing_snapshot(&aggregator.endpoint()).await?; + let digest = assertions::assert_node_producing_snapshot(aggregator).await?; let certificate_hash = assertions::assert_signer_is_signing_snapshot( - &aggregator.endpoint(), + aggregator, &digest, expected_epoch_min, ) .await?; assertions::assert_is_creating_certificate_with_enough_signers( - &aggregator.endpoint(), + aggregator, &certificate_hash, infrastructure.signers().len(), ) @@ -257,24 +252,22 @@ impl Spec { // Verify that Cardano database snapshot artifacts are produced and signed correctly if self.is_signing_cardano_database { let hash = - assertions::assert_node_producing_cardano_database_snapshot(&aggregator.endpoint()) - .await?; + assertions::assert_node_producing_cardano_database_snapshot(aggregator).await?; let certificate_hash = assertions::assert_signer_is_signing_cardano_database_snapshot( - &aggregator.endpoint(), + aggregator, &hash, expected_epoch_min, ) .await?; assertions::assert_is_creating_certificate_with_enough_signers( - &aggregator.endpoint(), + aggregator, &certificate_hash, infrastructure.signers().len(), ) .await?; - assertions::assert_node_producing_cardano_database_digests_map(&aggregator.endpoint()) - .await?; + assertions::assert_node_producing_cardano_database_digests_map(aggregator).await?; let mut client = infrastructure.build_client(aggregator).await?; assertions::assert_client_can_verify_cardano_database(&mut client, &hash).await?; @@ -282,18 +275,16 @@ impl Spec { // Verify that Cardano transactions artifacts are produced and signed correctly if self.is_signing_cardano_transactions { - let hash = - assertions::assert_node_producing_cardano_transactions(&aggregator.endpoint()) - .await?; + let hash = assertions::assert_node_producing_cardano_transactions(aggregator).await?; let certificate_hash = assertions::assert_signer_is_signing_cardano_transactions( - &aggregator.endpoint(), + aggregator, &hash, expected_epoch_min, ) .await?; assertions::assert_is_creating_certificate_with_enough_signers( - &aggregator.endpoint(), + aggregator, &certificate_hash, infrastructure.signers().len(), ) @@ -310,19 +301,18 @@ impl Spec { // Verify that Cardano stake distribution artifacts are produced and signed correctly if self.is_signing_cardano_stake_distribution { { - let (hash, epoch) = assertions::assert_node_producing_cardano_stake_distribution( - &aggregator.endpoint(), - ) - .await?; + let (hash, epoch) = + assertions::assert_node_producing_cardano_stake_distribution(aggregator) + .await?; let certificate_hash = assertions::assert_signer_is_signing_cardano_stake_distribution( - &aggregator.endpoint(), + aggregator, &hash, expected_epoch_min, ) .await?; assertions::assert_is_creating_certificate_with_enough_signers( - &aggregator.endpoint(), + aggregator, &certificate_hash, infrastructure.signers().len(), ) From 321c8093080226e51567a5d26e750bc43a707c1c Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 17 Mar 2025 12:19:48 +0100 Subject: [PATCH 11/33] fix(common): enhance Certificate display implementation --- mithril-common/src/entities/certificate.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mithril-common/src/entities/certificate.rs b/mithril-common/src/entities/certificate.rs index f94ce800fe6..c5214791b91 100644 --- a/mithril-common/src/entities/certificate.rs +++ b/mithril-common/src/entities/certificate.rs @@ -157,7 +157,7 @@ impl Debug for Certificate { true => debug .field( "aggregate_verification_key", - &format_args!("{:?}", self.aggregate_verification_key), + &format_args!("{:?}", self.aggregate_verification_key.to_json_hex()), ) .field("signature", &format_args!("{:?}", self.signature)) .finish(), From fe2592d5de0cedba9f098f464605702a7ae0c23c Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 17 Mar 2025 13:02:19 +0100 Subject: [PATCH 12/33] fix(aggregator): integration test for slave uses evolving Mithril stake distribution --- .../tests/create_certificate_slave.rs | 461 ++++++++++++++---- 1 file changed, 359 insertions(+), 102 deletions(-) diff --git a/mithril-aggregator/tests/create_certificate_slave.rs b/mithril-aggregator/tests/create_certificate_slave.rs index 68e70a3cb37..ab105a4044c 100644 --- a/mithril-aggregator/tests/create_certificate_slave.rs +++ b/mithril-aggregator/tests/create_certificate_slave.rs @@ -1,17 +1,84 @@ mod test_extensions; +use std::{collections::HashMap, ops::Range}; + use mithril_aggregator::Configuration; use mithril_common::{ entities::{ BlockNumber, CardanoTransactionsSigningConfig, ChainPoint, Epoch, ProtocolParameters, - SignedEntityType, SignedEntityTypeDiscriminants, SlotNumber, StakeDistribution, - StakeDistributionParty, TimePoint, + SignedEntityType, SignedEntityTypeDiscriminants, SlotNumber, StakeDistributionParty, + TimePoint, }, temp_dir, - test_utils::{MithrilFixtureBuilder, TempDir}, + test_utils::{ + MithrilFixture, MithrilFixtureBuilder, StakeDistributionGenerationMethod, TempDir, + }, }; use test_extensions::{utilities::get_test_dir, ExpectedCertificate, RuntimeTester}; +/// Epoch fixtures helps using the fixtures in the tests +struct EpochFixtures<'a> { + /// The fixture used for the registration of signers of the epoch + registering: &'a MithrilFixture, + /// The fixture used for the signing of the epoch + current_signing: Option<&'a MithrilFixture>, + /// The fixture used for the signing of the following epoch + next_signing: Option<&'a MithrilFixture>, +} + +/// Epoch fixtures map builders +struct EpochFixturesMapBuilder; + +impl EpochFixturesMapBuilder { + fn build_fixtures_sequence( + epochs_range: Range, + protocol_parameters: ProtocolParameters, + ) -> HashMap { + epochs_range + .map(|epoch| { + ( + Epoch(epoch as u64), + MithrilFixtureBuilder::default() + .with_signers(epoch + 1) + .with_protocol_parameters(protocol_parameters.clone()) + .with_stake_distribution( + StakeDistributionGenerationMethod::RandomDistribution { + seed: [epoch as u8; 32], + }, + ) + .build(), + ) + }) + .collect::>() + } + + fn build_epoch_fixtures_map( + fixtures: &HashMap, + ) -> HashMap> { + fixtures + .iter() + .map(|(Epoch(index), _fixture)| { + ( + Epoch(*index), + EpochFixtures { + registering: &fixtures[&Epoch(*index)], + next_signing: if *index >= 1 { + fixtures.get(&Epoch(*index - 1)) + } else { + None + }, + current_signing: if *index >= 2 { + fixtures.get(&Epoch(*index - 2)) + } else { + None + }, + }, + ) + }) + .collect::>() + } +} + #[tokio::test] async fn create_certificate_slave() { let protocol_parameters = ProtocolParameters { @@ -19,11 +86,9 @@ async fn create_certificate_slave() { m: 150, phi_f: 0.95, }; - let current_fixture = MithrilFixtureBuilder::default() - .with_signers(2) - .with_protocol_parameters(protocol_parameters.clone()) - .build(); - let current_avk = current_fixture.compute_and_encode_avk(); + let fixtures = + EpochFixturesMapBuilder::build_fixtures_sequence(1..10, protocol_parameters.clone()); + let epoch_fixtures_map = EpochFixturesMapBuilder::build_epoch_fixtures_map(&fixtures); let start_time_point = TimePoint { epoch: Epoch(1), immutable_file_number: 1, @@ -56,55 +121,180 @@ async fn create_certificate_slave() { comment!( "Epoch 1: - - the master aggregator bootstraps its genesis certificate - - the slave aggregator synchronizes signers from the master aggregator - - the slave aggregator stays in Idle state with an error as it doesn't have a genesis certificate yet - "); + - the master aggregator registers the first signers + - the master aggregator can't transition from 'Idle' to 'Ready' + - the slave aggregator can't transition from 'Idle' to 'Ready' + " + ); + let epoch_fixture = &epoch_fixtures_map[&Epoch(1)]; + + comment!("Master: update stake distribution source"); + master_tester + .update_stake_distribution(epoch_fixture.registering.stake_distribution()) + .await + .unwrap(); + + comment!("Slave: update stake distribution source"); + slave_tester + .update_stake_distribution(epoch_fixture.registering.stake_distribution()) + .await + .unwrap(); + + comment!("Master: start the runtime state machine"); + cycle_err!(master_tester, "idle"); + cycle_err!(master_tester, "idle"); - comment!("Master: create signers & declare stake distribution"); + comment!("Slave: start the runtime state machine"); + cycle_err!(slave_tester, "idle"); + cycle_err!(slave_tester, "idle"); + + comment!("Master: register signers"); master_tester - .init_state_from_fixture(¤t_fixture) + .register_signers(&epoch_fixture.registering.signers_fixture()) .await .unwrap(); + cycle_err!(master_tester, "idle"); - comment!("Slave: create signers & declare stake distribution"); + comment!( + "Epoch 2: + - the master aggregator creates its genesis certificate + - the master aggregator can't transition from 'Idle' to 'Ready' + - the slave aggregator can't transition from 'Idle' to 'Ready' + " + ); + let epoch_fixture = &epoch_fixtures_map[&Epoch(2)]; + + comment!("Master: update stake distribution source"); + master_tester + .update_stake_distribution(epoch_fixture.registering.stake_distribution()) + .await + .unwrap(); + + comment!("Slave: update stake distribution source"); slave_tester - .chain_observer - .set_signers(current_fixture.signers_with_stake()) - .await; + .update_stake_distribution(epoch_fixture.registering.stake_distribution()) + .await + .unwrap(); + + comment!("Master: change the epoch"); + master_tester.increase_epoch().await.unwrap(); + cycle_err!(master_tester, "idle"); + cycle_err!(master_tester, "idle"); + + comment!("Slave: change the epoch after master"); + slave_tester.increase_epoch().await.unwrap(); + cycle_err!(slave_tester, "idle"); + cycle_err!(slave_tester, "idle"); + + comment!("Master: register signers"); + master_tester + .register_signers(&epoch_fixture.registering.signers_fixture()) + .await + .unwrap(); + cycle_err!(master_tester, "idle"); comment!("Master: bootstrap the genesis certificate"); master_tester - .register_genesis_certificate(¤t_fixture) + .register_genesis_certificate(epoch_fixture.next_signing.unwrap()) .await .unwrap(); assert_last_certificate_eq!( master_tester, - ExpectedCertificate::new_genesis(Epoch(1), current_fixture.compute_and_encode_avk()) + ExpectedCertificate::new_genesis( + Epoch(2), + epoch_fixture.next_signing.unwrap().compute_and_encode_avk() + ) ); - comment!("Master: start the runtime state machine"); + comment!( + "Epoch 3: + - the master aggregator produces a new certificate + - the slave aggregator synchronizes signers from the master aggregator + - the slave aggregator can't transition from 'Idle' to 'Ready' + " + ); + let epoch_fixture = &epoch_fixtures_map[&Epoch(3)]; + + comment!("Master: update stake distribution source"); + master_tester + .update_stake_distribution(epoch_fixture.registering.stake_distribution()) + .await + .unwrap(); + + comment!("Slave: update stake distribution source"); + slave_tester + .update_stake_distribution(epoch_fixture.registering.stake_distribution()) + .await + .unwrap(); + + comment!("Master: change the epoch"); + master_tester.increase_epoch().await.unwrap(); cycle!(master_tester, "ready"); cycle!(master_tester, "signing"); - comment!("Slave: start the runtime state machine"); + comment!("Slave: change the epoch after master"); + slave_tester.increase_epoch().await.unwrap(); cycle_err!(slave_tester, "idle"); cycle_err!(slave_tester, "idle"); comment!("Master: register signers"); master_tester - .register_signers(¤t_fixture.signers_fixture()) + .register_signers(&epoch_fixture.registering.signers_fixture()) .await .unwrap(); + + comment!("Master: signers send their single signature"); + master_tester + .send_single_signatures( + SignedEntityTypeDiscriminants::MithrilStakeDistribution, + &epoch_fixture.current_signing.unwrap().signers_fixture(), + ) + .await + .unwrap(); + cycle!(master_tester, "ready"); + + comment!("Master: state machine should issue a certificate for the MithrilStakeDistribution"); + cycle!(master_tester, "signing"); + + assert_last_certificate_eq!( + master_tester, + ExpectedCertificate::new( + Epoch(3), + StakeDistributionParty::from_signers( + epoch_fixture.current_signing.unwrap().signers_with_stake() + ) + .as_slice(), + epoch_fixture + .current_signing + .unwrap() + .compute_and_encode_avk(), + SignedEntityType::MithrilStakeDistribution(Epoch(3)), + ExpectedCertificate::genesis_identifier(Epoch(2)), + ) + ); cycle_err!(master_tester, "signing"); comment!( - "Epoch 2: + "Epoch 4: - the master aggregator produces a new certificate - the slave aggregator synchronizes signers from the master aggregator - - the slave aggregator bootstraps its genesis certificate" + - the slave aggregator bootstraps its genesis certificate + - the slave aggregator can't transition from 'Idle' to 'Ready'" ); + let epoch_fixture = &epoch_fixtures_map[&Epoch(4)]; + + comment!("Master: update stake distribution source"); + master_tester + .update_stake_distribution(epoch_fixture.registering.stake_distribution()) + .await + .unwrap(); + + comment!("Slave: update stake distribution source"); + slave_tester + .update_stake_distribution(epoch_fixture.registering.stake_distribution()) + .await + .unwrap(); comment!("Master: change the epoch"); master_tester.increase_epoch().await.unwrap(); @@ -118,13 +308,13 @@ async fn create_certificate_slave() { comment!("Slave: bootstrap the genesis certificate"); slave_tester - .register_genesis_certificate(¤t_fixture) + .register_genesis_certificate(epoch_fixture.next_signing.unwrap()) .await .unwrap(); comment!("Master: register signers"); master_tester - .register_signers(¤t_fixture.signers_fixture()) + .register_signers(&epoch_fixture.registering.signers_fixture()) .await .unwrap(); cycle!(master_tester, "signing"); @@ -133,55 +323,49 @@ async fn create_certificate_slave() { master_tester .send_single_signatures( SignedEntityTypeDiscriminants::MithrilStakeDistribution, - ¤t_fixture.signers_fixture(), + &epoch_fixture.current_signing.unwrap().signers_fixture(), ) .await .unwrap(); comment!("Master: state machine should issue a certificate for the MithrilStakeDistribution"); cycle!(master_tester, "ready"); + assert_last_certificate_eq!( master_tester, ExpectedCertificate::new( - Epoch(2), - StakeDistributionParty::from_signers(current_fixture.signers_with_stake()).as_slice(), - current_fixture.compute_and_encode_avk(), - SignedEntityType::MithrilStakeDistribution(Epoch(2)), - ExpectedCertificate::genesis_identifier(Epoch(1)), + Epoch(4), + StakeDistributionParty::from_signers( + epoch_fixture.current_signing.unwrap().signers_with_stake() + ) + .as_slice(), + epoch_fixture + .current_signing + .unwrap() + .compute_and_encode_avk(), + SignedEntityType::MithrilStakeDistribution(Epoch(4)), + ExpectedCertificate::identifier(&SignedEntityType::MithrilStakeDistribution(Epoch(3))), ) ); cycle!(master_tester, "signing"); comment!( - "Epoch 3: + "Epoch 5: - the master aggregator produces a new certificate - - the master aggregator stake distribution is updated - - the slave aggregator can't transition from 'Idle' to 'Ready' when the master aggregator has not transitioned to a new epoch - - the slave aggregator can transition from 'Idle' to 'Ready' when the master aggregator has transitioned to a new epoch - the slave aggregator produces a new certificate - the slave aggregator new certificate uses the same avk as the master aggregator's new certificate - - the slave aggregator stake distribution is updated "); + let epoch_fixture = &epoch_fixtures_map[&Epoch(5)]; - comment!("Master: update stake distribution"); - let following_fixture = { - let updated_stake_distribution = StakeDistribution::from_iter( - current_fixture - .signers_with_stake() - .into_iter() - .map(|s| (s.party_id, s.stake + 1000)), - ); - - master_tester - .update_stake_distribution(updated_stake_distribution) - .await - .unwrap() - }; - let following_avk = following_fixture.compute_and_encode_avk(); + comment!("Master: update stake distribution source"); + master_tester + .update_stake_distribution(epoch_fixture.registering.stake_distribution()) + .await + .unwrap(); - comment!("Slave: update stake distribution"); + comment!("Slave: update stake distribution source"); slave_tester - .update_stake_distribution(following_fixture.stake_distribution()) + .update_stake_distribution(epoch_fixture.registering.stake_distribution()) .await .unwrap(); @@ -200,7 +384,7 @@ async fn create_certificate_slave() { comment!("Master: register signers"); master_tester - .register_signers(¤t_fixture.signers_fixture()) + .register_signers(&epoch_fixture.registering.signers_fixture()) .await .unwrap(); cycle!(master_tester, "signing"); @@ -209,7 +393,7 @@ async fn create_certificate_slave() { master_tester .send_single_signatures( SignedEntityTypeDiscriminants::MithrilStakeDistribution, - ¤t_fixture.signers_fixture(), + &epoch_fixture.current_signing.unwrap().signers_fixture(), ) .await .unwrap(); @@ -219,7 +403,7 @@ async fn create_certificate_slave() { slave_tester .send_single_signatures( SignedEntityTypeDiscriminants::MithrilStakeDistribution, - ¤t_fixture.signers_fixture(), + &epoch_fixture.current_signing.unwrap().signers_fixture(), ) .await .unwrap(); @@ -227,34 +411,63 @@ async fn create_certificate_slave() { comment!("Master: state machine should issue a certificate for the MithrilStakeDistribution"); cycle!(master_tester, "ready"); let master_expected_certificate = ExpectedCertificate::new( - Epoch(3), - StakeDistributionParty::from_signers(current_fixture.signers_with_stake()).as_slice(), - current_fixture.compute_and_encode_avk(), - SignedEntityType::MithrilStakeDistribution(Epoch(3)), - ExpectedCertificate::identifier(&SignedEntityType::MithrilStakeDistribution(Epoch(2))), + Epoch(5), + StakeDistributionParty::from_signers( + epoch_fixture.current_signing.unwrap().signers_with_stake(), + ) + .as_slice(), + epoch_fixture + .current_signing + .unwrap() + .compute_and_encode_avk(), + SignedEntityType::MithrilStakeDistribution(Epoch(5)), + ExpectedCertificate::identifier(&SignedEntityType::MithrilStakeDistribution(Epoch(4))), ); assert_last_certificate_eq!(master_tester, master_expected_certificate); comment!("Slave: state machine should issue a certificate for the MithrilStakeDistribution"); cycle!(slave_tester, "ready"); let slave_expected_certificate = ExpectedCertificate::new( - Epoch(3), - StakeDistributionParty::from_signers(current_fixture.signers_with_stake()).as_slice(), - current_fixture.compute_and_encode_avk(), - SignedEntityType::MithrilStakeDistribution(Epoch(3)), - ExpectedCertificate::genesis_identifier(Epoch(2)), + Epoch(5), + StakeDistributionParty::from_signers( + epoch_fixture.current_signing.unwrap().signers_with_stake(), + ) + .as_slice(), + epoch_fixture + .current_signing + .unwrap() + .compute_and_encode_avk(), + SignedEntityType::MithrilStakeDistribution(Epoch(5)), + ExpectedCertificate::genesis_identifier(Epoch(4)), ); assert_last_certificate_eq!(slave_tester, slave_expected_certificate); - let expected_avk = current_avk.clone(); + let expected_avk = epoch_fixture + .current_signing + .unwrap() + .compute_and_encode_avk() + .clone(); assert_eq!(expected_avk, master_expected_certificate.avk()); assert_eq!(expected_avk, slave_expected_certificate.avk()); comment!( - "Epoch 4: + "Epoch 6: - the master aggregator produces a new certificate - the slave aggregator produces a new certificate - the slave aggregator new certificate uses the same avk as the master aggregator's new certificate "); + let epoch_fixture = &epoch_fixtures_map[&Epoch(6)]; + + comment!("Master: update stake distribution source"); + master_tester + .update_stake_distribution(epoch_fixture.registering.stake_distribution()) + .await + .unwrap(); + + comment!("Slave: update stake distribution source"); + slave_tester + .update_stake_distribution(epoch_fixture.registering.stake_distribution()) + .await + .unwrap(); comment!("Master: change the epoch"); master_tester.increase_epoch().await.unwrap(); @@ -268,7 +481,7 @@ async fn create_certificate_slave() { comment!("Master: register signers"); master_tester - .register_signers(¤t_fixture.signers_fixture()) + .register_signers(&epoch_fixture.registering.signers_fixture()) .await .unwrap(); cycle!(master_tester, "signing"); @@ -277,7 +490,7 @@ async fn create_certificate_slave() { master_tester .send_single_signatures( SignedEntityTypeDiscriminants::MithrilStakeDistribution, - ¤t_fixture.signers_fixture(), + &epoch_fixture.current_signing.unwrap().signers_fixture(), ) .await .unwrap(); @@ -287,7 +500,7 @@ async fn create_certificate_slave() { slave_tester .send_single_signatures( SignedEntityTypeDiscriminants::MithrilStakeDistribution, - ¤t_fixture.signers_fixture(), + &epoch_fixture.current_signing.unwrap().signers_fixture(), ) .await .unwrap(); @@ -295,35 +508,63 @@ async fn create_certificate_slave() { comment!("Master: state machine should issue a certificate for the MithrilStakeDistribution"); cycle!(master_tester, "ready"); let master_expected_certificate = ExpectedCertificate::new( - Epoch(4), - StakeDistributionParty::from_signers(current_fixture.signers_with_stake()).as_slice(), - current_fixture.compute_and_encode_avk(), - SignedEntityType::MithrilStakeDistribution(Epoch(4)), - ExpectedCertificate::identifier(&SignedEntityType::MithrilStakeDistribution(Epoch(3))), + Epoch(6), + StakeDistributionParty::from_signers( + epoch_fixture.current_signing.unwrap().signers_with_stake(), + ) + .as_slice(), + epoch_fixture + .current_signing + .unwrap() + .compute_and_encode_avk(), + SignedEntityType::MithrilStakeDistribution(Epoch(6)), + ExpectedCertificate::identifier(&SignedEntityType::MithrilStakeDistribution(Epoch(5))), ); assert_last_certificate_eq!(master_tester, master_expected_certificate); comment!("Slave: state machine should issue a certificate for the MithrilStakeDistribution"); cycle!(slave_tester, "ready"); let slave_expected_certificate = ExpectedCertificate::new( - Epoch(4), - StakeDistributionParty::from_signers(current_fixture.signers_with_stake()).as_slice(), - current_fixture.compute_and_encode_avk(), - SignedEntityType::MithrilStakeDistribution(Epoch(4)), - ExpectedCertificate::identifier(&SignedEntityType::MithrilStakeDistribution(Epoch(3))), + Epoch(6), + StakeDistributionParty::from_signers( + epoch_fixture.current_signing.unwrap().signers_with_stake(), + ) + .as_slice(), + epoch_fixture + .current_signing + .unwrap() + .compute_and_encode_avk(), + SignedEntityType::MithrilStakeDistribution(Epoch(6)), + ExpectedCertificate::identifier(&SignedEntityType::MithrilStakeDistribution(Epoch(5))), ); assert_last_certificate_eq!(slave_tester, slave_expected_certificate); - let expected_avk = current_avk; + let expected_avk = epoch_fixture + .current_signing + .unwrap() + .compute_and_encode_avk() + .clone(); assert_eq!(expected_avk, master_expected_certificate.avk()); assert_eq!(expected_avk, slave_expected_certificate.avk()); comment!( - "Epoch 5: - - the master aggregator produces a new certificate with the new stake distribution from epoch 3 - - the slave aggregator produces a new certificate with the new stake distribution from epoch 3 + "Epoch 7: + - the master aggregator produces a new certificate + - the slave aggregator produces a new certificate - the slave aggregator new certificate uses the same avk as the master aggregator's new certificate "); - let fixture = following_fixture; + let epoch_fixture = &epoch_fixtures_map[&Epoch(7)]; + + comment!("Master: update stake distribution source"); + master_tester + .update_stake_distribution(epoch_fixture.registering.stake_distribution()) + .await + .unwrap(); + + comment!("Slave: update stake distribution source"); + slave_tester + .update_stake_distribution(epoch_fixture.registering.stake_distribution()) + .await + .unwrap(); comment!("Master: change the epoch"); master_tester.increase_epoch().await.unwrap(); @@ -337,7 +578,7 @@ async fn create_certificate_slave() { comment!("Master: register signers"); master_tester - .register_signers(&fixture.signers_fixture()) + .register_signers(&epoch_fixture.registering.signers_fixture()) .await .unwrap(); cycle!(master_tester, "signing"); @@ -346,7 +587,7 @@ async fn create_certificate_slave() { master_tester .send_single_signatures( SignedEntityTypeDiscriminants::MithrilStakeDistribution, - &fixture.signers_fixture(), + &epoch_fixture.current_signing.unwrap().signers_fixture(), ) .await .unwrap(); @@ -356,7 +597,7 @@ async fn create_certificate_slave() { slave_tester .send_single_signatures( SignedEntityTypeDiscriminants::MithrilStakeDistribution, - &fixture.signers_fixture(), + &epoch_fixture.current_signing.unwrap().signers_fixture(), ) .await .unwrap(); @@ -364,25 +605,41 @@ async fn create_certificate_slave() { comment!("Master: state machine should issue a certificate for the MithrilStakeDistribution"); cycle!(master_tester, "ready"); let master_expected_certificate = ExpectedCertificate::new( - Epoch(5), - StakeDistributionParty::from_signers(fixture.signers_with_stake()).as_slice(), - fixture.compute_and_encode_avk(), - SignedEntityType::MithrilStakeDistribution(Epoch(5)), - ExpectedCertificate::identifier(&SignedEntityType::MithrilStakeDistribution(Epoch(4))), + Epoch(7), + StakeDistributionParty::from_signers( + epoch_fixture.current_signing.unwrap().signers_with_stake(), + ) + .as_slice(), + epoch_fixture + .current_signing + .unwrap() + .compute_and_encode_avk(), + SignedEntityType::MithrilStakeDistribution(Epoch(7)), + ExpectedCertificate::identifier(&SignedEntityType::MithrilStakeDistribution(Epoch(6))), ); assert_last_certificate_eq!(master_tester, master_expected_certificate); comment!("Slave: state machine should issue a certificate for the MithrilStakeDistribution"); cycle!(slave_tester, "ready"); let slave_expected_certificate = ExpectedCertificate::new( - Epoch(5), - StakeDistributionParty::from_signers(fixture.signers_with_stake()).as_slice(), - fixture.compute_and_encode_avk(), - SignedEntityType::MithrilStakeDistribution(Epoch(5)), - ExpectedCertificate::identifier(&SignedEntityType::MithrilStakeDistribution(Epoch(4))), + Epoch(7), + StakeDistributionParty::from_signers( + epoch_fixture.current_signing.unwrap().signers_with_stake(), + ) + .as_slice(), + epoch_fixture + .current_signing + .unwrap() + .compute_and_encode_avk(), + SignedEntityType::MithrilStakeDistribution(Epoch(7)), + ExpectedCertificate::identifier(&SignedEntityType::MithrilStakeDistribution(Epoch(6))), ); assert_last_certificate_eq!(slave_tester, slave_expected_certificate); - let expected_avk = following_avk; + let expected_avk = epoch_fixture + .current_signing + .unwrap() + .compute_and_encode_avk() + .clone(); assert_eq!(expected_avk, master_expected_certificate.avk()); assert_eq!(expected_avk, slave_expected_certificate.avk()); } From 5cb1589b1f364d675c735fd834644d8f0f2340d7 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 17 Mar 2025 18:45:22 +0100 Subject: [PATCH 13/33] fix(common): avoid too low stake in random stake distribution Which could prevent signature from signers even with loose protocol parameters. --- .../tests/create_certificate_slave.rs | 1 + mithril-common/src/protocol/multi_signer.rs | 1 + mithril-common/src/protocol/signer_builder.rs | 1 + mithril-common/src/test_utils/fixture_builder.rs | 13 ++++++++++--- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/mithril-aggregator/tests/create_certificate_slave.rs b/mithril-aggregator/tests/create_certificate_slave.rs index ab105a4044c..9d7ed15c3dc 100644 --- a/mithril-aggregator/tests/create_certificate_slave.rs +++ b/mithril-aggregator/tests/create_certificate_slave.rs @@ -44,6 +44,7 @@ impl EpochFixturesMapBuilder { .with_stake_distribution( StakeDistributionGenerationMethod::RandomDistribution { seed: [epoch as u8; 32], + min_stake: 10, }, ) .build(), diff --git a/mithril-common/src/protocol/multi_signer.rs b/mithril-common/src/protocol/multi_signer.rs index 52c52240486..c1f11af44ef 100644 --- a/mithril-common/src/protocol/multi_signer.rs +++ b/mithril-common/src/protocol/multi_signer.rs @@ -171,6 +171,7 @@ mod test { .with_signers(1) .with_stake_distribution(StakeDistributionGenerationMethod::RandomDistribution { seed: [3u8; 32], + min_stake: 1, }) .build(), ); diff --git a/mithril-common/src/protocol/signer_builder.rs b/mithril-common/src/protocol/signer_builder.rs index 8bfeeb876e2..6e3c1080b69 100644 --- a/mithril-common/src/protocol/signer_builder.rs +++ b/mithril-common/src/protocol/signer_builder.rs @@ -218,6 +218,7 @@ mod test { .with_stake_distribution( crate::test_utils::StakeDistributionGenerationMethod::RandomDistribution { seed: [4u8; 32], + min_stake: 1, }, ) .build(); diff --git a/mithril-common/src/test_utils/fixture_builder.rs b/mithril-common/src/test_utils/fixture_builder.rs index 0e46b20ce1f..cbecfb5ae1d 100644 --- a/mithril-common/src/test_utils/fixture_builder.rs +++ b/mithril-common/src/test_utils/fixture_builder.rs @@ -29,7 +29,10 @@ impl Default for MithrilFixtureBuilder { enable_signers_certification: true, number_of_signers: 5, stake_distribution_generation_method: - StakeDistributionGenerationMethod::RandomDistribution { seed: [0u8; 32] }, + StakeDistributionGenerationMethod::RandomDistribution { + seed: [0u8; 32], + min_stake: 1, + }, party_id_seed: [0u8; 32], } } @@ -41,6 +44,8 @@ pub enum StakeDistributionGenerationMethod { RandomDistribution { /// The randomizer seed seed: [u8; 32], + /// The minimum stake + min_stake: Stake, }, /// Use a custom stake distribution @@ -106,13 +111,13 @@ impl MithrilFixtureBuilder { let signers_party_ids = self.generate_party_ids(); match &self.stake_distribution_generation_method { - StakeDistributionGenerationMethod::RandomDistribution { seed } => { + StakeDistributionGenerationMethod::RandomDistribution { seed, min_stake } => { let mut stake_rng = ChaCha20Rng::from_seed(*seed); signers_party_ids .into_iter() .map(|party_id| { - let stake = 1 + stake_rng.next_u64() % 999; + let stake = min_stake + stake_rng.next_u64() % 999; (party_id, stake) }) .collect::>() @@ -246,6 +251,7 @@ mod tests { let result = MithrilFixtureBuilder::default() .with_stake_distribution(StakeDistributionGenerationMethod::RandomDistribution { seed: [0u8; 32], + min_stake: 1, }) .with_signers(4) .build(); @@ -275,6 +281,7 @@ mod tests { let result = MithrilFixtureBuilder::default() .with_stake_distribution(StakeDistributionGenerationMethod::RandomDistribution { seed: [0u8; 32], + min_stake: 1, }) .with_signers(5) .build(); From ddcd10ea3f6411eec963d351149801a0f4716d88 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 17 Mar 2025 18:57:58 +0100 Subject: [PATCH 14/33] fix(aggregator): slave signer registration stabilization --- mithril-aggregator/src/runtime/state_machine.rs | 16 ++++++++-------- mithril-common/src/entities/epoch.rs | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/mithril-aggregator/src/runtime/state_machine.rs b/mithril-aggregator/src/runtime/state_machine.rs index 4669b5e89c1..6921cd3ba14 100644 --- a/mithril-aggregator/src/runtime/state_machine.rs +++ b/mithril-aggregator/src/runtime/state_machine.rs @@ -143,7 +143,6 @@ impl AggregatorRuntime { info!(self.logger, "→ Trying to transition to READY"; "last_time_point" => ?last_time_point); let can_try_transition_from_idle_to_ready = if self.config.is_slave { - println!("Checking if slave aggregator is at the same epoch as master"); self.runner .is_slave_aggregator_at_same_epoch_as_master(&last_time_point) .await? @@ -265,18 +264,19 @@ impl AggregatorRuntime { self.runner .update_stake_distribution(&new_time_point) .await?; - if self.config.is_slave { - self.runner - .synchronize_slave_aggregator_signer_registration() - .await?; - } self.runner.inform_new_epoch(new_time_point.epoch).await?; - self.runner.upkeep(new_time_point.epoch).await?; self.runner .open_signer_registration_round(&new_time_point) .await?; self.runner.update_epoch_settings().await?; + if self.config.is_slave { + self.runner + .synchronize_slave_aggregator_signer_registration() + .await?; + // Needed to recompute epoch data for the next signing round on the slave + self.runner.inform_new_epoch(new_time_point.epoch).await?; + } self.runner.precompute_epoch_data().await?; } @@ -940,7 +940,7 @@ mod tests { runner .expect_inform_new_epoch() .with(predicate::eq(new_time_point_clone.clone().epoch)) - .once() + .times(2) .returning(|_| Ok(())); runner .expect_update_epoch_settings() diff --git a/mithril-common/src/entities/epoch.rs b/mithril-common/src/entities/epoch.rs index cbc9133087c..e2c7469d800 100644 --- a/mithril-common/src/entities/epoch.rs +++ b/mithril-common/src/entities/epoch.rs @@ -38,7 +38,7 @@ impl Epoch { pub const CARDANO_STAKE_DISTRIBUTION_SNAPSHOT_OFFSET: u64 = 2; /// The epoch offset used to retrieve the epoch at which a signer has registered to the master aggregator. - pub const SIGNER_MASTER_SYNCHRONIZATION_OFFSET: u64 = 1; + pub const SIGNER_MASTER_SYNCHRONIZATION_OFFSET: u64 = 0; /// Computes a new Epoch by applying an epoch offset. /// From 9dd961f72acaa117205ca13699ef160c8f0bdee3 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 18 Mar 2025 12:14:54 +0100 Subject: [PATCH 15/33] feat(relay): implement signer relay modes Which can be 'Passthrough' or 'P2P'. --- mithril-relay/Cargo.toml | 1 + mithril-relay/src/commands/signer.rs | 14 +- mithril-relay/src/lib.rs | 1 + mithril-relay/src/relay/mod.rs | 1 + mithril-relay/src/relay/signer.rs | 172 +++++++++++++----- mithril-relay/src/repeater.rs | 1 + .../tests/register_signer_signature.rs | 6 +- 7 files changed, 148 insertions(+), 48 deletions(-) diff --git a/mithril-relay/Cargo.toml b/mithril-relay/Cargo.toml index f4298188c9e..a657716cb6b 100644 --- a/mithril-relay/Cargo.toml +++ b/mithril-relay/Cargo.toml @@ -48,6 +48,7 @@ slog = { version = "2.7.0", features = [ ] } slog-async = "2.8.0" slog-bunyan = "2.5.0" +strum = { version = "0.26.3", features = ["derive"] } thiserror = "2.0.11" tokio = { version = "1.43.0", features = ["full"] } warp = "0.3.7" diff --git a/mithril-relay/src/commands/signer.rs b/mithril-relay/src/commands/signer.rs index e54d968330e..7045f5c883c 100644 --- a/mithril-relay/src/commands/signer.rs +++ b/mithril-relay/src/commands/signer.rs @@ -6,7 +6,7 @@ use mithril_common::StdResult; use slog::error; use super::CommandContext; -use crate::SignerRelay; +use crate::{SignerRelay, SignerRelayMode}; #[derive(Parser, Debug, Clone)] pub struct SignerCommand { @@ -26,6 +26,14 @@ pub struct SignerCommand { #[clap(long, env = "AGGREGATOR_ENDPOINT")] aggregator_endpoint: String, + /// Signer registration relay mode + #[clap(value_enum, env = "SIGNER_REGISTRATION_MODE", default_value_t = SignerRelayMode::Passthrough)] + signer_registration_mode: SignerRelayMode, + + /// Signature registration relay mode + #[clap(value_enum, env = "SIGNATURE_REGISTRATION_MODE", default_value_t = SignerRelayMode::P2P)] + signature_registration_mode: SignerRelayMode, + /// Interval at which a signer registration should be repeated in milliseconds (defaults to 1 hour) #[clap(long, env = "SIGNER_REPEATER_DELAY", default_value_t = 3_600 * 1_000)] signer_repeater_delay: u64, @@ -38,12 +46,16 @@ impl SignerCommand { let server_port = self.server_port.to_owned(); let dial_to = self.dial_to.to_owned(); let addr: Multiaddr = format!("/ip4/0.0.0.0/tcp/{}", self.listen_port).parse()?; + let signer_registration_mode = &self.signer_registration_mode; + let signature_registration_mode = &self.signature_registration_mode; let aggregator_endpoint = self.aggregator_endpoint.to_owned(); let signer_repeater_delay = Duration::from_millis(self.signer_repeater_delay); let mut relay = SignerRelay::start( &addr, &server_port, + signer_registration_mode, + signature_registration_mode, &aggregator_endpoint, &signer_repeater_delay, logger, diff --git a/mithril-relay/src/lib.rs b/mithril-relay/src/lib.rs index a6ac2d52981..52e2ecc45e6 100644 --- a/mithril-relay/src/lib.rs +++ b/mithril-relay/src/lib.rs @@ -12,6 +12,7 @@ pub use commands::RelayCommands; pub use relay::AggregatorRelay; pub use relay::PassiveRelay; pub use relay::SignerRelay; +pub use relay::SignerRelayMode; /// The P2P topic names used by Mithril pub mod mithril_p2p_topic { diff --git a/mithril-relay/src/relay/mod.rs b/mithril-relay/src/relay/mod.rs index 0d177321999..3dffee0edb5 100644 --- a/mithril-relay/src/relay/mod.rs +++ b/mithril-relay/src/relay/mod.rs @@ -5,3 +5,4 @@ mod signer; pub use aggregator::AggregatorRelay; pub use passive::PassiveRelay; pub use signer::SignerRelay; +pub use signer::SignerRelayMode; diff --git a/mithril-relay/src/relay/signer.rs b/mithril-relay/src/relay/signer.rs index fc2ae98a037..db70bbd846f 100644 --- a/mithril-relay/src/relay/signer.rs +++ b/mithril-relay/src/relay/signer.rs @@ -2,6 +2,7 @@ use crate::{ p2p::{Peer, PeerEvent}, repeater::MessageRepeater, }; +use clap::ValueEnum; use libp2p::Multiaddr; use mithril_common::{ logging::LoggerExtensions, @@ -11,9 +12,37 @@ use mithril_common::{ }; use slog::{debug, info, Logger}; use std::{net::SocketAddr, sync::Arc, time::Duration}; +use strum::Display; use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; use warp::Filter; +/// Signer relay mode +/// +/// The relay mode defines how the relay will behave when it receives a message +#[derive(Debug, Clone, Display, ValueEnum)] +#[strum(serialize_all = "mixed_case")] +pub enum SignerRelayMode { + /// Passthrough relay mode + /// + /// In this mode, the relay will only call the aggregator with the message received + Passthrough, + /// P2P relay mode + /// + /// In this mode, the relay will publish the message received to the P2P network + P2P, +} + +struct HTTPServerConfiguration<'a> { + server_port: &'a u16, + signer_registration_mode: SignerRelayMode, + signature_registration_mode: SignerRelayMode, + aggregator_endpoint: &'a str, + signer_tx: UnboundedSender, + signature_tx: UnboundedSender, + signer_repeater: Arc>, + logger: &'a Logger, +} + /// A relay for a Mithril signer pub struct SignerRelay { server: TestHttpServer, @@ -29,12 +58,14 @@ impl SignerRelay { pub async fn start( address: &Multiaddr, server_port: &u16, + signer_registration_mode: &SignerRelayMode, + signature_registration_mode: &SignerRelayMode, aggregator_endpoint: &str, signer_repeater_delay: &Duration, logger: &Logger, ) -> StdResult { let relay_logger = logger.new_with_component_name::(); - debug!(relay_logger, "Starting..."); + debug!(relay_logger, "Starting..."; "signer_registration_mode" => ?signer_registration_mode, "signature_registration_mode" => ?signature_registration_mode); let (signature_tx, signature_rx) = unbounded_channel::(); let (signer_tx, signer_rx) = unbounded_channel::(); let signer_repeater = Arc::new(MessageRepeater::new( @@ -43,14 +74,16 @@ impl SignerRelay { logger, )); let peer = Peer::new(address).start().await?; - let server = Self::start_http_server( + let server = Self::start_http_server(&HTTPServerConfiguration { server_port, + signer_registration_mode: signer_registration_mode.to_owned(), + signature_registration_mode: signature_registration_mode.to_owned(), aggregator_endpoint, - signer_tx, - signature_tx, - signer_repeater.clone(), - logger, - ) + signer_tx: signer_tx.clone(), + signature_tx: signature_tx.clone(), + signer_repeater: signer_repeater.clone(), + logger: &relay_logger, + }) .await; info!(relay_logger, "Listening on"; "address" => ?server.address()); @@ -64,44 +97,55 @@ impl SignerRelay { }) } - async fn start_http_server( - server_port: &u16, - aggregator_endpoint: &str, - signer_tx: UnboundedSender, - signature_tx: UnboundedSender, - signer_repeater: Arc>, - logger: &Logger, - ) -> TestHttpServer { - let server_logger = logger.new_with_name("http_server"); + async fn start_http_server(configuration: &HTTPServerConfiguration<'_>) -> TestHttpServer { + let server_logger = configuration.logger.new_with_name("http_server"); test_http_server_with_socket_address( warp::path::end() .and(warp::get()) .and(middlewares::with_logger(&server_logger)) .and(middlewares::with_aggregator_endpoint( - aggregator_endpoint.to_string(), + configuration.aggregator_endpoint.to_string(), )) .and_then(handlers::aggregator_features_handler) .or(warp::path("register-signatures") .and(warp::post()) .and(warp::body::json()) + .and(middlewares::with_signer_relay_mode( + configuration.signature_registration_mode.clone(), + )) + .and(middlewares::with_aggregator_endpoint( + configuration.aggregator_endpoint.to_string(), + )) .and(middlewares::with_logger(&server_logger)) - .and(middlewares::with_transmitter(signature_tx)) + .and(middlewares::with_transmitter( + configuration.signature_tx.clone(), + )) .and_then(handlers::register_signatures_handler)) .or(warp::path("register-signer") .and(warp::post()) .and(warp::body::json()) + .and(middlewares::with_signer_relay_mode( + configuration.signer_registration_mode.clone(), + )) + .and(middlewares::with_aggregator_endpoint( + configuration.aggregator_endpoint.to_string(), + )) .and(middlewares::with_logger(&server_logger)) - .and(middlewares::with_transmitter(signer_tx)) - .and(middlewares::with_repeater(signer_repeater.clone())) + .and(middlewares::with_transmitter( + configuration.signer_tx.clone(), + )) + .and(middlewares::with_repeater( + configuration.signer_repeater.clone(), + )) .and_then(handlers::register_signer_handler)) .or(warp::path("epoch-settings") .and(warp::get()) .and(middlewares::with_logger(&server_logger)) .and(middlewares::with_aggregator_endpoint( - aggregator_endpoint.to_string(), + configuration.aggregator_endpoint.to_string(), )) .and_then(handlers::epoch_settings_handler)), - ([0, 0, 0, 0], *server_port).into(), + ([0, 0, 0, 0], *configuration.server_port).into(), ) } @@ -173,6 +217,8 @@ mod middlewares { use crate::repeater::MessageRepeater; + use super::SignerRelayMode; + pub fn with_logger( logger: &slog::Logger, ) -> impl Filter + Clone { @@ -197,6 +243,12 @@ mod middlewares { ) -> impl Filter + Clone { warp::any().map(move || aggregator_endpoint.clone()) } + + pub fn with_signer_relay_mode( + signer_relay_mode: SignerRelayMode, + ) -> impl Filter + Clone { + warp::any().map(move || signer_relay_mode.clone()) + } } mod handlers { @@ -205,10 +257,12 @@ mod handlers { use slog::{debug, Logger}; use std::{convert::Infallible, sync::Arc}; use tokio::sync::mpsc::UnboundedSender; - use warp::http::StatusCode; + use warp::{http::StatusCode, reply::WithStatus}; use crate::repeater; + use super::SignerRelayMode; + pub async fn aggregator_features_handler( logger: Logger, aggregator_endpoint: String, @@ -223,40 +277,66 @@ mod handlers { pub async fn register_signer_handler( register_signer_message: RegisterSignerMessage, + signer_relay_mode: SignerRelayMode, + aggregator_endpoint: String, logger: Logger, tx: UnboundedSender, repeater: Arc>, ) -> Result { - debug!(logger, "Serve HTTP route /register-signer"; "register_signer_message" => #?register_signer_message); - - repeater.set_message(register_signer_message.clone()).await; - match tx.send(register_signer_message) { - Ok(_) => Ok(Box::new(warp::reply::with_status( - "".to_string(), - StatusCode::CREATED, - ))), - Err(err) => Ok(Box::new(warp::reply::with_status( - format!("{err:?}"), - StatusCode::INTERNAL_SERVER_ERROR, - ))), + debug!(logger, "Serve HTTP route /register-signer"; "signer_relay_mode" => ?signer_relay_mode, "register_signer_message" => #?register_signer_message,); + match signer_relay_mode { + SignerRelayMode::P2P => { + repeater.set_message(register_signer_message.clone()).await; + match tx.send(register_signer_message) { + Ok(_) => Ok(Box::new(warp::reply::with_status( + "".to_string(), + StatusCode::CREATED, + ))), + Err(err) => Ok(Box::new(warp::reply::with_status( + format!("{err:?}"), + StatusCode::INTERNAL_SERVER_ERROR, + ))), + } + } + SignerRelayMode::Passthrough => { + let response = reqwest::Client::new() + .post(format!("{aggregator_endpoint}/register-signer")) + .json(®ister_signer_message) + .send() + .await; + reply_response(logger, response).await + } } } pub async fn register_signatures_handler( register_signature_message: RegisterSignatureMessage, + signer_relay_mode: SignerRelayMode, + aggregator_endpoint: String, logger: Logger, tx: UnboundedSender, ) -> Result { - debug!(logger, "Serve HTTP route /register-signatures"; "register_signature_message" => #?register_signature_message); - match tx.send(register_signature_message) { - Ok(_) => Ok(Box::new(warp::reply::with_status( - "".to_string(), - StatusCode::CREATED, - ))), - Err(err) => Ok(Box::new(warp::reply::with_status( - format!("{err:?}"), - StatusCode::INTERNAL_SERVER_ERROR, - ))), + debug!(logger, "Serve HTTP route /register-signatures"; "signer_relay_mode" => ?signer_relay_mode, "register_signature_message" => #?register_signature_message); + + match signer_relay_mode { + SignerRelayMode::P2P => match tx.send(register_signature_message) { + Ok(_) => Ok(Box::new(warp::reply::with_status( + "".to_string(), + StatusCode::CREATED, + ))), + Err(err) => Ok(Box::new(warp::reply::with_status( + format!("{err:?}"), + StatusCode::INTERNAL_SERVER_ERROR, + ))), + }, + SignerRelayMode::Passthrough => { + let response = reqwest::Client::new() + .post(format!("{aggregator_endpoint}/register-signatures")) + .json(®ister_signature_message) + .send() + .await; + reply_response(logger, response).await + } } } @@ -275,7 +355,7 @@ mod handlers { pub async fn reply_response( logger: Logger, response: Result, - ) -> Result { + ) -> Result>, Infallible> { match response { Ok(response) => match StatusCode::from_u16(response.status().into()) { Ok(status) => match response.text().await { diff --git a/mithril-relay/src/repeater.rs b/mithril-relay/src/repeater.rs index 33b2b5c47db..4b4073816d6 100644 --- a/mithril-relay/src/repeater.rs +++ b/mithril-relay/src/repeater.rs @@ -35,6 +35,7 @@ impl MessageRepeater { } /// Set the message to repeat + #[allow(dead_code)] pub async fn set_message(&self, message: M) { debug!(self.logger, "Set message"; "message" => #?message); *self.message.lock().await = Some(message); diff --git a/mithril-relay/tests/register_signer_signature.rs b/mithril-relay/tests/register_signer_signature.rs index 958fde0b1ca..9be58788594 100644 --- a/mithril-relay/tests/register_signer_signature.rs +++ b/mithril-relay/tests/register_signer_signature.rs @@ -4,7 +4,7 @@ use libp2p::{gossipsub, Multiaddr}; use mithril_common::messages::{RegisterSignatureMessage, RegisterSignerMessage}; use mithril_relay::{ p2p::{BroadcastMessage, PeerBehaviourEvent, PeerEvent}, - PassiveRelay, SignerRelay, + PassiveRelay, SignerRelay, SignerRelayMode, }; use reqwest::StatusCode; use slog::{Drain, Level, Logger}; @@ -35,11 +35,15 @@ async fn should_receive_registrations_from_signers_when_subscribed_to_pubsub() { let total_peers = 1 + total_p2p_client; let addr: Multiaddr = "/ip4/0.0.0.0/tcp/0".parse().unwrap(); let server_port = 0; + let signer_registration_mode = SignerRelayMode::P2P; + let signature_registration_mode = SignerRelayMode::P2P; let aggregator_endpoint = "http://0.0.0.0:1234".to_string(); let signer_repeater_delay = Duration::from_secs(100); let mut signer_relay = SignerRelay::start( &addr, &server_port, + &signer_registration_mode, + &signature_registration_mode, &aggregator_endpoint, &signer_repeater_delay, &logger, From 21673abab995ccb2b5bd75b0db9a6444a47abe99 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 18 Mar 2025 15:49:33 +0100 Subject: [PATCH 16/33] refactor(e2e): use signer relay modes in e2e test --- Cargo.lock | 2 + .../mithril-end-to-end/Cargo.toml | 1 + .../mithril-end-to-end/src/main.rs | 30 +++++++--- .../src/mithril/infrastructure.rs | 52 ++++++++++------ .../src/mithril/relay_signer.rs | 59 +++++++++++++------ 5 files changed, 99 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b8df062b4d1..fa079c9fc25 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3880,6 +3880,7 @@ dependencies = [ "indicatif", "mithril-common", "mithril-doc", + "mithril-relay", "reqwest 0.12.12", "serde", "serde_json", @@ -3945,6 +3946,7 @@ dependencies = [ "slog-bunyan", "slog-scope", "slog-term", + "strum", "thiserror 2.0.11", "tokio", "warp", diff --git a/mithril-test-lab/mithril-end-to-end/Cargo.toml b/mithril-test-lab/mithril-end-to-end/Cargo.toml index 78b12685861..8e4861e701e 100644 --- a/mithril-test-lab/mithril-end-to-end/Cargo.toml +++ b/mithril-test-lab/mithril-end-to-end/Cargo.toml @@ -21,6 +21,7 @@ clap = { version = "4.5.28", features = ["derive"] } indicatif = { version = "0.17.11", features = ["tokio"] } mithril-common = { path = "../../mithril-common", features = ["full"] } mithril-doc = { path = "../../internal/mithril-doc" } +mithril-relay = { path = "../../mithril-relay" } reqwest = { version = "0.12.12", features = ["json"] } serde = { version = "1.0.217", features = ["derive"] } serde_json = "1.0.138" diff --git a/mithril-test-lab/mithril-end-to-end/src/main.rs b/mithril-test-lab/mithril-end-to-end/src/main.rs index e4d478eec27..0d602a7a852 100644 --- a/mithril-test-lab/mithril-end-to-end/src/main.rs +++ b/mithril-test-lab/mithril-end-to-end/src/main.rs @@ -1,5 +1,6 @@ use anyhow::{anyhow, Context}; use clap::{CommandFactory, Parser, Subcommand}; +use mithril_relay::SignerRelayMode; use slog::{Drain, Level, Logger}; use slog_scope::{error, info}; use std::{ @@ -107,11 +108,19 @@ pub struct Args { #[clap(long)] run_only: bool, - /// Enable P2P network mode + /// Use Mithril relays #[clap(long)] - use_p2p_network: bool, + use_relays: bool, - /// Enable P2P passive relays in P2P mode + /// Signer registration relay mode (used only when 'use_relays' is set) + #[clap(long, value_enum, default_value_t = SignerRelayMode::Passthrough)] + relay_signer_registration_mode: SignerRelayMode, + + /// Signature registration relay mode (used only when 'use_relays' is set) + #[clap(long, value_enum, default_value_t = SignerRelayMode::P2P)] + relay_signature_registration_mode: SignerRelayMode, + + /// Enable P2P passive relays in P2P mode (used only when 'use_relays' is set) #[clap(long, default_value = "true")] use_p2p_passive_relays: bool, @@ -147,9 +156,9 @@ impl Args { if self.number_of_signers == 0 { return Err(anyhow!("At least one signer is required")); } - if !self.use_p2p_network && self.number_of_aggregators >= 2 { + if !self.use_relays && self.number_of_aggregators >= 2 { return Err(anyhow!( - "The P2P mode must be activated to run more than one aggregator" + "The 'use_relays' parameter must be activated to run more than one aggregator" )); } @@ -324,7 +333,10 @@ impl App { let server_port = 8080; args.validate()?; let run_only_mode = args.run_only; - let use_p2p_network_mode = args.use_p2p_network; + let use_relays = args.use_relays; + let relay_signer_registration_mode = args.relay_signer_registration_mode; + let relay_signature_registration_mode = args.relay_signature_registration_mode; + let use_p2p_passive_relays = args.use_p2p_passive_relays; let devnet = Devnet::bootstrap(&DevnetBootstrapArgs { @@ -355,7 +367,9 @@ impl App { mithril_era_reader_adapter: args.mithril_era_reader_adapter, signed_entity_types: args.signed_entity_types.clone(), run_only_mode, - use_p2p_network_mode, + use_relays, + relay_signer_registration_mode, + relay_signature_registration_mode, use_p2p_passive_relays, use_era_specific_work_dir: args.mithril_next_era.is_some(), }) @@ -519,7 +533,7 @@ mod tests { "validate should fail with more than one aggregator if p2p network is not used", ); - let args = Args::parse_from(["", "--use-p2p-network", "--number-of-aggregators", "2"]); + let args = Args::parse_from(["", "--use-relays", "--number-of-aggregators", "2"]); args.validate() .expect("validate should succeed with more than one aggregator if p2p network is used"); } diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs index a3905d35786..bb6adf08472 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs @@ -1,17 +1,18 @@ +use crate::mithril::relay_signer::RelaySignerConfiguration; +use crate::{ + assertions, Aggregator, AggregatorConfig, Client, Devnet, PoolNode, RelayAggregator, + RelayPassive, RelaySigner, Signer, DEVNET_MAGIC_ID, +}; use mithril_common::chain_observer::{ChainObserver, PallasChainObserver}; use mithril_common::entities::{Epoch, PartyId, ProtocolParameters}; use mithril_common::{CardanoNetwork, StdResult}; +use mithril_relay::SignerRelayMode; use slog_scope::info; use std::fs; use std::path::PathBuf; use std::sync::Arc; use tokio::sync::RwLock; -use crate::{ - assertions, Aggregator, AggregatorConfig, Client, Devnet, PoolNode, RelayAggregator, - RelayPassive, RelaySigner, Signer, DEVNET_MAGIC_ID, -}; - use super::signer::SignerConfig; pub struct MithrilInfrastructureConfig { @@ -29,7 +30,9 @@ pub struct MithrilInfrastructureConfig { pub mithril_era_reader_adapter: String, pub signed_entity_types: Vec, pub run_only_mode: bool, - pub use_p2p_network_mode: bool, + pub use_relays: bool, + pub relay_signer_registration_mode: SignerRelayMode, + pub relay_signature_registration_mode: SignerRelayMode, pub use_p2p_passive_relays: bool, pub use_era_specific_work_dir: bool, } @@ -64,6 +67,8 @@ impl MithrilInfrastructure { .iter() .map(|s| s.party_id()) .collect::>>()?; + let relay_signer_registration_mode = &config.relay_signer_registration_mode; + let relay_signature_registration_mode = &config.relay_signature_registration_mode; let aggregators = Self::start_aggregators(config, aggregator_cardano_nodes, chain_observer_type).await?; @@ -73,8 +78,13 @@ impl MithrilInfrastructure { .collect::>(); let master_aggregator_endpoint = aggregator_endpoints[0].to_owned(); - let (relay_aggregators, relay_signers, relay_passives) = - Self::start_relays(config, &aggregator_endpoints, &signer_party_ids)?; + let (relay_aggregators, relay_signers, relay_passives) = Self::start_relays( + config, + &aggregator_endpoints, + &signer_party_ids, + relay_signer_registration_mode.to_owned(), + relay_signature_registration_mode.to_owned(), + )?; let signers = Self::start_signers( config, @@ -219,8 +229,10 @@ impl MithrilInfrastructure { config: &MithrilInfrastructureConfig, aggregator_endpoints: &[String], signers_party_ids: &[PartyId], + relay_signer_registration_mode: SignerRelayMode, + relay_signature_registration_mode: SignerRelayMode, ) -> StdResult<(Vec, Vec, Vec)> { - if !config.use_p2p_network_mode { + if !config.use_relays { return Ok((vec![], vec![], vec![])); } @@ -249,15 +261,17 @@ impl MithrilInfrastructure { } for (index, party_id) in signers_party_ids.iter().enumerate() { - let mut relay_signer = RelaySigner::new( - config.server_port + index as u64 + 200, - config.server_port + index as u64 + 300, - bootstrap_peer_addr.clone(), - master_aggregator_endpoint, - party_id.clone(), - &config.work_dir, - &config.bin_dir, - )?; + let mut relay_signer = RelaySigner::new(&RelaySignerConfiguration { + listen_port: config.server_port + index as u64 + 200, + server_port: config.server_port + index as u64 + 300, + dial_to: bootstrap_peer_addr.clone(), + relay_signer_registration_mode: relay_signer_registration_mode.clone(), + relay_signature_registration_mode: relay_signature_registration_mode.clone(), + aggregator_endpoint: master_aggregator_endpoint, + party_id: party_id.clone(), + work_dir: &config.work_dir, + bin_dir: &config.bin_dir, + })?; relay_signer.start()?; relay_signers.push(relay_signer); @@ -308,7 +322,7 @@ impl MithrilInfrastructure { // Or 100% of signers otherwise let enable_certification = index % 2 == 0 || cfg!(not(feature = "allow_skip_signer_certification")); - let aggregator_endpoint = if config.use_p2p_network_mode { + let aggregator_endpoint = if config.use_relays { relay_signers[index].endpoint() } else { master_aggregator_endpoint.clone() diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/relay_signer.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/relay_signer.rs index 70c433abe5f..aa960c341aa 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/relay_signer.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/relay_signer.rs @@ -1,10 +1,23 @@ use crate::utils::MithrilCommand; use mithril_common::entities::PartyId; use mithril_common::StdResult; +use mithril_relay::SignerRelayMode; use std::collections::HashMap; use std::path::Path; use tokio::process::Child; +pub struct RelaySignerConfiguration<'a> { + pub listen_port: u64, + pub server_port: u64, + pub dial_to: Option, + pub relay_signer_registration_mode: SignerRelayMode, + pub relay_signature_registration_mode: SignerRelayMode, + pub aggregator_endpoint: &'a str, + pub party_id: PartyId, + pub work_dir: &'a Path, + pub bin_dir: &'a Path, +} + #[derive(Debug)] pub struct RelaySigner { listen_port: u64, @@ -15,35 +28,45 @@ pub struct RelaySigner { } impl RelaySigner { - pub fn new( - listen_port: u64, - server_port: u64, - dial_to: Option, - aggregator_endpoint: &str, - party_id: PartyId, - work_dir: &Path, - bin_dir: &Path, - ) -> StdResult { - let listen_port_str = format!("{listen_port}"); - let server_port_str = format!("{server_port}"); + pub fn new(configuration: &RelaySignerConfiguration) -> StdResult { + let listen_port_str = format!("{}", configuration.listen_port); + let server_port_str = format!("{}", configuration.server_port); + let relay_signer_registration_mode = + configuration.relay_signer_registration_mode.to_string(); + let relay_signature_registration_mode = + configuration.relay_signature_registration_mode.to_string(); let mut env = HashMap::from([ ("LISTEN_PORT", listen_port_str.as_str()), ("SERVER_PORT", server_port_str.as_str()), - ("AGGREGATOR_ENDPOINT", aggregator_endpoint), + ("AGGREGATOR_ENDPOINT", configuration.aggregator_endpoint), ("SIGNER_REPEATER_DELAY", "100"), + ( + "SIGNER_REGISTRATION_MODE", + relay_signer_registration_mode.as_str(), + ), + ( + "SIGNATURE_REGISTRATION_MODE", + relay_signature_registration_mode.as_str(), + ), ]); - if let Some(dial_to) = &dial_to { + if let Some(dial_to) = &configuration.dial_to { env.insert("DIAL_TO", dial_to); } let args = vec!["-vvv", "signer"]; - let mut command = MithrilCommand::new("mithril-relay", work_dir, bin_dir, env, &args)?; - command.set_log_name(format!("mithril-relay-signer-{party_id}").as_str()); + let mut command = MithrilCommand::new( + "mithril-relay", + configuration.work_dir, + configuration.bin_dir, + env, + &args, + )?; + command.set_log_name(format!("mithril-relay-signer-{}", configuration.party_id).as_str()); Ok(Self { - listen_port, - server_port, - party_id, + listen_port: configuration.listen_port, + server_port: configuration.server_port, + party_id: configuration.party_id.to_owned(), command, process: None, }) From ccae2ac4a86693e802c7e1bd9ef1140144ccf6ef Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 18 Mar 2025 15:50:21 +0100 Subject: [PATCH 17/33] refactor(ci): update e2e tests in CI to use the signer relay modes --- .github/workflows/ci.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01098c87275..8ed3ba9bbc2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -303,14 +303,22 @@ jobs: extra_args: [""] include: - # Include a test for the P2P mode - - mode: "p2p" + # Include a test for partial decentralization with master/slave signer registration and P2P signature registration + - mode: "master-slave" era: ${{ fromJSON(needs.build-ubuntu-X64.outputs.eras)[0] }} next_era: [""] cardano_node_version: "10.2.1" hard_fork_latest_era_at_epoch: 0 run_id: "#1" - extra_args: "--use-p2p-network" + extra_args: "--number-of-aggregators=2 --use-relays --relay-signer-registration-mode=passthrough --relay-signature-registration-mode=p2p" + # Include a test for full dedentralization P2P signer registration and P2P signature registration + - mode: "decentralized" + era: ${{ fromJSON(needs.build-ubuntu-X64.outputs.eras)[0] }} + next_era: [""] + cardano_node_version: "10.1.4" + hard_fork_latest_era_at_epoch: 0 + run_id: "#1" + extra_args: "--number-of-aggregators=2 --use-relays --relay-signer-registration-mode=p2p --relay-signature-registration-mode=p2p" # Include a test for the era switch without regenesis - mode: "std" era: ${{ fromJSON(needs.build-ubuntu-X64.outputs.eras)[0] }} From 24dac32dd64f5b4cc7d421a8b09f965e550a8f67 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 18 Mar 2025 16:22:25 +0100 Subject: [PATCH 18/33] refactor(e2e): better naming for aggregators in e2e tests --- mithril-relay/src/relay/signer.rs | 2 +- .../src/mithril/aggregator.rs | 40 ++-- .../src/mithril/infrastructure.rs | 179 ++++++++++++++++-- .../src/mithril/relay_aggregator.rs | 24 +-- .../src/stress_test/aggregator_helpers.rs | 3 +- 5 files changed, 183 insertions(+), 65 deletions(-) diff --git a/mithril-relay/src/relay/signer.rs b/mithril-relay/src/relay/signer.rs index db70bbd846f..135e9d12991 100644 --- a/mithril-relay/src/relay/signer.rs +++ b/mithril-relay/src/relay/signer.rs @@ -19,7 +19,7 @@ use warp::Filter; /// Signer relay mode /// /// The relay mode defines how the relay will behave when it receives a message -#[derive(Debug, Clone, Display, ValueEnum)] +#[derive(Debug, Clone, Display, PartialEq, Eq, ValueEnum)] #[strum(serialize_all = "mixed_case")] pub enum SignerRelayMode { /// Passthrough relay mode diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs index 5bac69843b0..18c90fca417 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs @@ -17,7 +17,8 @@ use tokio::sync::RwLock; #[derive(Debug)] pub struct AggregatorConfig<'a> { - pub index: usize, + pub is_master: bool, + pub name: &'a str, pub server_port: u64, pub pool_node: &'a PoolNode, pub cardano_cli_path: &'a Path, @@ -37,7 +38,8 @@ pub struct AggregatorConfig<'a> { #[derive(Debug)] pub struct Aggregator { - index: usize, + is_master: bool, + name_suffix: String, server_port: u64, db_directory: PathBuf, command: Arc>, @@ -121,20 +123,17 @@ impl Aggregator { "-vvv", ]; - let mut command = MithrilCommand::new( + let command = MithrilCommand::new( "mithril-aggregator", aggregator_config.work_dir, aggregator_config.bin_dir, env, &args, )?; - command.set_log_name(&format!( - "mithril-aggregator-{}", - Self::name_suffix(aggregator_config.index), - )); Ok(Self { - index: aggregator_config.index, + is_master: aggregator_config.is_master, + name_suffix: aggregator_config.name.to_string(), server_port: aggregator_config.server_port, db_directory: aggregator_config.pool_node.db_path.clone(), command: Arc::new(RwLock::new(command)), @@ -144,7 +143,8 @@ impl Aggregator { pub fn copy_configuration(other: &Aggregator) -> Self { Self { - index: other.index, + is_master: other.is_master, + name_suffix: other.name_suffix.clone(), server_port: other.server_port, db_directory: other.db_directory.clone(), command: other.command.clone(), @@ -152,24 +152,12 @@ impl Aggregator { } } - pub fn index(&self) -> usize { - self.index - } - pub fn is_master(&self) -> bool { - self.index == 0 + self.is_master } pub fn name(&self) -> String { - format!("mithril-aggregator-{}", Self::name_suffix(self.index)) - } - - pub fn name_suffix(index: usize) -> String { - if index == 0 { - "master".to_string() - } else { - format!("slave-{}", index) - } + format!("mithril-aggregator-{}", self.name_suffix) } pub fn endpoint(&self) -> String { @@ -182,6 +170,7 @@ impl Aggregator { pub async fn serve(&self) -> StdResult<()> { let mut command = self.command.write().await; + command.set_log_name(&format!("mithril-aggregator-{}", self.name_suffix)); let mut process = self.process.write().await; *process = Some(command.start(&["serve".to_string()])?); Ok(()) @@ -190,10 +179,7 @@ impl Aggregator { pub async fn bootstrap_genesis(&self) -> StdResult<()> { // Clone the command so we can alter it without affecting the original let mut command = self.command.write().await; - let command_name = &format!( - "mithril-aggregator-genesis-bootstrap-{}", - Self::name_suffix(self.index), - ); + let command_name = &format!("mithril-aggregator-genesis-bootstrap-{}", self.name_suffix,); command.set_log_name(command_name); let exit_status = command diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs index bb6adf08472..140f4d62dc0 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs @@ -37,6 +37,63 @@ pub struct MithrilInfrastructureConfig { pub use_era_specific_work_dir: bool, } +impl MithrilInfrastructureConfig { + pub fn is_master_aggregator(&self, index: usize) -> bool { + assert!( + index < self.number_of_aggregators as usize, + "Aggregator index out of bounds" + ); + + if self.relay_signer_registration_mode == SignerRelayMode::Passthrough { + self.number_of_aggregators > 1 && index == 0 + } else { + false + } + } + + pub fn aggregator_name_suffix(&self, index: usize) -> String { + assert!( + index < self.number_of_aggregators as usize, + "Aggregator index out of bounds" + ); + + if self.is_master_aggregator(0) { + if self.is_master_aggregator(index) { + "master".to_string() + } else { + format!("slave-{index}") + } + } else { + format!("{}", index + 1) + } + } + + #[cfg(test)] + pub fn dummy() -> Self { + Self { + number_of_aggregators: 1, + number_of_signers: 1, + server_port: 8080, + devnet: Devnet::default(), + work_dir: PathBuf::from("/tmp/work"), + store_dir: PathBuf::from("/tmp/store"), + artifacts_dir: PathBuf::from("/tmp/artifacts"), + bin_dir: PathBuf::from("/tmp/bin"), + cardano_node_version: "1.0.0".to_string(), + mithril_run_interval: 10, + mithril_era: "era1".to_string(), + mithril_era_reader_adapter: "adapter1".to_string(), + signed_entity_types: vec!["type1".to_string()], + run_only_mode: false, + use_relays: false, + relay_signer_registration_mode: SignerRelayMode::Passthrough, + relay_signature_registration_mode: SignerRelayMode::Passthrough, + use_p2p_passive_relays: false, + use_era_specific_work_dir: false, + } + } +} + pub struct MithrilInfrastructure { artifacts_dir: PathBuf, bin_dir: PathBuf, @@ -172,21 +229,16 @@ impl MithrilInfrastructure { let mut aggregators = vec![]; let mut master_aggregator_endpoint: Option = None; for (index, pool_node) in pool_nodes.iter().enumerate() { - let aggregator_artifacts_dir = if index == 0 { - config.artifacts_dir.join("mithril-aggregator") - } else { - config - .artifacts_dir - .join(format!("mithril-aggregator-slave-{index}")) - }; - let aggregator_store_dir = if index == 0 { - config.store_dir.join("aggregator") - } else { - config.store_dir.join(format!("aggregator-slave-{index}")) - }; - + let aggregator_name = config.aggregator_name_suffix(index); + let aggregator_artifacts_dir = config + .artifacts_dir + .join(format!("mithril-aggregator-{aggregator_name}")); + let aggregator_store_dir = config + .store_dir + .join(format!("aggregator-{aggregator_name}")); let aggregator = Aggregator::new(&AggregatorConfig { - index, + is_master: config.is_master_aggregator(index), + name: &aggregator_name, server_port: config.server_port + index as u64, pool_node, cardano_cli_path: &config.devnet.cardano_cli_path(), @@ -246,7 +298,7 @@ impl MithrilInfrastructure { let mut bootstrap_peer_addr = None; for (index, aggregator_endpoint) in aggregator_endpoints.iter().enumerate() { let mut relay_aggregator = RelayAggregator::new( - index, + config.aggregator_name_suffix(index), config.server_port + index as u64 + 100, bootstrap_peer_addr.clone(), aggregator_endpoint, @@ -424,10 +476,9 @@ impl MithrilInfrastructure { pub async fn build_client(&self, aggregator: &Aggregator) -> StdResult { let work_dir = { - let mut artifacts_dir = self.artifacts_dir.join(format!( - "mithril-client-aggregator-{}", - Aggregator::name_suffix(aggregator.index()) - )); + let mut artifacts_dir = self + .artifacts_dir + .join(format!("mithril-client-aggregator-{}", aggregator.name())); if self.use_era_specific_work_dir { let current_era = self.current_era.read().await; artifacts_dir = artifacts_dir.join(format!("era.{}", current_era)); @@ -479,3 +530,93 @@ impl MithrilInfrastructure { Ok(()) } } + +#[cfg(test)] +mod tests { + use crate::MithrilInfrastructureConfig; + + use super::*; + + #[test] + fn is_master_aggregator_succeeds() { + let config = MithrilInfrastructureConfig { + use_relays: true, + relay_signer_registration_mode: SignerRelayMode::Passthrough, + number_of_aggregators: 2, + ..MithrilInfrastructureConfig::dummy() + }; + + assert!(config.is_master_aggregator(0)); + assert!(!config.is_master_aggregator(1)); + + let config = MithrilInfrastructureConfig { + use_relays: true, + relay_signer_registration_mode: SignerRelayMode::P2P, + number_of_aggregators: 2, + ..MithrilInfrastructureConfig::dummy() + }; + + assert!(!config.is_master_aggregator(0)); + assert!(!config.is_master_aggregator(1)); + + let config = MithrilInfrastructureConfig { + use_relays: false, + number_of_aggregators: 1, + ..MithrilInfrastructureConfig::dummy() + }; + + assert!(!config.is_master_aggregator(0)); + } + + #[test] + #[should_panic] + fn is_master_aggregator_fails() { + let config = MithrilInfrastructureConfig { + number_of_aggregators: 1, + ..MithrilInfrastructureConfig::dummy() + }; + + config.is_master_aggregator(2); + } + + #[test] + fn aggregator_name_suffix_succeeds() { + let config = MithrilInfrastructureConfig { + relay_signer_registration_mode: SignerRelayMode::Passthrough, + number_of_aggregators: 3, + ..MithrilInfrastructureConfig::dummy() + }; + + assert_eq!(config.aggregator_name_suffix(0), "master"); + assert_eq!(config.aggregator_name_suffix(1), "slave-1"); + assert_eq!(config.aggregator_name_suffix(2), "slave-2"); + + let config = MithrilInfrastructureConfig { + number_of_aggregators: 3, + relay_signer_registration_mode: SignerRelayMode::P2P, + ..MithrilInfrastructureConfig::dummy() + }; + + assert_eq!(config.aggregator_name_suffix(0), "1"); + assert_eq!(config.aggregator_name_suffix(1), "2"); + assert_eq!(config.aggregator_name_suffix(2), "3"); + + let config = MithrilInfrastructureConfig { + number_of_aggregators: 1, + ..MithrilInfrastructureConfig::dummy() + }; + + assert_eq!(config.aggregator_name_suffix(0), "1"); + } + + #[test] + #[should_panic] + fn aggregator_name_suffix_fails() { + let config = MithrilInfrastructureConfig { + number_of_aggregators: 1, + ..MithrilInfrastructureConfig::dummy() + }; + + config.aggregator_name_suffix(2); + } +} diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/relay_aggregator.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/relay_aggregator.rs index 80228d597dc..f3b4bc2cf9c 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/relay_aggregator.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/relay_aggregator.rs @@ -6,7 +6,7 @@ use tokio::process::Child; #[derive(Debug)] pub struct RelayAggregator { - index: usize, + name_suffix: String, listen_port: u64, command: MithrilCommand, process: Option, @@ -14,7 +14,7 @@ pub struct RelayAggregator { impl RelayAggregator { pub fn new( - index: usize, + name: String, listen_port: u64, dial_to: Option, aggregator_endpoint: &str, @@ -32,13 +32,10 @@ impl RelayAggregator { let args = vec!["-vvv", "aggregator"]; let mut command = MithrilCommand::new("mithril-relay", work_dir, bin_dir, env, &args)?; - command.set_log_name(&format!( - "mithril-relay-aggregator-{}", - Self::name_suffix(index), - )); + command.set_log_name(&format!("mithril-relay-aggregator-{}", name,)); Ok(Self { - index, + name_suffix: name, listen_port, command, process: None, @@ -49,12 +46,8 @@ impl RelayAggregator { format!("/ip4/127.0.0.1/tcp/{}", self.listen_port) } - pub fn name_suffix(index: usize) -> String { - if index == 0 { - "master".to_string() - } else { - format!("slave-{}", index) - } + pub fn name_suffix(&self) -> String { + self.name_suffix.clone() } pub fn start(&mut self) -> StdResult<()> { @@ -65,10 +58,7 @@ impl RelayAggregator { pub async fn tail_logs(&self, number_of_line: u64) -> StdResult<()> { self.command .tail_logs( - Some(&format!( - "mithril-relay-aggregator-{}", - Self::name_suffix(self.index), - )), + Some(&format!("mithril-relay-aggregator-{}", self.name_suffix())), number_of_line, ) .await diff --git a/mithril-test-lab/mithril-end-to-end/src/stress_test/aggregator_helpers.rs b/mithril-test-lab/mithril-end-to-end/src/stress_test/aggregator_helpers.rs index ee1d6e69c1a..61117720a24 100644 --- a/mithril-test-lab/mithril-end-to-end/src/stress_test/aggregator_helpers.rs +++ b/mithril-test-lab/mithril-end-to-end/src/stress_test/aggregator_helpers.rs @@ -19,7 +19,8 @@ pub async fn bootstrap_aggregator( let chain_observer_type = "cardano-cli"; let mut aggregator = Aggregator::new(&AggregatorConfig { - index: 0, + is_master: false, + name: "genesis", server_port: args.server_port as u64, pool_node: &args.pool_node, cardano_cli_path: &args.cardano_cli_path, From eea39c8f693e736bb2e2478e6ad93fe1676b094c Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 18 Mar 2025 17:19:00 +0100 Subject: [PATCH 19/33] fix(e2e): delegate stakes only from the first aggregator --- .../mithril-end-to-end/src/end_to_end_spec.rs | 2 +- .../mithril-end-to-end/src/mithril/aggregator.rs | 8 ++++++++ .../mithril-end-to-end/src/mithril/infrastructure.rs | 12 ++++++++---- mithril-test-lab/mithril-end-to-end/src/run_only.rs | 2 +- .../src/stress_test/aggregator_helpers.rs | 1 + 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs index 4134471f1eb..4453252cce3 100644 --- a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs +++ b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs @@ -138,7 +138,7 @@ impl Spec { ) .await?; - if aggregator.is_master() { + if aggregator.index() == 0 { // Delegate some stakes to pools let delegation_round = 1; assertions::delegate_stakes_to_pools(infrastructure.devnet(), delegation_round).await?; diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs index 18c90fca417..8ede3fe0351 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs @@ -18,6 +18,7 @@ use tokio::sync::RwLock; #[derive(Debug)] pub struct AggregatorConfig<'a> { pub is_master: bool, + pub index: usize, pub name: &'a str, pub server_port: u64, pub pool_node: &'a PoolNode, @@ -39,6 +40,7 @@ pub struct AggregatorConfig<'a> { #[derive(Debug)] pub struct Aggregator { is_master: bool, + index: usize, name_suffix: String, server_port: u64, db_directory: PathBuf, @@ -133,6 +135,7 @@ impl Aggregator { Ok(Self { is_master: aggregator_config.is_master, + index: aggregator_config.index, name_suffix: aggregator_config.name.to_string(), server_port: aggregator_config.server_port, db_directory: aggregator_config.pool_node.db_path.clone(), @@ -144,6 +147,7 @@ impl Aggregator { pub fn copy_configuration(other: &Aggregator) -> Self { Self { is_master: other.is_master, + index: other.index, name_suffix: other.name_suffix.clone(), server_port: other.server_port, db_directory: other.db_directory.clone(), @@ -156,6 +160,10 @@ impl Aggregator { self.is_master } + pub fn index(&self) -> usize { + self.index + } + pub fn name(&self) -> String { format!("mithril-aggregator-{}", self.name_suffix) } diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs index 140f4d62dc0..9ba841ca985 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs @@ -238,6 +238,7 @@ impl MithrilInfrastructure { .join(format!("aggregator-{aggregator_name}")); let aggregator = Aggregator::new(&AggregatorConfig { is_master: config.is_master_aggregator(index), + index, name: &aggregator_name, server_port: config.server_port + index as u64, pool_node, @@ -264,16 +265,19 @@ impl MithrilInfrastructure { }) .await; - if master_aggregator_endpoint.is_none() { + if master_aggregator_endpoint.is_none() && config.is_master_aggregator(0) { master_aggregator_endpoint = Some(aggregator.endpoint()); - Self::register_startup_era(&aggregator, config).await?; } - aggregator.serve().await?; - aggregators.push(aggregator); } + Self::register_startup_era(&aggregators[0], config).await?; + + for aggregator in &aggregators { + aggregator.serve().await?; + } + Ok(aggregators) } diff --git a/mithril-test-lab/mithril-end-to-end/src/run_only.rs b/mithril-test-lab/mithril-end-to-end/src/run_only.rs index c624f1ff04f..8c5925395ec 100644 --- a/mithril-test-lab/mithril-end-to-end/src/run_only.rs +++ b/mithril-test-lab/mithril-end-to-end/src/run_only.rs @@ -94,7 +94,7 @@ impl RunOnly { assertions::bootstrap_genesis_certificate(aggregator).await?; assertions::wait_for_epoch_settings(&aggregator.endpoint()).await?; - if aggregator.is_master() { + if aggregator.index() == 0 { // Transfer some funds on the devnet to have some Cardano transactions to sign assertions::transfer_funds(infrastructure.devnet()).await?; } diff --git a/mithril-test-lab/mithril-end-to-end/src/stress_test/aggregator_helpers.rs b/mithril-test-lab/mithril-end-to-end/src/stress_test/aggregator_helpers.rs index 61117720a24..dd1442186e1 100644 --- a/mithril-test-lab/mithril-end-to-end/src/stress_test/aggregator_helpers.rs +++ b/mithril-test-lab/mithril-end-to-end/src/stress_test/aggregator_helpers.rs @@ -20,6 +20,7 @@ pub async fn bootstrap_aggregator( let mut aggregator = Aggregator::new(&AggregatorConfig { is_master: false, + index: 0, name: "genesis", server_port: args.server_port as u64, pool_node: &args.pool_node, From f71f6b0b936847e9c2a4d0ed5795dc98f15a9836 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 19 Mar 2025 18:15:21 +0100 Subject: [PATCH 20/33] refactor(e2e): remove distinction master/slave aggregator As master/slave signer registration is only one of the configurations to be tested. --- .../mithril-end-to-end/src/end_to_end_spec.rs | 56 +++---- .../src/mithril/aggregator.rs | 12 +- .../src/mithril/infrastructure.rs | 154 +++--------------- .../mithril-end-to-end/src/run_only.rs | 56 +++---- .../src/stress_test/aggregator_helpers.rs | 1 - 5 files changed, 73 insertions(+), 206 deletions(-) diff --git a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs index 4453252cce3..3382ec833bc 100644 --- a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs +++ b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs @@ -51,11 +51,16 @@ impl Spec { } pub async fn run(self) -> StdResult<()> { + let mut join_set = JoinSet::new(); let spec = Arc::new(self); let infrastructure_guard = spec.infrastructure.read().await; let infrastructure = infrastructure_guard .as_ref() .ok_or(anyhow!("No infrastructure found"))?; + let aggregators = infrastructure_guard + .as_ref() + .ok_or(anyhow!("No infrastructure found"))? + .aggregators(); // Transfer some funds on the devnet to have some Cardano transactions to sign. // This step needs to be executed early in the process so that the transactions are available @@ -63,13 +68,24 @@ impl Spec { // As we get closer to the tip of the chain when signing, we'll be able to relax this constraint. assertions::transfer_funds(infrastructure.devnet()).await?; - let mut join_set = JoinSet::new(); - let spec_clone = spec.clone(); - join_set.spawn(async move { spec_clone.start_master().await }); - for index in 0..infrastructure.slave_aggregators().len() { + for index in 0..aggregators.len() { let spec_clone = spec.clone(); - join_set.spawn(async move { spec_clone.start_slave(index).await }); + join_set.spawn(async move { + let infrastructure_guard = spec_clone.infrastructure.read().await; + let infrastructure = infrastructure_guard + .as_ref() + .ok_or(anyhow!("No infrastructure found"))?; + + spec_clone + .start_aggregator( + infrastructure.aggregator(index), + infrastructure.chain_observer(index), + infrastructure, + ) + .await + }); } + while let Some(res) = join_set.join_next().await { res??; } @@ -77,34 +93,6 @@ impl Spec { Ok(()) } - pub async fn start_master(&self) -> StdResult<()> { - let infrastructure_guard = self.infrastructure.read().await; - let infrastructure = infrastructure_guard - .as_ref() - .ok_or(anyhow!("No infrastructure found"))?; - - self.start_aggregator( - infrastructure.master_aggregator(), - infrastructure.master_chain_observer(), - infrastructure, - ) - .await - } - - pub async fn start_slave(&self, slave_index: usize) -> StdResult<()> { - let infrastructure_guard = self.infrastructure.read().await; - let infrastructure = infrastructure_guard - .as_ref() - .ok_or(anyhow!("No infrastructure found"))?; - - self.start_aggregator( - infrastructure.slave_aggregator(slave_index), - infrastructure.slave_chain_observer(slave_index), - infrastructure, - ) - .await - } - pub async fn start_aggregator( &self, aggregator: &Aggregator, @@ -138,7 +126,7 @@ impl Spec { ) .await?; - if aggregator.index() == 0 { + if aggregator.is_first() { // Delegate some stakes to pools let delegation_round = 1; assertions::delegate_stakes_to_pools(infrastructure.devnet(), delegation_round).await?; diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs index 8ede3fe0351..de0407b9ced 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs @@ -17,7 +17,6 @@ use tokio::sync::RwLock; #[derive(Debug)] pub struct AggregatorConfig<'a> { - pub is_master: bool, pub index: usize, pub name: &'a str, pub server_port: u64, @@ -39,7 +38,6 @@ pub struct AggregatorConfig<'a> { #[derive(Debug)] pub struct Aggregator { - is_master: bool, index: usize, name_suffix: String, server_port: u64, @@ -134,7 +132,6 @@ impl Aggregator { )?; Ok(Self { - is_master: aggregator_config.is_master, index: aggregator_config.index, name_suffix: aggregator_config.name.to_string(), server_port: aggregator_config.server_port, @@ -144,9 +141,12 @@ impl Aggregator { }) } + pub fn name_suffix(index: usize) -> String { + format!("{}", index + 1) + } + pub fn copy_configuration(other: &Aggregator) -> Self { Self { - is_master: other.is_master, index: other.index, name_suffix: other.name_suffix.clone(), server_port: other.server_port, @@ -156,8 +156,8 @@ impl Aggregator { } } - pub fn is_master(&self) -> bool { - self.is_master + pub fn is_first(&self) -> bool { + self.index == 0 } pub fn index(&self) -> usize { diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs index 9ba841ca985..0b175a1a50c 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs @@ -38,36 +38,14 @@ pub struct MithrilInfrastructureConfig { } impl MithrilInfrastructureConfig { - pub fn is_master_aggregator(&self, index: usize) -> bool { - assert!( - index < self.number_of_aggregators as usize, - "Aggregator index out of bounds" - ); - + pub fn has_master_slave_signer_registration(&self) -> bool { if self.relay_signer_registration_mode == SignerRelayMode::Passthrough { - self.number_of_aggregators > 1 && index == 0 + self.number_of_aggregators > 1 } else { false } } - pub fn aggregator_name_suffix(&self, index: usize) -> String { - assert!( - index < self.number_of_aggregators as usize, - "Aggregator index out of bounds" - ); - - if self.is_master_aggregator(0) { - if self.is_master_aggregator(index) { - "master".to_string() - } else { - format!("slave-{index}") - } - } else { - format!("{}", index + 1) - } - } - #[cfg(test)] pub fn dummy() -> Self { Self { @@ -200,20 +178,15 @@ impl MithrilInfrastructure { pub async fn register_switch_to_next_era(&self, next_era: &str) -> StdResult<()> { let next_era_epoch = self - .master_chain_observer() + .chain_observer(0) .get_current_epoch() .await? .unwrap_or_default() + 1; if self.era_reader_adapter == "cardano-chain" { let devnet = self.devnet.clone(); - assertions::register_era_marker( - self.master_aggregator(), - &devnet, - next_era, - next_era_epoch, - ) - .await?; + assertions::register_era_marker(self.aggregator(0), &devnet, next_era, next_era_epoch) + .await?; } let mut current_era = self.current_era.write().await; *current_era = next_era.to_owned(); @@ -229,7 +202,7 @@ impl MithrilInfrastructure { let mut aggregators = vec![]; let mut master_aggregator_endpoint: Option = None; for (index, pool_node) in pool_nodes.iter().enumerate() { - let aggregator_name = config.aggregator_name_suffix(index); + let aggregator_name = Aggregator::name_suffix(index); let aggregator_artifacts_dir = config .artifacts_dir .join(format!("mithril-aggregator-{aggregator_name}")); @@ -237,7 +210,6 @@ impl MithrilInfrastructure { .store_dir .join(format!("aggregator-{aggregator_name}")); let aggregator = Aggregator::new(&AggregatorConfig { - is_master: config.is_master_aggregator(index), index, name: &aggregator_name, server_port: config.server_port + index as u64, @@ -265,7 +237,8 @@ impl MithrilInfrastructure { }) .await; - if master_aggregator_endpoint.is_none() && config.is_master_aggregator(0) { + if master_aggregator_endpoint.is_none() && config.has_master_slave_signer_registration() + { master_aggregator_endpoint = Some(aggregator.endpoint()); } @@ -302,7 +275,7 @@ impl MithrilInfrastructure { let mut bootstrap_peer_addr = None; for (index, aggregator_endpoint) in aggregator_endpoints.iter().enumerate() { let mut relay_aggregator = RelayAggregator::new( - config.aggregator_name_suffix(index), + Aggregator::name_suffix(index), config.server_port + index as u64 + 100, bootstrap_peer_addr.clone(), aggregator_endpoint, @@ -426,20 +399,12 @@ impl MithrilInfrastructure { &self.devnet } - pub fn master_aggregator(&self) -> &Aggregator { - assert!( - !self.aggregators.is_empty(), - "No master aggregator available for this infrastructure" - ); - &self.aggregators[0] + pub fn aggregators(&self) -> &[Aggregator] { + &self.aggregators } - pub fn slave_aggregators(&self) -> &[Aggregator] { - &self.aggregators[1..] - } - - pub fn slave_aggregator(&self, index: usize) -> &Aggregator { - &self.aggregators[index + 1] + pub fn aggregator(&self, index: usize) -> &Aggregator { + &self.aggregators[index] } pub fn signers(&self) -> &[Signer] { @@ -462,20 +427,12 @@ impl MithrilInfrastructure { &self.relay_passives } - pub fn master_chain_observer(&self) -> Arc { - assert!( - !self.aggregators.is_empty(), - "No master chain observer available for this infrastructure" - ); - self.cardano_chain_observers[0].clone() + pub fn chain_observers(&self) -> &[Arc] { + &self.cardano_chain_observers } - pub fn slave_chain_observers(&self) -> &[Arc] { - &self.cardano_chain_observers[1..] - } - - pub fn slave_chain_observer(&self, index: usize) -> Arc { - self.cardano_chain_observers[index + 1].clone() + pub fn chain_observer(&self, index: usize) -> Arc { + self.cardano_chain_observers[index].clone() } pub async fn build_client(&self, aggregator: &Aggregator) -> StdResult { @@ -494,11 +451,7 @@ impl MithrilInfrastructure { artifacts_dir }; - Client::new( - self.master_aggregator().endpoint(), - &work_dir, - &self.bin_dir, - ) + Client::new(aggregator.endpoint(), &work_dir, &self.bin_dir) } pub fn run_only_mode(&self) -> bool { @@ -506,8 +459,7 @@ impl MithrilInfrastructure { } pub async fn tail_logs(&self, number_of_line: u64) -> StdResult<()> { - self.master_aggregator().tail_logs(number_of_line).await?; - for aggregator in self.slave_aggregators() { + for aggregator in self.aggregators() { aggregator.tail_logs(number_of_line).await?; } for signer in self.signers() { @@ -527,9 +479,9 @@ impl MithrilInfrastructure { } pub async fn last_error_in_logs(&self, number_of_error: u64) -> StdResult<()> { - self.master_aggregator() - .last_error_in_logs(number_of_error) - .await?; + for aggregator in self.aggregators() { + aggregator.last_error_in_logs(number_of_error).await?; + } Ok(()) } @@ -542,7 +494,7 @@ mod tests { use super::*; #[test] - fn is_master_aggregator_succeeds() { + fn has_master_aggregator_succeeds() { let config = MithrilInfrastructureConfig { use_relays: true, relay_signer_registration_mode: SignerRelayMode::Passthrough, @@ -550,8 +502,7 @@ mod tests { ..MithrilInfrastructureConfig::dummy() }; - assert!(config.is_master_aggregator(0)); - assert!(!config.is_master_aggregator(1)); + assert!(config.has_master_slave_signer_registration()); let config = MithrilInfrastructureConfig { use_relays: true, @@ -560,8 +511,7 @@ mod tests { ..MithrilInfrastructureConfig::dummy() }; - assert!(!config.is_master_aggregator(0)); - assert!(!config.is_master_aggregator(1)); + assert!(!config.has_master_slave_signer_registration()); let config = MithrilInfrastructureConfig { use_relays: false, @@ -569,58 +519,6 @@ mod tests { ..MithrilInfrastructureConfig::dummy() }; - assert!(!config.is_master_aggregator(0)); - } - - #[test] - #[should_panic] - fn is_master_aggregator_fails() { - let config = MithrilInfrastructureConfig { - number_of_aggregators: 1, - ..MithrilInfrastructureConfig::dummy() - }; - - config.is_master_aggregator(2); - } - - #[test] - fn aggregator_name_suffix_succeeds() { - let config = MithrilInfrastructureConfig { - relay_signer_registration_mode: SignerRelayMode::Passthrough, - number_of_aggregators: 3, - ..MithrilInfrastructureConfig::dummy() - }; - - assert_eq!(config.aggregator_name_suffix(0), "master"); - assert_eq!(config.aggregator_name_suffix(1), "slave-1"); - assert_eq!(config.aggregator_name_suffix(2), "slave-2"); - - let config = MithrilInfrastructureConfig { - number_of_aggregators: 3, - relay_signer_registration_mode: SignerRelayMode::P2P, - ..MithrilInfrastructureConfig::dummy() - }; - - assert_eq!(config.aggregator_name_suffix(0), "1"); - assert_eq!(config.aggregator_name_suffix(1), "2"); - assert_eq!(config.aggregator_name_suffix(2), "3"); - - let config = MithrilInfrastructureConfig { - number_of_aggregators: 1, - ..MithrilInfrastructureConfig::dummy() - }; - - assert_eq!(config.aggregator_name_suffix(0), "1"); - } - - #[test] - #[should_panic] - fn aggregator_name_suffix_fails() { - let config = MithrilInfrastructureConfig { - number_of_aggregators: 1, - ..MithrilInfrastructureConfig::dummy() - }; - - config.aggregator_name_suffix(2); + assert!(!config.has_master_slave_signer_registration()); } } diff --git a/mithril-test-lab/mithril-end-to-end/src/run_only.rs b/mithril-test-lab/mithril-end-to-end/src/run_only.rs index 8c5925395ec..81e0b3371cc 100644 --- a/mithril-test-lab/mithril-end-to-end/src/run_only.rs +++ b/mithril-test-lab/mithril-end-to-end/src/run_only.rs @@ -21,18 +21,28 @@ impl RunOnly { pub async fn run(self) -> StdResult<()> { let run_only = Arc::new(self); let mut join_set = JoinSet::new(); - - let run_only_clone = run_only.clone(); - join_set.spawn(async move { run_only_clone.start_master().await }); - let infrastructure_guard = run_only.infrastructure.read().await; - let slave_aggregators = infrastructure_guard + let aggregators = infrastructure_guard .as_ref() .ok_or(anyhow!("No infrastructure found"))? - .slave_aggregators(); - for index in 0..slave_aggregators.len() { + .aggregators(); + + for index in 0..aggregators.len() { let run_only_clone = run_only.clone(); - join_set.spawn(async move { run_only_clone.start_slave(index).await }); + join_set.spawn(async move { + let infrastructure_guard = run_only_clone.infrastructure.read().await; + let infrastructure = infrastructure_guard + .as_ref() + .ok_or(anyhow!("No infrastructure found"))?; + + run_only_clone + .start_aggregator( + infrastructure.aggregator(index), + infrastructure.chain_observer(index), + infrastructure, + ) + .await + }); } while let Some(res) = join_set.join_next().await { @@ -42,34 +52,6 @@ impl RunOnly { Ok(()) } - pub async fn start_master(&self) -> StdResult<()> { - let infrastructure_guard = self.infrastructure.read().await; - let infrastructure = infrastructure_guard - .as_ref() - .ok_or(anyhow!("No infrastructure found"))?; - - self.start_aggregator( - infrastructure.master_aggregator(), - infrastructure.master_chain_observer(), - infrastructure, - ) - .await - } - - pub async fn start_slave(&self, slave_index: usize) -> StdResult<()> { - let infrastructure_guard = self.infrastructure.read().await; - let infrastructure = infrastructure_guard - .as_ref() - .ok_or(anyhow!("No infrastructure found"))?; - - self.start_aggregator( - infrastructure.slave_aggregator(slave_index), - infrastructure.slave_chain_observer(slave_index), - infrastructure, - ) - .await - } - pub async fn start_aggregator( &self, aggregator: &Aggregator, @@ -94,7 +76,7 @@ impl RunOnly { assertions::bootstrap_genesis_certificate(aggregator).await?; assertions::wait_for_epoch_settings(&aggregator.endpoint()).await?; - if aggregator.index() == 0 { + if aggregator.is_first() { // Transfer some funds on the devnet to have some Cardano transactions to sign assertions::transfer_funds(infrastructure.devnet()).await?; } diff --git a/mithril-test-lab/mithril-end-to-end/src/stress_test/aggregator_helpers.rs b/mithril-test-lab/mithril-end-to-end/src/stress_test/aggregator_helpers.rs index dd1442186e1..9e23076443c 100644 --- a/mithril-test-lab/mithril-end-to-end/src/stress_test/aggregator_helpers.rs +++ b/mithril-test-lab/mithril-end-to-end/src/stress_test/aggregator_helpers.rs @@ -19,7 +19,6 @@ pub async fn bootstrap_aggregator( let chain_observer_type = "cardano-cli"; let mut aggregator = Aggregator::new(&AggregatorConfig { - is_master: false, index: 0, name: "genesis", server_port: args.server_port as u64, From bac426d0d4edf82c4028cff6ef01753ed1bf3389 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 19 Mar 2025 19:14:11 +0100 Subject: [PATCH 21/33] fix(e2e): make genesis bootstrap error retryable Until we can fix the source of flakiness. --- .../mithril-end-to-end/src/mithril/aggregator.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs index de0407b9ced..f8f1d3cd214 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs @@ -1,7 +1,7 @@ use crate::utils::MithrilCommand; use crate::{ - PoolNode, DEVNET_MAGIC_ID, ERA_MARKERS_SECRET_KEY, ERA_MARKERS_VERIFICATION_KEY, - GENESIS_SECRET_KEY, GENESIS_VERIFICATION_KEY, + PoolNode, RetryableDevnetError, DEVNET_MAGIC_ID, ERA_MARKERS_SECRET_KEY, + ERA_MARKERS_VERIFICATION_KEY, GENESIS_SECRET_KEY, GENESIS_VERIFICATION_KEY, }; use anyhow::{anyhow, Context}; use mithril_common::era::SupportedEra; @@ -209,6 +209,7 @@ impl Aggregator { anyhow!("`mithril-aggregator genesis bootstrap` was terminated with a signal") } }) + .map_err(|e| anyhow!(RetryableDevnetError(e.to_string()))) } } From 024fcf7f69e68cb3712622b8eb0b8997c1011115 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Thu, 20 Mar 2025 18:45:21 +0100 Subject: [PATCH 22/33] fix(e2e): flakiness in the genesis bootstrap of slave aggregators --- .../mithril-end-to-end/src/assertions/exec.rs | 9 +++++++++ .../mithril-end-to-end/src/mithril/aggregator.rs | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/mithril-test-lab/mithril-end-to-end/src/assertions/exec.rs b/mithril-test-lab/mithril-end-to-end/src/assertions/exec.rs index 2dcc0247d25..40a10f5601b 100644 --- a/mithril-test-lab/mithril-end-to-end/src/assertions/exec.rs +++ b/mithril-test-lab/mithril-end-to-end/src/assertions/exec.rs @@ -8,6 +8,15 @@ use slog_scope::info; pub async fn bootstrap_genesis_certificate(aggregator: &Aggregator) -> StdResult<()> { info!("Bootstrap genesis certificate"); + // A slave aggregator needs to wait few cycles of the state machine to be able to bootstrap + // This should be removed when the aggregator is able to synchronize its certificate chain from another aggregator + if !aggregator.is_first() { + tokio::time::sleep(std::time::Duration::from_millis( + 5 * aggregator.mithril_run_interval() as u64, + )) + .await; + } + info!("> stopping aggregator"); aggregator.stop().await?; info!("> bootstrapping genesis using signers registered two epochs ago..."); diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs index f8f1d3cd214..649f6e8678f 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs @@ -42,6 +42,7 @@ pub struct Aggregator { name_suffix: String, server_port: u64, db_directory: PathBuf, + mithril_run_interval: u32, command: Arc>, process: RwLock>, } @@ -136,6 +137,7 @@ impl Aggregator { name_suffix: aggregator_config.name.to_string(), server_port: aggregator_config.server_port, db_directory: aggregator_config.pool_node.db_path.clone(), + mithril_run_interval: aggregator_config.mithril_run_interval, command: Arc::new(RwLock::new(command)), process: RwLock::new(None), }) @@ -151,6 +153,7 @@ impl Aggregator { name_suffix: other.name_suffix.clone(), server_port: other.server_port, db_directory: other.db_directory.clone(), + mithril_run_interval: other.mithril_run_interval, command: other.command.clone(), process: RwLock::new(None), } @@ -176,6 +179,10 @@ impl Aggregator { &self.db_directory } + pub fn mithril_run_interval(&self) -> u32 { + self.mithril_run_interval + } + pub async fn serve(&self) -> StdResult<()> { let mut command = self.command.write().await; command.set_log_name(&format!("mithril-aggregator-{}", self.name_suffix)); From 320e695079f728e0a9f22c1abd0bf4fafae93d55 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Thu, 20 Mar 2025 19:07:25 +0100 Subject: [PATCH 23/33] refactor(e2e): enhance assertions logs with aggregator name --- .../src/assertions/check.rs | 27 ++++++++++--------- .../mithril-end-to-end/src/assertions/exec.rs | 21 +++++++-------- .../mithril-end-to-end/src/assertions/wait.rs | 21 +++++++++------ .../mithril-end-to-end/src/end_to_end_spec.rs | 10 +++++-- .../mithril-end-to-end/src/run_only.rs | 5 ++-- 5 files changed, 48 insertions(+), 36 deletions(-) diff --git a/mithril-test-lab/mithril-end-to-end/src/assertions/check.rs b/mithril-test-lab/mithril-end-to-end/src/assertions/check.rs index 85d4365fa78..d73c4e828f8 100644 --- a/mithril-test-lab/mithril-end-to-end/src/assertions/check.rs +++ b/mithril-test-lab/mithril-end-to-end/src/assertions/check.rs @@ -60,7 +60,7 @@ pub async fn assert_node_producing_mithril_stake_distribution( fetch_last_mithril_stake_distribution_hash(url.clone()).await }) { AttemptResult::Ok(hash) => { - info!("Aggregator produced a mithril stake distribution"; "hash" => &hash); + info!("Aggregator produced a mithril stake distribution"; "hash" => &hash, "aggregator" => &aggregator.name()); Ok(hash) } AttemptResult::Err(error) => Err(error), @@ -112,7 +112,7 @@ pub async fn assert_signer_is_signing_mithril_stake_distribution( fetch_mithril_stake_distribution_message(url.clone(), expected_epoch_min).await }) { AttemptResult::Ok(stake_distribution) => { - info!("Signer signed a mithril stake distribution"; "certificate_hash" => &stake_distribution.certificate_hash); + info!("Signer signed a mithril stake distribution"; "certificate_hash" => &stake_distribution.certificate_hash, "aggregator" => &aggregator.name()); Ok(stake_distribution.certificate_hash) } AttemptResult::Err(error) => Err(error), @@ -146,7 +146,7 @@ pub async fn assert_node_producing_snapshot(aggregator: &Aggregator) -> StdResul fetch_last_snapshot_digest(url.clone()).await }) { AttemptResult::Ok(digest) => { - info!("Aggregator produced a snapshot"; "digest" => &digest); + info!("Aggregator produced a snapshot"; "digest" => &digest, "aggregator" => &aggregator.name()); Ok(digest) } AttemptResult::Err(error) => Err(error), @@ -189,7 +189,7 @@ pub async fn assert_signer_is_signing_snapshot( fetch_snapshot_message(url.clone(), expected_epoch_min).await }) { AttemptResult::Ok(snapshot) => { - info!("Signer signed a snapshot"; "certificate_hash" => &snapshot.certificate_hash); + info!("Signer signed a snapshot"; "certificate_hash" => &snapshot.certificate_hash, "aggregator" => &aggregator.name()); Ok(snapshot.certificate_hash) } AttemptResult::Err(error) => { @@ -223,7 +223,7 @@ pub async fn assert_node_producing_cardano_database_snapshot( fetch_last_cardano_database_snapshot_hash(url.clone()).await }) { AttemptResult::Ok(hash) => { - info!("Aggregator produced a Cardano database snapshot"; "hash" => &hash); + info!("Aggregator produced a Cardano database snapshot"; "hash" => &hash, "aggregator" => &aggregator.name()); Ok(hash) } AttemptResult::Err(error) => Err(error), @@ -268,7 +268,7 @@ pub async fn assert_signer_is_signing_cardano_database_snapshot( fetch_cardano_database_snapshot_message(url.clone(), expected_epoch_min).await }) { AttemptResult::Ok(snapshot) => { - info!("Signer signed a snapshot"; "certificate_hash" => &snapshot.certificate_hash); + info!("Signer signed a snapshot"; "certificate_hash" => &snapshot.certificate_hash, "aggregator" => &aggregator.name()); Ok(snapshot.certificate_hash) } AttemptResult::Err(error) => Err(error), @@ -310,7 +310,7 @@ pub async fn assert_node_producing_cardano_database_digests_map( fetch_cardano_database_digests_map(url.clone()).await }) { AttemptResult::Ok(cardano_database_digests_map) => { - info!("Aggregator produced a Cardano database digests map"; "total_digests" => &cardano_database_digests_map.len()); + info!("Aggregator produced a Cardano database digests map"; "total_digests" => &cardano_database_digests_map.len(), "aggregator" => &aggregator.name()); Ok(cardano_database_digests_map) } AttemptResult::Err(error) => Err(error), @@ -329,7 +329,7 @@ pub async fn assert_node_producing_cardano_transactions( aggregator: &Aggregator, ) -> StdResult { let url = format!("{}/artifact/cardano-transactions", aggregator.endpoint()); - info!("Waiting for the aggregator to produce a Cardano transactions artifact"; "aggregator" => &aggregator.name()); + info!("Waiting for the aggregator to produce a Cardano transactions artifact"; "aggregator" => &aggregator.name(), "aggregator" => &aggregator.name()); async fn fetch_last_cardano_transaction_snapshot_hash( url: String, @@ -348,7 +348,7 @@ pub async fn assert_node_producing_cardano_transactions( fetch_last_cardano_transaction_snapshot_hash(url.clone()).await }) { AttemptResult::Ok(hash) => { - info!("Aggregator produced a Cardano transactions artifact"; "hash" => &hash); + info!("Aggregator produced a Cardano transactions artifact"; "hash" => &hash, "aggregator" => &aggregator.name()); Ok(hash) } AttemptResult::Err(error) => Err(error), @@ -394,7 +394,7 @@ pub async fn assert_signer_is_signing_cardano_transactions( fetch_cardano_transaction_snapshot_message(url.clone(), expected_epoch_min).await }) { AttemptResult::Ok(artifact) => { - info!("Signer signed a Cardano transactions artifact"; "certificate_hash" => &artifact.certificate_hash); + info!("Signer signed a Cardano transactions artifact"; "certificate_hash" => &artifact.certificate_hash, "aggregator" => &aggregator.name()); Ok(artifact.certificate_hash) } AttemptResult::Err(error) => Err(error), @@ -438,7 +438,7 @@ pub async fn assert_node_producing_cardano_stake_distribution( fetch_last_cardano_stake_distribution_message(url.clone()).await }) { AttemptResult::Ok((hash, epoch)) => { - info!("Aggregator produced a Cardano stake distribution"; "hash" => &hash, "epoch" => #?epoch); + info!("Aggregator produced a Cardano stake distribution"; "hash" => &hash, "epoch" => #?epoch, "aggregator" => &aggregator.name()); Ok((hash, epoch)) } AttemptResult::Err(error) => Err(error), @@ -490,7 +490,7 @@ pub async fn assert_signer_is_signing_cardano_stake_distribution( fetch_cardano_stake_distribution_message(url.clone(), expected_epoch_min).await }) { AttemptResult::Ok(cardano_stake_distribution) => { - info!("Signer signed a Cardano stake distribution"; "certificate_hash" => &cardano_stake_distribution.certificate_hash); + info!("Signer signed a Cardano stake distribution"; "certificate_hash" => &cardano_stake_distribution.certificate_hash, "aggregator" => &aggregator.name()); Ok(cardano_stake_distribution.certificate_hash) } AttemptResult::Err(error) => Err(error), @@ -529,7 +529,8 @@ pub async fn assert_is_creating_certificate_with_enough_signers( info!( "Certificate is signed by expected number of signers: {} >= {} ", certificate.metadata.signers.len(), - total_signers_expected + total_signers_expected ; + "aggregator" => &aggregator.name() ); Ok(()) } else { diff --git a/mithril-test-lab/mithril-end-to-end/src/assertions/exec.rs b/mithril-test-lab/mithril-end-to-end/src/assertions/exec.rs index 40a10f5601b..84325ea2f3a 100644 --- a/mithril-test-lab/mithril-end-to-end/src/assertions/exec.rs +++ b/mithril-test-lab/mithril-end-to-end/src/assertions/exec.rs @@ -6,7 +6,7 @@ use mithril_common::StdResult; use slog_scope::info; pub async fn bootstrap_genesis_certificate(aggregator: &Aggregator) -> StdResult<()> { - info!("Bootstrap genesis certificate"); + info!("Bootstrap genesis certificate"; "aggregator" => &aggregator.name()); // A slave aggregator needs to wait few cycles of the state machine to be able to bootstrap // This should be removed when the aggregator is able to synchronize its certificate chain from another aggregator @@ -17,11 +17,11 @@ pub async fn bootstrap_genesis_certificate(aggregator: &Aggregator) -> StdResult .await; } - info!("> stopping aggregator"); + info!("> stopping aggregator"; "aggregator" => &aggregator.name()); aggregator.stop().await?; - info!("> bootstrapping genesis using signers registered two epochs ago..."); + info!("> bootstrapping genesis using signers registered two epochs ago..."; "aggregator" => &aggregator.name()); aggregator.bootstrap_genesis().await?; - info!("> done, restarting aggregator"); + info!("> done, restarting aggregator"; "aggregator" => &aggregator.name()); aggregator.serve().await?; Ok(()) @@ -33,9 +33,9 @@ pub async fn register_era_marker( mithril_era: &str, era_epoch: Epoch, ) -> StdResult<()> { - info!("Register '{mithril_era}' era marker"); + info!("Register '{mithril_era}' era marker"; "aggregator" => &aggregator.name()); - info!("> generating era marker tx datum..."); + info!("> generating era marker tx datum..."; "aggregator" => &aggregator.name()); let tx_datum_file_path = devnet .artifacts_dir() .join(PathBuf::from("era-tx-datum.txt".to_string())); @@ -43,7 +43,7 @@ pub async fn register_era_marker( .era_generate_tx_datum(&tx_datum_file_path, mithril_era, era_epoch) .await?; - info!("> writing '{mithril_era}' era marker on the Cardano chain..."); + info!("> writing '{mithril_era}' era marker on the Cardano chain..."; "aggregator" => &aggregator.name()); devnet.write_era_marker(&tx_datum_file_path).await?; Ok(()) @@ -66,7 +66,7 @@ pub async fn transfer_funds(devnet: &Devnet) -> StdResult<()> { } pub async fn update_protocol_parameters(aggregator: &Aggregator) -> StdResult<()> { - info!("Update protocol parameters"); + info!("Update protocol parameters"; "aggregator" => &aggregator.name()); info!("> stopping aggregator"); aggregator.stop().await?; @@ -76,13 +76,12 @@ pub async fn update_protocol_parameters(aggregator: &Aggregator) -> StdResult<() phi_f: 0.80, }; info!( - "> updating protocol parameters to {:?}...", - protocol_parameters_new + "> updating protocol parameters to {protocol_parameters_new:?}..."; "aggregator" => &aggregator.name() ); aggregator .set_protocol_parameters(&protocol_parameters_new) .await; - info!("> done, restarting aggregator"); + info!("> done, restarting aggregator"; "aggregator" => &aggregator.name()); aggregator.serve().await?; Ok(()) diff --git a/mithril-test-lab/mithril-end-to-end/src/assertions/wait.rs b/mithril-test-lab/mithril-end-to-end/src/assertions/wait.rs index 2177940be2b..fa6aa114dee 100644 --- a/mithril-test-lab/mithril-end-to-end/src/assertions/wait.rs +++ b/mithril-test-lab/mithril-end-to-end/src/assertions/wait.rs @@ -1,4 +1,4 @@ -use crate::{attempt, utils::AttemptResult}; +use crate::{attempt, utils::AttemptResult, Aggregator}; use anyhow::{anyhow, Context}; use mithril_common::{ chain_observer::ChainObserver, digesters::ImmutableFile, entities::Epoch, @@ -6,11 +6,12 @@ use mithril_common::{ }; use reqwest::StatusCode; use slog_scope::{info, warn}; -use std::{path::Path, sync::Arc, time::Duration}; +use std::{sync::Arc, time::Duration}; -pub async fn wait_for_enough_immutable(db_directory: &Path) -> StdResult<()> { - info!("Waiting that enough immutable have been written in the devnet"); +pub async fn wait_for_enough_immutable(aggregator: &Aggregator) -> StdResult<()> { + info!("Waiting that enough immutable have been written in the devnet"; "aggregator" => aggregator.name()); + let db_directory = aggregator.db_directory(); match attempt!(24, Duration::from_secs(5), { match ImmutableFile::list_completed_in_dir(db_directory) .with_context(|| { @@ -34,9 +35,10 @@ pub async fn wait_for_enough_immutable(db_directory: &Path) -> StdResult<()> { } } -pub async fn wait_for_epoch_settings(aggregator_endpoint: &str) -> StdResult { +pub async fn wait_for_epoch_settings(aggregator: &Aggregator) -> StdResult { + let aggregator_endpoint = aggregator.endpoint(); let url = format!("{aggregator_endpoint}/epoch-settings"); - info!("Waiting for the aggregator to expose epoch settings"); + info!("Waiting for the aggregator to expose epoch settings"; "aggregator" => aggregator.name()); match attempt!(20, Duration::from_millis(1000), { match reqwest::get(url.clone()).await { @@ -52,7 +54,8 @@ pub async fn wait_for_epoch_settings(aggregator_endpoint: &str) -> StdResult { warn!( "Server error while waiting for the Aggregator, http code: {}", - s + s; + "aggregator" => aggregator.name() ); Ok(None) } @@ -70,12 +73,14 @@ pub async fn wait_for_epoch_settings(aggregator_endpoint: &str) -> StdResult, target_epoch: Epoch, wait_reason: String, ) -> StdResult<()> { info!( "Waiting for the cardano network to be at the target epoch: {}", wait_reason; + "aggregator" => aggregator.name(), "target_epoch" => ?target_epoch ); @@ -96,7 +101,7 @@ pub async fn wait_for_target_epoch( } }) { AttemptResult::Ok(_) => { - info!("Target epoch reached!"; "target_epoch" => ?target_epoch); + info!("Target epoch reached!"; "aggregator" => aggregator.name(), "target_epoch" => ?target_epoch); Ok(()) } AttemptResult::Err(error) => Err(error), diff --git a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs index 3382ec833bc..02cfde65e65 100644 --- a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs +++ b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs @@ -99,7 +99,7 @@ impl Spec { chain_observer: Arc, infrastructure: &MithrilInfrastructure, ) -> StdResult<()> { - assertions::wait_for_enough_immutable(aggregator.db_directory()).await?; + assertions::wait_for_enough_immutable(aggregator).await?; let start_epoch = chain_observer .get_current_epoch() .await? @@ -108,6 +108,7 @@ impl Spec { // Wait 4 epochs after start epoch for the aggregator to be able to bootstrap a genesis certificate let mut target_epoch = start_epoch + 4; assertions::wait_for_target_epoch( + aggregator, chain_observer.clone(), target_epoch, "minimal epoch for the aggregator to be able to bootstrap genesis certificate" @@ -115,11 +116,12 @@ impl Spec { ) .await?; assertions::bootstrap_genesis_certificate(aggregator).await?; - assertions::wait_for_epoch_settings(&aggregator.endpoint()).await?; + assertions::wait_for_epoch_settings(aggregator).await?; // Wait 2 epochs before changing stake distribution, so that we use at least one original stake distribution target_epoch += 2; assertions::wait_for_target_epoch( + aggregator, chain_observer.clone(), target_epoch, "epoch after which the stake distribution will change".to_string(), @@ -135,6 +137,7 @@ impl Spec { // Wait 2 epochs before changing protocol parameters target_epoch += 2; assertions::wait_for_target_epoch( + aggregator, chain_observer.clone(), target_epoch, "epoch after which the protocol parameters will change".to_string(), @@ -145,6 +148,7 @@ impl Spec { // Wait 6 epochs after protocol parameters update, so that we make sure that we use new protocol parameters as well as new stake distribution a few times target_epoch += 6; assertions::wait_for_target_epoch( + aggregator, chain_observer.clone(), target_epoch, "epoch after which the certificate chain will be long enough to catch most common troubles with stake distribution and protocol parameters".to_string(), @@ -162,6 +166,7 @@ impl Spec { infrastructure.register_switch_to_next_era(next_era).await?; target_epoch += 5; assertions::wait_for_target_epoch( + aggregator, chain_observer.clone(), target_epoch, "epoch after which the era switch will have triggered".to_string(), @@ -173,6 +178,7 @@ impl Spec { assertions::bootstrap_genesis_certificate(aggregator).await?; target_epoch += 5; assertions::wait_for_target_epoch( + aggregator, chain_observer.clone(), target_epoch, "epoch after which the re-genesis on era switch will be completed".to_string(), diff --git a/mithril-test-lab/mithril-end-to-end/src/run_only.rs b/mithril-test-lab/mithril-end-to-end/src/run_only.rs index 81e0b3371cc..15ecba86c1d 100644 --- a/mithril-test-lab/mithril-end-to-end/src/run_only.rs +++ b/mithril-test-lab/mithril-end-to-end/src/run_only.rs @@ -58,7 +58,7 @@ impl RunOnly { chain_observer: Arc, infrastructure: &MithrilInfrastructure, ) -> StdResult<()> { - assertions::wait_for_enough_immutable(aggregator.db_directory()).await?; + assertions::wait_for_enough_immutable(aggregator).await?; let start_epoch = chain_observer .get_current_epoch() .await? @@ -67,6 +67,7 @@ impl RunOnly { // Wait 3 epochs after start epoch for the aggregator to be able to bootstrap a genesis certificate let target_epoch = start_epoch + 3; assertions::wait_for_target_epoch( + aggregator, chain_observer, target_epoch, "minimal epoch for the aggregator to be able to bootstrap genesis certificate" @@ -74,7 +75,7 @@ impl RunOnly { ) .await?; assertions::bootstrap_genesis_certificate(aggregator).await?; - assertions::wait_for_epoch_settings(&aggregator.endpoint()).await?; + assertions::wait_for_epoch_settings(aggregator).await?; if aggregator.is_first() { // Transfer some funds on the devnet to have some Cardano transactions to sign From cc61c4261cf801b3a377d0a150136b569b04a357 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 21 Mar 2025 17:09:01 +0100 Subject: [PATCH 24/33] fix(ci): wrong format for next era in some e2e scenarios --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ed3ba9bbc2..853cd5bdc95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -314,7 +314,7 @@ jobs: # Include a test for full dedentralization P2P signer registration and P2P signature registration - mode: "decentralized" era: ${{ fromJSON(needs.build-ubuntu-X64.outputs.eras)[0] }} - next_era: [""] + next_era: "" cardano_node_version: "10.1.4" hard_fork_latest_era_at_epoch: 0 run_id: "#1" From 86f446009f8bfbc927dffdc160ccdec274eed800 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 21 Mar 2025 17:15:00 +0100 Subject: [PATCH 25/33] fix(e2e): era switch done on multiple aggregators --- mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs index 02cfde65e65..8252dad6971 100644 --- a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs +++ b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs @@ -163,7 +163,9 @@ impl Spec { // Verify that artifacts are produced and signed correctly after era switch if let Some(next_era) = &self.next_era { // Switch to next era - infrastructure.register_switch_to_next_era(next_era).await?; + if aggregator.is_first() { + infrastructure.register_switch_to_next_era(next_era).await?; + } target_epoch += 5; assertions::wait_for_target_epoch( aggregator, From 810440530be7ae6e0e44d48702827e8abbc44e52 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 21 Mar 2025 17:25:03 +0100 Subject: [PATCH 26/33] refactor(aggregator): simplify slave aggregator integration test - Removed last epoch which was not necessary - Removed unnecessary cycles - Reduced the number of signers per epoch - Use of 'checked_sub' in the 'EpochFixturesMapBuilder'. --- .../tests/create_certificate_slave.rs | 124 ++---------------- 1 file changed, 9 insertions(+), 115 deletions(-) diff --git a/mithril-aggregator/tests/create_certificate_slave.rs b/mithril-aggregator/tests/create_certificate_slave.rs index 9d7ed15c3dc..a54e01b5064 100644 --- a/mithril-aggregator/tests/create_certificate_slave.rs +++ b/mithril-aggregator/tests/create_certificate_slave.rs @@ -39,7 +39,7 @@ impl EpochFixturesMapBuilder { ( Epoch(epoch as u64), MithrilFixtureBuilder::default() - .with_signers(epoch + 1) + .with_signers(epoch) .with_protocol_parameters(protocol_parameters.clone()) .with_stake_distribution( StakeDistributionGenerationMethod::RandomDistribution { @@ -63,16 +63,14 @@ impl EpochFixturesMapBuilder { Epoch(*index), EpochFixtures { registering: &fixtures[&Epoch(*index)], - next_signing: if *index >= 1 { - fixtures.get(&Epoch(*index - 1)) - } else { - None - }, - current_signing: if *index >= 2 { - fixtures.get(&Epoch(*index - 2)) - } else { - None - }, + next_signing: index + .checked_sub(1) + .map(Epoch) + .and_then(|e| fixtures.get(&e)), + current_signing: index + .checked_sub(2) + .map(Epoch) + .and_then(|e| fixtures.get(&e)), }, ) }) @@ -143,11 +141,9 @@ async fn create_certificate_slave() { comment!("Master: start the runtime state machine"); cycle_err!(master_tester, "idle"); - cycle_err!(master_tester, "idle"); comment!("Slave: start the runtime state machine"); cycle_err!(slave_tester, "idle"); - cycle_err!(slave_tester, "idle"); comment!("Master: register signers"); master_tester @@ -180,12 +176,10 @@ async fn create_certificate_slave() { comment!("Master: change the epoch"); master_tester.increase_epoch().await.unwrap(); cycle_err!(master_tester, "idle"); - cycle_err!(master_tester, "idle"); comment!("Slave: change the epoch after master"); slave_tester.increase_epoch().await.unwrap(); cycle_err!(slave_tester, "idle"); - cycle_err!(slave_tester, "idle"); comment!("Master: register signers"); master_tester @@ -237,7 +231,6 @@ async fn create_certificate_slave() { comment!("Slave: change the epoch after master"); slave_tester.increase_epoch().await.unwrap(); cycle_err!(slave_tester, "idle"); - cycle_err!(slave_tester, "idle"); comment!("Master: register signers"); master_tester @@ -305,7 +298,6 @@ async fn create_certificate_slave() { comment!("Slave: change the epoch after master"); slave_tester.increase_epoch().await.unwrap(); cycle_err!(slave_tester, "idle"); - cycle_err!(slave_tester, "idle"); comment!("Slave: bootstrap the genesis certificate"); slave_tester @@ -373,7 +365,6 @@ async fn create_certificate_slave() { comment!("Slave: change the epoch before master"); slave_tester.increase_epoch().await.unwrap(); cycle!(slave_tester, "idle"); - cycle!(slave_tester, "idle"); comment!("Master: change the epoch"); master_tester.increase_epoch().await.unwrap(); @@ -546,101 +537,4 @@ async fn create_certificate_slave() { .clone(); assert_eq!(expected_avk, master_expected_certificate.avk()); assert_eq!(expected_avk, slave_expected_certificate.avk()); - - comment!( - "Epoch 7: - - the master aggregator produces a new certificate - - the slave aggregator produces a new certificate - - the slave aggregator new certificate uses the same avk as the master aggregator's new certificate - "); - let epoch_fixture = &epoch_fixtures_map[&Epoch(7)]; - - comment!("Master: update stake distribution source"); - master_tester - .update_stake_distribution(epoch_fixture.registering.stake_distribution()) - .await - .unwrap(); - - comment!("Slave: update stake distribution source"); - slave_tester - .update_stake_distribution(epoch_fixture.registering.stake_distribution()) - .await - .unwrap(); - - comment!("Master: change the epoch"); - master_tester.increase_epoch().await.unwrap(); - cycle!(master_tester, "idle"); - cycle!(master_tester, "ready"); - - comment!("Slave: change the epoch after master"); - slave_tester.increase_epoch().await.unwrap(); - cycle!(slave_tester, "idle"); - cycle!(slave_tester, "ready"); - - comment!("Master: register signers"); - master_tester - .register_signers(&epoch_fixture.registering.signers_fixture()) - .await - .unwrap(); - cycle!(master_tester, "signing"); - - comment!("Master: signers send their single signature"); - master_tester - .send_single_signatures( - SignedEntityTypeDiscriminants::MithrilStakeDistribution, - &epoch_fixture.current_signing.unwrap().signers_fixture(), - ) - .await - .unwrap(); - - comment!("Slave: signers send their single signature"); - cycle!(slave_tester, "signing"); - slave_tester - .send_single_signatures( - SignedEntityTypeDiscriminants::MithrilStakeDistribution, - &epoch_fixture.current_signing.unwrap().signers_fixture(), - ) - .await - .unwrap(); - - comment!("Master: state machine should issue a certificate for the MithrilStakeDistribution"); - cycle!(master_tester, "ready"); - let master_expected_certificate = ExpectedCertificate::new( - Epoch(7), - StakeDistributionParty::from_signers( - epoch_fixture.current_signing.unwrap().signers_with_stake(), - ) - .as_slice(), - epoch_fixture - .current_signing - .unwrap() - .compute_and_encode_avk(), - SignedEntityType::MithrilStakeDistribution(Epoch(7)), - ExpectedCertificate::identifier(&SignedEntityType::MithrilStakeDistribution(Epoch(6))), - ); - assert_last_certificate_eq!(master_tester, master_expected_certificate); - - comment!("Slave: state machine should issue a certificate for the MithrilStakeDistribution"); - cycle!(slave_tester, "ready"); - let slave_expected_certificate = ExpectedCertificate::new( - Epoch(7), - StakeDistributionParty::from_signers( - epoch_fixture.current_signing.unwrap().signers_with_stake(), - ) - .as_slice(), - epoch_fixture - .current_signing - .unwrap() - .compute_and_encode_avk(), - SignedEntityType::MithrilStakeDistribution(Epoch(7)), - ExpectedCertificate::identifier(&SignedEntityType::MithrilStakeDistribution(Epoch(6))), - ); - assert_last_certificate_eq!(slave_tester, slave_expected_certificate); - let expected_avk = epoch_fixture - .current_signing - .unwrap() - .compute_and_encode_avk() - .clone(); - assert_eq!(expected_avk, master_expected_certificate.avk()); - assert_eq!(expected_avk, slave_expected_certificate.avk()); } From 848825398ca0fb1189c3a9f692a240584fd54289 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 21 Mar 2025 18:11:52 +0100 Subject: [PATCH 27/33] refactor(e2e): better parameter handling with clap --- .../mithril-end-to-end/src/main.rs | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/mithril-test-lab/mithril-end-to-end/src/main.rs b/mithril-test-lab/mithril-end-to-end/src/main.rs index 0d602a7a852..9938c07b8f8 100644 --- a/mithril-test-lab/mithril-end-to-end/src/main.rs +++ b/mithril-test-lab/mithril-end-to-end/src/main.rs @@ -53,11 +53,11 @@ pub struct Args { bin_directory: PathBuf, /// Number of aggregators - #[clap(long, default_value_t = 1)] + #[clap(long, default_value_t = 1, value_parser = clap::value_parser!(u8).range(1..))] number_of_aggregators: u8, /// Number of signers - #[clap(long, default_value_t = 2)] + #[clap(long, default_value_t = 2, value_parser = clap::value_parser!(u8).range(1..))] number_of_signers: u8, /// Length of a Cardano slot in the devnet (in s) @@ -150,12 +150,6 @@ impl Args { } fn validate(&self) -> StdResult<()> { - if self.number_of_aggregators == 0 { - return Err(anyhow!("At least one aggregator is required")); - } - if self.number_of_signers == 0 { - return Err(anyhow!("At least one signer is required")); - } if !self.use_relays && self.number_of_aggregators >= 2 { return Err(anyhow!( "The 'use_relays' parameter must be activated to run more than one aggregator" @@ -520,14 +514,6 @@ mod tests { #[test] fn args_fails_validation() { - let args = Args::parse_from(["", "--number-of-signers", "0"]); - args.validate() - .expect_err("validate should fail without signers"); - - let args = Args::parse_from(["", "--number-of-aggregators", "0"]); - args.validate() - .expect_err("validate should fail without aggregators"); - let args = Args::parse_from(["", "--number-of-aggregators", "2"]); args.validate().expect_err( "validate should fail with more than one aggregator if p2p network is not used", From b1e0609ede6725070cc54f92a8255a13c6bfc906 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 21 Mar 2025 17:42:56 +0100 Subject: [PATCH 28/33] chore(e2e): apply review comments --- mithril-relay/src/repeater.rs | 1 - .../mithril-end-to-end/src/assertions/wait.rs | 6 +----- .../mithril-end-to-end/src/end_to_end_spec.rs | 4 ++-- .../mithril-end-to-end/src/main.rs | 8 ++++---- .../src/mithril/infrastructure.rs | 18 ++++++++++++------ 5 files changed, 19 insertions(+), 18 deletions(-) diff --git a/mithril-relay/src/repeater.rs b/mithril-relay/src/repeater.rs index 4b4073816d6..33b2b5c47db 100644 --- a/mithril-relay/src/repeater.rs +++ b/mithril-relay/src/repeater.rs @@ -35,7 +35,6 @@ impl MessageRepeater { } /// Set the message to repeat - #[allow(dead_code)] pub async fn set_message(&self, message: M) { debug!(self.logger, "Set message"; "message" => #?message); *self.message.lock().await = Some(message); diff --git a/mithril-test-lab/mithril-end-to-end/src/assertions/wait.rs b/mithril-test-lab/mithril-end-to-end/src/assertions/wait.rs index fa6aa114dee..5ca0ccfe3fc 100644 --- a/mithril-test-lab/mithril-end-to-end/src/assertions/wait.rs +++ b/mithril-test-lab/mithril-end-to-end/src/assertions/wait.rs @@ -52,11 +52,7 @@ pub async fn wait_for_epoch_settings(aggregator: &Aggregator) -> StdResult { - warn!( - "Server error while waiting for the Aggregator, http code: {}", - s; - "aggregator" => aggregator.name() - ); + warn!( "Server error while waiting for the Aggregator, http code: {s}"; "aggregator" => aggregator.name()); Ok(None) } _ => Ok(None), diff --git a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs index 8252dad6971..39814666972 100644 --- a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs +++ b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs @@ -77,7 +77,7 @@ impl Spec { .ok_or(anyhow!("No infrastructure found"))?; spec_clone - .start_aggregator( + .run_scenario( infrastructure.aggregator(index), infrastructure.chain_observer(index), infrastructure, @@ -93,7 +93,7 @@ impl Spec { Ok(()) } - pub async fn start_aggregator( + pub async fn run_scenario( &self, aggregator: &Aggregator, chain_observer: Arc, diff --git a/mithril-test-lab/mithril-end-to-end/src/main.rs b/mithril-test-lab/mithril-end-to-end/src/main.rs index 9938c07b8f8..8c08e647c67 100644 --- a/mithril-test-lab/mithril-end-to-end/src/main.rs +++ b/mithril-test-lab/mithril-end-to-end/src/main.rs @@ -201,19 +201,19 @@ async fn main_exec() -> StdResult<()> { }; let artifacts_dir = { let path = work_dir.join("artifacts"); - fs::create_dir(&path).expect("Artifacts dir creation failure"); + fs::create_dir(&path).with_context(|| "Artifacts dir creation failure")?; path }; let store_dir = { let path = work_dir.join("stores"); - fs::create_dir(&path).expect("Stores dir creation failure"); + fs::create_dir(&path).with_context(|| "Stores dir creation failure")?; path }; let mut app = App::new(); let mut app_stopper = AppStopper::new(&app); let mut join_set = JoinSet::new(); - with_gracefull_shutdown(&mut join_set); + with_graceful_shutdown(&mut join_set); join_set.spawn(async move { app.run(args, work_dir, store_dir, artifacts_dir).await }); @@ -446,7 +446,7 @@ fn create_workdir_if_not_exist_clean_otherwise(work_dir: &Path) { #[error("Signal received: `{0}`")] pub struct SignalError(pub String); -fn with_gracefull_shutdown(join_set: &mut JoinSet>) { +fn with_graceful_shutdown(join_set: &mut JoinSet>) { join_set.spawn(async move { let mut sigterm = signal(SignalKind::terminate()).expect("Failed to create SIGTERM signal"); sigterm.recv().await; diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs index 0b175a1a50c..05f14661395 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs @@ -494,9 +494,16 @@ mod tests { use super::*; #[test] - fn has_master_aggregator_succeeds() { + fn has_master_slave_signer_registration_succeeds() { + let config = MithrilInfrastructureConfig { + relay_signer_registration_mode: SignerRelayMode::Passthrough, + number_of_aggregators: 1, + ..MithrilInfrastructureConfig::dummy() + }; + + assert!(!config.has_master_slave_signer_registration()); + let config = MithrilInfrastructureConfig { - use_relays: true, relay_signer_registration_mode: SignerRelayMode::Passthrough, number_of_aggregators: 2, ..MithrilInfrastructureConfig::dummy() @@ -505,17 +512,16 @@ mod tests { assert!(config.has_master_slave_signer_registration()); let config = MithrilInfrastructureConfig { - use_relays: true, relay_signer_registration_mode: SignerRelayMode::P2P, - number_of_aggregators: 2, + number_of_aggregators: 1, ..MithrilInfrastructureConfig::dummy() }; assert!(!config.has_master_slave_signer_registration()); let config = MithrilInfrastructureConfig { - use_relays: false, - number_of_aggregators: 1, + relay_signer_registration_mode: SignerRelayMode::P2P, + number_of_aggregators: 2, ..MithrilInfrastructureConfig::dummy() }; From 39ab5461422654725a84b0569478aa648ede045b Mon Sep 17 00:00:00 2001 From: DJO <790521+Alenar@users.noreply.github.com> Date: Fri, 21 Mar 2025 16:16:07 +0100 Subject: [PATCH 29/33] refactor(e2e): avoid rwlock of option in specs --- .../mithril-end-to-end/src/end_to_end_spec.rs | 23 ++----- .../mithril-end-to-end/src/main.rs | 66 ++++++++++--------- .../src/mithril/infrastructure.rs | 19 +++--- .../mithril-end-to-end/src/mithril/signer.rs | 25 ++++--- .../mithril-end-to-end/src/run_only.rs | 18 ++--- 5 files changed, 67 insertions(+), 84 deletions(-) diff --git a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs index 39814666972..dbb36c54960 100644 --- a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs +++ b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs @@ -1,7 +1,5 @@ use std::sync::Arc; -use anyhow::anyhow; -use tokio::sync::RwLock; use tokio::task::JoinSet; use mithril_common::{ @@ -13,7 +11,7 @@ use mithril_common::{ use crate::{assertions, Aggregator, MithrilInfrastructure}; pub struct Spec { - pub infrastructure: Arc>>, + pub infrastructure: Arc, is_signing_cardano_transactions: bool, is_signing_cardano_stake_distribution: bool, is_signing_cardano_database: bool, @@ -23,7 +21,7 @@ pub struct Spec { impl Spec { pub fn new( - infrastructure: Arc>>, + infrastructure: Arc, signed_entity_types: Vec, next_era: Option, regenesis_on_era_switch: bool, @@ -53,28 +51,17 @@ impl Spec { pub async fn run(self) -> StdResult<()> { let mut join_set = JoinSet::new(); let spec = Arc::new(self); - let infrastructure_guard = spec.infrastructure.read().await; - let infrastructure = infrastructure_guard - .as_ref() - .ok_or(anyhow!("No infrastructure found"))?; - let aggregators = infrastructure_guard - .as_ref() - .ok_or(anyhow!("No infrastructure found"))? - .aggregators(); // Transfer some funds on the devnet to have some Cardano transactions to sign. // This step needs to be executed early in the process so that the transactions are available // for signing in the penultimate immutable chunk before the end of the test. // As we get closer to the tip of the chain when signing, we'll be able to relax this constraint. - assertions::transfer_funds(infrastructure.devnet()).await?; + assertions::transfer_funds(spec.infrastructure.devnet()).await?; - for index in 0..aggregators.len() { + for index in 0..spec.infrastructure.aggregators().len() { let spec_clone = spec.clone(); join_set.spawn(async move { - let infrastructure_guard = spec_clone.infrastructure.read().await; - let infrastructure = infrastructure_guard - .as_ref() - .ok_or(anyhow!("No infrastructure found"))?; + let infrastructure = &spec_clone.infrastructure; spec_clone .run_scenario( diff --git a/mithril-test-lab/mithril-end-to-end/src/main.rs b/mithril-test-lab/mithril-end-to-end/src/main.rs index 8c08e647c67..a581dd22f88 100644 --- a/mithril-test-lab/mithril-end-to-end/src/main.rs +++ b/mithril-test-lab/mithril-end-to-end/src/main.rs @@ -13,7 +13,7 @@ use std::{ use thiserror::Error; use tokio::{ signal::unix::{signal, SignalKind}, - sync::{Mutex, RwLock}, + sync::Mutex, task::JoinSet, }; @@ -290,19 +290,19 @@ impl From> for AppResult { struct App { devnet: Arc>>, - infrastructure: Arc>>, + infrastructure: Arc>>>, } impl App { fn new() -> Self { Self { devnet: Arc::new(Mutex::new(None)), - infrastructure: Arc::new(RwLock::new(None)), + infrastructure: Arc::new(Mutex::new(None)), } } async fn tail_logs(&self) { - if let Some(infrastructure) = self.infrastructure.read().await.as_ref() { + if let Some(infrastructure) = self.infrastructure.lock().await.as_ref() { let _ = infrastructure.tail_logs(40).await.inspect_err(|e| { error!("Failed to tail logs: {}", e); }); @@ -310,7 +310,7 @@ impl App { } async fn last_error_in_logs(&self) { - if let Some(infrastructure) = self.infrastructure.read().await.as_ref() { + if let Some(infrastructure) = self.infrastructure.lock().await.as_ref() { let _ = infrastructure.last_error_in_logs(1).await.inspect_err(|e| { error!("Failed to grep error in logs: {}", e); }); @@ -346,35 +346,37 @@ impl App { .await?; *self.devnet.lock().await = Some(devnet.clone()); - let infrastructure = MithrilInfrastructure::start(&MithrilInfrastructureConfig { - number_of_aggregators: args.number_of_aggregators, - number_of_signers: args.number_of_signers, - server_port, - devnet: devnet.clone(), - work_dir, - store_dir, - artifacts_dir, - bin_dir: args.bin_directory, - cardano_node_version: args.cardano_node_version, - mithril_run_interval: args.mithril_run_interval, - mithril_era: args.mithril_era, - mithril_era_reader_adapter: args.mithril_era_reader_adapter, - signed_entity_types: args.signed_entity_types.clone(), - run_only_mode, - use_relays, - relay_signer_registration_mode, - relay_signature_registration_mode, - use_p2p_passive_relays, - use_era_specific_work_dir: args.mithril_next_era.is_some(), - }) - .await?; - *self.infrastructure.write().await = Some(infrastructure); + let infrastructure = Arc::new( + MithrilInfrastructure::start(&MithrilInfrastructureConfig { + number_of_aggregators: args.number_of_aggregators, + number_of_signers: args.number_of_signers, + server_port, + devnet: devnet.clone(), + work_dir, + store_dir, + artifacts_dir, + bin_dir: args.bin_directory, + cardano_node_version: args.cardano_node_version, + mithril_run_interval: args.mithril_run_interval, + mithril_era: args.mithril_era, + mithril_era_reader_adapter: args.mithril_era_reader_adapter, + signed_entity_types: args.signed_entity_types.clone(), + run_only_mode, + use_relays, + relay_signer_registration_mode, + relay_signature_registration_mode, + use_p2p_passive_relays, + use_era_specific_work_dir: args.mithril_next_era.is_some(), + }) + .await?, + ); + *self.infrastructure.lock().await = Some(infrastructure.clone()); let runner: StdResult<()> = match run_only_mode { - true => RunOnly::new(self.infrastructure.clone()).run().await, + true => RunOnly::new(infrastructure).run().await, false => { Spec::new( - self.infrastructure.clone(), + infrastructure, args.signed_entity_types, args.mithril_next_era, args.mithril_era_regenesis_on_switch, @@ -401,7 +403,7 @@ impl App { struct AppStopper { devnet: Arc>>, - infrastructure: Arc>>, + infrastructure: Arc>>>, } impl AppStopper { @@ -413,7 +415,7 @@ impl AppStopper { } pub async fn stop(&mut self) { - if let Some(infrastructure) = self.infrastructure.write().await.as_mut() { + if let Some(infrastructure) = self.infrastructure.lock().await.as_mut() { let _ = infrastructure.stop_nodes().await.inspect_err(|e| { error!("Failed to stop nodes: {}", e); }); diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs index 05f14661395..3cd8e33d6b1 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs @@ -126,7 +126,8 @@ impl MithrilInfrastructure { master_aggregator_endpoint, signer_cardano_nodes, &relay_signers, - )?; + ) + .await?; fn build_chain_observer( cardano_node: &PoolNode, @@ -338,7 +339,7 @@ impl MithrilInfrastructure { Ok((relay_aggregators, relay_signers, relay_passives)) } - fn start_signers( + async fn start_signers( config: &MithrilInfrastructureConfig, master_aggregator_endpoint: String, pool_nodes: &[PoolNode], @@ -357,7 +358,7 @@ impl MithrilInfrastructure { master_aggregator_endpoint.clone() }; - let mut signer = Signer::new(&SignerConfig { + let signer = Signer::new(&SignerConfig { signer_number: index + 1, aggregator_endpoint, pool_node, @@ -373,7 +374,7 @@ impl MithrilInfrastructure { mithril_era_marker_address: &config.devnet.mithril_era_marker_address()?, enable_certification, })?; - signer.start()?; + signer.start().await?; signers.push(signer); } @@ -381,14 +382,14 @@ impl MithrilInfrastructure { Ok(signers) } - pub async fn stop_nodes(&mut self) -> StdResult<()> { + pub async fn stop_nodes(&self) -> StdResult<()> { // Note: The aggregators should be stopped *last* since signers depends on it info!("Stopping Mithril infrastructure"); - for signer in self.signers.as_mut_slice() { + for signer in &self.signers { signer.stop().await?; } - for aggregator in self.aggregators.as_mut_slice() { + for aggregator in &self.aggregators { aggregator.stop().await?; } @@ -411,10 +412,6 @@ impl MithrilInfrastructure { &self.signers } - pub fn signers_mut(&mut self) -> &mut [Signer] { - self.signers.as_mut_slice() - } - pub fn relay_aggregators(&self) -> &[RelayAggregator] { &self.relay_aggregators } diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/signer.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/signer.rs index d92633c1e76..53a8ad79c75 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/signer.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/signer.rs @@ -7,7 +7,9 @@ use mithril_common::StdResult; use slog_scope::info; use std::collections::HashMap; use std::path::Path; +use std::sync::Arc; use tokio::process::Child; +use tokio::sync::RwLock; #[derive(Debug)] pub struct SignerConfig<'a> { @@ -29,8 +31,8 @@ pub struct SignerConfig<'a> { pub struct Signer { name: String, party_id: PartyId, - command: MithrilCommand, - process: Option, + command: Arc>, + process: RwLock>, } impl Signer { @@ -115,30 +117,35 @@ impl Signer { Ok(Self { name, party_id, - command, - process: None, + command: Arc::new(RwLock::new(command)), + process: RwLock::new(None), }) } - pub fn start(&mut self) -> StdResult<()> { - self.process = Some(self.command.start(&[])?); + pub async fn start(&self) -> StdResult<()> { + let mut command = self.command.write().await; + let mut process = self.process.write().await; + *process = Some(command.start(&[])?); Ok(()) } - pub async fn stop(&mut self) -> StdResult<()> { - if let Some(process) = self.process.as_mut() { + pub async fn stop(&self) -> StdResult<()> { + let mut process_option = self.process.write().await; + if let Some(process) = process_option.as_mut() { let name = self.name.as_str(); info!("Stopping {name}"); process .kill() .await .with_context(|| "Could not kill signer")?; - self.process = None; + *process_option = None; } Ok(()) } pub async fn tail_logs(&self, number_of_line: u64) -> StdResult<()> { self.command + .read() + .await .tail_logs( Some(format!("mithril-signer-{}", self.party_id).as_str()), number_of_line, diff --git a/mithril-test-lab/mithril-end-to-end/src/run_only.rs b/mithril-test-lab/mithril-end-to-end/src/run_only.rs index 15ecba86c1d..eeed763b3d6 100644 --- a/mithril-test-lab/mithril-end-to-end/src/run_only.rs +++ b/mithril-test-lab/mithril-end-to-end/src/run_only.rs @@ -1,8 +1,6 @@ use std::sync::Arc; -use anyhow::anyhow; use mithril_common::chain_observer::ChainObserver; -use tokio::sync::RwLock; use tokio::task::JoinSet; use mithril_common::StdResult; @@ -10,30 +8,22 @@ use mithril_common::StdResult; use crate::{assertions, Aggregator, MithrilInfrastructure}; pub struct RunOnly { - pub infrastructure: Arc>>, + pub infrastructure: Arc, } impl RunOnly { - pub fn new(infrastructure: Arc>>) -> Self { + pub fn new(infrastructure: Arc) -> Self { Self { infrastructure } } pub async fn run(self) -> StdResult<()> { let run_only = Arc::new(self); let mut join_set = JoinSet::new(); - let infrastructure_guard = run_only.infrastructure.read().await; - let aggregators = infrastructure_guard - .as_ref() - .ok_or(anyhow!("No infrastructure found"))? - .aggregators(); - for index in 0..aggregators.len() { + for index in 0..run_only.infrastructure.aggregators().len() { let run_only_clone = run_only.clone(); join_set.spawn(async move { - let infrastructure_guard = run_only_clone.infrastructure.read().await; - let infrastructure = infrastructure_guard - .as_ref() - .ok_or(anyhow!("No infrastructure found"))?; + let infrastructure = &run_only_clone.infrastructure; run_only_clone .start_aggregator( From 4c3dded76b33331c8133318ad2e1eb5ddd8cf50c Mon Sep 17 00:00:00 2001 From: DJO <790521+Alenar@users.noreply.github.com> Date: Fri, 21 Mar 2025 16:32:55 +0100 Subject: [PATCH 30/33] refactor(e2e): simplify chain observer usage --- .../mithril-end-to-end/src/assertions/wait.rs | 11 ++++--- .../mithril-end-to-end/src/end_to_end_spec.rs | 27 +++++------------ .../src/mithril/aggregator.rs | 16 ++++++++-- .../src/mithril/infrastructure.rs | 30 ++++++------------- .../mithril-end-to-end/src/run_only.rs | 14 +++------ 5 files changed, 40 insertions(+), 58 deletions(-) diff --git a/mithril-test-lab/mithril-end-to-end/src/assertions/wait.rs b/mithril-test-lab/mithril-end-to-end/src/assertions/wait.rs index 5ca0ccfe3fc..39fa50a9d0d 100644 --- a/mithril-test-lab/mithril-end-to-end/src/assertions/wait.rs +++ b/mithril-test-lab/mithril-end-to-end/src/assertions/wait.rs @@ -1,12 +1,11 @@ use crate::{attempt, utils::AttemptResult, Aggregator}; use anyhow::{anyhow, Context}; use mithril_common::{ - chain_observer::ChainObserver, digesters::ImmutableFile, entities::Epoch, - messages::EpochSettingsMessage, StdResult, + digesters::ImmutableFile, entities::Epoch, messages::EpochSettingsMessage, StdResult, }; use reqwest::StatusCode; use slog_scope::{info, warn}; -use std::{sync::Arc, time::Duration}; +use std::time::Duration; pub async fn wait_for_enough_immutable(aggregator: &Aggregator) -> StdResult<()> { info!("Waiting that enough immutable have been written in the devnet"; "aggregator" => aggregator.name()); @@ -68,9 +67,8 @@ pub async fn wait_for_epoch_settings(aggregator: &Aggregator) -> StdResult, target_epoch: Epoch, wait_reason: String, ) -> StdResult<()> { @@ -81,7 +79,8 @@ pub async fn wait_for_target_epoch( ); match attempt!(90, Duration::from_millis(1000), { - match chain_observer + match aggregator + .chain_observer() .get_current_epoch() .await .with_context(|| "Could not query current epoch")? diff --git a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs index dbb36c54960..397aee961d6 100644 --- a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs +++ b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs @@ -3,7 +3,6 @@ use std::sync::Arc; use tokio::task::JoinSet; use mithril_common::{ - chain_observer::ChainObserver, entities::{Epoch, SignedEntityTypeDiscriminants}, StdResult, }; @@ -64,11 +63,7 @@ impl Spec { let infrastructure = &spec_clone.infrastructure; spec_clone - .run_scenario( - infrastructure.aggregator(index), - infrastructure.chain_observer(index), - infrastructure, - ) + .run_scenario(infrastructure.aggregator(index), infrastructure) .await }); } @@ -83,10 +78,10 @@ impl Spec { pub async fn run_scenario( &self, aggregator: &Aggregator, - chain_observer: Arc, infrastructure: &MithrilInfrastructure, ) -> StdResult<()> { assertions::wait_for_enough_immutable(aggregator).await?; + let chain_observer = aggregator.chain_observer(); let start_epoch = chain_observer .get_current_epoch() .await? @@ -94,9 +89,8 @@ impl Spec { // Wait 4 epochs after start epoch for the aggregator to be able to bootstrap a genesis certificate let mut target_epoch = start_epoch + 4; - assertions::wait_for_target_epoch( + assertions::wait_for_aggregator_at_target_epoch( aggregator, - chain_observer.clone(), target_epoch, "minimal epoch for the aggregator to be able to bootstrap genesis certificate" .to_string(), @@ -107,9 +101,8 @@ impl Spec { // Wait 2 epochs before changing stake distribution, so that we use at least one original stake distribution target_epoch += 2; - assertions::wait_for_target_epoch( + assertions::wait_for_aggregator_at_target_epoch( aggregator, - chain_observer.clone(), target_epoch, "epoch after which the stake distribution will change".to_string(), ) @@ -123,9 +116,8 @@ impl Spec { // Wait 2 epochs before changing protocol parameters target_epoch += 2; - assertions::wait_for_target_epoch( + assertions::wait_for_aggregator_at_target_epoch( aggregator, - chain_observer.clone(), target_epoch, "epoch after which the protocol parameters will change".to_string(), ) @@ -134,9 +126,8 @@ impl Spec { // Wait 6 epochs after protocol parameters update, so that we make sure that we use new protocol parameters as well as new stake distribution a few times target_epoch += 6; - assertions::wait_for_target_epoch( + assertions::wait_for_aggregator_at_target_epoch( aggregator, - chain_observer.clone(), target_epoch, "epoch after which the certificate chain will be long enough to catch most common troubles with stake distribution and protocol parameters".to_string(), ) @@ -154,9 +145,8 @@ impl Spec { infrastructure.register_switch_to_next_era(next_era).await?; } target_epoch += 5; - assertions::wait_for_target_epoch( + assertions::wait_for_aggregator_at_target_epoch( aggregator, - chain_observer.clone(), target_epoch, "epoch after which the era switch will have triggered".to_string(), ) @@ -166,9 +156,8 @@ impl Spec { if self.regenesis_on_era_switch { assertions::bootstrap_genesis_certificate(aggregator).await?; target_epoch += 5; - assertions::wait_for_target_epoch( + assertions::wait_for_aggregator_at_target_epoch( aggregator, - chain_observer.clone(), target_epoch, "epoch after which the re-genesis on era switch will be completed".to_string(), ) diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs index 649f6e8678f..c76654c98b3 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/aggregator.rs @@ -4,11 +4,13 @@ use crate::{ ERA_MARKERS_VERIFICATION_KEY, GENESIS_SECRET_KEY, GENESIS_VERIFICATION_KEY, }; use anyhow::{anyhow, Context}; +use mithril_common::chain_observer::{ChainObserver, PallasChainObserver}; use mithril_common::era::SupportedEra; -use mithril_common::{entities, StdResult}; +use mithril_common::{entities, CardanoNetwork, StdResult}; use slog_scope::info; use std::cmp; use std::collections::HashMap; +use std::fmt::Debug; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; @@ -36,7 +38,6 @@ pub struct AggregatorConfig<'a> { pub master_aggregator_endpoint: &'a Option, } -#[derive(Debug)] pub struct Aggregator { index: usize, name_suffix: String, @@ -45,6 +46,7 @@ pub struct Aggregator { mithril_run_interval: u32, command: Arc>, process: RwLock>, + chain_observer: Arc, } impl Aggregator { @@ -131,6 +133,10 @@ impl Aggregator { env, &args, )?; + let chain_observer = Arc::new(PallasChainObserver::new( + &aggregator_config.pool_node.socket_path, + CardanoNetwork::DevNet(DEVNET_MAGIC_ID), + )); Ok(Self { index: aggregator_config.index, @@ -140,6 +146,7 @@ impl Aggregator { mithril_run_interval: aggregator_config.mithril_run_interval, command: Arc::new(RwLock::new(command)), process: RwLock::new(None), + chain_observer, }) } @@ -156,6 +163,7 @@ impl Aggregator { mithril_run_interval: other.mithril_run_interval, command: other.command.clone(), process: RwLock::new(None), + chain_observer: other.chain_observer.clone(), } } @@ -183,6 +191,10 @@ impl Aggregator { self.mithril_run_interval } + pub fn chain_observer(&self) -> Arc { + self.chain_observer.clone() + } + pub async fn serve(&self) -> StdResult<()> { let mut command = self.command.write().await; command.set_log_name(&format!("mithril-aggregator-{}", self.name_suffix)); diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs index 3cd8e33d6b1..6a3931707c1 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs @@ -81,7 +81,7 @@ pub struct MithrilInfrastructure { relay_aggregators: Vec, relay_signers: Vec, relay_passives: Vec, - cardano_chain_observers: Vec>, + cardano_chain_observer: Arc, run_only_mode: bool, current_era: RwLock, era_reader_adapter: String, @@ -129,18 +129,10 @@ impl MithrilInfrastructure { ) .await?; - fn build_chain_observer( - cardano_node: &PoolNode, - network: CardanoNetwork, - ) -> Arc { - Arc::new(PallasChainObserver::new(&cardano_node.socket_path, network)) - } - let cardano_chain_observers: Vec> = aggregator_cardano_nodes - .iter() - .map(|cardano_node| { - build_chain_observer(cardano_node, CardanoNetwork::DevNet(DEVNET_MAGIC_ID)) - }) - .collect(); + let cardano_chain_observer = Arc::new(PallasChainObserver::new( + &aggregator_cardano_nodes[0].socket_path, + CardanoNetwork::DevNet(DEVNET_MAGIC_ID), + )); Ok(Self { bin_dir: config.bin_dir.to_path_buf(), @@ -151,7 +143,7 @@ impl MithrilInfrastructure { relay_aggregators, relay_signers, relay_passives, - cardano_chain_observers, + cardano_chain_observer, run_only_mode: config.run_only_mode, current_era: RwLock::new(config.mithril_era.clone()), era_reader_adapter: config.mithril_era_reader_adapter.clone(), @@ -179,7 +171,7 @@ impl MithrilInfrastructure { pub async fn register_switch_to_next_era(&self, next_era: &str) -> StdResult<()> { let next_era_epoch = self - .chain_observer(0) + .cardano_chain_observer .get_current_epoch() .await? .unwrap_or_default() @@ -424,12 +416,8 @@ impl MithrilInfrastructure { &self.relay_passives } - pub fn chain_observers(&self) -> &[Arc] { - &self.cardano_chain_observers - } - - pub fn chain_observer(&self, index: usize) -> Arc { - self.cardano_chain_observers[index].clone() + pub fn chain_observer(&self) -> Arc { + self.cardano_chain_observer.clone() } pub async fn build_client(&self, aggregator: &Aggregator) -> StdResult { diff --git a/mithril-test-lab/mithril-end-to-end/src/run_only.rs b/mithril-test-lab/mithril-end-to-end/src/run_only.rs index eeed763b3d6..2103710139d 100644 --- a/mithril-test-lab/mithril-end-to-end/src/run_only.rs +++ b/mithril-test-lab/mithril-end-to-end/src/run_only.rs @@ -1,6 +1,5 @@ use std::sync::Arc; -use mithril_common::chain_observer::ChainObserver; use tokio::task::JoinSet; use mithril_common::StdResult; @@ -26,11 +25,7 @@ impl RunOnly { let infrastructure = &run_only_clone.infrastructure; run_only_clone - .start_aggregator( - infrastructure.aggregator(index), - infrastructure.chain_observer(index), - infrastructure, - ) + .bootstrap_aggregator(infrastructure.aggregator(index), infrastructure) .await }); } @@ -42,13 +37,13 @@ impl RunOnly { Ok(()) } - pub async fn start_aggregator( + pub async fn bootstrap_aggregator( &self, aggregator: &Aggregator, - chain_observer: Arc, infrastructure: &MithrilInfrastructure, ) -> StdResult<()> { assertions::wait_for_enough_immutable(aggregator).await?; + let chain_observer = aggregator.chain_observer(); let start_epoch = chain_observer .get_current_epoch() .await? @@ -56,9 +51,8 @@ impl RunOnly { // Wait 3 epochs after start epoch for the aggregator to be able to bootstrap a genesis certificate let target_epoch = start_epoch + 3; - assertions::wait_for_target_epoch( + assertions::wait_for_aggregator_at_target_epoch( aggregator, - chain_observer, target_epoch, "minimal epoch for the aggregator to be able to bootstrap genesis certificate" .to_string(), From 922d28a0af3c849ae9a92c306b6ecafd958efbb7 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 24 Mar 2025 17:52:04 +0100 Subject: [PATCH 31/33] refactor(e2e): remove dependency to 'mithril-relay' --- Cargo.lock | 1 - .../mithril-end-to-end/Cargo.toml | 1 - .../mithril-end-to-end/src/main.rs | 13 +++++----- .../src/mithril/infrastructure.rs | 25 ++++++++----------- .../src/mithril/relay_signer.rs | 5 ++-- 5 files changed, 19 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fa079c9fc25..01f243a176d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3880,7 +3880,6 @@ dependencies = [ "indicatif", "mithril-common", "mithril-doc", - "mithril-relay", "reqwest 0.12.12", "serde", "serde_json", diff --git a/mithril-test-lab/mithril-end-to-end/Cargo.toml b/mithril-test-lab/mithril-end-to-end/Cargo.toml index 8e4861e701e..78b12685861 100644 --- a/mithril-test-lab/mithril-end-to-end/Cargo.toml +++ b/mithril-test-lab/mithril-end-to-end/Cargo.toml @@ -21,7 +21,6 @@ clap = { version = "4.5.28", features = ["derive"] } indicatif = { version = "0.17.11", features = ["tokio"] } mithril-common = { path = "../../mithril-common", features = ["full"] } mithril-doc = { path = "../../internal/mithril-doc" } -mithril-relay = { path = "../../mithril-relay" } reqwest = { version = "0.12.12", features = ["json"] } serde = { version = "1.0.217", features = ["derive"] } serde_json = "1.0.138" diff --git a/mithril-test-lab/mithril-end-to-end/src/main.rs b/mithril-test-lab/mithril-end-to-end/src/main.rs index a581dd22f88..ed8450159c1 100644 --- a/mithril-test-lab/mithril-end-to-end/src/main.rs +++ b/mithril-test-lab/mithril-end-to-end/src/main.rs @@ -1,6 +1,5 @@ use anyhow::{anyhow, Context}; use clap::{CommandFactory, Parser, Subcommand}; -use mithril_relay::SignerRelayMode; use slog::{Drain, Level, Logger}; use slog_scope::{error, info}; use std::{ @@ -112,13 +111,13 @@ pub struct Args { #[clap(long)] use_relays: bool, - /// Signer registration relay mode (used only when 'use_relays' is set) - #[clap(long, value_enum, default_value_t = SignerRelayMode::Passthrough)] - relay_signer_registration_mode: SignerRelayMode, + /// Signer registration relay mode (used only when 'use_relays' is set, can be 'passthrough' or 'p2p') + #[clap(long, default_value = "passthrough")] + relay_signer_registration_mode: String, - /// Signature registration relay mode (used only when 'use_relays' is set) - #[clap(long, value_enum, default_value_t = SignerRelayMode::P2P)] - relay_signature_registration_mode: SignerRelayMode, + /// Signature registration relay mode (used only when 'use_relays' is set, can be 'passthrough' or 'p2p') + #[clap(long, default_value = "p2p")] + relay_signature_registration_mode: String, /// Enable P2P passive relays in P2P mode (used only when 'use_relays' is set) #[clap(long, default_value = "true")] diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs index 6a3931707c1..791d7792fcb 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs @@ -6,7 +6,6 @@ use crate::{ use mithril_common::chain_observer::{ChainObserver, PallasChainObserver}; use mithril_common::entities::{Epoch, PartyId, ProtocolParameters}; use mithril_common::{CardanoNetwork, StdResult}; -use mithril_relay::SignerRelayMode; use slog_scope::info; use std::fs; use std::path::PathBuf; @@ -31,15 +30,15 @@ pub struct MithrilInfrastructureConfig { pub signed_entity_types: Vec, pub run_only_mode: bool, pub use_relays: bool, - pub relay_signer_registration_mode: SignerRelayMode, - pub relay_signature_registration_mode: SignerRelayMode, + pub relay_signer_registration_mode: String, + pub relay_signature_registration_mode: String, pub use_p2p_passive_relays: bool, pub use_era_specific_work_dir: bool, } impl MithrilInfrastructureConfig { pub fn has_master_slave_signer_registration(&self) -> bool { - if self.relay_signer_registration_mode == SignerRelayMode::Passthrough { + if &self.relay_signer_registration_mode == "passthrough" { self.number_of_aggregators > 1 } else { false @@ -64,8 +63,8 @@ impl MithrilInfrastructureConfig { signed_entity_types: vec!["type1".to_string()], run_only_mode: false, use_relays: false, - relay_signer_registration_mode: SignerRelayMode::Passthrough, - relay_signature_registration_mode: SignerRelayMode::Passthrough, + relay_signer_registration_mode: "passthrough".to_string(), + relay_signature_registration_mode: "passthrough".to_string(), use_p2p_passive_relays: false, use_era_specific_work_dir: false, } @@ -251,8 +250,8 @@ impl MithrilInfrastructure { config: &MithrilInfrastructureConfig, aggregator_endpoints: &[String], signers_party_ids: &[PartyId], - relay_signer_registration_mode: SignerRelayMode, - relay_signature_registration_mode: SignerRelayMode, + relay_signer_registration_mode: String, + relay_signature_registration_mode: String, ) -> StdResult<(Vec, Vec, Vec)> { if !config.use_relays { return Ok((vec![], vec![], vec![])); @@ -476,12 +475,10 @@ impl MithrilInfrastructure { mod tests { use crate::MithrilInfrastructureConfig; - use super::*; - #[test] fn has_master_slave_signer_registration_succeeds() { let config = MithrilInfrastructureConfig { - relay_signer_registration_mode: SignerRelayMode::Passthrough, + relay_signer_registration_mode: "passthrough".to_string(), number_of_aggregators: 1, ..MithrilInfrastructureConfig::dummy() }; @@ -489,7 +486,7 @@ mod tests { assert!(!config.has_master_slave_signer_registration()); let config = MithrilInfrastructureConfig { - relay_signer_registration_mode: SignerRelayMode::Passthrough, + relay_signer_registration_mode: "passthrough".to_string(), number_of_aggregators: 2, ..MithrilInfrastructureConfig::dummy() }; @@ -497,7 +494,7 @@ mod tests { assert!(config.has_master_slave_signer_registration()); let config = MithrilInfrastructureConfig { - relay_signer_registration_mode: SignerRelayMode::P2P, + relay_signer_registration_mode: "p2p".to_string(), number_of_aggregators: 1, ..MithrilInfrastructureConfig::dummy() }; @@ -505,7 +502,7 @@ mod tests { assert!(!config.has_master_slave_signer_registration()); let config = MithrilInfrastructureConfig { - relay_signer_registration_mode: SignerRelayMode::P2P, + relay_signer_registration_mode: "p2p".to_string(), number_of_aggregators: 2, ..MithrilInfrastructureConfig::dummy() }; diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/relay_signer.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/relay_signer.rs index aa960c341aa..4f87f531fba 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/relay_signer.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/relay_signer.rs @@ -1,7 +1,6 @@ use crate::utils::MithrilCommand; use mithril_common::entities::PartyId; use mithril_common::StdResult; -use mithril_relay::SignerRelayMode; use std::collections::HashMap; use std::path::Path; use tokio::process::Child; @@ -10,8 +9,8 @@ pub struct RelaySignerConfiguration<'a> { pub listen_port: u64, pub server_port: u64, pub dial_to: Option, - pub relay_signer_registration_mode: SignerRelayMode, - pub relay_signature_registration_mode: SignerRelayMode, + pub relay_signer_registration_mode: String, + pub relay_signature_registration_mode: String, pub aggregator_endpoint: &'a str, pub party_id: PartyId, pub work_dir: &'a Path, From fef7e169de2c5dc546a1b34743c5d4059aa25f01 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 24 Mar 2025 18:58:21 +0100 Subject: [PATCH 32/33] refactor(relay): create function converting mpsc transmission result to HTTP response in signer --- mithril-relay/src/relay/signer.rs | 41 +++++++++++++++---------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/mithril-relay/src/relay/signer.rs b/mithril-relay/src/relay/signer.rs index 135e9d12991..f98f56c5d81 100644 --- a/mithril-relay/src/relay/signer.rs +++ b/mithril-relay/src/relay/signer.rs @@ -256,7 +256,7 @@ mod handlers { use reqwest::{Error, Response}; use slog::{debug, Logger}; use std::{convert::Infallible, sync::Arc}; - use tokio::sync::mpsc::UnboundedSender; + use tokio::sync::mpsc::{error::SendError, UnboundedSender}; use warp::{http::StatusCode, reply::WithStatus}; use crate::repeater; @@ -287,16 +287,7 @@ mod handlers { match signer_relay_mode { SignerRelayMode::P2P => { repeater.set_message(register_signer_message.clone()).await; - match tx.send(register_signer_message) { - Ok(_) => Ok(Box::new(warp::reply::with_status( - "".to_string(), - StatusCode::CREATED, - ))), - Err(err) => Ok(Box::new(warp::reply::with_status( - format!("{err:?}"), - StatusCode::INTERNAL_SERVER_ERROR, - ))), - } + reply_response_from_tx_send_result(tx.send(register_signer_message)) } SignerRelayMode::Passthrough => { let response = reqwest::Client::new() @@ -319,16 +310,9 @@ mod handlers { debug!(logger, "Serve HTTP route /register-signatures"; "signer_relay_mode" => ?signer_relay_mode, "register_signature_message" => #?register_signature_message); match signer_relay_mode { - SignerRelayMode::P2P => match tx.send(register_signature_message) { - Ok(_) => Ok(Box::new(warp::reply::with_status( - "".to_string(), - StatusCode::CREATED, - ))), - Err(err) => Ok(Box::new(warp::reply::with_status( - format!("{err:?}"), - StatusCode::INTERNAL_SERVER_ERROR, - ))), - }, + SignerRelayMode::P2P => { + reply_response_from_tx_send_result(tx.send(register_signature_message)) + } SignerRelayMode::Passthrough => { let response = reqwest::Client::new() .post(format!("{aggregator_endpoint}/register-signatures")) @@ -340,6 +324,21 @@ mod handlers { } } + fn reply_response_from_tx_send_result( + result: Result<(), SendError>, + ) -> Result>, Infallible> { + match result { + Ok(_) => Ok(Box::new(warp::reply::with_status( + "".to_string(), + StatusCode::CREATED, + ))), + Err(err) => Ok(Box::new(warp::reply::with_status( + format!("{err:?}"), + StatusCode::INTERNAL_SERVER_ERROR, + ))), + } + } + pub async fn epoch_settings_handler( logger: Logger, aggregator_endpoint: String, From b48abad704308d323e06797cbc8c135bad51b895 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 21 Mar 2025 17:19:35 +0100 Subject: [PATCH 33/33] wip(ci): DO NOT MERGE --- .github/workflows/ci.yml | 9 ++++++--- .../mithril-end-to-end/src/assertions/exec.rs | 2 +- .../mithril-end-to-end/src/assertions/wait.rs | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 853cd5bdc95..73cb4e626b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -294,13 +294,16 @@ jobs: strategy: fail-fast: false matrix: - mode: ["std"] + mode: ["master-slave"] era: ${{ fromJSON(needs.build-ubuntu-X64.outputs.eras) }} next_era: [""] cardano_node_version: ["10.1.3", "10.1.4", "10.2.1"] hard_fork_latest_era_at_epoch: [0] - run_id: ["#1", "#2"] - extra_args: [""] + run_id: ["#1", "#2", "#3", "#4", "#5", "#6", "#7", "#8", "#9", "#10"] + extra_args: + [ + "--number-of-aggregators=2 --use-relays --relay-signer-registration-mode=passthrough --relay-signature-registration-mode=p2p", + ] include: # Include a test for partial decentralization with master/slave signer registration and P2P signature registration diff --git a/mithril-test-lab/mithril-end-to-end/src/assertions/exec.rs b/mithril-test-lab/mithril-end-to-end/src/assertions/exec.rs index 84325ea2f3a..1a6a437acc2 100644 --- a/mithril-test-lab/mithril-end-to-end/src/assertions/exec.rs +++ b/mithril-test-lab/mithril-end-to-end/src/assertions/exec.rs @@ -12,7 +12,7 @@ pub async fn bootstrap_genesis_certificate(aggregator: &Aggregator) -> StdResult // This should be removed when the aggregator is able to synchronize its certificate chain from another aggregator if !aggregator.is_first() { tokio::time::sleep(std::time::Duration::from_millis( - 5 * aggregator.mithril_run_interval() as u64, + 2 * aggregator.mithril_run_interval() as u64, )) .await; } diff --git a/mithril-test-lab/mithril-end-to-end/src/assertions/wait.rs b/mithril-test-lab/mithril-end-to-end/src/assertions/wait.rs index 39fa50a9d0d..9178a8b4f11 100644 --- a/mithril-test-lab/mithril-end-to-end/src/assertions/wait.rs +++ b/mithril-test-lab/mithril-end-to-end/src/assertions/wait.rs @@ -78,7 +78,7 @@ pub async fn wait_for_aggregator_at_target_epoch( "target_epoch" => ?target_epoch ); - match attempt!(90, Duration::from_millis(1000), { + match attempt!(450, Duration::from_millis(200), { match aggregator .chain_observer() .get_current_epoch()