squashing localnet-v2 commits (again)

fixes to localnet purge

provide path in the error message

output args

log failed exec

print based on tty

check-prerequisites cmd

checked iptables update

basic kernel features check

enable ipv6 rules

add forwarding rules

squashing localnet-v2 commits

additional changes

propagate custom-dns flag to all run containers

remove is_mock from EcashManager

another localnet squash

unused import

chore: remove redundant testnet manager

missing impl

additional linux fixes

command to rebuild container image

wait for at least 2 blocks

additional node startup fixes

added --custom-dns flag to nym node setup

add gateway probe + wait for DKG magic file

fixed localnet down on linux

container ls

re-enable state resync

additional feature locking

macos adjustments

working nyxd startup on linux

wip linux box

wip

separating network inspect betweewn macos and linux

initial linux feature locking

moved all container commands into a single location

finally working initial node performance

squashing orchestrator commits

cleanup

fixed condition for naive rearrangement

added cache of cosmwasm contracts for speed up on subsequent runs

'down' command

refreshing described cache after nodes are bonded

nym nodes setup + wip on nym api refresh

nodes setup WIP

first pass cleanup

placeholder for nym-node setup

bypassing the dkg

further progress on nym-api setup

wip: api setup

up/down/purge placeholders

persisting contract setup data

fix contract upload by forcing amd64 container platform

wip: contracts setup4

wip: contracts setup3

wip: contracts setup2

wip: contracts setup

include network setup

init and spawn nyxd

build nyxd image in dedicated orchestrator

build nyxd image

Author: jstuczyn

Cargo.lock

@@ -153,10 +153,10 @@ members = [
     "tools/internal/contract-state-importer/importer-cli",
     "tools/internal/contract-state-importer/importer-contract",
     "tools/internal/mixnet-connectivity-check",
-#    "tools/internal/sdk-version-bump",
+    #    "tools/internal/sdk-version-bump",
     "tools/internal/ssl-inject",
-    "tools/internal/testnet-manager",
-    "tools/internal/testnet-manager/dkg-bypass-contract",
+    "tools/internal/localnet-orchestrator",
+    "tools/internal/localnet-orchestrator/dkg-bypass-contract",
     "tools/internal/validator-status-check",
     "tools/nym-cli",
     "tools/nym-id-cli",
@@ -186,6 +186,7 @@ default-members = [
     "service-providers/ip-packet-router",
     "service-providers/network-requester",
     "tools/nymvisor",
+    "tools/internal/localnet-orchestrator"
 ]
 
 exclude = ["contracts", "nym-wallet", "cpu-cycles"]
@@ -225,6 +226,7 @@ bloomfilter = "3.0.1"
 bs58 = "0.5.1"
 bytecodec = "0.4.15"
 bytes = "1.10.1"
+cargo-edit = "0.13.8"
 cargo_metadata = "0.19.2"
 celes = "2.6.0"
 cfg-if = "1.0.0"

Cargo.toml

@@ -2,7 +2,7 @@
 name = "nym-client-core"
 version = "1.1.15"
 authors = ["Dave Hrycyszyn <[email protected]>"]
-edition = "2021"
+edition = "2024"
 rust-version = "1.85"
 license.workspace = true
 

common/client-core/Cargo.toml

@@ -32,6 +32,7 @@ const DEFAULT_MIN_MIXNODE_PERFORMANCE: u8 = 50;
 const DEFAULT_MIN_GATEWAY_PERFORMANCE: u8 = 50;
 
 const DEFAULT_MAX_STARTUP_GATEWAY_WAITING_PERIOD: Duration = Duration::from_secs(70 * 60); // 70min -> full epoch (1h) + a bit of overhead
+const DEFAULT_MAX_STARTUP_TOPOLOGY_WAITING_PERIOD: Duration = Duration::from_secs(70 * 60); // 70min -> full epoch (1h) + a bit of overhead
 
 // Set this to a high value for now, so that we don't risk sporadic timeouts that might cause
 // bought bandwidth tokens to not have time to be spent; Once we remove the gateway from the
@@ -555,6 +556,11 @@ pub struct Topology {
     #[serde(with = "humantime_serde")]
     pub max_startup_gateway_waiting_period: Duration,
 
+    /// Defines how long the client is going to wait on startup for minimal topology to become online,
+    /// before abandoning the procedure.
+    #[serde(with = "humantime_serde")]
+    pub max_startup_network_waiting_period: Duration,
+
     /// Specifies a minimum performance of a mixnode that is used on route construction.
     /// This setting is only applicable when `NymApi` topology is used.
     pub minimum_mixnode_performance: u8,
@@ -583,6 +589,7 @@ impl Default for Topology {
             topology_resolution_timeout: DEFAULT_TOPOLOGY_RESOLUTION_TIMEOUT,
             disable_refreshing: false,
             max_startup_gateway_waiting_period: DEFAULT_MAX_STARTUP_GATEWAY_WAITING_PERIOD,
+            max_startup_network_waiting_period: DEFAULT_MAX_STARTUP_TOPOLOGY_WAITING_PERIOD,
             minimum_mixnode_performance: DEFAULT_MIN_MIXNODE_PERFORMANCE,
             minimum_gateway_performance: DEFAULT_MIN_GATEWAY_PERFORMANCE,
             use_extended_topology: false,

common/client-core/config-types/src/lib.rs

@@ -159,6 +159,7 @@ impl From<ConfigV6> for Config {
                     use_extended_topology: value.debug.topology.use_extended_topology,
                     ignore_egress_epoch_role: value.debug.topology.ignore_egress_epoch_role,
                     ignore_ingress_epoch_role: value.debug.topology.ignore_ingress_epoch_role,
+                    ..Default::default()
                 },
                 reply_surbs: ReplySurbs {
                     minimum_reply_surb_storage_threshold: value

common/client-core/config-types/src/old/v6.rs

@@ -160,7 +160,10 @@ where
         )
         .await?;
     } else {
-        info!("registered with new gateway {} (under address {address}), but this will not be our default address", gateway_details.gateway_id);
+        info!(
+            "registered with new gateway {} (under address {address}), but this will not be our default address",
+            gateway_details.gateway_id
+        );
     }
 
     Ok(GatewayInfo {

common/client-core/src/cli_helpers/client_add_gateway.rs

@@ -4,13 +4,13 @@
 use super::mix_traffic::ClientRequestSender;
 use super::received_buffer::ReceivedBufferMessage;
 use super::statistics_control::StatisticsControl;
-use crate::client::base_client::storage::helpers::store_client_keys;
 use crate::client::base_client::storage::MixnetClientStorage;
+use crate::client::base_client::storage::helpers::store_client_keys;
 use crate::client::cover_traffic_stream::LoopCoverTrafficStream;
 use crate::client::event_control::EventControl;
 use crate::client::inbound_messages::{InputMessage, InputMessageReceiver, InputMessageSender};
-use crate::client::key_manager::persistence::KeyStore;
 use crate::client::key_manager::ClientKeys;
+use crate::client::key_manager::persistence::KeyStore;
 use crate::client::mix_traffic::transceiver::{GatewayReceiver, GatewayTransceiver, RemoteGateway};
 use crate::client::mix_traffic::{BatchMixMessageSender, MixTrafficController, MixTrafficEvent};
 use crate::client::real_messages_control;
@@ -52,12 +52,12 @@ use nym_sphinx::addressing::nodes::NodeIdentity;
 use nym_sphinx::receiver::{ReconstructedMessage, SphinxMessageReceiver};
 use nym_statistics_common::clients::ClientStatsSender;
 use nym_statistics_common::generate_client_stats_id;
-use nym_task::connections::{ConnectionCommandReceiver, ConnectionCommandSender, LaneQueueLengths};
 use nym_task::ShutdownTracker;
-use nym_topology::provider_trait::TopologyProvider;
+use nym_task::connections::{ConnectionCommandReceiver, ConnectionCommandSender, LaneQueueLengths};
 use nym_topology::HardcodedTopologyProvider;
+use nym_topology::provider_trait::TopologyProvider;
 use nym_validator_client::nym_api::NymApiClientExt;
-use nym_validator_client::{nyxd::contract_traits::DkgQueryClient, UserAgent};
+use nym_validator_client::{UserAgent, nyxd::contract_traits::DkgQueryClient};
 use rand::prelude::SliceRandom;
 use rand::rngs::OsRng;
 use rand::thread_rng;
@@ -220,6 +220,7 @@ pub struct BaseClientBuilder<C, S: MixnetClientStorage> {
     nym_api_urls: Option<Vec<nym_network_defaults::ApiUrl>>,
 
     wait_for_gateway: bool,
+    wait_for_initial_topology: bool,
     custom_topology_provider: Option<Box<dyn TopologyProvider + Send + Sync>>,
     custom_gateway_transceiver: Option<Box<dyn GatewayTransceiver + Send>>,
     shutdown: Option<ShutdownTracker>,
@@ -250,6 +251,7 @@ where
             dkg_query_client,
             nym_api_urls: None,
             wait_for_gateway: false,
+            wait_for_initial_topology: false,
             custom_topology_provider: None,
             custom_gateway_transceiver: None,
             shutdown: None,
@@ -305,6 +307,12 @@ where
         self
     }
 
+    #[must_use]
+    pub fn with_wait_for_initial_topology(mut self, wait_for_initial_topology: bool) -> Self {
+        self.wait_for_initial_topology = wait_for_initial_topology;
+        self
+    }
+
     #[must_use]
     pub fn with_topology_provider(
         mut self,
@@ -674,6 +682,7 @@ where
         topology_accessor: TopologyAccessor,
         local_gateway: NodeIdentity,
         wait_for_gateway: bool,
+        wait_for_initial_topology: bool,
         shutdown_tracker: &ShutdownTracker,
     ) -> Result<(), ClientCoreError> {
         let topology_refresher_config =
@@ -694,6 +703,46 @@ where
         tracing::info!("Obtaining initial network topology");
         topology_refresher.try_refresh().await;
 
+        // 1. wait for the minimum topology (if applicable)
+        if topology_refresher
+            .ensure_topology_is_routable()
+            .await
+            .is_err()
+            && wait_for_initial_topology
+        {
+            if let Err(err) = topology_refresher
+                .wait_for_initial_network(topology_config.max_startup_network_waiting_period)
+                .await
+            {
+                tracing::error!(
+                    "the network did not come become online within the specified timeout: {err}"
+                );
+                return Err(err.into());
+            }
+        }
+
+        // 2. wait for our gateway (if applicable)
+        if topology_refresher
+            .ensure_contains_routable_egress(local_gateway)
+            .await
+            .is_err()
+            && wait_for_gateway
+        {
+            if let Err(err) = topology_refresher
+                .wait_for_gateway(
+                    local_gateway,
+                    topology_config.max_startup_gateway_waiting_period,
+                )
+                .await
+            {
+                tracing::error!(
+                    "the gateway did not come back online within the specified timeout: {err}"
+                );
+                return Err(err.into());
+            }
+        }
+
+        // 3. check if the topology is routable (in case we were NOT waiting for it)
         if let Err(err) = topology_refresher.ensure_topology_is_routable().await {
             tracing::error!(
                 "The current network topology seem to be insufficient to route any packets through \
@@ -702,30 +751,15 @@ where
             return Err(ClientCoreError::InsufficientNetworkTopology(err));
         }
 
-        let gateway_wait_timeout = if wait_for_gateway {
-            Some(topology_config.max_startup_gateway_waiting_period)
-        } else {
-            None
-        };
-
+        // 4. check if the gateway exists (in case we were NOT waiting for it)
         if let Err(err) = topology_refresher
             .ensure_contains_routable_egress(local_gateway)
             .await
         {
-            if let Some(waiting_timeout) = gateway_wait_timeout {
-                if let Err(err) = topology_refresher
-                    .wait_for_gateway(local_gateway, waiting_timeout)
-                    .await
-                {
-                    tracing::error!(
-                        "the gateway did not come back online within the specified timeout: {err}"
-                    );
-                    return Err(err.into());
-                }
-            } else {
-                tracing::error!("the gateway we're supposedly connected to does not exist. We'll not be able to send any packets to ourselves: {err}");
-                return Err(err.into());
-            }
+            tracing::error!(
+                "the gateway we're supposedly connected to does not exist. We'll not be able to send any packets to ourselves: {err}"
+            );
+            return Err(err.into());
         }
 
         if !topology_config.disable_refreshing {
@@ -1024,6 +1058,7 @@ where
             shared_topology_accessor.clone(),
             self_address.gateway(),
             self.wait_for_gateway,
+            self.wait_for_initial_topology,
             &shutdown_tracker.clone(),
         )
         .await?;
@@ -1195,9 +1230,11 @@ mod tests {
         ]);
 
         assert_eq!(network_details.nym_api_urls.as_ref().unwrap().len(), 2);
-        assert!(network_details.nym_api_urls.as_ref().unwrap()[1]
-            .front_hosts
-            .is_some());
+        assert!(
+            network_details.nym_api_urls.as_ref().unwrap()[1]
+                .front_hosts
+                .is_some()
+        );
     }
 
     #[test]
@@ -1210,11 +1247,13 @@ mod tests {
 
         assert_eq!(api_url.url, "https://nym-frontdoor.vercel.app/api/");
         assert_eq!(api_url.front_hosts.as_ref().unwrap().len(), 2);
-        assert!(api_url
-            .front_hosts
-            .as_ref()
-            .unwrap()
-            .contains(&"vercel.app".to_string()));
+        assert!(
+            api_url
+                .front_hosts
+                .as_ref()
+                .unwrap()
+                .contains(&"vercel.app".to_string())
+        );
     }
 
     #[test]

common/client-core/src/client/base_client/mod.rs

@@ -2,15 +2,15 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use crate::{
-    client::replies::reply_storage::{fs_backend, CombinedReplyStorage, ReplyStorageBackend},
+    client::replies::reply_storage::{CombinedReplyStorage, ReplyStorageBackend, fs_backend},
     config,
     config::Config,
     error::ClientCoreError,
 };
 use nym_bandwidth_controller::BandwidthController;
 use nym_client_core_gateways_storage::OnDiskGatewaysDetails;
 use nym_credential_storage::storage::Storage as CredentialStorage;
-use nym_validator_client::{nyxd, QueryHttpRpcNyxdClient};
+use nym_validator_client::{QueryHttpRpcNyxdClient, nyxd};
 use std::{io, path::Path};
 use time::OffsetDateTime;
 use tracing::{error, info, trace};
@@ -24,7 +24,9 @@ async fn setup_fresh_backend<P: AsRef<Path>>(
     let mut storage_backend = match fs_backend::Backend::init(db_path).await {
         Ok(backend) => backend,
         Err(err) => {
-            error!("setup_fresh_backend: Failed to setup persistent storage backend for our reply needs: {err}");
+            error!(
+                "setup_fresh_backend: Failed to setup persistent storage backend for our reply needs: {err}"
+            );
             return Err(ClientCoreError::SurbStorageError {
                 source: Box::new(err),
             });
@@ -93,7 +95,9 @@ pub async fn setup_fs_reply_surb_backend<P: AsRef<Path>>(
         match fs_backend::Backend::try_load(db_path).await {
             Ok(backend) => Ok(backend),
             Err(err) => {
-                error!("setup_fs_reply_surb_backend: Failed to setup persistent storage backend for our reply needs: {err}. We're going to create a fresh database instead. This behaviour might change in the future");
+                error!(
+                    "setup_fs_reply_surb_backend: Failed to setup persistent storage backend for our reply needs: {err}. We're going to create a fresh database instead. This behaviour might change in the future"
+                );
                 archive_corrupted_database(db_path).await?;
                 setup_fresh_backend(db_path, surb_config).await
             }

common/client-core/src/client/base_client/non_wasm_helpers.rs

@@ -1,8 +1,8 @@
 // Copyright 2024 - Nym Technologies SA <[email protected]>
 // SPDX-License-Identifier: Apache-2.0
 
-use crate::client::key_manager::persistence::KeyStore;
 use crate::client::key_manager::ClientKeys;
+use crate::client::key_manager::persistence::KeyStore;
 use crate::error::ClientCoreError;
 use nym_client_core_gateways_storage::{
     ActiveGateway, GatewayPublishedData, GatewayRegistration, GatewaysDetailsStore,

common/client-core/src/client/base_client/storage/helpers.rs

@@ -11,19 +11,19 @@ use nym_sphinx::addressing::clients::Recipient;
 use nym_sphinx::cover::generate_loop_cover_packet;
 use nym_sphinx::params::{PacketSize, PacketType};
 use nym_sphinx::utils::sample_poisson_duration;
-use nym_statistics_common::clients::{packet_statistics::PacketStatisticsEvent, ClientStatsSender};
-use rand::{rngs::OsRng, CryptoRng, Rng};
+use nym_statistics_common::clients::{ClientStatsSender, packet_statistics::PacketStatisticsEvent};
+use rand::{CryptoRng, Rng, rngs::OsRng};
 use std::pin::Pin;
 use std::sync::Arc;
 use std::time::Duration;
 use tokio::sync::mpsc::error::TrySendError;
 use tracing::*;
 
 #[cfg(not(target_arch = "wasm32"))]
-use tokio::time::{sleep, Sleep};
+use tokio::time::{Sleep, sleep};
 
 #[cfg(target_arch = "wasm32")]
-use wasmtimer::tokio::{sleep, Sleep};
+use wasmtimer::tokio::{Sleep, sleep};
 
 pub struct LoopCoverTrafficStream<R>
 where
@@ -179,7 +179,9 @@ impl LoopCoverTrafficStream<OsRng> {
         ) {
             Ok(topology) => topology,
             Err(err) => {
-                warn!("We're not going to send any loop cover message this time, as the current topology seem to be invalid - {err}");
+                warn!(
+                    "We're not going to send any loop cover message this time, as the current topology seem to be invalid - {err}"
+                );
                 return;
             }
         };