From a089f971315b9bad7e8fe2ad0de8048c00e54da5 Mon Sep 17 00:00:00 2001 From: Marios Christou Date: Thu, 16 Jul 2026 16:52:37 +0300 Subject: [PATCH 01/10] More v3 candidates zombienet-sdk tests --- .../zombienet_polkadot_tests.yml | 25 +++ .../doesnt_break_parachains.rs | 91 ++++++-- .../tests/functional/mod.rs | 1 + .../v3_mixed_validators_disputes.rs | 194 ++++++++++++++++++ 4 files changed, 289 insertions(+), 22 deletions(-) create mode 100644 polkadot/zombienet-sdk-tests/tests/functional/v3_mixed_validators_disputes.rs diff --git a/.github/zombienet-tests/zombienet_polkadot_tests.yml b/.github/zombienet-tests/zombienet_polkadot_tests.yml index aab5b0161eb7..8490d289fb17 100644 --- a/.github/zombienet-tests/zombienet_polkadot_tests.yml +++ b/.github/zombienet-tests/zombienet_polkadot_tests.yml @@ -132,6 +132,8 @@ - job-name: "zombienet-polkadot-elastic-scaling-doesnt-break-parachains" test-filter: "elastic_scaling::doesnt_break_parachains::doesnt_break_parachains_test" runner-type: "default" + use-zombienet-sdk: true + cumulus-image: "test-parachain" - job-name: "zombienet-polkadot-elastic-scaling-basic-3cores" @@ -258,3 +260,26 @@ additional-env: OLD_POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:master-187cddde" OLD_POLKADOT_COMMAND: "polkadot-old" + +- job-name: "zombienet-polkadot-functional-v3-mixed-validators-disputes" + test-filter: "functional::v3_mixed_validators_disputes::v3_mixed_validators_disputes" + runner-type: "default" + use-zombienet-sdk: true + cumulus-image: "test-parachain" + additional-setup: | + BIN_DIR="$(pwd)/bin_old" + mkdir -p $BIN_DIR + echo "downloading polkadot as polkadot-old in $BIN_DIR" + curl --retry 5 --retry-delay 10 --retry-max-time 120 -L -o $BIN_DIR/polkadot-old https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-stable2512/polkadot + chmod 755 $BIN_DIR/polkadot-old + for bin in polkadot-execute-worker polkadot-prepare-worker; do + echo "downloading $bin in $BIN_DIR" + curl --retry 5 --retry-delay 10 --retry-max-time 120 -L -o $BIN_DIR/$bin https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-stable2512/$bin + chmod 755 $BIN_DIR/$bin + done + ls -ltr $BIN_DIR + export PATH=$BIN_DIR:$PATH + echo "PATH=$PATH" >> $GITHUB_ENV + additional-env: + OLD_POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:master-187cddde" + OLD_POLKADOT_COMMAND: "polkadot-old" diff --git a/polkadot/zombienet-sdk-tests/tests/elastic_scaling/doesnt_break_parachains.rs b/polkadot/zombienet-sdk-tests/tests/elastic_scaling/doesnt_break_parachains.rs index a04d1b015902..4f4b300f9054 100644 --- a/polkadot/zombienet-sdk-tests/tests/elastic_scaling/doesnt_break_parachains.rs +++ b/polkadot/zombienet-sdk-tests/tests/elastic_scaling/doesnt_break_parachains.rs @@ -9,22 +9,62 @@ use codec::Decode; use cumulus_zombienet_sdk_helpers::{ assert_finality_lag, assert_para_throughput, assign_cores, wait_for_pvf_prepare, }; -use polkadot_primitives::{CoreIndex, Id as ParaId}; +use polkadot_primitives::{CandidateDescriptorVersion, CoreIndex, Id as ParaId}; +use rstest::rstest; use serde_json::json; -use std::collections::{BTreeMap, VecDeque}; +use std::collections::{BTreeMap, HashMap, VecDeque}; use zombienet_sdk::{ subxt::{OnlineClient, PolkadotConfig}, NetworkConfigBuilder, }; +#[rstest] +#[case::v2("test-parachain", None, false)] +#[case::v3("test-parachain", Some("v3"), true)] #[tokio::test(flavor = "multi_thread")] -async fn doesnt_break_parachains_test() -> Result<(), anyhow::Error> { +async fn doesnt_break_parachains_test( + #[case] collator_command: &str, + #[case] collator_chain: Option<&str>, + #[case] use_v3: bool, +) -> Result<(), anyhow::Error> { let _ = env_logger::try_init_from_env( env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"), ); let images = zombienet_sdk::environment::get_images_from_env(); + // V3 case additionally sets node-features bits 3+4 so the collator emits V3 descriptors. + let genesis_overrides = if use_v3 { + json!({ + "configuration": { + "config": { + "scheduler_params": { + "num_cores": 1, + "max_validators_per_core": 2, + }, + "node_features": {"bits": 8, "data": [0b00011000]} + } + } + }) + } else { + json!({ + "configuration": { + "config": { + "scheduler_params": { + "num_cores": 1, + "max_validators_per_core": 2, + } + } + } + }) + }; + + // `--authoring=slot-based` only for the V3 case. + let mut collator_args = vec![("-lparachain=debug,aura=debug").into()]; + if use_v3 { + collator_args.push("--authoring=slot-based".into()); + } + let config = NetworkConfigBuilder::new() .with_relaychain(|r| { let r = r @@ -32,16 +72,7 @@ async fn doesnt_break_parachains_test() -> Result<(), anyhow::Error> { .with_default_command("polkadot") .with_default_image(images.polkadot.as_str()) .with_default_args(vec![("-lparachain=debug").into()]) - .with_genesis_overrides(json!({ - "configuration": { - "config": { - "scheduler_params": { - "num_cores": 1, - "max_validators_per_core": 2, - } - } - } - })) + .with_genesis_overrides(genesis_overrides) // Have to set a `with_validator` outside of the loop below, so that `r` has the // right type. .with_validator(|node| node.with_name("validator-0")); @@ -51,12 +82,16 @@ async fn doesnt_break_parachains_test() -> Result<(), anyhow::Error> { }) }) .with_parachain(|p| { - // Use default, which has 6 second slot time. Also, don't use slot-based collator. - p.with_id(2000) - .with_default_command("polkadot-parachain") + let p = p + .with_id(2000) + .with_default_command(collator_command) .with_default_image(images.cumulus.as_str()) - .with_default_args(vec![("-lparachain=debug,aura=debug").into()]) - .with_collator(|n| n.with_name("collator-2000")) + .with_default_args(collator_args); + let p = match collator_chain { + Some(chain) => p.with_chain(chain), + None => p, + }; + p.with_collator(|n| n.with_name("collator-2000")) }) .build() .map_err(|e| { @@ -77,9 +112,21 @@ async fn doesnt_break_parachains_test() -> Result<(), anyhow::Error> { let para_id = ParaId::from(2000); // Wait for PVF preparation to complete. wait_for_pvf_prepare(&network, 1).await?; - // Expect the parachain to be making normal progress, 1 candidate backed per relay chain block. - // Lowering to 12 to make sure CI passes. - assert_para_throughput(&relay_client, 15, [(para_id, 12..16)], []).await?; + + if use_v3 { + // V3 candidates at single-core throughput. + crate::utils::assert_candidates_version( + &relay_client, + CandidateDescriptorVersion::V3, + HashMap::from([(para_id, 12..16)]), + 15, + ) + .await?; + } else { + // Expect the parachain to be making normal progress, 1 candidate backed per relay chain + // block. Lowering to 12 to make sure CI passes. + assert_para_throughput(&relay_client, 15, [(para_id, 12..16)], []).await?; + } let para_client = para_node.wait_client().await?; // Assert the parachain finalized block height is also on par with the number of backed @@ -97,7 +144,7 @@ async fn doesnt_break_parachains_test() -> Result<(), anyhow::Error> { .await?[..], )?; - // Get looakahead config + // Get lookahead config let lookahead = u32::decode( &mut &relay_client .runtime_api() diff --git a/polkadot/zombienet-sdk-tests/tests/functional/mod.rs b/polkadot/zombienet-sdk-tests/tests/functional/mod.rs index 0e404b6abd22..eeee498a068f 100644 --- a/polkadot/zombienet-sdk-tests/tests/functional/mod.rs +++ b/polkadot/zombienet-sdk-tests/tests/functional/mod.rs @@ -23,5 +23,6 @@ mod spam_statement_distribution_requests; mod sync_backing; mod systematic_chunk_recovery; mod v3_dynamic_enablement; +mod v3_mixed_validators_disputes; mod v3_rolling_upgrade; mod validator_disabling; diff --git a/polkadot/zombienet-sdk-tests/tests/functional/v3_mixed_validators_disputes.rs b/polkadot/zombienet-sdk-tests/tests/functional/v3_mixed_validators_disputes.rs new file mode 100644 index 000000000000..a1d0ddceb79d --- /dev/null +++ b/polkadot/zombienet-sdk-tests/tests/functional/v3_mixed_validators_disputes.rs @@ -0,0 +1,194 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +//! A V3 candidate is backed by v3-capable validators; a single pre-v3 validator that then takes +//! part in approval voting raises a losing dispute and is disabled. + +use anyhow::anyhow; +use codec::Decode; +use cumulus_zombienet_sdk_helpers::assert_finality_lag; +use polkadot_primitives::{CandidateDescriptorVersion, Id as ParaId, ValidatorIndex}; +use serde_json::json; +use std::collections::HashMap; +use zombienet_sdk::{ + subxt::{OnlineClient, PolkadotConfig}, + NetworkConfigBuilder, +}; + +use crate::utils::assert_candidates_version; + +#[tokio::test(flavor = "multi_thread")] +async fn v3_mixed_validators_disputes() -> Result<(), anyhow::Error> { + let _ = env_logger::try_init_from_env( + env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"), + ); + + let images = zombienet_sdk::environment::get_images_from_env(); + + let native = std::env::var("ZOMBIE_PROVIDER").as_deref() == Ok("native"); + let old_image = std::env::var("OLD_POLKADOT_IMAGE").ok(); + let old_command = std::env::var("OLD_POLKADOT_COMMAND").unwrap_or("polkadot".into()); + + // V2 (bit 3) and V3 (bit 4) enabled. + let node_features_with_v3 = json!({"bits": 8, "data": [0b00011000]}); + + let config = NetworkConfigBuilder::new() + .with_relaychain(|r| { + // westend-local so staking-based disabling takes effect. + let r = r + .with_chain("westend-local") + .with_default_command("polkadot") + .with_default_image(images.polkadot.as_str()) + .with_default_args(vec![("-lparachain=debug,runtime::staking=debug").into()]) + .with_genesis_overrides(json!({ + "configuration": { + "config": { + "scheduler_params": { + // Small groups so validators remain outside the backing group to do + // real approval checking (approval-voting insta-approves when + // `n_validators - backing_group_size < needed_approvals`). + "group_rotation_frequency": 100, + "max_validators_per_core": 3 + }, + "needed_approvals": 2, + "node_features": node_features_with_v3 + } + } + })) + // validator-0..5: v3-capable backers. + .with_validator(|node| { + node.with_name("validator-0") + .with_args(vec!["-lparachain=debug,runtime::staking=debug".into()]) + .invulnerable(false) + }); + + let r = (1..6usize).fold(r, |acc, i| { + acc.with_validator(|node| { + node.with_name(&format!("validator-{i}")) + .with_args(vec!["-lparachain=debug,runtime::staking=debug".into()]) + .invulnerable(false) + }) + }); + + // old-validator-6: pre-v3 node; never a backer, but takes part in approval voting. + (6..7usize).fold(r, |acc, i| { + acc.with_validator(|node| { + let node = node + .with_name(&format!("old-validator-{i}")) + .with_command(old_command.as_str()); + // Native ignores (and rejects) an image path; only set it for container + // providers when OLD_POLKADOT_IMAGE is given. + let node = match (native, old_image.as_deref()) { + (false, Some(img)) => node.with_image(img), + _ => node, + }; + node.invulnerable(false) + }) + }) + }) + // Para 3000: V3-capable collator ("v3" chain spec + slot-based authoring). + .with_parachain(|p| { + p.with_id(3000) + .with_default_command("test-parachain") + .with_default_image(images.cumulus.as_str()) + .with_chain("v3") + .with_default_args(vec![ + ("--authoring=slot-based").into(), + ("-lparachain=debug,aura=debug").into(), + ]) + .with_collator(|n| n.with_name("collator-3000")) + }) + // Keep the network alive on failure for post-mortem logs. + .with_global_settings(|s| s.with_tear_down_on_failure(false)) + .build() + .map_err(|e| { + let errs = e.into_iter().map(|e| e.to_string()).collect::>().join(" "); + anyhow!("config errs: {errs}") + })?; + + let spawn_fn = zombienet_sdk::environment::get_spawn_fn(); + log::info!("Spawning network"); + let network = spawn_fn(config).await?; + + let relay_node = network.get_node("validator-0")?; + let para_node = network.get_node("collator-3000")?; + let relay_client: OnlineClient = relay_node.wait_client().await?; + + + log::info!("Waiting for V3 candidates to be backed by the v3 backing group"); + assert_candidates_version( + &relay_client, + CandidateDescriptorVersion::V3, + HashMap::from([(ParaId::from(3000), 5..21)]), + 20, + None, + ) + .await?; + + log::info!("Waiting for a dispute to be raised and concluded"); + let parachain_candidate_dispute_metric = "parachain_candidate_disputes_total"; + let concluded_dispute_metric = + "polkadot_parachain_candidate_dispute_concluded{validity=\"valid\"}"; + + relay_node + .wait_metric_with_timeout(parachain_candidate_dispute_metric, |d| d >= 1.0, 600_u64) + .await?; + relay_node + .wait_metric_with_timeout(concluded_dispute_metric, |d| d >= 1.0, 200_u64) + .await?; + + log::info!("Waiting for the pre-v3 disputer to be disabled"); + let mut best_blocks = relay_client.blocks().subscribe_best().await?; + let mut disabled_validators: Vec = Vec::new(); + let mut blocks_checked = 0u32; + while let Some(block) = best_blocks.next().await { + let hash = block?.hash(); + disabled_validators = Vec::::decode( + &mut &relay_client + .runtime_api() + .at(hash) + .call_raw("ParachainHost_disabled_validators", None) + .await?[..], + )?; + if !disabled_validators.is_empty() { + break; + } + blocks_checked += 1; + + if blocks_checked >= 50 { + break; + } + } + + assert!( + !disabled_validators.is_empty(), + "expected the pre-v3 disputer to be disabled after the concluded dispute (checked {blocks_checked} blocks)", + ); + log::info!("Disabled validators: {:?}", disabled_validators); + + log::info!("Verifying the parachain keeps producing blocks after the dispute"); + let para_client: OnlineClient = para_node.wait_client().await?; + let start_height = para_client.blocks().at_latest().await?.header().number; + let target_height = (start_height + 8) as f64; + para_node + .wait_metric_with_timeout( + "substrate_block_height{status=\"best\"}", + |h| h >= target_height, + 120u64, + ) + .await?; + + log::info!("Waiting for relay-chain finality to recover after the dispute"); + relay_node + .wait_metric_with_timeout( + "polkadot_parachain_approval_checking_finality_lag", + |lag| lag <= 5.0, + 180u64, + ) + .await?; + + assert_finality_lag(¶_client, 8).await?; + + log::info!("Mixed validators disputes test finished successfully"); + Ok(()) +} From a4022760a9de6bbea39feaa7bfea8bc67561d9fd Mon Sep 17 00:00:00 2001 From: Marios Christou Date: Wed, 22 Jul 2026 15:49:28 +0300 Subject: [PATCH 02/10] more tests --- .../zombienet_polkadot_tests.yml | 23 + cumulus/test/runtime/build.rs | 9 + cumulus/test/runtime/src/flavors.rs | 1 + .../doesnt_break_parachains.rs | 1 + .../tests/functional/mod.rs | 1 + .../v3_mixed_validators_disputes.rs | 3 +- .../v3_old_validator_dispute_storm.rs | 518 ++++++++++++++++++ 7 files changed, 554 insertions(+), 2 deletions(-) create mode 100644 polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs diff --git a/.github/zombienet-tests/zombienet_polkadot_tests.yml b/.github/zombienet-tests/zombienet_polkadot_tests.yml index 8490d289fb17..5976c24309fb 100644 --- a/.github/zombienet-tests/zombienet_polkadot_tests.yml +++ b/.github/zombienet-tests/zombienet_polkadot_tests.yml @@ -283,3 +283,26 @@ additional-env: OLD_POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:master-187cddde" OLD_POLKADOT_COMMAND: "polkadot-old" + +- job-name: "zombienet-polkadot-functional-v3-old-validator-dispute-storm" + test-filter: "functional::v3_old_validator_dispute_storm::v3_old_validator_dispute_storm" + runner-type: "default" + use-zombienet-sdk: true + cumulus-image: "test-parachain" + additional-setup: | + BIN_DIR="$(pwd)/bin_old" + mkdir -p $BIN_DIR + echo "downloading polkadot as polkadot-old in $BIN_DIR" + curl --retry 5 --retry-delay 10 --retry-max-time 120 -L -o $BIN_DIR/polkadot-old https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-stable2512/polkadot + chmod 755 $BIN_DIR/polkadot-old + for bin in polkadot-execute-worker polkadot-prepare-worker; do + echo "downloading $bin in $BIN_DIR" + curl --retry 5 --retry-delay 10 --retry-max-time 120 -L -o $BIN_DIR/$bin https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-stable2512/$bin + chmod 755 $BIN_DIR/$bin + done + ls -ltr $BIN_DIR + export PATH=$BIN_DIR:$PATH + echo "PATH=$PATH" >> $GITHUB_ENV + additional-env: + OLD_POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:master-187cddde" + OLD_POLKADOT_COMMAND: "polkadot-old" diff --git a/cumulus/test/runtime/build.rs b/cumulus/test/runtime/build.rs index b0a3728bebd9..cf0fe883523f 100644 --- a/cumulus/test/runtime/build.rs +++ b/cumulus/test/runtime/build.rs @@ -75,6 +75,15 @@ fn main() { .set_file_name(v3::WASM_FILE_NAME) .build(); + // A V3-descriptor runtime with an incremented spec version, so it can be enacted as a genuine + // runtime upgrade on top of a plain (V2, spec_version 2) parachain runtime. Used by the + // old-validator dispute-storm test to flip a running parachain from V2 to V3 descriptors. + WasmBuilder::init_with_defaults() + .enable_feature("v3-descriptor") + .enable_feature("spec-version-3") + .set_file_name(v3_spec_version_incremented::WASM_FILE_NAME) + .build(); + WasmBuilder::init_with_defaults() .enable_feature("v3-descriptor") .enable_feature("relay-parent-offset-2") diff --git a/cumulus/test/runtime/src/flavors.rs b/cumulus/test/runtime/src/flavors.rs index af654c6cd8be..bb161c9f3b89 100644 --- a/cumulus/test/runtime/src/flavors.rs +++ b/cumulus/test/runtime/src/flavors.rs @@ -41,6 +41,7 @@ macro_rules! define_flavors { define_flavor!(block_bundling, $($capabilities)*); define_flavor!(sync_backing, $($capabilities)*); define_flavor!(v3, $($capabilities)*); + define_flavor!(v3_spec_version_incremented, $($capabilities)*); define_flavor!(v3_rpo_2, $($capabilities)*); define_flavor!(v3_rpo_4, $($capabilities)*); define_flavor!(v3_rpo_6, $($capabilities)*); diff --git a/polkadot/zombienet-sdk-tests/tests/elastic_scaling/doesnt_break_parachains.rs b/polkadot/zombienet-sdk-tests/tests/elastic_scaling/doesnt_break_parachains.rs index 4f4b300f9054..bd581af4efec 100644 --- a/polkadot/zombienet-sdk-tests/tests/elastic_scaling/doesnt_break_parachains.rs +++ b/polkadot/zombienet-sdk-tests/tests/elastic_scaling/doesnt_break_parachains.rs @@ -120,6 +120,7 @@ async fn doesnt_break_parachains_test( CandidateDescriptorVersion::V3, HashMap::from([(para_id, 12..16)]), 15, + None, ) .await?; } else { diff --git a/polkadot/zombienet-sdk-tests/tests/functional/mod.rs b/polkadot/zombienet-sdk-tests/tests/functional/mod.rs index eeee498a068f..1192ac76137b 100644 --- a/polkadot/zombienet-sdk-tests/tests/functional/mod.rs +++ b/polkadot/zombienet-sdk-tests/tests/functional/mod.rs @@ -24,5 +24,6 @@ mod sync_backing; mod systematic_chunk_recovery; mod v3_dynamic_enablement; mod v3_mixed_validators_disputes; +mod v3_old_validator_dispute_storm; mod v3_rolling_upgrade; mod validator_disabling; diff --git a/polkadot/zombienet-sdk-tests/tests/functional/v3_mixed_validators_disputes.rs b/polkadot/zombienet-sdk-tests/tests/functional/v3_mixed_validators_disputes.rs index a1d0ddceb79d..853d46ab36dd 100644 --- a/polkadot/zombienet-sdk-tests/tests/functional/v3_mixed_validators_disputes.rs +++ b/polkadot/zombienet-sdk-tests/tests/functional/v3_mixed_validators_disputes.rs @@ -114,7 +114,6 @@ async fn v3_mixed_validators_disputes() -> Result<(), anyhow::Error> { let para_node = network.get_node("collator-3000")?; let relay_client: OnlineClient = relay_node.wait_client().await?; - log::info!("Waiting for V3 candidates to be backed by the v3 backing group"); assert_candidates_version( &relay_client, @@ -155,7 +154,7 @@ async fn v3_mixed_validators_disputes() -> Result<(), anyhow::Error> { } blocks_checked += 1; - if blocks_checked >= 50 { + if blocks_checked >= 50 { break; } } diff --git a/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs b/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs new file mode 100644 index 000000000000..962a4dd4fdcc --- /dev/null +++ b/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs @@ -0,0 +1,518 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +//! Enabling V3 while part of the validator set still runs a pre-V3 binary. +//! +//! The network is `westend-local` with 15 validators. Dispute conclusion needs a +//! `n - (n - 1) / 3 = 11` vote supermajority, which is exactly what the two variants pivot on: +//! +//! * `4/15` old: the valid side reaches `15 - 4 = 11` → disputes conclude valid → the 4 old +//! validators collect against-valid offences and are disabled (4 is the disabling cap for n = 15) +//! → with every disputer disabled the storm self-extinguishes. This proves the safety net holds. +//! * `5/15` old: the valid side maxes out at `15 - 5 = 10 < 11` → disputes never conclude, nobody +//! is slashed or disabled, and the degradation persists. This documents the boundary: 2/3 is the +//! soundness floor, but the operational bar is that ~all validators are upgraded. +//! +//! Two single-collator parachains are used: A stays plain V2 throughout (a canary proving that V3 +//! gossip does not poison the old validators on an unrelated para), and B starts V2 and is upgraded +//! mid-test to a v3-descriptor runtime. + +use crate::utils::assert_candidates_version; +use anyhow::anyhow; +use codec::Decode; +use cumulus_zombienet_sdk_helpers::{assert_finality_lag, wait_for_runtime_upgrade}; +use polkadot_primitives::{ + BlockNumber, CandidateDescriptorVersion, CandidateHash, CandidateReceiptV2, DisputeState, + Id as ParaId, SessionIndex, ValidatorId, ValidatorIndex, +}; +use rstest::rstest; +use serde_json::json; +use std::collections::{HashMap, HashSet}; +use zombienet_sdk::{ + subxt::{ + dynamic::Value, ext::scale_value::value, tx::dynamic, utils::H256, OnlineClient, + PolkadotConfig, + }, + subxt_signer::sr25519::dev, + NetworkConfigBuilder, +}; + +/// Total validators in the set. Chosen so the dispute supermajority (`n - (n - 1) / 3`) lands at +/// 11, straddled by the two variants (11 valid voters vs 10). +const TOTAL_VALIDATORS: usize = 15; + +const PARA_A: u32 = 2000; // plain V2 canary. +const PARA_B: u32 = 2001; // upgraded from V2 to V3 mid-test. + +#[rstest] +// 4 old validators: valid side reaches 11 → disputes conclude → all 4 disabled → storm dies out. +#[case::four_old(4)] +// 5 old validators: valid side stuck at 10 → disputes never conclude → nobody disabled. +#[case::five_old(5)] +#[tokio::test(flavor = "multi_thread")] +async fn v3_old_validator_dispute_storm(#[case] num_old: usize) -> Result<(), anyhow::Error> { + let _ = env_logger::try_init_from_env( + env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"), + ); + + let num_new = TOTAL_VALIDATORS - num_old; + assert!(num_new >= 1, "need at least one v3-capable validator to back candidates"); + + let images = zombienet_sdk::environment::get_images_from_env(); + + let v3_wasm = + cumulus_test_runtime::v3_spec_version_incremented::WASM_BINARY.ok_or_else(|| { + anyhow!("cumulus-test-runtime v3_spec_version_incremented WASM not built (needs WASM build)") + })?; + + let native = std::env::var("ZOMBIE_PROVIDER").as_deref() == Ok("native"); + let old_image = std::env::var("OLD_POLKADOT_IMAGE").ok(); + let old_command = std::env::var("OLD_POLKADOT_COMMAND").unwrap_or("polkadot".into()); + + // V3 (bit 4) is enabled from genesis alongside V2 (bit 3). Para B still runs a V2 runtime, so + // it keeps emitting V2 until its mid-test upgrade — the storm is triggered by that upgrade, + // not by the feature bit. + let node_features_with_v3 = json!({"bits": 8, "data": [0b00011000]}); + + let old_names: Vec = + (num_new..TOTAL_VALIDATORS).map(|i| format!("old-validator-{i}")).collect(); + + let config = NetworkConfigBuilder::new() + .with_relaychain(|r| { + // westend-local so staking-based disabling can take effect. + let r = r + .with_chain("westend-local") + .with_default_command("polkadot") + .with_default_image(images.polkadot.as_str()) + .with_default_args(vec![("-lparachain=debug,runtime::staking=debug").into()]) + .with_genesis_overrides(json!({ + "configuration": { + "config": { + "scheduler_params": { + "max_validators_per_core": 3, + "group_rotation_frequency": 10 + }, + "minimum_backing_votes": 2, + "node_features": node_features_with_v3 + } + } + })) + // validator-0: v3-capable, used as the query node. + .with_validator(|node| { + node.with_name("validator-0") + .with_args(vec!["-lparachain=debug,runtime::staking=debug".into()]) + .invulnerable(false) + }); + + let r = (1..num_new).fold(r, |acc, i| { + acc.with_validator(|node| { + node.with_name(&format!("validator-{i}")) + .with_args(vec!["-lparachain=debug,runtime::staking=debug".into()]) + .invulnerable(false) + }) + }); + + (num_new..TOTAL_VALIDATORS).fold(r, |acc, i| { + acc.with_validator(|node| { + let node = node + .with_name(&format!("old-validator-{i}")) + .with_command(old_command.as_str()) + .with_args(vec!["-lparachain=debug,runtime::staking=debug".into()]); + let node = match (native, old_image.as_deref()) { + (false, Some(img)) => node.with_image(img), + _ => node, + }; + node.invulnerable(false) + }) + }) + }) + .with_parachain(|p| { + p.with_id(PARA_A) + .with_default_command("test-parachain") + .with_default_image(images.cumulus.as_str()) + .with_default_args(vec![("-lparachain=debug,aura=debug").into()]) + .with_collator(|n| n.with_name("collator-a")) + }) + // Para B: starts V2 ("async-backing"), upgraded to the v3 runtime mid-test. + .with_parachain(|p| { + p.with_id(PARA_B) + .with_default_command("test-parachain") + .with_default_image(images.cumulus.as_str()) + .with_chain("async-backing") + .with_default_args(vec![ + ("--authoring=slot-based").into(), + ("-lparachain=debug,aura=debug").into(), + ]) + .with_collator(|n| n.with_name("collator-b")) + }) + // Keep the network alive on failure for post-mortem logs. + .with_global_settings(|s| s.with_tear_down_on_failure(false)) + .build() + .map_err(|e| { + let errs = e.into_iter().map(|e| e.to_string()).collect::>().join(" "); + anyhow!("config errs: {errs}") + })?; + + let spawn_fn = zombienet_sdk::environment::get_spawn_fn(); + log::info!("Spawning network: {num_new} v3-capable + {num_old} pre-v3 validators"); + let network = spawn_fn(config).await?; + + let relay_node = network.get_node("validator-0")?; + let para_b_node = network.get_node("collator-b")?; + let relay_client: OnlineClient = relay_node.wait_client().await?; + let para_b_client: OnlineClient = para_b_node.wait_client().await?; + + // Collect the pre-V3 validators' public keys so we can tell them apart in the disabled set. + let mut old_pubkeys: HashSet = HashSet::new(); + for name in &old_names { + let spec = serde_json::to_value(network.get_node(name)?.spec())?; + let pk = spec + .pointer("/accounts/accounts/sr/public_key") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow!("no sr public_key in spec for {name}"))?; + old_pubkeys.insert(normalize_hex(pk)); + } + assert_eq!(old_pubkeys.len(), num_old, "collected {} old keys", old_pubkeys.len()); + + let para_a = ParaId::from(PARA_A); + let para_b = ParaId::from(PARA_B); + + // Baseline: V3 is enabled on the relay from genesis, but para B still runs a V2 runtime, so + // both paras emit V2 and there are no disputes. This pins the later storm on the para upgrade, + // not on the feature bit. + log::info!("baseline: V3 enabled on the relay, both paras still emit V2"); + assert_candidates_version( + &relay_client, + CandidateDescriptorVersion::V2, + HashMap::from([(para_a, 7..10), (para_b, 7..10)]), + 8, + None, + ) + .await?; + assert!( + relay_node + .assert_with("parachain_candidate_disputes_total", |d| d == 0.0) + .await?, + "no disputes expected before the para upgrade" + ); + + log::info!("upgrading para B to the v3 runtime"); + let upgrade_call = dynamic( + "Sudo", + "sudo_unchecked_weight", + vec![ + value! { System(set_code_without_checks { code: Value::from_bytes(v3_wasm) }) }, + value! { { ref_time: 1u64, proof_size: 1u64 } }, + ], + ); + para_b_client + .tx() + .sign_and_submit_then_watch_default(&upgrade_call, &dev::alice()) + .await? + .wait_for_finalized_success() + .await?; + wait_for_runtime_upgrade(¶_b_client).await?; + + // Para B now emits V3 descriptors. We deliberately do NOT assert this over finalized blocks + // here: the dispute storm we wait for next only happens on V3 candidates (para A is V2 and the + // pre-V3 validators validate it fine), so a rising dispute counter is itself proof that B went + // V3. Counting over finalized blocks would in any case be unreliable now, since the disputes + // start stalling finality. Para B's V3 band is asserted explicitly in the 4/15 path once + // finality has recovered. + + // The storm: every V3 candidate a pre-V3 validator approval-checks raises a dispute. + log::info!("waiting for the dispute storm to start (proves para B emits V3)"); + relay_node + .wait_metric_with_timeout("parachain_candidate_disputes_total", |d| d >= 1.0, 300u64) + .await?; + + if num_old == 4 { + assert_four_old_self_extinguishes(relay_node, &relay_client, &old_pubkeys, num_old).await?; + + // Immediate recovery: disabling takes effect within the session (not at a boundary), so the + // moment the last disputer is disabled the storm ends and para B resumes. We measure over + // the *best* chain starting right after disabling — same session, no finality/session + // wait — which verifies the V3 throughput band and proves the effect is immediate. The + // window spans two group rotations so a transiently short-handed group (disabled members) + // can't skew it. Over it: para B holds a healthy V3 band, the canary para A a healthy V2 + // band, and the dispute counter must not keep growing (no disputer left). + log::info!( + "verifying immediate recovery on the best chain (V3 band + canary + no new disputes)" + ); + let disputes_before = read_metric(relay_node, "parachain_candidate_disputes_total").await?; + let counts = count_backed_over_best_blocks( + &relay_client, + 20, + &[(para_b, CandidateDescriptorVersion::V3), (para_a, CandidateDescriptorVersion::V2)], + ) + .await?; + let disputes_after = read_metric(relay_node, "parachain_candidate_disputes_total").await?; + log::info!( + "over 20 best blocks post-disabling: para B V3={}, para A V2={}, disputes {disputes_before}→{disputes_after}", + counts[¶_b], + counts[¶_a], + ); + assert!( + counts[¶_b] >= 8, + "para B did not recover to a healthy V3 band immediately after disabling: {} in 20 best blocks", + counts[¶_b], + ); + assert!( + counts[¶_a] >= 8, + "canary para A degraded: {} V2 in 20 best blocks", + counts[¶_a], + ); + assert!( + disputes_after - disputes_before <= 3.0, + "dispute storm should self-extinguish after disabling, but grew {disputes_before}→{disputes_after}", + ); + + log::info!("waiting for relay-chain finality to recover after the storm"); + relay_node + .wait_metric_with_timeout( + "polkadot_parachain_approval_checking_finality_lag", + |lag| lag <= 5.0, + 180u64, + ) + .await?; + assert_finality_lag(¶_b_client, 12).await?; + } else { + let collator_a = network.get_node("collator-a")?; + assert_five_old_persists(relay_node, &relay_client, collator_a).await?; + } + + log::info!("V3 old-validator dispute storm test ({num_old}/{TOTAL_VALIDATORS} old) finished"); + Ok(()) +} + +/// `4/15`: the valid supermajority (11) concludes the disputes, the 4 pre-V3 validators are +/// disabled, and the storm self-extinguishes. +async fn assert_four_old_self_extinguishes( + relay_node: &zombienet_sdk::NetworkNode, + relay_client: &OnlineClient, + old_pubkeys: &HashSet, + num_old: usize, +) -> Result<(), anyhow::Error> { + log::info!("waiting for disputes to conclude valid"); + relay_node + .wait_metric_with_timeout( + "polkadot_parachain_candidate_dispute_concluded{validity=\"valid\"}", + |d| d >= 1.0, + 300u64, + ) + .await?; + + log::info!("waiting for all {num_old} pre-v3 validators to be disabled"); + let mut best_blocks = relay_client.blocks().subscribe_best().await?; + let mut disabled_old = 0usize; + let mut blocks_checked = 0u32; + while let Some(block) = best_blocks.next().await { + let hash = block?.hash(); + let disabled = disabled_validators_at(relay_client, hash).await?; + if !disabled.is_empty() { + let validators = session_validators_at(relay_client, hash).await?; + disabled_old = count_disabled_old(&disabled, &validators, old_pubkeys); + log::info!("disabled {} validators, {disabled_old} of them pre-v3", disabled.len()); + if disabled_old >= num_old { + break; + } + } + blocks_checked += 1; + if blocks_checked >= 90 { + break; + } + } + assert!( + disabled_old >= num_old, + "expected all {num_old} pre-v3 validators disabled, got {disabled_old} after \ + {blocks_checked} blocks", + ); + Ok(()) +} + +/// `5/15`: the valid side never reaches 11, so disputes never conclude, nobody is disabled, and the +/// degradation persists. The canary para A must stay healthy V2 throughout. +async fn assert_five_old_persists( + relay_node: &zombienet_sdk::NetworkNode, + relay_client: &OnlineClient, + collator_a: &zombienet_sdk::NetworkNode, +) -> Result<(), anyhow::Error> { + log::info!("observing that disputes persist unconcluded and nobody is disabled"); + let mut best_blocks = relay_client.blocks().subscribe_best().await?; + let mut blocks_checked = 0u32; + let mut saw_unconcluded = false; + while let Some(block) = best_blocks.next().await { + let hash = block?.hash(); + + // Nobody may be disabled: the valid side cannot muster the 11-vote supermajority. + let disabled = disabled_validators_at(relay_client, hash).await?; + assert!( + disabled.is_empty(), + "no validator should be disabled in the 5/15 case, found {disabled:?}", + ); + + // And at least one dispute must remain open with the valid side capped at 10. + for (_, _, state) in disputes_at(relay_client, hash).await? { + if state.concluded_at.is_none() { + saw_unconcluded = true; + assert!( + state.validators_for.count_ones() <= 10, + "valid side should be capped at 10 votes, got {}", + state.validators_for.count_ones(), + ); + } + } + + blocks_checked += 1; + if blocks_checked >= 20 { + break; + } + } + assert!(saw_unconcluded, "expected persistent unconcluded disputes in the 5/15 case"); + + // No dispute may have concluded on the valid side. + assert!( + relay_node + .assert_with( + "polkadot_parachain_candidate_dispute_concluded{validity=\"valid\"}", + |d| d == 0.0, + ) + .await?, + "no dispute should conclude valid with only 10 valid voters", + ); + + // Canary: the unrelated plain-V2 para A keeps making progress — V3 gossip does not poison the + // pre-v3 validators on paras they can actually validate. Relay finality is stalled here, so we + // cannot count candidates over finalized blocks; instead we track para A's collator best block + // height, which advances as its candidates are backed on the (still-progressing) best chain. + log::info!("asserting canary para A keeps producing despite the stalled relay finality"); + let best_height_metric = "substrate_block_height{status=\"best\"}"; + let start_height = read_metric(collator_a, best_height_metric).await?; + let target_height = start_height + 8.0; + collator_a + .wait_metric_with_timeout(best_height_metric, |h| h >= target_height, 180u64) + .await + .map_err(|e| { + anyhow!( + "canary para A stalled (best height {start_height} → target {target_height}): {e}" + ) + })?; + + // The degradation persists; we record the finality lag rather than asserting a bound. + let lag = read_metric(relay_node, "polkadot_parachain_approval_checking_finality_lag").await?; + log::info!("finality lag with an unresolvable dispute (5/15 case): {lag}"); + Ok(()) +} + +/// Normalize a hex public key to lowercase without a `0x` prefix. +fn normalize_hex(s: &str) -> String { + s.trim_start_matches("0x").to_ascii_lowercase() +} + +/// Count how many disabled validator indices belong to a pre-V3 (old) validator. +fn count_disabled_old( + disabled: &[ValidatorIndex], + validators: &[ValidatorId], + old_pubkeys: &HashSet, +) -> usize { + disabled + .iter() + .filter(|idx| { + validators + .get(idx.0 as usize) + .map(|id| { + let hex = id + .clone() + .into_inner() + .0 + .iter() + .map(|b| format!("{b:02x}")) + .collect::(); + old_pubkeys.contains(&hex) + }) + .unwrap_or(false) + }) + .count() +} + +async fn disabled_validators_at( + relay_client: &OnlineClient, + hash: H256, +) -> Result, anyhow::Error> { + Ok(Vec::::decode( + &mut &relay_client + .runtime_api() + .at(hash) + .call_raw("ParachainHost_disabled_validators", None) + .await?[..], + )?) +} + +async fn session_validators_at( + relay_client: &OnlineClient, + hash: H256, +) -> Result, anyhow::Error> { + Ok(Vec::::decode( + &mut &relay_client + .runtime_api() + .at(hash) + .call_raw("ParachainHost_validators", None) + .await?[..], + )?) +} + +async fn disputes_at( + relay_client: &OnlineClient, + hash: H256, +) -> Result)>, anyhow::Error> { + Ok(Vec::<(SessionIndex, CandidateHash, DisputeState)>::decode( + &mut &relay_client + .runtime_api() + .at(hash) + .call_raw("ParachainHost_disputes", None) + .await?[..], + )?) +} + +/// Count backed candidates matching the requested `(para, version)` targets over the next +/// `num_blocks` *best* relay blocks. Best (not finalized) blocks keep advancing while relay +/// finality is still catching up, and starting from the current best means the count reflects what +/// happens immediately, not after a session or finality boundary. +async fn count_backed_over_best_blocks( + relay_client: &OnlineClient, + num_blocks: u32, + targets: &[(ParaId, CandidateDescriptorVersion)], +) -> Result, anyhow::Error> { + let mut best_blocks = relay_client.blocks().subscribe_best().await?; + let mut counts: HashMap = targets.iter().map(|(p, _)| (*p, 0)).collect(); + let mut blocks_checked = 0u32; + while let Some(block) = best_blocks.next().await { + let events = block?.events().await?; + for event in events.iter() { + let event = event?; + if event.pallet_name() == "ParaInclusion" && event.variant_name() == "CandidateBacked" { + let receipt = CandidateReceiptV2::::decode(&mut &event.field_bytes()[..])?; + let para_id = receipt.descriptor.para_id(); + let version = receipt.descriptor.version(); + if targets.iter().any(|(p, v)| *p == para_id && *v == version) { + *counts.get_mut(¶_id).expect("para_id is a target; qed") += 1; + } + } + } + blocks_checked += 1; + if blocks_checked >= num_blocks { + break; + } + } + Ok(counts) +} + +/// Read a single metric value from a node. +async fn read_metric( + node: &zombienet_sdk::NetworkNode, + metric: &str, +) -> Result { + node.reports(metric) + .await + .map_err(|e| anyhow!("failed to read metric {metric}: {e}")) +} From b345cf721fb7bd2c5f9286e3062591029ed3ce7b Mon Sep 17 00:00:00 2001 From: Marios Christou Date: Thu, 23 Jul 2026 12:57:57 +0300 Subject: [PATCH 03/10] fix flaky disputes test --- .../v3_old_validator_dispute_storm.rs | 267 +++++++----------- 1 file changed, 96 insertions(+), 171 deletions(-) diff --git a/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs b/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs index 962a4dd4fdcc..b90b90145571 100644 --- a/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs +++ b/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs @@ -3,27 +3,27 @@ //! Enabling V3 while part of the validator set still runs a pre-V3 binary. //! -//! The network is `westend-local` with 15 validators. Dispute conclusion needs a -//! `n - (n - 1) / 3 = 11` vote supermajority, which is exactly what the two variants pivot on: +//! `westend-local` with 15 validators. Dispute conclusion needs an `n - (n - 1) / 3 = 11` vote +//! supermajority, which is what the two variants pivot on: //! -//! * `4/15` old: the valid side reaches `15 - 4 = 11` → disputes conclude valid → the 4 old -//! validators collect against-valid offences and are disabled (4 is the disabling cap for n = 15) -//! → with every disputer disabled the storm self-extinguishes. This proves the safety net holds. -//! * `5/15` old: the valid side maxes out at `15 - 5 = 10 < 11` → disputes never conclude, nobody -//! is slashed or disabled, and the degradation persists. This documents the boundary: 2/3 is the -//! soundness floor, but the operational bar is that ~all validators are upgraded. +//! * `4/15` old: the valid side reaches 11 → disputes conclude valid → the 4 old validators are +//! disabled (4 is the disabling cap for n = 15) → the storm self-extinguishes. Proves the safety +//! net holds. +//! * `5/15` old: the valid side maxes out at 10 < 11, so no dispute ever concludes and nobody is +//! disabled. But 5 invalid votes exceed the byzantine threshold, reverting each disputed block +//! and clamping GRANDPA to the undisputed chain → finality is degraded. Documents the boundary: +//! 2/3 is the soundness floor, but the operational bar is that ~all validators are upgraded. //! -//! Two single-collator parachains are used: A stays plain V2 throughout (a canary proving that V3 -//! gossip does not poison the old validators on an unrelated para), and B starts V2 and is upgraded -//! mid-test to a v3-descriptor runtime. +//! Two single-collator parachains: A stays plain V2 (canary that V3 gossip does not poison the old +//! validators on an unrelated para), B starts V2 and is upgraded to a v3-descriptor runtime +//! mid-test. use crate::utils::assert_candidates_version; use anyhow::anyhow; use codec::Decode; use cumulus_zombienet_sdk_helpers::{assert_finality_lag, wait_for_runtime_upgrade}; use polkadot_primitives::{ - BlockNumber, CandidateDescriptorVersion, CandidateHash, CandidateReceiptV2, DisputeState, - Id as ParaId, SessionIndex, ValidatorId, ValidatorIndex, + CandidateDescriptorVersion, CandidateReceiptV2, Id as ParaId, ValidatorId, ValidatorIndex, }; use rstest::rstest; use serde_json::json; @@ -37,17 +37,17 @@ use zombienet_sdk::{ NetworkConfigBuilder, }; -/// Total validators in the set. Chosen so the dispute supermajority (`n - (n - 1) / 3`) lands at -/// 11, straddled by the two variants (11 valid voters vs 10). const TOTAL_VALIDATORS: usize = 15; -const PARA_A: u32 = 2000; // plain V2 canary. -const PARA_B: u32 = 2001; // upgraded from V2 to V3 mid-test. +const PARA_A: u32 = 2000; +const PARA_B: u32 = 2001; + +const GROUP_ROTATION: u32 = 10; + +const HEALTHY_BACKING_FLOOR: u32 = 7; #[rstest] -// 4 old validators: valid side reaches 11 → disputes conclude → all 4 disabled → storm dies out. #[case::four_old(4)] -// 5 old validators: valid side stuck at 10 → disputes never conclude → nobody disabled. #[case::five_old(5)] #[tokio::test(flavor = "multi_thread")] async fn v3_old_validator_dispute_storm(#[case] num_old: usize) -> Result<(), anyhow::Error> { @@ -69,9 +69,6 @@ async fn v3_old_validator_dispute_storm(#[case] num_old: usize) -> Result<(), an let old_image = std::env::var("OLD_POLKADOT_IMAGE").ok(); let old_command = std::env::var("OLD_POLKADOT_COMMAND").unwrap_or("polkadot".into()); - // V3 (bit 4) is enabled from genesis alongside V2 (bit 3). Para B still runs a V2 runtime, so - // it keeps emitting V2 until its mid-test upgrade — the storm is triggered by that upgrade, - // not by the feature bit. let node_features_with_v3 = json!({"bits": 8, "data": [0b00011000]}); let old_names: Vec = @@ -79,7 +76,6 @@ async fn v3_old_validator_dispute_storm(#[case] num_old: usize) -> Result<(), an let config = NetworkConfigBuilder::new() .with_relaychain(|r| { - // westend-local so staking-based disabling can take effect. let r = r .with_chain("westend-local") .with_default_command("polkadot") @@ -97,7 +93,6 @@ async fn v3_old_validator_dispute_storm(#[case] num_old: usize) -> Result<(), an } } })) - // validator-0: v3-capable, used as the query node. .with_validator(|node| { node.with_name("validator-0") .with_args(vec!["-lparachain=debug,runtime::staking=debug".into()]) @@ -133,7 +128,6 @@ async fn v3_old_validator_dispute_storm(#[case] num_old: usize) -> Result<(), an .with_default_args(vec![("-lparachain=debug,aura=debug").into()]) .with_collator(|n| n.with_name("collator-a")) }) - // Para B: starts V2 ("async-backing"), upgraded to the v3 runtime mid-test. .with_parachain(|p| { p.with_id(PARA_B) .with_default_command("test-parachain") @@ -145,7 +139,6 @@ async fn v3_old_validator_dispute_storm(#[case] num_old: usize) -> Result<(), an ]) .with_collator(|n| n.with_name("collator-b")) }) - // Keep the network alive on failure for post-mortem logs. .with_global_settings(|s| s.with_tear_down_on_failure(false)) .build() .map_err(|e| { @@ -162,7 +155,6 @@ async fn v3_old_validator_dispute_storm(#[case] num_old: usize) -> Result<(), an let relay_client: OnlineClient = relay_node.wait_client().await?; let para_b_client: OnlineClient = para_b_node.wait_client().await?; - // Collect the pre-V3 validators' public keys so we can tell them apart in the disabled set. let mut old_pubkeys: HashSet = HashSet::new(); for name in &old_names { let spec = serde_json::to_value(network.get_node(name)?.spec())?; @@ -177,10 +169,7 @@ async fn v3_old_validator_dispute_storm(#[case] num_old: usize) -> Result<(), an let para_a = ParaId::from(PARA_A); let para_b = ParaId::from(PARA_B); - // Baseline: V3 is enabled on the relay from genesis, but para B still runs a V2 runtime, so - // both paras emit V2 and there are no disputes. This pins the later storm on the para upgrade, - // not on the feature bit. - log::info!("baseline: V3 enabled on the relay, both paras still emit V2"); + log::info!("baseline: both paras emit V2"); assert_candidates_version( &relay_client, CandidateDescriptorVersion::V2, @@ -213,15 +202,7 @@ async fn v3_old_validator_dispute_storm(#[case] num_old: usize) -> Result<(), an .await?; wait_for_runtime_upgrade(¶_b_client).await?; - // Para B now emits V3 descriptors. We deliberately do NOT assert this over finalized blocks - // here: the dispute storm we wait for next only happens on V3 candidates (para A is V2 and the - // pre-V3 validators validate it fine), so a rising dispute counter is itself proof that B went - // V3. Counting over finalized blocks would in any case be unreliable now, since the disputes - // start stalling finality. Para B's V3 band is asserted explicitly in the 4/15 path once - // finality has recovered. - - // The storm: every V3 candidate a pre-V3 validator approval-checks raises a dispute. - log::info!("waiting for the dispute storm to start (proves para B emits V3)"); + log::info!("waiting for dispute storm to start"); relay_node .wait_metric_with_timeout("parachain_candidate_disputes_total", |d| d >= 1.0, 300u64) .await?; @@ -229,64 +210,39 @@ async fn v3_old_validator_dispute_storm(#[case] num_old: usize) -> Result<(), an if num_old == 4 { assert_four_old_self_extinguishes(relay_node, &relay_client, &old_pubkeys, num_old).await?; - // Immediate recovery: disabling takes effect within the session (not at a boundary), so the - // moment the last disputer is disabled the storm ends and para B resumes. We measure over - // the *best* chain starting right after disabling — same session, no finality/session - // wait — which verifies the V3 throughput band and proves the effect is immediate. The - // window spans two group rotations so a transiently short-handed group (disabled members) - // can't skew it. Over it: para B holds a healthy V3 band, the canary para A a healthy V2 - // band, and the dispute counter must not keep growing (no disputer left). - log::info!( - "verifying immediate recovery on the best chain (V3 band + canary + no new disputes)" - ); - let disputes_before = read_metric(relay_node, "parachain_candidate_disputes_total").await?; - let counts = count_backed_over_best_blocks( + log::info!("waiting for dispute storm to self-extinguish"); + let total_disputes = wait_for_dispute_storm_to_settle(relay_node, 240).await?; + log::info!("storm self-extinguished at {total_disputes} disputes"); + + assert_finality_lag(¶_b_client, 6).await?; + + let series = backed_series_over_best_blocks( &relay_client, - 20, + GROUP_ROTATION, &[(para_b, CandidateDescriptorVersion::V3), (para_a, CandidateDescriptorVersion::V2)], ) .await?; - let disputes_after = read_metric(relay_node, "parachain_candidate_disputes_total").await?; - log::info!( - "over 20 best blocks post-disabling: para B V3={}, para A V2={}, disputes {disputes_before}→{disputes_after}", - counts[¶_b], - counts[¶_a], - ); + let b_backed: u32 = series[¶_b].iter().sum(); + let a_backed: u32 = series[¶_a].iter().sum(); + log::info!("post-recovery: para B V3={b_backed}, para A V2={a_backed} (floor {HEALTHY_BACKING_FLOOR})"); assert!( - counts[¶_b] >= 8, - "para B did not recover to a healthy V3 band immediately after disabling: {} in 20 best blocks", - counts[¶_b], + b_backed >= HEALTHY_BACKING_FLOOR, + "para B did not recover to a healthy V3 band: {b_backed} in {GROUP_ROTATION} blocks \ + (floor {HEALTHY_BACKING_FLOOR})", ); assert!( - counts[¶_a] >= 8, - "canary para A degraded: {} V2 in 20 best blocks", - counts[¶_a], + a_backed >= HEALTHY_BACKING_FLOOR, + "canary para A degraded: {a_backed} V2 in {GROUP_ROTATION} blocks (floor \ + {HEALTHY_BACKING_FLOOR})", ); - assert!( - disputes_after - disputes_before <= 3.0, - "dispute storm should self-extinguish after disabling, but grew {disputes_before}→{disputes_after}", - ); - - log::info!("waiting for relay-chain finality to recover after the storm"); - relay_node - .wait_metric_with_timeout( - "polkadot_parachain_approval_checking_finality_lag", - |lag| lag <= 5.0, - 180u64, - ) - .await?; - assert_finality_lag(¶_b_client, 12).await?; } else { - let collator_a = network.get_node("collator-a")?; - assert_five_old_persists(relay_node, &relay_client, collator_a).await?; + assert_five_old_persists(relay_node, &relay_client).await?; } log::info!("V3 old-validator dispute storm test ({num_old}/{TOTAL_VALIDATORS} old) finished"); Ok(()) } -/// `4/15`: the valid supermajority (11) concludes the disputes, the 4 pre-V3 validators are -/// disabled, and the storm self-extinguishes. async fn assert_four_old_self_extinguishes( relay_node: &zombienet_sdk::NetworkNode, relay_client: &OnlineClient, @@ -330,86 +286,44 @@ async fn assert_four_old_self_extinguishes( Ok(()) } -/// `5/15`: the valid side never reaches 11, so disputes never conclude, nobody is disabled, and the -/// degradation persists. The canary para A must stay healthy V2 throughout. async fn assert_five_old_persists( relay_node: &zombienet_sdk::NetworkNode, relay_client: &OnlineClient, - collator_a: &zombienet_sdk::NetworkNode, ) -> Result<(), anyhow::Error> { - log::info!("observing that disputes persist unconcluded and nobody is disabled"); - let mut best_blocks = relay_client.blocks().subscribe_best().await?; - let mut blocks_checked = 0u32; - let mut saw_unconcluded = false; - while let Some(block) = best_blocks.next().await { - let hash = block?.hash(); - - // Nobody may be disabled: the valid side cannot muster the 11-vote supermajority. - let disabled = disabled_validators_at(relay_client, hash).await?; - assert!( - disabled.is_empty(), - "no validator should be disabled in the 5/15 case, found {disabled:?}", - ); - - // And at least one dispute must remain open with the valid side capped at 10. - for (_, _, state) in disputes_at(relay_client, hash).await? { - if state.concluded_at.is_none() { - saw_unconcluded = true; - assert!( - state.validators_for.count_ones() <= 10, - "valid side should be capped at 10 votes, got {}", - state.validators_for.count_ones(), - ); - } - } - - blocks_checked += 1; - if blocks_checked >= 20 { - break; - } - } - assert!(saw_unconcluded, "expected persistent unconcluded disputes in the 5/15 case"); - - // No dispute may have concluded on the valid side. + let finalized_metric = "substrate_block_height{status=\"finalized\"}"; + let f0 = read_metric(relay_node, finalized_metric).await?; + tokio::time::sleep(std::time::Duration::from_secs(60)).await; + let f1 = read_metric(relay_node, finalized_metric).await?; + let finalized_progress = f1 - f0; + log::info!("finalized over 60s: {f0} → {f1} (+{finalized_progress}, healthy ~10)"); assert!( - relay_node - .assert_with( - "polkadot_parachain_candidate_dispute_concluded{validity=\"valid\"}", - |d| d == 0.0, - ) - .await?, - "no dispute should conclude valid with only 10 valid voters", + finalized_progress == 0.0, + "expected finality to be degraded behind the unresolvable dispute, but finalized advanced \ + {f0} → {f1} (+{finalized_progress}; healthy would be ~10 per 60s)", ); - // Canary: the unrelated plain-V2 para A keeps making progress — V3 gossip does not poison the - // pre-v3 validators on paras they can actually validate. Relay finality is stalled here, so we - // cannot count candidates over finalized blocks; instead we track para A's collator best block - // height, which advances as its candidates are backed on the (still-progressing) best chain. - log::info!("asserting canary para A keeps producing despite the stalled relay finality"); - let best_height_metric = "substrate_block_height{status=\"best\"}"; - let start_height = read_metric(collator_a, best_height_metric).await?; - let target_height = start_height + 8.0; - collator_a - .wait_metric_with_timeout(best_height_metric, |h| h >= target_height, 180u64) - .await - .map_err(|e| { - anyhow!( - "canary para A stalled (best height {start_height} → target {target_height}): {e}" - ) - })?; + let finalized_hash = relay_client.blocks().at_latest().await?.hash(); + let disabled = disabled_validators_at(relay_client, finalized_hash).await?; + assert!( + disabled.is_empty(), + "no validator should be disabled in the 5/15 case, found {disabled:?}", + ); - // The degradation persists; we record the finality lag rather than asserting a bound. - let lag = read_metric(relay_node, "polkadot_parachain_approval_checking_finality_lag").await?; - log::info!("finality lag with an unresolvable dispute (5/15 case): {lag}"); + for validity in ["valid", "invalid"] { + let metric = + format!("polkadot_parachain_candidate_dispute_concluded{{validity=\"{validity}\"}}"); + assert!( + relay_node.assert_with(metric, |d| d == 0.0).await?, + "no dispute should conclude ({validity}) without an 11-vote supermajority", + ); + } Ok(()) } -/// Normalize a hex public key to lowercase without a `0x` prefix. fn normalize_hex(s: &str) -> String { s.trim_start_matches("0x").to_ascii_lowercase() } -/// Count how many disabled validator indices belong to a pre-V3 (old) validator. fn count_disabled_old( disabled: &[ValidatorIndex], validators: &[ValidatorId], @@ -461,33 +375,17 @@ async fn session_validators_at( )?) } -async fn disputes_at( - relay_client: &OnlineClient, - hash: H256, -) -> Result)>, anyhow::Error> { - Ok(Vec::<(SessionIndex, CandidateHash, DisputeState)>::decode( - &mut &relay_client - .runtime_api() - .at(hash) - .call_raw("ParachainHost_disputes", None) - .await?[..], - )?) -} - -/// Count backed candidates matching the requested `(para, version)` targets over the next -/// `num_blocks` *best* relay blocks. Best (not finalized) blocks keep advancing while relay -/// finality is still catching up, and starting from the current best means the count reflects what -/// happens immediately, not after a session or finality boundary. -async fn count_backed_over_best_blocks( +async fn backed_series_over_best_blocks( relay_client: &OnlineClient, num_blocks: u32, targets: &[(ParaId, CandidateDescriptorVersion)], -) -> Result, anyhow::Error> { +) -> Result>, anyhow::Error> { let mut best_blocks = relay_client.blocks().subscribe_best().await?; - let mut counts: HashMap = targets.iter().map(|(p, _)| (*p, 0)).collect(); + let mut series: HashMap> = targets.iter().map(|(p, _)| (*p, vec![])).collect(); let mut blocks_checked = 0u32; while let Some(block) = best_blocks.next().await { let events = block?.events().await?; + let mut per_block: HashMap = targets.iter().map(|(p, _)| (*p, 0)).collect(); for event in events.iter() { let event = event?; if event.pallet_name() == "ParaInclusion" && event.variant_name() == "CandidateBacked" { @@ -495,19 +393,21 @@ async fn count_backed_over_best_blocks( let para_id = receipt.descriptor.para_id(); let version = receipt.descriptor.version(); if targets.iter().any(|(p, v)| *p == para_id && *v == version) { - *counts.get_mut(¶_id).expect("para_id is a target; qed") += 1; + *per_block.get_mut(¶_id).expect("para_id is a target; qed") += 1; } } } + for (para_id, count) in per_block { + series.get_mut(¶_id).expect("para_id is a target; qed").push(count); + } blocks_checked += 1; if blocks_checked >= num_blocks { break; } } - Ok(counts) + Ok(series) } -/// Read a single metric value from a node. async fn read_metric( node: &zombienet_sdk::NetworkNode, metric: &str, @@ -516,3 +416,28 @@ async fn read_metric( .await .map_err(|e| anyhow!("failed to read metric {metric}: {e}")) } + +async fn wait_for_dispute_storm_to_settle( + node: &zombienet_sdk::NetworkNode, + timeout_secs: u64, +) -> Result { + const STEP_SECS: u64 = 25; + let metric = "parachain_candidate_disputes_total"; + let mut prev = read_metric(node, metric).await?; + let mut elapsed = 0u64; + loop { + tokio::time::sleep(std::time::Duration::from_secs(STEP_SECS)).await; + elapsed += STEP_SECS; + let cur = read_metric(node, metric).await?; + log::info!("dispute counter {prev} → {cur} after {elapsed}s"); + if cur - prev == 0.0 { + return Ok(cur); + } + prev = cur; + if elapsed >= timeout_secs { + return Err(anyhow!( + "dispute storm did not settle within {timeout_secs}s (still growing, last {cur})" + )); + } + } +} From 14fc7475b3ea1d4c7419280b444382a4b94a52be Mon Sep 17 00:00:00 2001 From: Marios Christou Date: Fri, 24 Jul 2026 11:24:12 +0300 Subject: [PATCH 04/10] polish --- .../zombienet_polkadot_tests.yml | 23 --- .../tests/functional/mod.rs | 1 - .../v3_mixed_validators_disputes.rs | 193 ------------------ 3 files changed, 217 deletions(-) delete mode 100644 polkadot/zombienet-sdk-tests/tests/functional/v3_mixed_validators_disputes.rs diff --git a/.github/zombienet-tests/zombienet_polkadot_tests.yml b/.github/zombienet-tests/zombienet_polkadot_tests.yml index 5976c24309fb..e493d1a52f6b 100644 --- a/.github/zombienet-tests/zombienet_polkadot_tests.yml +++ b/.github/zombienet-tests/zombienet_polkadot_tests.yml @@ -261,29 +261,6 @@ OLD_POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:master-187cddde" OLD_POLKADOT_COMMAND: "polkadot-old" -- job-name: "zombienet-polkadot-functional-v3-mixed-validators-disputes" - test-filter: "functional::v3_mixed_validators_disputes::v3_mixed_validators_disputes" - runner-type: "default" - use-zombienet-sdk: true - cumulus-image: "test-parachain" - additional-setup: | - BIN_DIR="$(pwd)/bin_old" - mkdir -p $BIN_DIR - echo "downloading polkadot as polkadot-old in $BIN_DIR" - curl --retry 5 --retry-delay 10 --retry-max-time 120 -L -o $BIN_DIR/polkadot-old https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-stable2512/polkadot - chmod 755 $BIN_DIR/polkadot-old - for bin in polkadot-execute-worker polkadot-prepare-worker; do - echo "downloading $bin in $BIN_DIR" - curl --retry 5 --retry-delay 10 --retry-max-time 120 -L -o $BIN_DIR/$bin https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-stable2512/$bin - chmod 755 $BIN_DIR/$bin - done - ls -ltr $BIN_DIR - export PATH=$BIN_DIR:$PATH - echo "PATH=$PATH" >> $GITHUB_ENV - additional-env: - OLD_POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:master-187cddde" - OLD_POLKADOT_COMMAND: "polkadot-old" - - job-name: "zombienet-polkadot-functional-v3-old-validator-dispute-storm" test-filter: "functional::v3_old_validator_dispute_storm::v3_old_validator_dispute_storm" runner-type: "default" diff --git a/polkadot/zombienet-sdk-tests/tests/functional/mod.rs b/polkadot/zombienet-sdk-tests/tests/functional/mod.rs index 1192ac76137b..e88c1e094f7a 100644 --- a/polkadot/zombienet-sdk-tests/tests/functional/mod.rs +++ b/polkadot/zombienet-sdk-tests/tests/functional/mod.rs @@ -23,7 +23,6 @@ mod spam_statement_distribution_requests; mod sync_backing; mod systematic_chunk_recovery; mod v3_dynamic_enablement; -mod v3_mixed_validators_disputes; mod v3_old_validator_dispute_storm; mod v3_rolling_upgrade; mod validator_disabling; diff --git a/polkadot/zombienet-sdk-tests/tests/functional/v3_mixed_validators_disputes.rs b/polkadot/zombienet-sdk-tests/tests/functional/v3_mixed_validators_disputes.rs deleted file mode 100644 index 853d46ab36dd..000000000000 --- a/polkadot/zombienet-sdk-tests/tests/functional/v3_mixed_validators_disputes.rs +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -//! A V3 candidate is backed by v3-capable validators; a single pre-v3 validator that then takes -//! part in approval voting raises a losing dispute and is disabled. - -use anyhow::anyhow; -use codec::Decode; -use cumulus_zombienet_sdk_helpers::assert_finality_lag; -use polkadot_primitives::{CandidateDescriptorVersion, Id as ParaId, ValidatorIndex}; -use serde_json::json; -use std::collections::HashMap; -use zombienet_sdk::{ - subxt::{OnlineClient, PolkadotConfig}, - NetworkConfigBuilder, -}; - -use crate::utils::assert_candidates_version; - -#[tokio::test(flavor = "multi_thread")] -async fn v3_mixed_validators_disputes() -> Result<(), anyhow::Error> { - let _ = env_logger::try_init_from_env( - env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"), - ); - - let images = zombienet_sdk::environment::get_images_from_env(); - - let native = std::env::var("ZOMBIE_PROVIDER").as_deref() == Ok("native"); - let old_image = std::env::var("OLD_POLKADOT_IMAGE").ok(); - let old_command = std::env::var("OLD_POLKADOT_COMMAND").unwrap_or("polkadot".into()); - - // V2 (bit 3) and V3 (bit 4) enabled. - let node_features_with_v3 = json!({"bits": 8, "data": [0b00011000]}); - - let config = NetworkConfigBuilder::new() - .with_relaychain(|r| { - // westend-local so staking-based disabling takes effect. - let r = r - .with_chain("westend-local") - .with_default_command("polkadot") - .with_default_image(images.polkadot.as_str()) - .with_default_args(vec![("-lparachain=debug,runtime::staking=debug").into()]) - .with_genesis_overrides(json!({ - "configuration": { - "config": { - "scheduler_params": { - // Small groups so validators remain outside the backing group to do - // real approval checking (approval-voting insta-approves when - // `n_validators - backing_group_size < needed_approvals`). - "group_rotation_frequency": 100, - "max_validators_per_core": 3 - }, - "needed_approvals": 2, - "node_features": node_features_with_v3 - } - } - })) - // validator-0..5: v3-capable backers. - .with_validator(|node| { - node.with_name("validator-0") - .with_args(vec!["-lparachain=debug,runtime::staking=debug".into()]) - .invulnerable(false) - }); - - let r = (1..6usize).fold(r, |acc, i| { - acc.with_validator(|node| { - node.with_name(&format!("validator-{i}")) - .with_args(vec!["-lparachain=debug,runtime::staking=debug".into()]) - .invulnerable(false) - }) - }); - - // old-validator-6: pre-v3 node; never a backer, but takes part in approval voting. - (6..7usize).fold(r, |acc, i| { - acc.with_validator(|node| { - let node = node - .with_name(&format!("old-validator-{i}")) - .with_command(old_command.as_str()); - // Native ignores (and rejects) an image path; only set it for container - // providers when OLD_POLKADOT_IMAGE is given. - let node = match (native, old_image.as_deref()) { - (false, Some(img)) => node.with_image(img), - _ => node, - }; - node.invulnerable(false) - }) - }) - }) - // Para 3000: V3-capable collator ("v3" chain spec + slot-based authoring). - .with_parachain(|p| { - p.with_id(3000) - .with_default_command("test-parachain") - .with_default_image(images.cumulus.as_str()) - .with_chain("v3") - .with_default_args(vec![ - ("--authoring=slot-based").into(), - ("-lparachain=debug,aura=debug").into(), - ]) - .with_collator(|n| n.with_name("collator-3000")) - }) - // Keep the network alive on failure for post-mortem logs. - .with_global_settings(|s| s.with_tear_down_on_failure(false)) - .build() - .map_err(|e| { - let errs = e.into_iter().map(|e| e.to_string()).collect::>().join(" "); - anyhow!("config errs: {errs}") - })?; - - let spawn_fn = zombienet_sdk::environment::get_spawn_fn(); - log::info!("Spawning network"); - let network = spawn_fn(config).await?; - - let relay_node = network.get_node("validator-0")?; - let para_node = network.get_node("collator-3000")?; - let relay_client: OnlineClient = relay_node.wait_client().await?; - - log::info!("Waiting for V3 candidates to be backed by the v3 backing group"); - assert_candidates_version( - &relay_client, - CandidateDescriptorVersion::V3, - HashMap::from([(ParaId::from(3000), 5..21)]), - 20, - None, - ) - .await?; - - log::info!("Waiting for a dispute to be raised and concluded"); - let parachain_candidate_dispute_metric = "parachain_candidate_disputes_total"; - let concluded_dispute_metric = - "polkadot_parachain_candidate_dispute_concluded{validity=\"valid\"}"; - - relay_node - .wait_metric_with_timeout(parachain_candidate_dispute_metric, |d| d >= 1.0, 600_u64) - .await?; - relay_node - .wait_metric_with_timeout(concluded_dispute_metric, |d| d >= 1.0, 200_u64) - .await?; - - log::info!("Waiting for the pre-v3 disputer to be disabled"); - let mut best_blocks = relay_client.blocks().subscribe_best().await?; - let mut disabled_validators: Vec = Vec::new(); - let mut blocks_checked = 0u32; - while let Some(block) = best_blocks.next().await { - let hash = block?.hash(); - disabled_validators = Vec::::decode( - &mut &relay_client - .runtime_api() - .at(hash) - .call_raw("ParachainHost_disabled_validators", None) - .await?[..], - )?; - if !disabled_validators.is_empty() { - break; - } - blocks_checked += 1; - - if blocks_checked >= 50 { - break; - } - } - - assert!( - !disabled_validators.is_empty(), - "expected the pre-v3 disputer to be disabled after the concluded dispute (checked {blocks_checked} blocks)", - ); - log::info!("Disabled validators: {:?}", disabled_validators); - - log::info!("Verifying the parachain keeps producing blocks after the dispute"); - let para_client: OnlineClient = para_node.wait_client().await?; - let start_height = para_client.blocks().at_latest().await?.header().number; - let target_height = (start_height + 8) as f64; - para_node - .wait_metric_with_timeout( - "substrate_block_height{status=\"best\"}", - |h| h >= target_height, - 120u64, - ) - .await?; - - log::info!("Waiting for relay-chain finality to recover after the dispute"); - relay_node - .wait_metric_with_timeout( - "polkadot_parachain_approval_checking_finality_lag", - |lag| lag <= 5.0, - 180u64, - ) - .await?; - - assert_finality_lag(¶_client, 8).await?; - - log::info!("Mixed validators disputes test finished successfully"); - Ok(()) -} From 46f5f1aa311d8b90e9e02683e36e78635939c2ef Mon Sep 17 00:00:00 2001 From: Marios Christou Date: Fri, 24 Jul 2026 11:24:12 +0300 Subject: [PATCH 05/10] polish --- cumulus/test/runtime/build.rs | 3 - .../doesnt_break_parachains.rs | 1 - .../v3_old_validator_dispute_storm.rs | 227 +++++++++--------- 3 files changed, 113 insertions(+), 118 deletions(-) diff --git a/cumulus/test/runtime/build.rs b/cumulus/test/runtime/build.rs index cf0fe883523f..e305a6c0d7c6 100644 --- a/cumulus/test/runtime/build.rs +++ b/cumulus/test/runtime/build.rs @@ -75,9 +75,6 @@ fn main() { .set_file_name(v3::WASM_FILE_NAME) .build(); - // A V3-descriptor runtime with an incremented spec version, so it can be enacted as a genuine - // runtime upgrade on top of a plain (V2, spec_version 2) parachain runtime. Used by the - // old-validator dispute-storm test to flip a running parachain from V2 to V3 descriptors. WasmBuilder::init_with_defaults() .enable_feature("v3-descriptor") .enable_feature("spec-version-3") diff --git a/polkadot/zombienet-sdk-tests/tests/elastic_scaling/doesnt_break_parachains.rs b/polkadot/zombienet-sdk-tests/tests/elastic_scaling/doesnt_break_parachains.rs index bd581af4efec..4f4b300f9054 100644 --- a/polkadot/zombienet-sdk-tests/tests/elastic_scaling/doesnt_break_parachains.rs +++ b/polkadot/zombienet-sdk-tests/tests/elastic_scaling/doesnt_break_parachains.rs @@ -120,7 +120,6 @@ async fn doesnt_break_parachains_test( CandidateDescriptorVersion::V3, HashMap::from([(para_id, 12..16)]), 15, - None, ) .await?; } else { diff --git a/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs b/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs index b90b90145571..15f4c1c5585e 100644 --- a/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs +++ b/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs @@ -21,10 +21,10 @@ use crate::utils::assert_candidates_version; use anyhow::anyhow; use codec::Decode; -use cumulus_zombienet_sdk_helpers::{assert_finality_lag, wait_for_runtime_upgrade}; -use polkadot_primitives::{ - CandidateDescriptorVersion, CandidateReceiptV2, Id as ParaId, ValidatorId, ValidatorIndex, +use cumulus_zombienet_sdk_helpers::{ + assert_finality_lag, assert_para_throughput_with, wait_for_runtime_upgrade, }; +use polkadot_primitives::{CandidateDescriptorVersion, Id as ParaId, ValidatorId, ValidatorIndex}; use rstest::rstest; use serde_json::json; use std::collections::{HashMap, HashSet}; @@ -42,10 +42,6 @@ const TOTAL_VALIDATORS: usize = 15; const PARA_A: u32 = 2000; const PARA_B: u32 = 2001; -const GROUP_ROTATION: u32 = 10; - -const HEALTHY_BACKING_FLOOR: u32 = 7; - #[rstest] #[case::four_old(4)] #[case::five_old(5)] @@ -93,17 +89,11 @@ async fn v3_old_validator_dispute_storm(#[case] num_old: usize) -> Result<(), an } } })) - .with_validator(|node| { - node.with_name("validator-0") - .with_args(vec!["-lparachain=debug,runtime::staking=debug".into()]) - .invulnerable(false) - }); + .with_validator(|node| node.with_name("validator-0").invulnerable(false)); let r = (1..num_new).fold(r, |acc, i| { acc.with_validator(|node| { - node.with_name(&format!("validator-{i}")) - .with_args(vec!["-lparachain=debug,runtime::staking=debug".into()]) - .invulnerable(false) + node.with_name(&format!("validator-{i}")).invulnerable(false) }) }); @@ -111,8 +101,7 @@ async fn v3_old_validator_dispute_storm(#[case] num_old: usize) -> Result<(), an acc.with_validator(|node| { let node = node .with_name(&format!("old-validator-{i}")) - .with_command(old_command.as_str()) - .with_args(vec!["-lparachain=debug,runtime::staking=debug".into()]); + .with_command(old_command.as_str()); let node = match (native, old_image.as_deref()) { (false, Some(img)) => node.with_image(img), _ => node, @@ -175,7 +164,6 @@ async fn v3_old_validator_dispute_storm(#[case] num_old: usize) -> Result<(), an CandidateDescriptorVersion::V2, HashMap::from([(para_a, 7..10), (para_b, 7..10)]), 8, - None, ) .await?; assert!( @@ -210,31 +198,35 @@ async fn v3_old_validator_dispute_storm(#[case] num_old: usize) -> Result<(), an if num_old == 4 { assert_four_old_self_extinguishes(relay_node, &relay_client, &old_pubkeys, num_old).await?; - log::info!("waiting for dispute storm to self-extinguish"); - let total_disputes = wait_for_dispute_storm_to_settle(relay_node, 240).await?; - log::info!("storm self-extinguished at {total_disputes} disputes"); + log::info!("waiting for every raised dispute to conclude valid"); + let total_disputes = wait_for_disputes_resolved_valid(relay_node, 240).await?; + log::info!("storm resolved: all {total_disputes} disputes concluded valid, none invalid"); assert_finality_lag(¶_b_client, 6).await?; - let series = backed_series_over_best_blocks( + const RECOVERY_WINDOW: u32 = 20; + const PARA_A_FLOOR: u32 = 15; + const PARA_B_V3_FLOOR: u32 = 8; + assert_para_throughput_with( &relay_client, - GROUP_ROTATION, - &[(para_b, CandidateDescriptorVersion::V3), (para_a, CandidateDescriptorVersion::V2)], + RECOVERY_WINDOW, + HashMap::from([ + (para_b, PARA_B_V3_FLOOR..RECOVERY_WINDOW + 1), + (para_a, PARA_A_FLOOR..RECOVERY_WINDOW + 1), + ]), + |receipt| { + let para_id = receipt.descriptor.para_id(); + let version = receipt.descriptor.version(); + if para_id == para_b && version != CandidateDescriptorVersion::V3 { + return Err(anyhow!("para B backed non-V3 post-recovery: {version:?}")); + } + if para_id == para_a && version != CandidateDescriptorVersion::V2 { + return Err(anyhow!("canary para A backed non-V2: {version:?}")); + } + Ok(true) + }, ) .await?; - let b_backed: u32 = series[¶_b].iter().sum(); - let a_backed: u32 = series[¶_a].iter().sum(); - log::info!("post-recovery: para B V3={b_backed}, para A V2={a_backed} (floor {HEALTHY_BACKING_FLOOR})"); - assert!( - b_backed >= HEALTHY_BACKING_FLOOR, - "para B did not recover to a healthy V3 band: {b_backed} in {GROUP_ROTATION} blocks \ - (floor {HEALTHY_BACKING_FLOOR})", - ); - assert!( - a_backed >= HEALTHY_BACKING_FLOOR, - "canary para A degraded: {a_backed} V2 in {GROUP_ROTATION} blocks (floor \ - {HEALTHY_BACKING_FLOOR})", - ); } else { assert_five_old_persists(relay_node, &relay_client).await?; } @@ -290,17 +282,7 @@ async fn assert_five_old_persists( relay_node: &zombienet_sdk::NetworkNode, relay_client: &OnlineClient, ) -> Result<(), anyhow::Error> { - let finalized_metric = "substrate_block_height{status=\"finalized\"}"; - let f0 = read_metric(relay_node, finalized_metric).await?; - tokio::time::sleep(std::time::Duration::from_secs(60)).await; - let f1 = read_metric(relay_node, finalized_metric).await?; - let finalized_progress = f1 - f0; - log::info!("finalized over 60s: {f0} → {f1} (+{finalized_progress}, healthy ~10)"); - assert!( - finalized_progress == 0.0, - "expected finality to be degraded behind the unresolvable dispute, but finalized advanced \ - {f0} → {f1} (+{finalized_progress}; healthy would be ~10 per 60s)", - ); + assert_finality_stalls(relay_node, 120).await?; let finalized_hash = relay_client.blocks().at_latest().await?.hash(); let disabled = disabled_validators_at(relay_client, finalized_hash).await?; @@ -334,78 +316,32 @@ fn count_disabled_old( .filter(|idx| { validators .get(idx.0 as usize) - .map(|id| { - let hex = id - .clone() - .into_inner() - .0 - .iter() - .map(|b| format!("{b:02x}")) - .collect::(); - old_pubkeys.contains(&hex) - }) - .unwrap_or(false) + .is_some_and(|id| old_pubkeys.contains(&hex::encode(id.clone().into_inner().0))) }) .count() } +/// Call a raw `ParachainHost` runtime API at `hash` and SCALE-decode its result. +async fn runtime_api_decode( + relay_client: &OnlineClient, + hash: H256, + method: &str, +) -> Result { + Ok(T::decode(&mut &relay_client.runtime_api().at(hash).call_raw(method, None).await?[..])?) +} + async fn disabled_validators_at( relay_client: &OnlineClient, hash: H256, ) -> Result, anyhow::Error> { - Ok(Vec::::decode( - &mut &relay_client - .runtime_api() - .at(hash) - .call_raw("ParachainHost_disabled_validators", None) - .await?[..], - )?) + runtime_api_decode(relay_client, hash, "ParachainHost_disabled_validators").await } async fn session_validators_at( relay_client: &OnlineClient, hash: H256, ) -> Result, anyhow::Error> { - Ok(Vec::::decode( - &mut &relay_client - .runtime_api() - .at(hash) - .call_raw("ParachainHost_validators", None) - .await?[..], - )?) -} - -async fn backed_series_over_best_blocks( - relay_client: &OnlineClient, - num_blocks: u32, - targets: &[(ParaId, CandidateDescriptorVersion)], -) -> Result>, anyhow::Error> { - let mut best_blocks = relay_client.blocks().subscribe_best().await?; - let mut series: HashMap> = targets.iter().map(|(p, _)| (*p, vec![])).collect(); - let mut blocks_checked = 0u32; - while let Some(block) = best_blocks.next().await { - let events = block?.events().await?; - let mut per_block: HashMap = targets.iter().map(|(p, _)| (*p, 0)).collect(); - for event in events.iter() { - let event = event?; - if event.pallet_name() == "ParaInclusion" && event.variant_name() == "CandidateBacked" { - let receipt = CandidateReceiptV2::::decode(&mut &event.field_bytes()[..])?; - let para_id = receipt.descriptor.para_id(); - let version = receipt.descriptor.version(); - if targets.iter().any(|(p, v)| *p == para_id && *v == version) { - *per_block.get_mut(¶_id).expect("para_id is a target; qed") += 1; - } - } - } - for (para_id, count) in per_block { - series.get_mut(¶_id).expect("para_id is a target; qed").push(count); - } - blocks_checked += 1; - if blocks_checked >= num_blocks { - break; - } - } - Ok(series) + runtime_api_decode(relay_client, hash, "ParachainHost_validators").await } async fn read_metric( @@ -417,26 +353,89 @@ async fn read_metric( .map_err(|e| anyhow!("failed to read metric {metric}: {e}")) } -async fn wait_for_dispute_storm_to_settle( +async fn assert_finality_stalls( node: &zombienet_sdk::NetworkNode, timeout_secs: u64, -) -> Result { - const STEP_SECS: u64 = 25; - let metric = "parachain_candidate_disputes_total"; +) -> Result<(), anyhow::Error> { + const STEP_SECS: u64 = 20; + let metric = "substrate_block_height{status=\"finalized\"}"; let mut prev = read_metric(node, metric).await?; let mut elapsed = 0u64; loop { tokio::time::sleep(std::time::Duration::from_secs(STEP_SECS)).await; elapsed += STEP_SECS; let cur = read_metric(node, metric).await?; - log::info!("dispute counter {prev} → {cur} after {elapsed}s"); - if cur - prev == 0.0 { - return Ok(cur); + log::info!( + "finalized {prev} → {cur} after {elapsed}s (waiting for the dispute to freeze it)" + ); + if cur == prev { + tokio::time::sleep(std::time::Duration::from_secs(STEP_SECS)).await; + let after = read_metric(node, metric).await?; + assert!( + after == cur, + "finality resumed after appearing to stall ({cur} → {after}); the unresolved \ + dispute should keep it frozen in the 5/15 case", + ); + log::info!("finality frozen at {cur} (degraded as expected)"); + return Ok(()); } prev = cur; if elapsed >= timeout_secs { return Err(anyhow!( - "dispute storm did not settle within {timeout_secs}s (still growing, last {cur})" + "finality kept advancing (last {cur}) for {timeout_secs}s; expected the unresolved \ + dispute to freeze it in the 5/15 case", + )); + } + } +} + +/// Wait for the dispute storm to fully resolve: every raised dispute must conclude *valid* (the +/// honest supermajority wins) and none may conclude *invalid*. +async fn wait_for_disputes_resolved_valid( + node: &zombienet_sdk::NetworkNode, + timeout_secs: u64, +) -> Result { + const STEP_SECS: u64 = 20; + let total_metric = "parachain_candidate_disputes_total"; + let valid_metric = "polkadot_parachain_candidate_dispute_concluded{validity=\"valid\"}"; + let invalid_metric = "polkadot_parachain_candidate_dispute_concluded{validity=\"invalid\"}"; + let mut elapsed = 0u64; + loop { + let total = read_metric(node, total_metric).await?; + let valid = read_metric(node, valid_metric).await?; + // The invalid-conclusion counter is absent until first incremented; treat absent as zero. + let invalid = read_metric(node, invalid_metric).await.unwrap_or(0.0); + log::info!( + "disputes after {elapsed}s: total={total}, concluded_valid={valid}, \ + concluded_invalid={invalid}" + ); + assert!( + invalid == 0.0, + "a dispute concluded invalid ({invalid}); the honest supermajority must never lose", + ); + + if total >= 1.0 && valid >= total { + // All raised disputes concluded valid. Confirm no new dispute is still in flight. + tokio::time::sleep(std::time::Duration::from_secs(STEP_SECS)).await; + elapsed += STEP_SECS; + let total_after = read_metric(node, total_metric).await?; + let invalid_after = read_metric(node, invalid_metric).await.unwrap_or(0.0); + assert!( + invalid_after == 0.0, + "a dispute concluded invalid ({invalid_after}); the honest supermajority must never lose", + ); + if total_after <= total { + return Ok(total_after); + } + } else { + tokio::time::sleep(std::time::Duration::from_secs(STEP_SECS)).await; + elapsed += STEP_SECS; + } + + if elapsed >= timeout_secs { + return Err(anyhow!( + "disputes did not all conclude valid within {timeout_secs}s \ + (total={total}, concluded_valid={valid})" )); } } From f64f12b2c19bee17a4eb9ea295b9f79c7c2caa43 Mon Sep 17 00:00:00 2001 From: Marios Christou Date: Mon, 27 Jul 2026 11:39:01 +0300 Subject: [PATCH 06/10] test with v2 and v3 collator on same para --- .../zombienet_polkadot_tests.yml | 5 +++ .../tests/functional/v3_rolling_upgrade.rs | 45 ++++++++++++++++--- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/.github/zombienet-tests/zombienet_polkadot_tests.yml b/.github/zombienet-tests/zombienet_polkadot_tests.yml index e493d1a52f6b..b04f6338a461 100644 --- a/.github/zombienet-tests/zombienet_polkadot_tests.yml +++ b/.github/zombienet-tests/zombienet_polkadot_tests.yml @@ -249,6 +249,9 @@ echo "downloading polkadot as polkadot-old in $BIN_DIR" curl --retry 5 --retry-delay 10 --retry-max-time 120 -L -o $BIN_DIR/polkadot-old https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-stable2512/polkadot chmod 755 $BIN_DIR/polkadot-old + echo "downloading polkadot-parachain as polkadot-parachain-old in $BIN_DIR" + curl --retry 5 --retry-delay 10 --retry-max-time 120 -L -o $BIN_DIR/polkadot-parachain-old https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-stable2603-4/polkadot-parachain + chmod 755 $BIN_DIR/polkadot-parachain-old for bin in polkadot-execute-worker polkadot-prepare-worker; do echo "downloading $bin in $BIN_DIR" curl --retry 5 --retry-delay 10 --retry-max-time 120 -L -o $BIN_DIR/$bin https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-stable2512/$bin @@ -260,6 +263,8 @@ additional-env: OLD_POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:master-187cddde" OLD_POLKADOT_COMMAND: "polkadot-old" + OLD_PARACHAIN_COMMAND: "polkadot-parachain-old" + OLD_PARACHAIN_IMAGE: "docker.io/parity/polkadot-parachain:stable2603-4" - job-name: "zombienet-polkadot-functional-v3-old-validator-dispute-storm" test-filter: "functional::v3_old_validator_dispute_storm::v3_old_validator_dispute_storm" diff --git a/polkadot/zombienet-sdk-tests/tests/functional/v3_rolling_upgrade.rs b/polkadot/zombienet-sdk-tests/tests/functional/v3_rolling_upgrade.rs index bc2390b4dbca..d52e1f6a56bb 100644 --- a/polkadot/zombienet-sdk-tests/tests/functional/v3_rolling_upgrade.rs +++ b/polkadot/zombienet-sdk-tests/tests/functional/v3_rolling_upgrade.rs @@ -1,14 +1,22 @@ // Copyright (C) Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 -//! Rolling upgrade test: mixed V2/V3 validator fleet. +//! Rolling upgrade test: mixed V2/V3 validator *and* collator fleets. //! //! Runs a network where some validators use the current binary (V3-capable) and others use an -//! older binary (`OLD_POLKADOT_IMAGE`) that does not understand V3 descriptors. V3 is enabled -//! at the runtime level. The collator produces V2 candidates throughout. +//! older binary (`OLD_POLKADOT_IMAGE`) that does not understand V3 descriptors. The V3 +//! `CandidateReceiptV3` node feature is enabled on the relay chain, but the parachain runtime has +//! V3 scheduling **disabled** (`async-backing`). +//! +//! The parachain is served by a mixed collator fleet, both authoring slot-based: +//! - a **V3-capable** collator (the current `test-parachain`), and +//! - a **V2-only** collator (an older `polkadot-parachain` release, predating V3, supplied via +//! `OLD_PARACHAIN_COMMAND` / `OLD_PARACHAIN_IMAGE`). //! //! Verifies that: -//! - V2 candidates are backed by the mixed fleet. +//! - V2 candidates are backed by the mixed validator fleet, from both collators. +//! - A V3-capable collator does not emit V3 to a V3-disabled parachain (a V3 candidate would be +//! rejected by `validate_block`, collapsing throughput). //! - Statement and availability distribution work across binary versions. //! - GRANDPA finality does not stall. //! - Parachain throughput is sustained. @@ -38,6 +46,13 @@ async fn v3_rolling_upgrade() -> Result<(), anyhow::Error> { .expect("OLD_POLKADOT_IMAGE must be set for rolling upgrade test"); let old_command = std::env::var("OLD_POLKADOT_COMMAND").unwrap_or("polkadot".into()); + // Old, V2-only `polkadot-parachain` (predates the cumulus V3 changes, PR #10742 / first in + // `polkadot-stable2606`). Under the native provider only the command (resolved from PATH) + // matters; the image is used by the k8s provider. + let old_parachain_command = + std::env::var("OLD_PARACHAIN_COMMAND").unwrap_or_else(|_| "polkadot-parachain-old".into()); + let old_parachain_image = std::env::var("OLD_PARACHAIN_IMAGE").ok(); + let config = NetworkConfigBuilder::new() .with_relaychain(|r| { let r = r @@ -70,7 +85,8 @@ async fn v3_rolling_upgrade() -> Result<(), anyhow::Error> { }) }) .with_parachain(|p| { - p.with_id(3000) + let p = p + .with_id(3000) .with_default_command("test-parachain") .with_default_image(images.cumulus.as_str()) .with_chain("async-backing") @@ -78,7 +94,22 @@ async fn v3_rolling_upgrade() -> Result<(), anyhow::Error> { ("--authoring=slot-based").into(), ("-lparachain=debug,aura=debug").into(), ]) - .with_collator(|n| n.with_name("collator-3000")) + // V3-capable collator (current binary); on a V3-disabled para it emits V2. + .with_collator(|n| n.with_name("collator-3000")); + // V2-only collator: an older `polkadot-parachain` that predates V3. + p.with_collator(|n| { + let n = n + .with_name("old-collator-3000") + .with_command(old_parachain_command.as_str()) + .with_args(vec![ + ("--authoring=slot-based").into(), + ("-lparachain=debug,aura=debug").into(), + ]); + match old_parachain_image.as_deref() { + Some(img) => n.with_image(img), + None => n, + } + }) }) .build() .map_err(|e| { @@ -91,6 +122,7 @@ async fn v3_rolling_upgrade() -> Result<(), anyhow::Error> { let relay_node = network.get_node("validator-0")?; let para_node = network.get_node("collator-3000")?; + let old_para_node = network.get_node("old-collator-3000")?; let relay_client: OnlineClient = relay_node.wait_client().await?; // enabling v3 here does not overwrite all node_features @@ -127,6 +159,7 @@ async fn v3_rolling_upgrade() -> Result<(), anyhow::Error> { } assert_finality_lag(¶_node.wait_client().await?, 6).await?; + assert_finality_lag(&old_para_node.wait_client().await?, 6).await?; log::info!("Rolling upgrade test finished successfully"); From 428b150155f8f32326fc01cedea7727dc11b99df Mon Sep 17 00:00:00 2001 From: Marios Christou Date: Mon, 27 Jul 2026 15:07:39 +0300 Subject: [PATCH 07/10] changes --- .../zombienet_polkadot_tests.yml | 23 --- .../doesnt_break_parachains.rs | 7 +- .../v3_old_validator_dispute_storm.rs | 195 +++++++++++------- .../tests/functional/v3_rolling_upgrade.rs | 25 ++- 4 files changed, 145 insertions(+), 105 deletions(-) diff --git a/.github/zombienet-tests/zombienet_polkadot_tests.yml b/.github/zombienet-tests/zombienet_polkadot_tests.yml index b04f6338a461..c602c7119928 100644 --- a/.github/zombienet-tests/zombienet_polkadot_tests.yml +++ b/.github/zombienet-tests/zombienet_polkadot_tests.yml @@ -265,26 +265,3 @@ OLD_POLKADOT_COMMAND: "polkadot-old" OLD_PARACHAIN_COMMAND: "polkadot-parachain-old" OLD_PARACHAIN_IMAGE: "docker.io/parity/polkadot-parachain:stable2603-4" - -- job-name: "zombienet-polkadot-functional-v3-old-validator-dispute-storm" - test-filter: "functional::v3_old_validator_dispute_storm::v3_old_validator_dispute_storm" - runner-type: "default" - use-zombienet-sdk: true - cumulus-image: "test-parachain" - additional-setup: | - BIN_DIR="$(pwd)/bin_old" - mkdir -p $BIN_DIR - echo "downloading polkadot as polkadot-old in $BIN_DIR" - curl --retry 5 --retry-delay 10 --retry-max-time 120 -L -o $BIN_DIR/polkadot-old https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-stable2512/polkadot - chmod 755 $BIN_DIR/polkadot-old - for bin in polkadot-execute-worker polkadot-prepare-worker; do - echo "downloading $bin in $BIN_DIR" - curl --retry 5 --retry-delay 10 --retry-max-time 120 -L -o $BIN_DIR/$bin https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-stable2512/$bin - chmod 755 $BIN_DIR/$bin - done - ls -ltr $BIN_DIR - export PATH=$BIN_DIR:$PATH - echo "PATH=$PATH" >> $GITHUB_ENV - additional-env: - OLD_POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:master-187cddde" - OLD_POLKADOT_COMMAND: "polkadot-old" diff --git a/polkadot/zombienet-sdk-tests/tests/elastic_scaling/doesnt_break_parachains.rs b/polkadot/zombienet-sdk-tests/tests/elastic_scaling/doesnt_break_parachains.rs index 4f4b300f9054..42c7fa20e23c 100644 --- a/polkadot/zombienet-sdk-tests/tests/elastic_scaling/doesnt_break_parachains.rs +++ b/polkadot/zombienet-sdk-tests/tests/elastic_scaling/doesnt_break_parachains.rs @@ -19,11 +19,10 @@ use zombienet_sdk::{ }; #[rstest] -#[case::v2("test-parachain", None, false)] -#[case::v3("test-parachain", Some("v3"), true)] +#[case::v2(None, false)] +#[case::v3(Some("v3"), true)] #[tokio::test(flavor = "multi_thread")] async fn doesnt_break_parachains_test( - #[case] collator_command: &str, #[case] collator_chain: Option<&str>, #[case] use_v3: bool, ) -> Result<(), anyhow::Error> { @@ -84,7 +83,7 @@ async fn doesnt_break_parachains_test( .with_parachain(|p| { let p = p .with_id(2000) - .with_default_command(collator_command) + .with_default_command("test-parachain") .with_default_image(images.cumulus.as_str()) .with_default_args(collator_args); let p = match collator_chain { diff --git a/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs b/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs index 15f4c1c5585e..9103e0130ee6 100644 --- a/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs +++ b/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs @@ -168,11 +168,17 @@ async fn v3_old_validator_dispute_storm(#[case] num_old: usize) -> Result<(), an .await?; assert!( relay_node - .assert_with("parachain_candidate_disputes_total", |d| d == 0.0) + .assert_with("polkadot_parachain_candidate_disputes_total", |d| d == 0.0) .await?, "no disputes expected before the para upgrade" ); + // Capture finalized height before the upgrade so assert_finality_stalls (5/15 path) can + // require real forward progress before declaring a stall. + let baseline_finalized = + read_metric(relay_node, "substrate_block_height{status=\"finalized\"}").await?; + log::info!("baseline relay finalized height (before para-B upgrade): {baseline_finalized}"); + log::info!("upgrading para B to the v3 runtime"); let upgrade_call = dynamic( "Sudo", @@ -192,43 +198,46 @@ async fn v3_old_validator_dispute_storm(#[case] num_old: usize) -> Result<(), an log::info!("waiting for dispute storm to start"); relay_node - .wait_metric_with_timeout("parachain_candidate_disputes_total", |d| d >= 1.0, 300u64) - .await?; - - if num_old == 4 { - assert_four_old_self_extinguishes(relay_node, &relay_client, &old_pubkeys, num_old).await?; - - log::info!("waiting for every raised dispute to conclude valid"); - let total_disputes = wait_for_disputes_resolved_valid(relay_node, 240).await?; - log::info!("storm resolved: all {total_disputes} disputes concluded valid, none invalid"); - - assert_finality_lag(¶_b_client, 6).await?; - - const RECOVERY_WINDOW: u32 = 20; - const PARA_A_FLOOR: u32 = 15; - const PARA_B_V3_FLOOR: u32 = 8; - assert_para_throughput_with( - &relay_client, - RECOVERY_WINDOW, - HashMap::from([ - (para_b, PARA_B_V3_FLOOR..RECOVERY_WINDOW + 1), - (para_a, PARA_A_FLOOR..RECOVERY_WINDOW + 1), - ]), - |receipt| { - let para_id = receipt.descriptor.para_id(); - let version = receipt.descriptor.version(); - if para_id == para_b && version != CandidateDescriptorVersion::V3 { - return Err(anyhow!("para B backed non-V3 post-recovery: {version:?}")); - } - if para_id == para_a && version != CandidateDescriptorVersion::V2 { - return Err(anyhow!("canary para A backed non-V2: {version:?}")); - } - Ok(true) - }, + .wait_metric_with_timeout( + "polkadot_parachain_candidate_disputes_total", + |d| d >= 1.0, + 300u64, ) .await?; - } else { - assert_five_old_persists(relay_node, &relay_client).await?; + + match num_old { + 4 => { + assert_four_old_self_extinguishes(relay_node, &relay_client, &old_pubkeys).await?; + + log::info!("waiting for every raised dispute to conclude valid"); + let total_disputes = wait_for_disputes_resolved_valid(relay_node, 300).await?; + log::info!( + "storm resolved: all {total_disputes} disputes concluded valid, none invalid" + ); + + assert_finality_lag(¶_b_client, 6).await?; + assert_para_throughput_with( + &relay_client, + 20, + HashMap::from([(para_b, 15..21), (para_a, 15..21)]), + |receipt| { + let para_id = receipt.descriptor.para_id(); + let version = receipt.descriptor.version(); + if para_id == para_b && version != CandidateDescriptorVersion::V3 { + return Err(anyhow!("para B backed non-V3 post-recovery: {version:?}")); + } + if para_id == para_a && version != CandidateDescriptorVersion::V2 { + return Err(anyhow!("canary para A backed non-V2: {version:?}")); + } + Ok(true) + }, + ) + .await?; + }, + 5 => { + assert_five_old_persists(relay_node, &relay_client, baseline_finalized).await?; + }, + other => unreachable!("unexpected num_old: {other}"), } log::info!("V3 old-validator dispute storm test ({num_old}/{TOTAL_VALIDATORS} old) finished"); @@ -239,8 +248,8 @@ async fn assert_four_old_self_extinguishes( relay_node: &zombienet_sdk::NetworkNode, relay_client: &OnlineClient, old_pubkeys: &HashSet, - num_old: usize, ) -> Result<(), anyhow::Error> { + const NUM_OLD: usize = 4; log::info!("waiting for disputes to conclude valid"); relay_node .wait_metric_with_timeout( @@ -250,7 +259,7 @@ async fn assert_four_old_self_extinguishes( ) .await?; - log::info!("waiting for all {num_old} pre-v3 validators to be disabled"); + log::info!("waiting for all {NUM_OLD} pre-v3 validators to be disabled"); let mut best_blocks = relay_client.blocks().subscribe_best().await?; let mut disabled_old = 0usize; let mut blocks_checked = 0u32; @@ -261,7 +270,7 @@ async fn assert_four_old_self_extinguishes( let validators = session_validators_at(relay_client, hash).await?; disabled_old = count_disabled_old(&disabled, &validators, old_pubkeys); log::info!("disabled {} validators, {disabled_old} of them pre-v3", disabled.len()); - if disabled_old >= num_old { + if disabled_old >= NUM_OLD { break; } } @@ -271,8 +280,8 @@ async fn assert_four_old_self_extinguishes( } } assert!( - disabled_old >= num_old, - "expected all {num_old} pre-v3 validators disabled, got {disabled_old} after \ + disabled_old >= NUM_OLD, + "expected all {NUM_OLD} pre-v3 validators disabled, got {disabled_old} after \ {blocks_checked} blocks", ); Ok(()) @@ -281,14 +290,18 @@ async fn assert_four_old_self_extinguishes( async fn assert_five_old_persists( relay_node: &zombienet_sdk::NetworkNode, relay_client: &OnlineClient, + baseline_finalized: f64, ) -> Result<(), anyhow::Error> { - assert_finality_stalls(relay_node, 120).await?; + assert_finality_stalls(relay_node, 120, baseline_finalized).await?; - let finalized_hash = relay_client.blocks().at_latest().await?.hash(); - let disabled = disabled_validators_at(relay_client, finalized_hash).await?; + // `at_latest()` returns the best imported block, not the finalized head; in the 5/15 case + // they diverge by design. We check for disabled validators at the best block, which is the + // correct thing to check here. + let best_hash = relay_client.blocks().at_latest().await?.hash(); + let disabled = disabled_validators_at(relay_client, best_hash).await?; assert!( disabled.is_empty(), - "no validator should be disabled in the 5/15 case, found {disabled:?}", + "no validator should be disabled in the 5/15 case (at best block), found {disabled:?}", ); for validity in ["valid", "invalid"] { @@ -356,11 +369,40 @@ async fn read_metric( async fn assert_finality_stalls( node: &zombienet_sdk::NetworkNode, timeout_secs: u64, + baseline_finalized: f64, ) -> Result<(), anyhow::Error> { const STEP_SECS: u64 = 20; + // Require finality to have advanced at least this many blocks from the pre-upgrade baseline + // before we start watching for a stall. This guards against a vacuous pass if the function + // is entered before GRANDPA has had a chance to advance (slow pod, subsystem init). + const ADVANCE_MARGIN: f64 = 3.0; let metric = "substrate_block_height{status=\"finalized\"}"; - let mut prev = read_metric(node, metric).await?; let mut elapsed = 0u64; + + // wait for finality to advance past baseline + margin, proving GRANDPA was running. + loop { + let cur = read_metric(node, metric).await?; + if cur >= baseline_finalized + ADVANCE_MARGIN { + log::info!( + "finality at {cur} ({:.0} blocks past baseline {baseline_finalized}); \ + now watching for stall", + cur - baseline_finalized + ); + break; + } + tokio::time::sleep(std::time::Duration::from_secs(STEP_SECS)).await; + elapsed += STEP_SECS; + if elapsed >= timeout_secs { + return Err(anyhow!( + "finality at {cur} never advanced {ADVANCE_MARGIN} blocks above baseline \ + {baseline_finalized} within {timeout_secs}s; expected GRANDPA to make progress \ + before the dispute storm clamps it", + )); + } + } + + // look for the stall — two consecutive equal samples confirm finality is frozen. + let mut prev = read_metric(node, metric).await?; loop { tokio::time::sleep(std::time::Duration::from_secs(STEP_SECS)).await; elapsed += STEP_SECS; @@ -391,51 +433,64 @@ async fn assert_finality_stalls( /// Wait for the dispute storm to fully resolve: every raised dispute must conclude *valid* (the /// honest supermajority wins) and none may conclude *invalid*. +/// +/// A single quiet sample is not enough. Para B keeps authoring V3 candidates the whole time, so +/// the pre-v3 validators keep raising fresh disputes until they are disabled and the last of their +/// candidates drains out. The storm only counts as extinguished once the *raised* count has stopped +/// growing — with everything raised so far concluded valid — for a sustained `QUIET_WINDOW_SECS`. async fn wait_for_disputes_resolved_valid( node: &zombienet_sdk::NetworkNode, timeout_secs: u64, ) -> Result { const STEP_SECS: u64 = 20; - let total_metric = "parachain_candidate_disputes_total"; + // How long the raised count must stay flat before we believe the storm is over. + const QUIET_WINDOW_SECS: u64 = 120; + const QUIET_STEPS: u32 = (QUIET_WINDOW_SECS / STEP_SECS) as u32; + + let total_metric = "polkadot_parachain_candidate_disputes_total"; let valid_metric = "polkadot_parachain_candidate_dispute_concluded{validity=\"valid\"}"; let invalid_metric = "polkadot_parachain_candidate_dispute_concluded{validity=\"invalid\"}"; + let mut elapsed = 0u64; + let mut quiet_steps = 0u32; + let mut prev_total = f64::NEG_INFINITY; + loop { let total = read_metric(node, total_metric).await?; let valid = read_metric(node, valid_metric).await?; - // The invalid-conclusion counter is absent until first incremented; treat absent as zero. - let invalid = read_metric(node, invalid_metric).await.unwrap_or(0.0); - log::info!( - "disputes after {elapsed}s: total={total}, concluded_valid={valid}, \ - concluded_invalid={invalid}" - ); + let invalid = read_metric(node, invalid_metric).await?; assert!( invalid == 0.0, "a dispute concluded invalid ({invalid}); the honest supermajority must never lose", ); - if total >= 1.0 && valid >= total { - // All raised disputes concluded valid. Confirm no new dispute is still in flight. - tokio::time::sleep(std::time::Duration::from_secs(STEP_SECS)).await; - elapsed += STEP_SECS; - let total_after = read_metric(node, total_metric).await?; - let invalid_after = read_metric(node, invalid_metric).await.unwrap_or(0.0); - assert!( - invalid_after == 0.0, - "a dispute concluded invalid ({invalid_after}); the honest supermajority must never lose", - ); - if total_after <= total { - return Ok(total_after); - } + // A step is quiet when nothing new was raised since the previous sample and everything + // raised so far has concluded valid. Any new dispute resets the window. + if total >= 1.0 && valid >= total && total <= prev_total { + quiet_steps += 1; } else { - tokio::time::sleep(std::time::Duration::from_secs(STEP_SECS)).await; - elapsed += STEP_SECS; + quiet_steps = 0; + } + prev_total = total; + + log::info!( + "disputes after {elapsed}s: total={total}, concluded_valid={valid}, \ + concluded_invalid={invalid}, quiet {}/{QUIET_STEPS}", + quiet_steps + ); + + if quiet_steps >= QUIET_STEPS { + return Ok(total); } + tokio::time::sleep(std::time::Duration::from_secs(STEP_SECS)).await; + elapsed += STEP_SECS; + if elapsed >= timeout_secs { return Err(anyhow!( - "disputes did not all conclude valid within {timeout_secs}s \ - (total={total}, concluded_valid={valid})" + "disputes did not settle within {timeout_secs}s (total={total}, \ + concluded_valid={valid}, quiet for {}s of the required {QUIET_WINDOW_SECS}s)", + u64::from(quiet_steps) * STEP_SECS )); } } diff --git a/polkadot/zombienet-sdk-tests/tests/functional/v3_rolling_upgrade.rs b/polkadot/zombienet-sdk-tests/tests/functional/v3_rolling_upgrade.rs index d52e1f6a56bb..5bb6e9d87203 100644 --- a/polkadot/zombienet-sdk-tests/tests/functional/v3_rolling_upgrade.rs +++ b/polkadot/zombienet-sdk-tests/tests/functional/v3_rolling_upgrade.rs @@ -8,15 +8,11 @@ //! `CandidateReceiptV3` node feature is enabled on the relay chain, but the parachain runtime has //! V3 scheduling **disabled** (`async-backing`). //! -//! The parachain is served by a mixed collator fleet, both authoring slot-based: +//! The parachain is served by a mixed collator fleet: //! - a **V3-capable** collator (the current `test-parachain`), and //! - a **V2-only** collator (an older `polkadot-parachain` release, predating V3, supplied via //! `OLD_PARACHAIN_COMMAND` / `OLD_PARACHAIN_IMAGE`). //! -//! Verifies that: -//! - V2 candidates are backed by the mixed validator fleet, from both collators. -//! - A V3-capable collator does not emit V3 to a V3-disabled parachain (a V3 candidate would be -//! rejected by `validate_block`, collapsing throughput). //! - Statement and availability distribution work across binary versions. //! - GRANDPA finality does not stall. //! - Parachain throughput is sustained. @@ -46,9 +42,7 @@ async fn v3_rolling_upgrade() -> Result<(), anyhow::Error> { .expect("OLD_POLKADOT_IMAGE must be set for rolling upgrade test"); let old_command = std::env::var("OLD_POLKADOT_COMMAND").unwrap_or("polkadot".into()); - // Old, V2-only `polkadot-parachain` (predates the cumulus V3 changes, PR #10742 / first in - // `polkadot-stable2606`). Under the native provider only the command (resolved from PATH) - // matters; the image is used by the k8s provider. + // Old, V2-only `polkadot-parachain` let old_parachain_command = std::env::var("OLD_PARACHAIN_COMMAND").unwrap_or_else(|_| "polkadot-parachain-old".into()); let old_parachain_image = std::env::var("OLD_PARACHAIN_IMAGE").ok(); @@ -161,6 +155,21 @@ async fn v3_rolling_upgrade() -> Result<(), anyhow::Error> { assert_finality_lag(¶_node.wait_client().await?, 6).await?; assert_finality_lag(&old_para_node.wait_client().await?, 6).await?; + for (name, node) in [("collator-3000", ¶_node), ("old-collator-3000", &old_para_node)] { + node.wait_metric_with_timeout( + "substrate_proposer_block_constructed_count", + |v| v >= 1.0, + 30u64, + ) + .await + .map_err(|e| { + anyhow!( + "Collator {name} did not author any parachain blocks \ + (metric substrate_proposer_block_constructed_count): {e}" + ) + })?; + } + log::info!("Rolling upgrade test finished successfully"); Ok(()) From 3e5b1e9c180521449bae6e4be54b818945313d05 Mon Sep 17 00:00:00 2001 From: Marios Christou Date: Tue, 28 Jul 2026 14:28:56 +0300 Subject: [PATCH 08/10] fix --- .../v3_old_validator_dispute_storm.rs | 343 +++++++++++++++--- 1 file changed, 295 insertions(+), 48 deletions(-) diff --git a/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs b/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs index 9103e0130ee6..aafb1fd1da0d 100644 --- a/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs +++ b/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs @@ -7,8 +7,8 @@ //! supermajority, which is what the two variants pivot on: //! //! * `4/15` old: the valid side reaches 11 → disputes conclude valid → the 4 old validators are -//! disabled (4 is the disabling cap for n = 15) → the storm self-extinguishes. Proves the safety -//! net holds. +//! disabled (4 is the disabling cap for n = 15). Proves the safety net holds. +//! //! * `5/15` old: the valid side maxes out at 10 < 11, so no dispute ever concludes and nobody is //! disabled. But 5 invalid votes exceed the byzantine threshold, reverting each disputed block //! and clamping GRANDPA to the undisputed chain → finality is degraded. Documents the boundary: @@ -24,14 +24,17 @@ use codec::Decode; use cumulus_zombienet_sdk_helpers::{ assert_finality_lag, assert_para_throughput_with, wait_for_runtime_upgrade, }; -use polkadot_primitives::{CandidateDescriptorVersion, Id as ParaId, ValidatorId, ValidatorIndex}; +use polkadot_primitives::{ + CandidateDescriptorVersion, CandidateReceiptV2, CoreIndex, GroupRotationInfo, Id as ParaId, + ValidatorId, ValidatorIndex, +}; use rstest::rstest; use serde_json::json; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use zombienet_sdk::{ subxt::{ - dynamic::Value, ext::scale_value::value, tx::dynamic, utils::H256, OnlineClient, - PolkadotConfig, + blocks::Block, dynamic::Value, ext::scale_value::value, tx::dynamic, utils::H256, + OnlineClient, PolkadotConfig, }, subxt_signer::sr25519::dev, NetworkConfigBuilder, @@ -42,6 +45,13 @@ const TOTAL_VALIDATORS: usize = 15; const PARA_A: u32 = 2000; const PARA_B: u32 = 2001; +/// Relay blocks to measure para B over once a capable group holds its core. Must stay below +/// `group_rotation_frequency` (10) so the whole window falls inside one group assignment. +const PARA_B_WINDOW: u32 = 6; + +/// How many relay blocks to spend looking for a capable group before giving up. +const PARA_B_SEARCH_LIMIT: u32 = 150; + #[rstest] #[case::four_old(4)] #[case::five_old(5)] @@ -200,26 +210,24 @@ async fn v3_old_validator_dispute_storm(#[case] num_old: usize) -> Result<(), an relay_node .wait_metric_with_timeout( "polkadot_parachain_candidate_disputes_total", - |d| d >= 1.0, + |d| d >= 4.0, 300u64, ) .await?; match num_old { 4 => { - assert_four_old_self_extinguishes(relay_node, &relay_client, &old_pubkeys).await?; + assert_four_old_disabled(relay_node, &relay_client, &old_pubkeys).await?; - log::info!("waiting for every raised dispute to conclude valid"); - let total_disputes = wait_for_disputes_resolved_valid(relay_node, 300).await?; - log::info!( - "storm resolved: all {total_disputes} disputes concluded valid, none invalid" - ); + log::info!("waiting for every dispute raised so far to conclude valid"); + let total_disputes = wait_for_pending_disputes_resolved_valid(relay_node, 300).await?; + log::info!("all {total_disputes} disputes raised so far concluded valid, none invalid"); assert_finality_lag(¶_b_client, 6).await?; assert_para_throughput_with( &relay_client, - 20, - HashMap::from([(para_b, 15..21), (para_a, 15..21)]), + 50, + HashMap::from([(para_b, 20..51), (para_a, 45..51)]), |receipt| { let para_id = receipt.descriptor.para_id(); let version = receipt.descriptor.version(); @@ -244,7 +252,7 @@ async fn v3_old_validator_dispute_storm(#[case] num_old: usize) -> Result<(), an Ok(()) } -async fn assert_four_old_self_extinguishes( +async fn assert_four_old_disabled( relay_node: &zombienet_sdk::NetworkNode, relay_client: &OnlineClient, old_pubkeys: &HashSet, @@ -315,6 +323,254 @@ async fn assert_five_old_persists( Ok(()) } +/// What para B's backing pipeline looks like at one relay block. +struct BackingContext { + /// Group membership is redrawn here; used to notice a reshuffle mid-window. + session_start: u32, + group_index: u32, + group_size: usize, + /// Group members that can actually back a V3 candidate: neither pre-v3 nor disabled. + usable: usize, + min_backing: usize, + blocks_left_in_rotation: u32, +} + +impl BackingContext { + fn is_capable(&self) -> bool { + self.usable >= self.min_backing + } +} + +/// Measure para B's backing rate while a group that can actually back it holds its core. +/// +/// A pre-v3 validator cannot validate a V3 candidate at all — `validate_block` panics on the +/// missing V3 extension — so it is dead weight in a backing group, whether or not it has been +/// disabled yet. +/// +/// That makes para B's candidate count over a fixed window a *draw*, not a rate. Group membership +/// is redrawn from on-chain randomness at every session change, and the group-to-core assignment +/// rotates every `group_rotation_frequency` blocks, so a 20-block window samples only about two +/// assignments. Observed runs produced 4 and 9 candidates in 20 blocks while para A produced 20 in +/// both — same storm, same disabled validators, just a different shuffle. +/// +/// So rather than widening the bound until it asserts nothing, condition on the layout: wait for a +/// capable group with room left in its rotation, then require para B to be backed in essentially +/// every block of the window. This terminates because `num_old` pre-v3 validators can starve at +/// most `num_old / (group_size - min_backing + 1)` groups — 2 of 5 here — so a capable group always +/// comes round. +async fn assert_para_b_backed_under_capable_group( + relay_client: &OnlineClient, + para_b: ParaId, + old_pubkeys: &HashSet, +) -> Result<(), anyhow::Error> { + struct Window { + session_start: u32, + group_index: u32, + counted: u32, + backed: u32, + } + + log::info!("looking for a v3-capable backing group on para B's core"); + let mut blocks_sub = relay_client.blocks().subscribe_finalized().await?; + let mut window: Option = None; + let mut searched = 0u32; + + while let Some(block) = blocks_sub.next().await { + let block = block?; + let number = block.number(); + + // Session changes redraw group membership and never carry backed candidates. + if is_session_change(&block).await? { + if window.take().is_some() { + log::info!("session change at relay block {number}, restarting the measurement"); + } + continue; + } + + let Some(ctx) = + para_b_backing_context(relay_client, block.hash(), para_b, old_pubkeys).await? + else { + log::debug!("relay block {number}: para B has no core assigned"); + continue; + }; + + if window.is_none() { + searched += 1; + if searched > PARA_B_SEARCH_LIMIT { + return Err(anyhow!( + "no v3-capable group held para B's core for {PARA_B_WINDOW} consecutive blocks \ + within {PARA_B_SEARCH_LIMIT} relay blocks; at most 2 of 5 groups should be \ + starved by 4 pre-v3 validators, so this points at a real backing failure", + )); + } + if !ctx.is_capable() || ctx.blocks_left_in_rotation < PARA_B_WINDOW { + log::debug!( + "relay block {number}: group {} has {}/{} usable backers (need {}), \ + {} blocks left in rotation - not measuring", + ctx.group_index, + ctx.usable, + ctx.group_size, + ctx.min_backing, + ctx.blocks_left_in_rotation, + ); + continue; + } + log::info!( + "relay block {number}: group {} on para B's core has {}/{} usable backers \ + (need {}), {} blocks left in rotation - measuring {PARA_B_WINDOW} blocks", + ctx.group_index, + ctx.usable, + ctx.group_size, + ctx.min_backing, + ctx.blocks_left_in_rotation, + ); + window = Some(Window { + session_start: ctx.session_start, + group_index: ctx.group_index, + counted: 0, + backed: 0, + }); + } + + let w = window.as_mut().expect("set directly above or on an earlier iteration; qed"); + + // The layout must not shift under us, or we are no longer measuring what we selected for. + if ctx.session_start != w.session_start || + ctx.group_index != w.group_index || + !ctx.is_capable() + { + log::info!( + "relay block {number}: para B's backing context changed mid-window \ + (group {} -> {}, usable {}/{}), restarting", + w.group_index, + ctx.group_index, + ctx.usable, + ctx.group_size, + ); + window = None; + continue; + } + + let backed = para_b_backed_in(&block, para_b).await?; + w.counted += 1; + if backed { + w.backed += 1; + } + log::info!( + "relay block {number}: para B backed={backed} ({}/{} blocks measured)", + w.backed, + w.counted, + ); + + if w.counted >= PARA_B_WINDOW { + let backed = w.backed; + // One tolerated miss absorbs a single dropped collation; anything more means the V3 + // para is not being backed even when the validators serving it are able to. + assert!( + backed + 1 >= PARA_B_WINDOW, + "para B was backed in only {backed} of {PARA_B_WINDOW} relay blocks while a group \ + that can back it held its core; under a capable group it should be backed in \ + essentially every block, dispute storm or not", + ); + log::info!("para B backed in {backed}/{PARA_B_WINDOW} blocks under a v3-capable group"); + return Ok(()); + } + } + + Err(anyhow!("finalized block subscription ended while measuring para B")) +} + +/// Read the group currently serving para B's core and how much of it can back a V3 candidate. +/// +/// Returns `None` if para B has no core in the claim queue at this block. +async fn para_b_backing_context( + relay_client: &OnlineClient, + hash: H256, + para_b: ParaId, + old_pubkeys: &HashSet, +) -> Result, anyhow::Error> { + let claim_queue: BTreeMap> = + runtime_api_decode(relay_client, hash, "ParachainHost_claim_queue").await?; + let Some(core) = claim_queue + .iter() + .find(|(_, queue)| queue.contains(¶_b)) + .map(|(core, _)| *core) + else { + return Ok(None); + }; + + let (groups, rotation): (Vec>, GroupRotationInfo) = + runtime_api_decode(relay_client, hash, "ParachainHost_validator_groups").await?; + let min_backing: u32 = + runtime_api_decode(relay_client, hash, "ParachainHost_minimum_backing_votes").await?; + + // The rotation modulus is the number of *groups*, not the number of occupied cores — see + // `n_cores` in `polkadot/node/core/backing/src/lib.rs`. There are more groups (5) than paras + // (2) here, so using the claim queue length would resolve to the wrong group. + let group_index = rotation.group_for_core(core, groups.len()); + let group = groups.get(group_index.0 as usize).ok_or_else(|| { + anyhow!("group {} out of range, only {} groups", group_index.0, groups.len()) + })?; + + let validators = session_validators_at(relay_client, hash).await?; + let disabled = disabled_validators_at(relay_client, hash).await?; + let usable = group + .iter() + .filter(|idx| { + !disabled.contains(idx) && + validators.get(idx.0 as usize).is_some_and(|id| { + !old_pubkeys.contains(&hex::encode(id.clone().into_inner().0)) + }) + }) + .count(); + + Ok(Some(BackingContext { + session_start: rotation.session_start_block, + group_index: group_index.0, + group_size: group.len(), + usable, + min_backing: min_backing as usize, + blocks_left_in_rotation: rotation.next_rotation_at().saturating_sub(rotation.now), + })) +} + +/// Whether `block` backed a candidate for para B, erroring if one of them is not V3. +async fn para_b_backed_in( + block: &Block>, + para_b: ParaId, +) -> Result { + let events = block.events().await?; + let mut backed = false; + for event in events.iter() { + let event = event?; + if event.pallet_name() != "ParaInclusion" || event.variant_name() != "CandidateBacked" { + continue; + } + let receipt = CandidateReceiptV2::::decode(&mut &event.field_bytes()[..])?; + if receipt.descriptor.para_id() != para_b { + continue; + } + let version = receipt.descriptor.version(); + if version != CandidateDescriptorVersion::V3 { + return Err(anyhow!("para B backed a non-V3 candidate post-recovery: {version:?}")); + } + backed = true; + } + Ok(backed) +} + +/// Returns `true` if `block` contains a session change. +async fn is_session_change( + block: &Block>, +) -> Result { + let events = block.events().await?; + Ok(events.iter().any(|event| { + event.as_ref().is_ok_and(|event| { + event.pallet_name() == "Session" && event.variant_name() == "NewSession" + }) + })) +} + fn normalize_hex(s: &str) -> String { s.trim_start_matches("0x").to_ascii_lowercase() } @@ -431,32 +687,33 @@ async fn assert_finality_stalls( } } -/// Wait for the dispute storm to fully resolve: every raised dispute must conclude *valid* (the -/// honest supermajority wins) and none may conclude *invalid*. +/// Wait until every dispute raised *up to the moment this is called* has concluded valid, with none +/// concluding invalid. /// -/// A single quiet sample is not enough. Para B keeps authoring V3 candidates the whole time, so -/// the pre-v3 validators keep raising fresh disputes until they are disabled and the last of their -/// candidates drains out. The storm only counts as extinguished once the *raised* count has stopped -/// growing — with everything raised so far concluded valid — for a sustained `QUIET_WINDOW_SECS`. -async fn wait_for_disputes_resolved_valid( +/// Deliberately snapshot-based rather than waiting for the storm to burn out. Disabling only holds +/// for the session it is applied in: each new session re-enables the pre-v3 validators, which then +/// dispute the next batch of V3 candidates. As long as para B keeps authoring V3 candidates the +/// raised counter therefore keeps climbing for the lifetime of the network, so any condition of the +/// form "the raised count stopped growing" is a race against the session length and will flake. +/// +/// What actually holds — and what is worth asserting — is that the honest supermajority resolves +/// every round it is given, and never loses one. +async fn wait_for_pending_disputes_resolved_valid( node: &zombienet_sdk::NetworkNode, timeout_secs: u64, ) -> Result { - const STEP_SECS: u64 = 20; - // How long the raised count must stay flat before we believe the storm is over. - const QUIET_WINDOW_SECS: u64 = 120; - const QUIET_STEPS: u32 = (QUIET_WINDOW_SECS / STEP_SECS) as u32; + const STEP_SECS: u64 = 10; let total_metric = "polkadot_parachain_candidate_disputes_total"; let valid_metric = "polkadot_parachain_candidate_dispute_concluded{validity=\"valid\"}"; let invalid_metric = "polkadot_parachain_candidate_dispute_concluded{validity=\"invalid\"}"; - let mut elapsed = 0u64; - let mut quiet_steps = 0u32; - let mut prev_total = f64::NEG_INFINITY; + let target = read_metric(node, total_metric).await?; + assert!(target >= 1.0, "expected at least one dispute to have been raised by now"); + log::info!("{target} disputes raised so far; waiting for all of them to conclude valid"); + let mut elapsed = 0u64; loop { - let total = read_metric(node, total_metric).await?; let valid = read_metric(node, valid_metric).await?; let invalid = read_metric(node, invalid_metric).await?; assert!( @@ -464,33 +721,23 @@ async fn wait_for_disputes_resolved_valid( "a dispute concluded invalid ({invalid}); the honest supermajority must never lose", ); - // A step is quiet when nothing new was raised since the previous sample and everything - // raised so far has concluded valid. Any new dispute resets the window. - if total >= 1.0 && valid >= total && total <= prev_total { - quiet_steps += 1; - } else { - quiet_steps = 0; + if valid >= target { + return Ok(target); } - prev_total = total; log::info!( - "disputes after {elapsed}s: total={total}, concluded_valid={valid}, \ - concluded_invalid={invalid}, quiet {}/{QUIET_STEPS}", - quiet_steps + "after {elapsed}s: {valid}/{target} of the snapshotted disputes concluded valid \ + (raised so far: {})", + read_metric(node, total_metric).await?, ); - if quiet_steps >= QUIET_STEPS { - return Ok(total); - } - tokio::time::sleep(std::time::Duration::from_secs(STEP_SECS)).await; elapsed += STEP_SECS; if elapsed >= timeout_secs { return Err(anyhow!( - "disputes did not settle within {timeout_secs}s (total={total}, \ - concluded_valid={valid}, quiet for {}s of the required {QUIET_WINDOW_SECS}s)", - u64::from(quiet_steps) * STEP_SECS + "only {valid} of the {target} disputes raised at snapshot time concluded valid \ + within {timeout_secs}s", )); } } From 36b244f589160b9f27e12ddca45f66f440d3ec7e Mon Sep 17 00:00:00 2001 From: Marios Christou Date: Tue, 28 Jul 2026 14:36:44 +0300 Subject: [PATCH 09/10] clippy --- .../v3_old_validator_dispute_storm.rs | 266 +----------------- 1 file changed, 4 insertions(+), 262 deletions(-) diff --git a/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs b/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs index aafb1fd1da0d..e1f0962866b8 100644 --- a/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs +++ b/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs @@ -24,17 +24,14 @@ use codec::Decode; use cumulus_zombienet_sdk_helpers::{ assert_finality_lag, assert_para_throughput_with, wait_for_runtime_upgrade, }; -use polkadot_primitives::{ - CandidateDescriptorVersion, CandidateReceiptV2, CoreIndex, GroupRotationInfo, Id as ParaId, - ValidatorId, ValidatorIndex, -}; +use polkadot_primitives::{CandidateDescriptorVersion, Id as ParaId, ValidatorId, ValidatorIndex}; use rstest::rstest; use serde_json::json; -use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; +use std::collections::{HashMap, HashSet}; use zombienet_sdk::{ subxt::{ - blocks::Block, dynamic::Value, ext::scale_value::value, tx::dynamic, utils::H256, - OnlineClient, PolkadotConfig, + dynamic::Value, ext::scale_value::value, tx::dynamic, utils::H256, OnlineClient, + PolkadotConfig, }, subxt_signer::sr25519::dev, NetworkConfigBuilder, @@ -45,13 +42,6 @@ const TOTAL_VALIDATORS: usize = 15; const PARA_A: u32 = 2000; const PARA_B: u32 = 2001; -/// Relay blocks to measure para B over once a capable group holds its core. Must stay below -/// `group_rotation_frequency` (10) so the whole window falls inside one group assignment. -const PARA_B_WINDOW: u32 = 6; - -/// How many relay blocks to spend looking for a capable group before giving up. -const PARA_B_SEARCH_LIMIT: u32 = 150; - #[rstest] #[case::four_old(4)] #[case::five_old(5)] @@ -323,254 +313,6 @@ async fn assert_five_old_persists( Ok(()) } -/// What para B's backing pipeline looks like at one relay block. -struct BackingContext { - /// Group membership is redrawn here; used to notice a reshuffle mid-window. - session_start: u32, - group_index: u32, - group_size: usize, - /// Group members that can actually back a V3 candidate: neither pre-v3 nor disabled. - usable: usize, - min_backing: usize, - blocks_left_in_rotation: u32, -} - -impl BackingContext { - fn is_capable(&self) -> bool { - self.usable >= self.min_backing - } -} - -/// Measure para B's backing rate while a group that can actually back it holds its core. -/// -/// A pre-v3 validator cannot validate a V3 candidate at all — `validate_block` panics on the -/// missing V3 extension — so it is dead weight in a backing group, whether or not it has been -/// disabled yet. -/// -/// That makes para B's candidate count over a fixed window a *draw*, not a rate. Group membership -/// is redrawn from on-chain randomness at every session change, and the group-to-core assignment -/// rotates every `group_rotation_frequency` blocks, so a 20-block window samples only about two -/// assignments. Observed runs produced 4 and 9 candidates in 20 blocks while para A produced 20 in -/// both — same storm, same disabled validators, just a different shuffle. -/// -/// So rather than widening the bound until it asserts nothing, condition on the layout: wait for a -/// capable group with room left in its rotation, then require para B to be backed in essentially -/// every block of the window. This terminates because `num_old` pre-v3 validators can starve at -/// most `num_old / (group_size - min_backing + 1)` groups — 2 of 5 here — so a capable group always -/// comes round. -async fn assert_para_b_backed_under_capable_group( - relay_client: &OnlineClient, - para_b: ParaId, - old_pubkeys: &HashSet, -) -> Result<(), anyhow::Error> { - struct Window { - session_start: u32, - group_index: u32, - counted: u32, - backed: u32, - } - - log::info!("looking for a v3-capable backing group on para B's core"); - let mut blocks_sub = relay_client.blocks().subscribe_finalized().await?; - let mut window: Option = None; - let mut searched = 0u32; - - while let Some(block) = blocks_sub.next().await { - let block = block?; - let number = block.number(); - - // Session changes redraw group membership and never carry backed candidates. - if is_session_change(&block).await? { - if window.take().is_some() { - log::info!("session change at relay block {number}, restarting the measurement"); - } - continue; - } - - let Some(ctx) = - para_b_backing_context(relay_client, block.hash(), para_b, old_pubkeys).await? - else { - log::debug!("relay block {number}: para B has no core assigned"); - continue; - }; - - if window.is_none() { - searched += 1; - if searched > PARA_B_SEARCH_LIMIT { - return Err(anyhow!( - "no v3-capable group held para B's core for {PARA_B_WINDOW} consecutive blocks \ - within {PARA_B_SEARCH_LIMIT} relay blocks; at most 2 of 5 groups should be \ - starved by 4 pre-v3 validators, so this points at a real backing failure", - )); - } - if !ctx.is_capable() || ctx.blocks_left_in_rotation < PARA_B_WINDOW { - log::debug!( - "relay block {number}: group {} has {}/{} usable backers (need {}), \ - {} blocks left in rotation - not measuring", - ctx.group_index, - ctx.usable, - ctx.group_size, - ctx.min_backing, - ctx.blocks_left_in_rotation, - ); - continue; - } - log::info!( - "relay block {number}: group {} on para B's core has {}/{} usable backers \ - (need {}), {} blocks left in rotation - measuring {PARA_B_WINDOW} blocks", - ctx.group_index, - ctx.usable, - ctx.group_size, - ctx.min_backing, - ctx.blocks_left_in_rotation, - ); - window = Some(Window { - session_start: ctx.session_start, - group_index: ctx.group_index, - counted: 0, - backed: 0, - }); - } - - let w = window.as_mut().expect("set directly above or on an earlier iteration; qed"); - - // The layout must not shift under us, or we are no longer measuring what we selected for. - if ctx.session_start != w.session_start || - ctx.group_index != w.group_index || - !ctx.is_capable() - { - log::info!( - "relay block {number}: para B's backing context changed mid-window \ - (group {} -> {}, usable {}/{}), restarting", - w.group_index, - ctx.group_index, - ctx.usable, - ctx.group_size, - ); - window = None; - continue; - } - - let backed = para_b_backed_in(&block, para_b).await?; - w.counted += 1; - if backed { - w.backed += 1; - } - log::info!( - "relay block {number}: para B backed={backed} ({}/{} blocks measured)", - w.backed, - w.counted, - ); - - if w.counted >= PARA_B_WINDOW { - let backed = w.backed; - // One tolerated miss absorbs a single dropped collation; anything more means the V3 - // para is not being backed even when the validators serving it are able to. - assert!( - backed + 1 >= PARA_B_WINDOW, - "para B was backed in only {backed} of {PARA_B_WINDOW} relay blocks while a group \ - that can back it held its core; under a capable group it should be backed in \ - essentially every block, dispute storm or not", - ); - log::info!("para B backed in {backed}/{PARA_B_WINDOW} blocks under a v3-capable group"); - return Ok(()); - } - } - - Err(anyhow!("finalized block subscription ended while measuring para B")) -} - -/// Read the group currently serving para B's core and how much of it can back a V3 candidate. -/// -/// Returns `None` if para B has no core in the claim queue at this block. -async fn para_b_backing_context( - relay_client: &OnlineClient, - hash: H256, - para_b: ParaId, - old_pubkeys: &HashSet, -) -> Result, anyhow::Error> { - let claim_queue: BTreeMap> = - runtime_api_decode(relay_client, hash, "ParachainHost_claim_queue").await?; - let Some(core) = claim_queue - .iter() - .find(|(_, queue)| queue.contains(¶_b)) - .map(|(core, _)| *core) - else { - return Ok(None); - }; - - let (groups, rotation): (Vec>, GroupRotationInfo) = - runtime_api_decode(relay_client, hash, "ParachainHost_validator_groups").await?; - let min_backing: u32 = - runtime_api_decode(relay_client, hash, "ParachainHost_minimum_backing_votes").await?; - - // The rotation modulus is the number of *groups*, not the number of occupied cores — see - // `n_cores` in `polkadot/node/core/backing/src/lib.rs`. There are more groups (5) than paras - // (2) here, so using the claim queue length would resolve to the wrong group. - let group_index = rotation.group_for_core(core, groups.len()); - let group = groups.get(group_index.0 as usize).ok_or_else(|| { - anyhow!("group {} out of range, only {} groups", group_index.0, groups.len()) - })?; - - let validators = session_validators_at(relay_client, hash).await?; - let disabled = disabled_validators_at(relay_client, hash).await?; - let usable = group - .iter() - .filter(|idx| { - !disabled.contains(idx) && - validators.get(idx.0 as usize).is_some_and(|id| { - !old_pubkeys.contains(&hex::encode(id.clone().into_inner().0)) - }) - }) - .count(); - - Ok(Some(BackingContext { - session_start: rotation.session_start_block, - group_index: group_index.0, - group_size: group.len(), - usable, - min_backing: min_backing as usize, - blocks_left_in_rotation: rotation.next_rotation_at().saturating_sub(rotation.now), - })) -} - -/// Whether `block` backed a candidate for para B, erroring if one of them is not V3. -async fn para_b_backed_in( - block: &Block>, - para_b: ParaId, -) -> Result { - let events = block.events().await?; - let mut backed = false; - for event in events.iter() { - let event = event?; - if event.pallet_name() != "ParaInclusion" || event.variant_name() != "CandidateBacked" { - continue; - } - let receipt = CandidateReceiptV2::::decode(&mut &event.field_bytes()[..])?; - if receipt.descriptor.para_id() != para_b { - continue; - } - let version = receipt.descriptor.version(); - if version != CandidateDescriptorVersion::V3 { - return Err(anyhow!("para B backed a non-V3 candidate post-recovery: {version:?}")); - } - backed = true; - } - Ok(backed) -} - -/// Returns `true` if `block` contains a session change. -async fn is_session_change( - block: &Block>, -) -> Result { - let events = block.events().await?; - Ok(events.iter().any(|event| { - event.as_ref().is_ok_and(|event| { - event.pallet_name() == "Session" && event.variant_name() == "NewSession" - }) - })) -} - fn normalize_hex(s: &str) -> String { s.trim_start_matches("0x").to_ascii_lowercase() } From 740fe35405bc583d74fcb74cbeb01fb194649486 Mon Sep 17 00:00:00 2001 From: Marios Christou Date: Tue, 28 Jul 2026 18:01:46 +0300 Subject: [PATCH 10/10] review pass --- .../zombienet_polkadot_tests.yml | 5 +- .../v3_old_validator_dispute_storm.rs | 72 +++++++++++++------ .../tests/functional/v3_rolling_upgrade.rs | 2 + 3 files changed, 55 insertions(+), 24 deletions(-) diff --git a/.github/zombienet-tests/zombienet_polkadot_tests.yml b/.github/zombienet-tests/zombienet_polkadot_tests.yml index c602c7119928..0f682ff2cac0 100644 --- a/.github/zombienet-tests/zombienet_polkadot_tests.yml +++ b/.github/zombienet-tests/zombienet_polkadot_tests.yml @@ -132,7 +132,6 @@ - job-name: "zombienet-polkadot-elastic-scaling-doesnt-break-parachains" test-filter: "elastic_scaling::doesnt_break_parachains::doesnt_break_parachains_test" runner-type: "default" - use-zombienet-sdk: true cumulus-image: "test-parachain" @@ -250,7 +249,7 @@ curl --retry 5 --retry-delay 10 --retry-max-time 120 -L -o $BIN_DIR/polkadot-old https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-stable2512/polkadot chmod 755 $BIN_DIR/polkadot-old echo "downloading polkadot-parachain as polkadot-parachain-old in $BIN_DIR" - curl --retry 5 --retry-delay 10 --retry-max-time 120 -L -o $BIN_DIR/polkadot-parachain-old https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-stable2603-4/polkadot-parachain + curl --retry 5 --retry-delay 10 --retry-max-time 120 -L -o $BIN_DIR/polkadot-parachain-old https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-stable2512/polkadot-parachain chmod 755 $BIN_DIR/polkadot-parachain-old for bin in polkadot-execute-worker polkadot-prepare-worker; do echo "downloading $bin in $BIN_DIR" @@ -264,4 +263,4 @@ OLD_POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:master-187cddde" OLD_POLKADOT_COMMAND: "polkadot-old" OLD_PARACHAIN_COMMAND: "polkadot-parachain-old" - OLD_PARACHAIN_IMAGE: "docker.io/parity/polkadot-parachain:stable2603-4" + OLD_PARACHAIN_IMAGE: "docker.io/parity/polkadot-parachain:stable2512" diff --git a/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs b/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs index e1f0962866b8..880202b5e1c5 100644 --- a/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs +++ b/polkadot/zombienet-sdk-tests/tests/functional/v3_old_validator_dispute_storm.rs @@ -22,7 +22,8 @@ use crate::utils::assert_candidates_version; use anyhow::anyhow; use codec::Decode; use cumulus_zombienet_sdk_helpers::{ - assert_finality_lag, assert_para_throughput_with, wait_for_runtime_upgrade, + assert_finality_lag, assert_para_throughput_with, wait_for_pvf_prepare, + wait_for_runtime_upgrade, }; use polkadot_primitives::{CandidateDescriptorVersion, Id as ParaId, ValidatorId, ValidatorIndex}; use rstest::rstest; @@ -140,6 +141,7 @@ async fn v3_old_validator_dispute_storm(#[case] num_old: usize) -> Result<(), an let network = spawn_fn(config).await?; let relay_node = network.get_node("validator-0")?; + let para_a_node = network.get_node("collator-a")?; let para_b_node = network.get_node("collator-b")?; let relay_client: OnlineClient = relay_node.wait_client().await?; let para_b_client: OnlineClient = para_b_node.wait_client().await?; @@ -195,6 +197,7 @@ async fn v3_old_validator_dispute_storm(#[case] num_old: usize) -> Result<(), an .wait_for_finalized_success() .await?; wait_for_runtime_upgrade(¶_b_client).await?; + wait_for_pvf_prepare(&network, 2).await?; log::info!("waiting for dispute storm to start"); relay_node @@ -207,7 +210,8 @@ async fn v3_old_validator_dispute_storm(#[case] num_old: usize) -> Result<(), an match num_old { 4 => { - assert_four_old_disabled(relay_node, &relay_client, &old_pubkeys).await?; + assert_old_validators_disabled(relay_node, &relay_client, &old_pubkeys, num_old) + .await?; log::info!("waiting for every dispute raised so far to conclude valid"); let total_disputes = wait_for_pending_disputes_resolved_valid(relay_node, 300).await?; @@ -233,7 +237,8 @@ async fn v3_old_validator_dispute_storm(#[case] num_old: usize) -> Result<(), an .await?; }, 5 => { - assert_five_old_persists(relay_node, &relay_client, baseline_finalized).await?; + assert_five_old_persists(relay_node, para_a_node, &relay_client, baseline_finalized) + .await?; }, other => unreachable!("unexpected num_old: {other}"), } @@ -242,12 +247,12 @@ async fn v3_old_validator_dispute_storm(#[case] num_old: usize) -> Result<(), an Ok(()) } -async fn assert_four_old_disabled( +async fn assert_old_validators_disabled( relay_node: &zombienet_sdk::NetworkNode, relay_client: &OnlineClient, old_pubkeys: &HashSet, + num_old: usize, ) -> Result<(), anyhow::Error> { - const NUM_OLD: usize = 4; log::info!("waiting for disputes to conclude valid"); relay_node .wait_metric_with_timeout( @@ -257,18 +262,22 @@ async fn assert_four_old_disabled( ) .await?; - log::info!("waiting for all {NUM_OLD} pre-v3 validators to be disabled"); + log::info!("waiting for all {num_old} pre-v3 validators to be disabled"); let mut best_blocks = relay_client.blocks().subscribe_best().await?; - let mut disabled_old = 0usize; + let mut seen_disabled_old: HashSet = HashSet::new(); let mut blocks_checked = 0u32; while let Some(block) = best_blocks.next().await { let hash = block?.hash(); let disabled = disabled_validators_at(relay_client, hash).await?; if !disabled.is_empty() { let validators = session_validators_at(relay_client, hash).await?; - disabled_old = count_disabled_old(&disabled, &validators, old_pubkeys); - log::info!("disabled {} validators, {disabled_old} of them pre-v3", disabled.len()); - if disabled_old >= NUM_OLD { + seen_disabled_old.extend(disabled_old_keys(&disabled, &validators, old_pubkeys)); + log::info!( + "disabled {} validators, {} distinct pre-v3 seen disabled so far", + disabled.len(), + seen_disabled_old.len(), + ); + if seen_disabled_old.len() >= num_old { break; } } @@ -278,15 +287,16 @@ async fn assert_four_old_disabled( } } assert!( - disabled_old >= NUM_OLD, - "expected all {NUM_OLD} pre-v3 validators disabled, got {disabled_old} after \ - {blocks_checked} blocks", + seen_disabled_old.len() >= num_old, + "expected all {num_old} pre-v3 validators disabled, got {} after {blocks_checked} blocks", + seen_disabled_old.len(), ); Ok(()) } async fn assert_five_old_persists( relay_node: &zombienet_sdk::NetworkNode, + para_a_node: &zombienet_sdk::NetworkNode, relay_client: &OnlineClient, baseline_finalized: f64, ) -> Result<(), anyhow::Error> { @@ -302,6 +312,15 @@ async fn assert_five_old_persists( "no validator should be disabled in the 5/15 case (at best block), found {disabled:?}", ); + let invalid_votes = + read_metric(relay_node, "polkadot_parachain_candidate_dispute_votes{validity=\"invalid\"}") + .await?; + assert!( + invalid_votes >= 1.0, + "expected the pre-v3 validators to have cast invalid votes; read {invalid_votes}, which \ + means the dispute vote metrics are not being reported and the checks below are vacuous", + ); + for validity in ["valid", "invalid"] { let metric = format!("polkadot_parachain_candidate_dispute_concluded{{validity=\"{validity}\"}}"); @@ -310,6 +329,18 @@ async fn assert_five_old_persists( "no dispute should conclude ({validity}) without an 11-vote supermajority", ); } + + let best_metric = "substrate_block_height{status=\"best\"}"; + let para_a_best = read_metric(para_a_node, best_metric).await?; + para_a_node + .wait_metric_with_timeout(best_metric, |v| v > para_a_best, 120u64) + .await + .map_err(|e| { + anyhow!( + "canary para A stopped extending its best chain at height {para_a_best}; the V3 \ + dispute storm on para B must not stall an unrelated V2 para: {e}" + ) + })?; Ok(()) } @@ -317,19 +348,17 @@ fn normalize_hex(s: &str) -> String { s.trim_start_matches("0x").to_ascii_lowercase() } -fn count_disabled_old( +fn disabled_old_keys( disabled: &[ValidatorIndex], validators: &[ValidatorId], old_pubkeys: &HashSet, -) -> usize { +) -> HashSet { disabled .iter() - .filter(|idx| { - validators - .get(idx.0 as usize) - .is_some_and(|id| old_pubkeys.contains(&hex::encode(id.clone().into_inner().0))) - }) - .count() + .filter_map(|idx| validators.get(idx.0 as usize)) + .map(|id| hex::encode(id.clone().into_inner().0)) + .filter(|key| old_pubkeys.contains(key)) + .collect() } /// Call a raw `ParachainHost` runtime API at `hash` and SCALE-decode its result. @@ -400,6 +429,7 @@ async fn assert_finality_stalls( } // look for the stall — two consecutive equal samples confirm finality is frozen. + let mut elapsed = 0u64; let mut prev = read_metric(node, metric).await?; loop { tokio::time::sleep(std::time::Duration::from_secs(STEP_SECS)).await; diff --git a/polkadot/zombienet-sdk-tests/tests/functional/v3_rolling_upgrade.rs b/polkadot/zombienet-sdk-tests/tests/functional/v3_rolling_upgrade.rs index 5bb6e9d87203..9dfd4fb61b5c 100644 --- a/polkadot/zombienet-sdk-tests/tests/functional/v3_rolling_upgrade.rs +++ b/polkadot/zombienet-sdk-tests/tests/functional/v3_rolling_upgrade.rs @@ -13,6 +13,8 @@ //! - a **V2-only** collator (an older `polkadot-parachain` release, predating V3, supplied via //! `OLD_PARACHAIN_COMMAND` / `OLD_PARACHAIN_IMAGE`). //! +//! Verifies that: +//! - V2 candidates are backed by the mixed fleet. //! - Statement and availability distribution work across binary versions. //! - GRANDPA finality does not stall. //! - Parachain throughput is sustained.