From 4d9c2df38c327d16f7f6903d6a4f7c3a51054c90 Mon Sep 17 00:00:00 2001 From: akildemir <34187742+akildemir@users.noreply.github.com> Date: Fri, 21 Jun 2024 15:39:17 +0300 Subject: [PATCH] Add coordinator rotation test (#535) * add node side unit test * complete rotation test for all networks * set up the fast-epoch docker file * fix pr comments * add coordinator side rotation test * bug fixes * Remove EPOCH_INTERVAL * Minor nits * Add note on origin of publish_tx function in tests/coordinator * Correct ThresholdParams assert_eq * fmt * Correct detection of handover completion * Restore key gen message match from develop It was modified in response to the handover completion bug, which has now been resolved. * bug fixes * Correct invalid constant * Typo fixes * remove selecting participant to remove at random --------- Co-authored-by: Luke Parker --- substrate/abi/src/grandpa.rs | 3 +- substrate/client/src/serai/mod.rs | 4 +- .../client/tests/common/in_instructions.rs | 2 +- .../client/tests/common/validator_sets.rs | 47 +-- substrate/client/tests/validator_sets.rs | 218 +++++++++---- substrate/validator-sets/pallet/src/lib.rs | 33 +- tests/coordinator/src/lib.rs | 18 +- tests/coordinator/src/tests/batch.rs | 40 ++- tests/coordinator/src/tests/key_gen.rs | 31 +- tests/coordinator/src/tests/mod.rs | 10 +- tests/coordinator/src/tests/rotation.rs | 169 ++++++++++ tests/coordinator/src/tests/sign.rs | 289 +++++++++--------- tests/full-stack/src/tests/mod.rs | 2 +- 13 files changed, 601 insertions(+), 265 deletions(-) create mode 100644 tests/coordinator/src/tests/rotation.rs diff --git a/substrate/abi/src/grandpa.rs b/substrate/abi/src/grandpa.rs index ead8dfc2..376b0b1d 100644 --- a/substrate/abi/src/grandpa.rs +++ b/substrate/abi/src/grandpa.rs @@ -1,4 +1,3 @@ -use sp_core::{ConstU32, bounded::BoundedVec}; use sp_consensus_grandpa::EquivocationProof; use serai_primitives::{BlockNumber, SeraiAddress}; @@ -19,7 +18,7 @@ pub enum Call { #[cfg_attr(feature = "serde", derive(serde::Serialize))] #[cfg_attr(all(feature = "std", feature = "serde"), derive(serde::Deserialize))] pub enum Event { - NewAuthorities { authority_set: BoundedVec<(SeraiAddress, u64), ConstU32<0>> }, + NewAuthorities { authority_set: alloc::vec::Vec<(SeraiAddress, u64)> }, // TODO: Remove these Paused, Resumed, diff --git a/substrate/client/src/serai/mod.rs b/substrate/client/src/serai/mod.rs index 63a44cf4..b1b8b041 100644 --- a/substrate/client/src/serai/mod.rs +++ b/substrate/client/src/serai/mod.rs @@ -195,10 +195,10 @@ impl Serai { } async fn active_network_validators(&self, network: NetworkId) -> Result, SeraiError> { - let hash: String = self + let validators: String = self .call("state_call", ["SeraiRuntimeApi_validators".to_string(), hex::encode(network.encode())]) .await?; - let bytes = Self::hex_decode(hash)?; + let bytes = Self::hex_decode(validators)?; let r = Vec::::decode(&mut bytes.as_slice()) .map_err(|e| SeraiError::ErrorInResponse(e.to_string()))?; Ok(r) diff --git a/substrate/client/tests/common/in_instructions.rs b/substrate/client/tests/common/in_instructions.rs index b4c24898..e335244a 100644 --- a/substrate/client/tests/common/in_instructions.rs +++ b/substrate/client/tests/common/in_instructions.rs @@ -31,7 +31,7 @@ pub async fn provide_batch(serai: &Serai, batch: Batch) -> [u8; 32] { keys } else { let keys = KeyPair(pair.public(), vec![].try_into().unwrap()); - set_keys(serai, set, keys.clone()).await; + set_keys(serai, set, keys.clone(), &[insecure_pair_from_name("Alice")]).await; keys }; assert_eq!(keys.0, pair.public()); diff --git a/substrate/client/tests/common/validator_sets.rs b/substrate/client/tests/common/validator_sets.rs index 7924d05c..3238501a 100644 --- a/substrate/client/tests/common/validator_sets.rs +++ b/substrate/client/tests/common/validator_sets.rs @@ -14,7 +14,6 @@ use frost::dkg::musig::musig; use schnorrkel::Schnorrkel; use serai_client::{ - primitives::insecure_pair_from_name, validator_sets::{ primitives::{ValidatorSet, KeyPair, musig_context, set_keys_message}, ValidatorSetsEvent, @@ -25,26 +24,40 @@ use serai_client::{ use crate::common::tx::publish_tx; #[allow(dead_code)] -pub async fn set_keys(serai: &Serai, set: ValidatorSet, key_pair: KeyPair) -> [u8; 32] { - let pair = insecure_pair_from_name("Alice"); - let public = pair.public(); +pub async fn set_keys( + serai: &Serai, + set: ValidatorSet, + key_pair: KeyPair, + pairs: &[Pair], +) -> [u8; 32] { + let mut pub_keys = vec![]; + for pair in pairs { + let public_key = + ::read_G::<&[u8]>(&mut pair.public().0.as_ref()).unwrap(); + pub_keys.push(public_key); + } - let public_key = ::read_G::<&[u8]>(&mut public.0.as_ref()).unwrap(); - let secret_key = ::read_F::<&[u8]>( - &mut pair.as_ref().secret.to_bytes()[.. 32].as_ref(), - ) - .unwrap(); - assert_eq!(Ristretto::generator() * secret_key, public_key); - let threshold_keys = - musig::(&musig_context(set), &Zeroizing::new(secret_key), &[public_key]).unwrap(); + let mut threshold_keys = vec![]; + for i in 0 .. pairs.len() { + let secret_key = ::read_F::<&[u8]>( + &mut pairs[i].as_ref().secret.to_bytes()[.. 32].as_ref(), + ) + .unwrap(); + assert_eq!(Ristretto::generator() * secret_key, pub_keys[i]); + + threshold_keys.push( + musig::(&musig_context(set), &Zeroizing::new(secret_key), &pub_keys).unwrap(), + ); + } + + let mut musig_keys = HashMap::new(); + for tk in threshold_keys { + musig_keys.insert(tk.params().i(), tk.into()); + } let sig = frost::tests::sign_without_caching( &mut OsRng, - frost::tests::algorithm_machines( - &mut OsRng, - &Schnorrkel::new(b"substrate"), - &HashMap::from([(threshold_keys.params().i(), threshold_keys.into())]), - ), + frost::tests::algorithm_machines(&mut OsRng, &Schnorrkel::new(b"substrate"), &musig_keys), &set_keys_message(&set, &[], &key_pair), ); diff --git a/substrate/client/tests/validator_sets.rs b/substrate/client/tests/validator_sets.rs index 8ae150ec..8aa8174f 100644 --- a/substrate/client/tests/validator_sets.rs +++ b/substrate/client/tests/validator_sets.rs @@ -1,36 +1,71 @@ use rand_core::{RngCore, OsRng}; -use sp_core::{sr25519::Public, Pair}; +use sp_core::{ + sr25519::{Public, Pair}, + Pair as PairTrait, +}; use serai_client::{ - primitives::{NETWORKS, NetworkId, insecure_pair_from_name}, + primitives::{NETWORKS, NetworkId, BlockHash, insecure_pair_from_name}, validator_sets::{ primitives::{Session, ValidatorSet, KeyPair}, ValidatorSetsEvent, }, + in_instructions::{ + primitives::{Batch, SignedBatch, batch_message}, + SeraiInInstructions, + }, Amount, Serai, }; mod common; -use common::validator_sets::{set_keys, allocate_stake, deallocate_stake}; +use common::{ + tx::publish_tx, + validator_sets::{allocate_stake, deallocate_stake, set_keys}, +}; -const EPOCH_INTERVAL: u64 = 5; +fn get_random_key_pair() -> KeyPair { + let mut ristretto_key = [0; 32]; + OsRng.fill_bytes(&mut ristretto_key); + let mut external_key = vec![0; 33]; + OsRng.fill_bytes(&mut external_key); + KeyPair(Public(ristretto_key), external_key.try_into().unwrap()) +} + +async fn get_ordered_keys(serai: &Serai, network: NetworkId, accounts: &[Pair]) -> Vec { + // retrieve the current session validators so that we know the order of the keys + // that is necessary for the correct musig signature. + let validators = serai + .as_of_latest_finalized_block() + .await + .unwrap() + .validator_sets() + .active_network_validators(network) + .await + .unwrap(); + + // collect the pairs of the validators + let mut pairs = vec![]; + for v in validators { + let p = accounts.iter().find(|pair| pair.public() == v).unwrap().clone(); + pairs.push(p); + } + + pairs +} serai_test!( set_keys_test: (|serai: Serai| async move { let network = NetworkId::Bitcoin; let set = ValidatorSet { session: Session(0), network }; - let public = insecure_pair_from_name("Alice").public(); + let pair = insecure_pair_from_name("Alice"); + let public = pair.public(); // Neither of these keys are validated // The external key is infeasible to validate on-chain, the Ristretto key is feasible // TODO: Should the Ristretto key be validated? - let mut ristretto_key = [0; 32]; - OsRng.fill_bytes(&mut ristretto_key); - let mut external_key = vec![0; 33]; - OsRng.fill_bytes(&mut external_key); - let key_pair = KeyPair(Public(ristretto_key), external_key.try_into().unwrap()); + let key_pair = get_random_key_pair(); // Make sure the genesis is as expected assert_eq!( @@ -62,7 +97,7 @@ serai_test!( assert_eq!(participants_ref, [public].as_ref()); } - let block = set_keys(&serai, set, key_pair.clone()).await; + let block = set_keys(&serai, set, key_pair.clone(), &[pair]).await; // While the set_keys function should handle this, it's beneficial to // independently test it @@ -149,11 +184,13 @@ async fn validator_set_rotation() { ); // genesis accounts - let pair1 = insecure_pair_from_name("Alice"); - let pair2 = insecure_pair_from_name("Bob"); - let pair3 = insecure_pair_from_name("Charlie"); - let pair4 = insecure_pair_from_name("Dave"); - let pair5 = insecure_pair_from_name("Eve"); + let accounts = vec![ + insecure_pair_from_name("Alice"), + insecure_pair_from_name("Bob"), + insecure_pair_from_name("Charlie"), + insecure_pair_from_name("Dave"), + insecure_pair_from_name("Eve"), + ]; // amounts for single key share per network let key_shares = HashMap::from([ @@ -164,8 +201,9 @@ async fn validator_set_rotation() { ]); // genesis participants per network + #[allow(clippy::redundant_closure_for_method_calls)] let default_participants = - vec![pair1.public(), pair2.public(), pair3.public(), pair4.public()]; + accounts[.. 4].to_vec().iter().map(|pair| pair.public()).collect::>(); let mut participants = HashMap::from([ (NetworkId::Serai, default_participants.clone()), (NetworkId::Bitcoin, default_participants.clone()), @@ -181,28 +219,83 @@ async fn validator_set_rotation() { participants.sort(); verify_session_and_active_validators(&serai, network, 0, participants).await; - // add 1 participant & verify - let hash = - allocate_stake(&serai, network, key_shares[&network], &pair5, i.try_into().unwrap()) - .await; - participants.push(pair5.public()); - participants.sort(); - verify_session_and_active_validators( + // add 1 participant + let last_participant = accounts[4].clone(); + let hash = allocate_stake( &serai, network, - get_active_session(&serai, network, hash).await, - participants, + key_shares[&network], + &last_participant, + i.try_into().unwrap(), ) .await; + participants.push(last_participant.public()); + // the session at which set changes becomes active + let activation_session = get_session_at_which_changes_activate(&serai, network, hash).await; - // remove 1 participant & verify - let hash = - deallocate_stake(&serai, network, key_shares[&network], &pair2, i.try_into().unwrap()) - .await; - participants.swap_remove(participants.iter().position(|k| *k == pair2.public()).unwrap()); - let active_session = get_active_session(&serai, network, hash).await; + // set the keys if it is an external set + if network != NetworkId::Serai { + let set = ValidatorSet { session: Session(0), network }; + let key_pair = get_random_key_pair(); + let pairs = get_ordered_keys(&serai, network, &accounts).await; + set_keys(&serai, set, key_pair, &pairs).await; + } + + // verify participants.sort(); - verify_session_and_active_validators(&serai, network, active_session, participants).await; + verify_session_and_active_validators(&serai, network, activation_session, participants) + .await; + + // remove 1 participant + let participant_to_remove = accounts[1].clone(); + let hash = deallocate_stake( + &serai, + network, + key_shares[&network], + &participant_to_remove, + i.try_into().unwrap(), + ) + .await; + participants.swap_remove( + participants.iter().position(|k| *k == participant_to_remove.public()).unwrap(), + ); + let activation_session = get_session_at_which_changes_activate(&serai, network, hash).await; + + if network != NetworkId::Serai { + // set the keys if it is an external set + let set = ValidatorSet { session: Session(1), network }; + + // we need the whole substrate key pair to sign the batch + let (substrate_pair, key_pair) = { + let pair = insecure_pair_from_name("session-1-key-pair"); + let public = pair.public(); + + let mut external_key = vec![0; 33]; + OsRng.fill_bytes(&mut external_key); + + (pair, KeyPair(public, external_key.try_into().unwrap())) + }; + let pairs = get_ordered_keys(&serai, network, &accounts).await; + set_keys(&serai, set, key_pair, &pairs).await; + + // provide a batch to complete the handover and retire the previous set + let mut block_hash = BlockHash([0; 32]); + OsRng.fill_bytes(&mut block_hash.0); + let batch = Batch { network, id: 0, block: block_hash, instructions: vec![] }; + publish_tx( + &serai, + &SeraiInInstructions::execute_batch(SignedBatch { + batch: batch.clone(), + signature: substrate_pair.sign(&batch_message(&batch)), + }), + ) + .await; + } + + // verify + participants.sort(); + verify_session_and_active_validators(&serai, network, activation_session, participants) + .await; // check pending deallocations let pending = serai @@ -212,8 +305,8 @@ async fn validator_set_rotation() { .validator_sets() .pending_deallocations( network, - pair2.public(), - Session(u32::try_from(active_session + 1).unwrap()), + participant_to_remove.public(), + Session(activation_session + 1), ) .await .unwrap(); @@ -223,24 +316,39 @@ async fn validator_set_rotation() { .await; } +async fn session_for_block(serai: &Serai, block: [u8; 32], network: NetworkId) -> u32 { + serai.as_of(block).validator_sets().session(network).await.unwrap().unwrap().0 +} + async fn verify_session_and_active_validators( serai: &Serai, network: NetworkId, - session: u64, + session: u32, participants: &[Public], ) { - // wait untill the epoch block finalized - let epoch_block = (session * EPOCH_INTERVAL) + 1; - while serai.finalized_block_by_number(epoch_block).await.unwrap().is_none() { - // sleep 1 block - tokio::time::sleep(tokio::time::Duration::from_secs(6)).await; - } - let serai_for_block = - serai.as_of(serai.finalized_block_by_number(epoch_block).await.unwrap().unwrap().hash()); + // wait until the active session. This wait should be max 30 secs since the epoch time. + let block = tokio::time::timeout(core::time::Duration::from_secs(2 * 60), async move { + loop { + let mut block = serai.latest_finalized_block_hash().await.unwrap(); + if session_for_block(serai, block, network).await < session { + // Sleep a block + tokio::time::sleep(core::time::Duration::from_secs(6)).await; + continue; + } + while session_for_block(serai, block, network).await > session { + block = serai.block(block).await.unwrap().unwrap().header.parent_hash.0; + } + assert_eq!(session_for_block(serai, block, network).await, session); + break block; + } + }) + .await + .unwrap(); + let serai_for_block = serai.as_of(block); // verify session let s = serai_for_block.validator_sets().session(network).await.unwrap().unwrap(); - assert_eq!(u64::from(s.0), session); + assert_eq!(s.0, session); // verify participants let mut validators = @@ -249,10 +357,11 @@ async fn verify_session_and_active_validators( assert_eq!(validators, participants); // make sure finalization continues as usual after the changes - tokio::time::timeout(tokio::time::Duration::from_secs(60), async move { + let current_finalized_block = serai.latest_finalized_block().await.unwrap().header.number; + tokio::time::timeout(core::time::Duration::from_secs(60), async move { let mut finalized_block = serai.latest_finalized_block().await.unwrap().header.number; - while finalized_block <= epoch_block + 2 { - tokio::time::sleep(tokio::time::Duration::from_secs(6)).await; + while finalized_block <= current_finalized_block + 2 { + tokio::time::sleep(core::time::Duration::from_secs(6)).await; finalized_block = serai.latest_finalized_block().await.unwrap().header.number; } }) @@ -262,15 +371,18 @@ async fn verify_session_and_active_validators( // TODO: verify key shares as well? } -async fn get_active_session(serai: &Serai, network: NetworkId, hash: [u8; 32]) -> u64 { - let block_number = serai.block(hash).await.unwrap().unwrap().header.number; - let epoch = block_number / EPOCH_INTERVAL; +async fn get_session_at_which_changes_activate( + serai: &Serai, + network: NetworkId, + hash: [u8; 32], +) -> u32 { + let session = session_for_block(serai, hash, network).await; // changes should be active in the next session if network == NetworkId::Serai { // it takes 1 extra session for serai net to make the changes active. - epoch + 2 + session + 2 } else { - epoch + 1 + session + 1 } } diff --git a/substrate/validator-sets/pallet/src/lib.rs b/substrate/validator-sets/pallet/src/lib.rs index 6ea89764..7191b3bd 100644 --- a/substrate/validator-sets/pallet/src/lib.rs +++ b/substrate/validator-sets/pallet/src/lib.rs @@ -643,8 +643,9 @@ pub mod pallet { // Checks if this session has completed the handover from the prior session. fn handover_completed(network: NetworkId, session: Session) -> bool { let Some(current_session) = Self::session(network) else { return false }; - // No handover occurs on genesis - if current_session.0 == 0 { + + // If the session we've been queried about is old, it must have completed its handover + if current_session.0 > session.0 { return true; } // If the session we've been queried about has yet to start, it can't have completed its @@ -652,19 +653,21 @@ pub mod pallet { if current_session.0 < session.0 { return false; } - if current_session.0 == session.0 { - // Handover is automatically complete for Serai as it doesn't have a handover protocol - // If not Serai, check the prior session had its keys cleared, which happens once its - // retired - return (network == NetworkId::Serai) || - (!Keys::::contains_key(ValidatorSet { - network, - session: Session(current_session.0 - 1), - })); + + // Handover is automatically complete for Serai as it doesn't have a handover protocol + if network == NetworkId::Serai { + return true; } - // We're currently in a future session, meaning this session definitely performed itself - // handover - true + + // The current session must have set keys for its handover to be completed + if !Keys::::contains_key(ValidatorSet { network, session }) { + return false; + } + + // This must be the first session (which has set keys) OR the prior session must have been + // retired (signified by its keys no longer being present) + (session.0 == 0) || + (!Keys::::contains_key(ValidatorSet { network, session: Session(session.0 - 1) })) } fn new_session() { @@ -682,6 +685,8 @@ pub mod pallet { } } + // TODO: This is called retire_set, yet just starts retiring the set + // Update the nomenclature within this function pub fn retire_set(set: ValidatorSet) { // If the prior prior set didn't report, emit they're retired now if PendingSlashReport::::get(set.network).is_some() { diff --git a/tests/coordinator/src/lib.rs b/tests/coordinator/src/lib.rs index e6b0324d..c364128c 100644 --- a/tests/coordinator/src/lib.rs +++ b/tests/coordinator/src/lib.rs @@ -60,12 +60,18 @@ pub fn coordinator_instance( ) } -pub fn serai_composition(name: &str) -> TestBodySpecification { - serai_docker_tests::build("serai".to_string()); - - TestBodySpecification::with_image( - Image::with_repository("serai-dev-serai").pull_policy(PullPolicy::Never), - ) +pub fn serai_composition(name: &str, fast_epoch: bool) -> TestBodySpecification { + (if fast_epoch { + serai_docker_tests::build("serai-fast-epoch".to_string()); + TestBodySpecification::with_image( + Image::with_repository("serai-dev-serai-fast-epoch").pull_policy(PullPolicy::Never), + ) + } else { + serai_docker_tests::build("serai".to_string()); + TestBodySpecification::with_image( + Image::with_repository("serai-dev-serai").pull_policy(PullPolicy::Never), + ) + }) .replace_env( [("SERAI_NAME".to_string(), name.to_lowercase()), ("KEY".to_string(), " ".to_string())].into(), ) diff --git a/tests/coordinator/src/tests/batch.rs b/tests/coordinator/src/tests/batch.rs index ebba957b..bfe4e36e 100644 --- a/tests/coordinator/src/tests/batch.rs +++ b/tests/coordinator/src/tests/batch.rs @@ -260,21 +260,29 @@ pub async fn batch( #[tokio::test] async fn batch_test() { - new_test(|mut processors: Vec| async move { - let (processor_is, substrate_key, _) = key_gen::(&mut processors).await; - batch( - &mut processors, - &processor_is, - Session(0), - &substrate_key, - Batch { - network: NetworkId::Bitcoin, - id: 0, - block: BlockHash([0x22; 32]), - instructions: vec![], - }, - ) - .await; - }) + new_test( + |mut processors: Vec| async move { + // pop the last participant since genesis keygen has only 4 participants + processors.pop().unwrap(); + assert_eq!(processors.len(), COORDINATORS); + + let (processor_is, substrate_key, _) = + key_gen::(&mut processors, Session(0)).await; + batch( + &mut processors, + &processor_is, + Session(0), + &substrate_key, + Batch { + network: NetworkId::Bitcoin, + id: 0, + block: BlockHash([0x22; 32]), + instructions: vec![], + }, + ) + .await; + }, + false, + ) .await; } diff --git a/tests/coordinator/src/tests/key_gen.rs b/tests/coordinator/src/tests/key_gen.rs index 8250b3bf..8ea14cbc 100644 --- a/tests/coordinator/src/tests/key_gen.rs +++ b/tests/coordinator/src/tests/key_gen.rs @@ -23,10 +23,12 @@ use crate::tests::*; pub async fn key_gen( processors: &mut [Processor], + session: Session, ) -> (Vec, Zeroizing<::F>, Zeroizing) { + let coordinators = processors.len(); let mut participant_is = vec![]; - let set = ValidatorSet { session: Session(0), network: NetworkId::Bitcoin }; + let set = ValidatorSet { session, network: NetworkId::Bitcoin }; let id = KeyGenId { session: set.session, attempt: 0 }; for (i, processor) in processors.iter_mut().enumerate() { @@ -46,8 +48,8 @@ pub async fn key_gen( CoordinatorMessage::KeyGen(messages::key_gen::CoordinatorMessage::GenerateKey { id, params: ThresholdParams::new( - u16::try_from(((COORDINATORS * 2) / 3) + 1).unwrap(), - u16::try_from(COORDINATORS).unwrap(), + u16::try_from(((coordinators * 2) / 3) + 1).unwrap(), + u16::try_from(coordinators).unwrap(), participant_is[i], ) .unwrap(), @@ -65,7 +67,7 @@ pub async fn key_gen( wait_for_tributary().await; for (i, processor) in processors.iter_mut().enumerate() { - let mut commitments = (0 .. u8::try_from(COORDINATORS).unwrap()) + let mut commitments = (0 .. u8::try_from(coordinators).unwrap()) .map(|l| { ( participant_is[usize::from(l)], @@ -83,7 +85,7 @@ pub async fn key_gen( ); // Recipient it's for -> (Sender i, Recipient i) - let mut shares = (0 .. u8::try_from(COORDINATORS).unwrap()) + let mut shares = (0 .. u8::try_from(coordinators).unwrap()) .map(|l| { ( participant_is[usize::from(l)], @@ -118,7 +120,7 @@ pub async fn key_gen( CoordinatorMessage::KeyGen(messages::key_gen::CoordinatorMessage::Shares { id, shares: { - let mut shares = (0 .. u8::try_from(COORDINATORS).unwrap()) + let mut shares = (0 .. u8::try_from(coordinators).unwrap()) .map(|l| { ( participant_is[usize::from(l)], @@ -182,14 +184,14 @@ pub async fn key_gen( .unwrap() .as_secs() .abs_diff(context.serai_time) < - 70 + (60 * 60 * 3) // 3 hours, which should exceed the length of any test we run ); assert_eq!(context.network_latest_finalized_block.0, [0; 32]); assert_eq!(set.session, session); assert_eq!(key_pair.0 .0, substrate_key); assert_eq!(&key_pair.1, &network_key); } - _ => panic!("coordinator didn't respond with ConfirmKeyPair"), + _ => panic!("coordinator didn't respond with ConfirmKeyPair. msg: {msg:?}"), } message = Some(msg); } else { @@ -220,8 +222,15 @@ pub async fn key_gen( #[tokio::test] async fn key_gen_test() { - new_test(|mut processors: Vec| async move { - key_gen::(&mut processors).await; - }) + new_test( + |mut processors: Vec| async move { + // pop the last participant since genesis keygen has only 4 participants + processors.pop().unwrap(); + assert_eq!(processors.len(), COORDINATORS); + + key_gen::(&mut processors, Session(0)).await; + }, + false, + ) .await; } diff --git a/tests/coordinator/src/tests/mod.rs b/tests/coordinator/src/tests/mod.rs index b564a26b..ef67b0ac 100644 --- a/tests/coordinator/src/tests/mod.rs +++ b/tests/coordinator/src/tests/mod.rs @@ -22,6 +22,8 @@ mod sign; #[allow(unused_imports)] pub use sign::sign; +mod rotation; + pub(crate) const COORDINATORS: usize = 4; pub(crate) const THRESHOLD: usize = ((COORDINATORS * 2) / 3) + 1; @@ -39,13 +41,15 @@ impl) -> F> Test } } -pub(crate) async fn new_test(test_body: impl TestBody) { +pub(crate) async fn new_test(test_body: impl TestBody, fast_epoch: bool) { let mut unique_id_lock = UNIQUE_ID.get_or_init(|| Mutex::new(0)).lock().await; let mut coordinators = vec![]; let mut test = DockerTest::new().with_network(dockertest::Network::Isolated); let mut coordinator_compositions = vec![]; - for i in 0 .. COORDINATORS { + // Spawn one extra coordinator which isn't in-set + #[allow(clippy::range_plus_one)] + for i in 0 .. (COORDINATORS + 1) { let name = match i { 0 => "Alice", 1 => "Bob", @@ -55,7 +59,7 @@ pub(crate) async fn new_test(test_body: impl TestBody) { 5 => "Ferdie", _ => panic!("needed a 7th name for a serai node"), }; - let serai_composition = serai_composition(name); + let serai_composition = serai_composition(name, fast_epoch); let (processor_key, message_queue_keys, message_queue_composition) = serai_message_queue_tests::instance(); diff --git a/tests/coordinator/src/tests/rotation.rs b/tests/coordinator/src/tests/rotation.rs new file mode 100644 index 00000000..1ebeec16 --- /dev/null +++ b/tests/coordinator/src/tests/rotation.rs @@ -0,0 +1,169 @@ +use tokio::time::{sleep, Duration}; + +use ciphersuite::Secp256k1; + +use serai_client::{ + primitives::{insecure_pair_from_name, NetworkId}, + validator_sets::{ + self, + primitives::{Session, ValidatorSet}, + ValidatorSetsEvent, + }, + Amount, Pair, Transaction, +}; + +use crate::{*, tests::*}; + +// TODO: This is duplicated with serai-client's tests +async fn publish_tx(serai: &Serai, tx: &Transaction) -> [u8; 32] { + let mut latest = serai + .block(serai.latest_finalized_block_hash().await.unwrap()) + .await + .unwrap() + .unwrap() + .number(); + + serai.publish(tx).await.unwrap(); + + // Get the block it was included in + // TODO: Add an RPC method for this/check the guarantee on the subscription + let mut ticks = 0; + loop { + latest += 1; + + let block = { + let mut block; + while { + block = serai.finalized_block_by_number(latest).await.unwrap(); + block.is_none() + } { + sleep(Duration::from_secs(1)).await; + ticks += 1; + + if ticks > 60 { + panic!("60 seconds without inclusion in a finalized block"); + } + } + block.unwrap() + }; + + for transaction in &block.transactions { + if transaction == tx { + return block.hash(); + } + } + } +} + +#[allow(dead_code)] +async fn allocate_stake( + serai: &Serai, + network: NetworkId, + amount: Amount, + pair: &Pair, + nonce: u32, +) -> [u8; 32] { + // get the call + let tx = + serai.sign(pair, validator_sets::SeraiValidatorSets::allocate(network, amount), nonce, 0); + publish_tx(serai, &tx).await +} + +#[allow(dead_code)] +async fn deallocate_stake( + serai: &Serai, + network: NetworkId, + amount: Amount, + pair: &Pair, + nonce: u32, +) -> [u8; 32] { + // get the call + let tx = + serai.sign(pair, validator_sets::SeraiValidatorSets::deallocate(network, amount), nonce, 0); + publish_tx(serai, &tx).await +} + +async fn get_session(serai: &Serai, network: NetworkId) -> Session { + serai + .as_of_latest_finalized_block() + .await + .unwrap() + .validator_sets() + .session(network) + .await + .unwrap() + .unwrap() +} + +async fn wait_till_session_1(serai: &Serai, network: NetworkId) { + let mut current_session = get_session(serai, network).await; + + while current_session.0 < 1 { + sleep(Duration::from_secs(6)).await; + current_session = get_session(serai, network).await; + } +} + +async fn most_recent_new_set_event(serai: &Serai, network: NetworkId) -> ValidatorSetsEvent { + let mut current_block = serai.latest_finalized_block().await.unwrap(); + loop { + let events = serai.as_of(current_block.hash()).validator_sets().new_set_events().await.unwrap(); + for event in events { + match event { + ValidatorSetsEvent::NewSet { set } => { + if set.network == network { + return event; + } + } + _ => panic!("new_set_events gave non-NewSet event: {event:?}"), + } + } + current_block = serai.block(current_block.header.parent_hash.0).await.unwrap().unwrap(); + } +} + +#[tokio::test] +async fn set_rotation_test() { + new_test( + |mut processors: Vec| async move { + // exclude the last processor from keygen since we will add him later + let mut excluded = processors.pop().unwrap(); + assert_eq!(processors.len(), COORDINATORS); + + // excluded participant + let pair5 = insecure_pair_from_name("Eve"); + let network = NetworkId::Bitcoin; + let amount = Amount(1_000_000 * 10_u64.pow(8)); + let serai = processors[0].serai().await; + + // allocate now for the last participant so that it is guaranteed to be included into session + // 1 set. This doesn't affect the genesis set at all since that is a predetermined set. + allocate_stake(&serai, network, amount, &pair5, 0).await; + + // genesis keygen + let _ = key_gen::(&mut processors, Session(0)).await; + // Even the excluded processor should receive the key pair confirmation + match excluded.recv_message().await { + CoordinatorMessage::Substrate( + messages::substrate::CoordinatorMessage::ConfirmKeyPair { session, .. }, + ) => assert_eq!(session, Session(0)), + _ => panic!("excluded got message other than ConfirmKeyPair"), + } + + // wait until next session to see the effect on coordinator + wait_till_session_1(&serai, network).await; + + // Ensure the new validator was included in the new set + assert_eq!( + most_recent_new_set_event(&serai, network).await, + ValidatorSetsEvent::NewSet { set: ValidatorSet { session: Session(1), network } }, + ); + + // add the last participant & do the keygen + processors.push(excluded); + let _ = key_gen::(&mut processors, Session(1)).await; + }, + true, + ) + .await; +} diff --git a/tests/coordinator/src/tests/sign.rs b/tests/coordinator/src/tests/sign.rs index e46e8890..db8a7203 100644 --- a/tests/coordinator/src/tests/sign.rs +++ b/tests/coordinator/src/tests/sign.rs @@ -168,161 +168,172 @@ pub async fn sign( #[tokio::test] async fn sign_test() { - new_test(|mut processors: Vec| async move { - let (participant_is, substrate_key, _) = key_gen::(&mut processors).await; + new_test( + |mut processors: Vec| async move { + // pop the last participant since genesis keygen has only 4 participant. + processors.pop().unwrap(); + assert_eq!(processors.len(), COORDINATORS); - // 'Send' external coins into Serai - let serai = processors[0].serai().await; - let (serai_pair, serai_addr) = { - let mut name = [0; 4]; - OsRng.fill_bytes(&mut name); - let pair = insecure_pair_from_name(&hex::encode(name)); - let address = SeraiAddress::from(pair.public()); + let (participant_is, substrate_key, _) = + key_gen::(&mut processors, Session(0)).await; - // Fund the new account to pay for fees - let balance = Balance { coin: Coin::Serai, amount: Amount(1_000_000_000) }; + // 'Send' external coins into Serai + let serai = processors[0].serai().await; + let (serai_pair, serai_addr) = { + let mut name = [0; 4]; + OsRng.fill_bytes(&mut name); + let pair = insecure_pair_from_name(&hex::encode(name)); + let address = SeraiAddress::from(pair.public()); + + // Fund the new account to pay for fees + let balance = Balance { coin: Coin::Serai, amount: Amount(1_000_000_000) }; + serai + .publish(&serai.sign( + &insecure_pair_from_name("Ferdie"), + SeraiCoins::transfer(address, balance), + 0, + Default::default(), + )) + .await + .unwrap(); + + (pair, address) + }; + + #[allow(clippy::inconsistent_digit_grouping)] + let amount = Amount(1_000_000_00); + let balance = Balance { coin: Coin::Bitcoin, amount }; + + let coin_block = BlockHash([0x33; 32]); + let block_included_in = batch( + &mut processors, + &participant_is, + Session(0), + &substrate_key, + Batch { + network: NetworkId::Bitcoin, + id: 0, + block: coin_block, + instructions: vec![InInstructionWithBalance { + instruction: InInstruction::Transfer(serai_addr), + balance, + }], + }, + ) + .await; + + { + let block_included_in_hash = + serai.finalized_block_by_number(block_included_in).await.unwrap().unwrap().hash(); + + let serai = serai.as_of(block_included_in_hash); + let serai = serai.coins(); + assert_eq!( + serai.coin_balance(Coin::Serai, serai_addr).await.unwrap(), + Amount(1_000_000_000) + ); + + // Verify the mint occurred as expected + assert_eq!( + serai.mint_events().await.unwrap(), + vec![CoinsEvent::Mint { to: serai_addr, balance }] + ); + assert_eq!(serai.coin_supply(Coin::Bitcoin).await.unwrap(), amount); + assert_eq!(serai.coin_balance(Coin::Bitcoin, serai_addr).await.unwrap(), amount); + } + + // Trigger a burn + let out_instruction = OutInstructionWithBalance { + balance, + instruction: OutInstruction { + address: ExternalAddress::new(b"external".to_vec()).unwrap(), + data: None, + }, + }; serai .publish(&serai.sign( - &insecure_pair_from_name("Ferdie"), - SeraiCoins::transfer(address, balance), + &serai_pair, + SeraiCoins::burn_with_instruction(out_instruction.clone()), 0, Default::default(), )) .await .unwrap(); - (pair, address) - }; - - #[allow(clippy::inconsistent_digit_grouping)] - let amount = Amount(1_000_000_00); - let balance = Balance { coin: Coin::Bitcoin, amount }; - - let coin_block = BlockHash([0x33; 32]); - let block_included_in = batch( - &mut processors, - &participant_is, - Session(0), - &substrate_key, - Batch { - network: NetworkId::Bitcoin, - id: 0, - block: coin_block, - instructions: vec![InInstructionWithBalance { - instruction: InInstruction::Transfer(serai_addr), - balance, - }], - }, - ) - .await; - - { - let block_included_in_hash = - serai.finalized_block_by_number(block_included_in).await.unwrap().unwrap().hash(); - - let serai = serai.as_of(block_included_in_hash); - let serai = serai.coins(); - assert_eq!(serai.coin_balance(Coin::Serai, serai_addr).await.unwrap(), Amount(1_000_000_000)); - - // Verify the mint occurred as expected - assert_eq!( - serai.mint_events().await.unwrap(), - vec![CoinsEvent::Mint { to: serai_addr, balance }] - ); - assert_eq!(serai.coin_supply(Coin::Bitcoin).await.unwrap(), amount); - assert_eq!(serai.coin_balance(Coin::Bitcoin, serai_addr).await.unwrap(), amount); - } - - // Trigger a burn - let out_instruction = OutInstructionWithBalance { - balance, - instruction: OutInstruction { - address: ExternalAddress::new(b"external".to_vec()).unwrap(), - data: None, - }, - }; - serai - .publish(&serai.sign( - &serai_pair, - SeraiCoins::burn_with_instruction(out_instruction.clone()), - 0, - Default::default(), - )) - .await - .unwrap(); - - // TODO: We *really* need a helper for this pattern - let mut last_serai_block = block_included_in; - 'outer: for _ in 0 .. 20 { - tokio::time::sleep(Duration::from_secs(6)).await; - if std::env::var("GITHUB_CI") == Ok("true".to_string()) { + // TODO: We *really* need a helper for this pattern + let mut last_serai_block = block_included_in; + 'outer: for _ in 0 .. 20 { tokio::time::sleep(Duration::from_secs(6)).await; - } - - while last_serai_block <= serai.latest_finalized_block().await.unwrap().number() { - let burn_events = serai - .as_of(serai.finalized_block_by_number(last_serai_block).await.unwrap().unwrap().hash()) - .coins() - .burn_with_instruction_events() - .await - .unwrap(); - - if !burn_events.is_empty() { - assert_eq!(burn_events.len(), 1); - assert_eq!( - burn_events[0], - CoinsEvent::BurnWithInstruction { - from: serai_addr, - instruction: out_instruction.clone() - } - ); - break 'outer; + if std::env::var("GITHUB_CI") == Ok("true".to_string()) { + tokio::time::sleep(Duration::from_secs(6)).await; } - last_serai_block += 1; - } - } - let last_serai_block = - serai.finalized_block_by_number(last_serai_block).await.unwrap().unwrap(); - let last_serai_block_hash = last_serai_block.hash(); - let serai = serai.as_of(last_serai_block_hash); - let serai = serai.coins(); - assert_eq!(serai.coin_supply(Coin::Bitcoin).await.unwrap(), Amount(0)); - assert_eq!(serai.coin_balance(Coin::Bitcoin, serai_addr).await.unwrap(), Amount(0)); + while last_serai_block <= serai.latest_finalized_block().await.unwrap().number() { + let burn_events = serai + .as_of(serai.finalized_block_by_number(last_serai_block).await.unwrap().unwrap().hash()) + .coins() + .burn_with_instruction_events() + .await + .unwrap(); - let mut plan_id = [0; 32]; - OsRng.fill_bytes(&mut plan_id); - let plan_id = plan_id; - - // We should now get a SubstrateBlock - for processor in &mut processors { - assert_eq!( - processor.recv_message().await, - messages::CoordinatorMessage::Substrate( - messages::substrate::CoordinatorMessage::SubstrateBlock { - context: SubstrateContext { - serai_time: last_serai_block.time().unwrap() / 1000, - network_latest_finalized_block: coin_block, - }, - block: last_serai_block.number(), - burns: vec![out_instruction.clone()], - batches: vec![], + if !burn_events.is_empty() { + assert_eq!(burn_events.len(), 1); + assert_eq!( + burn_events[0], + CoinsEvent::BurnWithInstruction { + from: serai_addr, + instruction: out_instruction.clone() + } + ); + break 'outer; } - ) - ); + last_serai_block += 1; + } + } - // Send the ACK, claiming there's a plan to sign - processor - .send_message(messages::ProcessorMessage::Coordinator( - messages::coordinator::ProcessorMessage::SubstrateBlockAck { - block: last_serai_block.number(), - plans: vec![PlanMeta { session: Session(0), id: plan_id }], - }, - )) - .await; - } + let last_serai_block = + serai.finalized_block_by_number(last_serai_block).await.unwrap().unwrap(); + let last_serai_block_hash = last_serai_block.hash(); + let serai = serai.as_of(last_serai_block_hash); + let serai = serai.coins(); + assert_eq!(serai.coin_supply(Coin::Bitcoin).await.unwrap(), Amount(0)); + assert_eq!(serai.coin_balance(Coin::Bitcoin, serai_addr).await.unwrap(), Amount(0)); - sign(&mut processors, &participant_is, Session(0), plan_id).await; - }) + let mut plan_id = [0; 32]; + OsRng.fill_bytes(&mut plan_id); + let plan_id = plan_id; + + // We should now get a SubstrateBlock + for processor in &mut processors { + assert_eq!( + processor.recv_message().await, + messages::CoordinatorMessage::Substrate( + messages::substrate::CoordinatorMessage::SubstrateBlock { + context: SubstrateContext { + serai_time: last_serai_block.time().unwrap() / 1000, + network_latest_finalized_block: coin_block, + }, + block: last_serai_block.number(), + burns: vec![out_instruction.clone()], + batches: vec![], + } + ) + ); + + // Send the ACK, claiming there's a plan to sign + processor + .send_message(messages::ProcessorMessage::Coordinator( + messages::coordinator::ProcessorMessage::SubstrateBlockAck { + block: last_serai_block.number(), + plans: vec![PlanMeta { session: Session(0), id: plan_id }], + }, + )) + .await; + } + + sign(&mut processors, &participant_is, Session(0), plan_id).await; + }, + false, + ) .await; } diff --git a/tests/full-stack/src/tests/mod.rs b/tests/full-stack/src/tests/mod.rs index 7aaad832..7d92070e 100644 --- a/tests/full-stack/src/tests/mod.rs +++ b/tests/full-stack/src/tests/mod.rs @@ -69,7 +69,7 @@ pub(crate) async fn new_test(test_body: impl TestBody) { let monero_processor_composition = monero_processor_composition.swap_remove(0); let coordinator_composition = coordinator_instance(name, coord_key); - let serai_composition = serai_composition(name); + let serai_composition = serai_composition(name, false); // Give every item in this stack a unique ID // Uses a Mutex as we can't generate a 8-byte random ID without hitting hostname length limits