From 4de1a5804dde95aef30d64c1935de3349967fddb Mon Sep 17 00:00:00 2001 From: Luke Parker Date: Sun, 22 Dec 2024 06:41:55 -0500 Subject: [PATCH] Dedicated library for intending and evaluating cosigns Not only cleans the existing cosign code but enables non-Serai-coordinators to evaluate cosigns if they gain access to a feed of them (such as over an RPC). This would let centralized services not only track the finalized chain yet the cosigned chain without directly running a coordinator. Still being wrapped up. --- .github/workflows/msrv.yml | 1 + .github/workflows/tests.yml | 1 + Cargo.lock | 17 ++ Cargo.toml | 3 +- coordinator/cosign/Cargo.toml | 36 +++ coordinator/cosign/LICENSE | 15 + coordinator/cosign/README.md | 121 ++++++++ coordinator/cosign/src/evaluator.rs | 188 +++++++++++++ coordinator/cosign/src/intend.rs | 116 ++++++++ coordinator/cosign/src/lib.rs | 416 ++++++++++++++++++++++++++++ coordinator/src/cosign_evaluator.rs | 336 ---------------------- coordinator/src/substrate/cosign.rs | 332 ---------------------- deny.toml | 1 + 13 files changed, 914 insertions(+), 669 deletions(-) create mode 100644 coordinator/cosign/Cargo.toml create mode 100644 coordinator/cosign/LICENSE create mode 100644 coordinator/cosign/README.md create mode 100644 coordinator/cosign/src/evaluator.rs create mode 100644 coordinator/cosign/src/intend.rs create mode 100644 coordinator/cosign/src/lib.rs delete mode 100644 coordinator/src/cosign_evaluator.rs delete mode 100644 coordinator/src/substrate/cosign.rs diff --git a/.github/workflows/msrv.yml b/.github/workflows/msrv.yml index 409f5a9b..75fcdd79 100644 --- a/.github/workflows/msrv.yml +++ b/.github/workflows/msrv.yml @@ -175,6 +175,7 @@ jobs: run: | cargo msrv verify --manifest-path coordinator/tributary/tendermint/Cargo.toml cargo msrv verify --manifest-path coordinator/tributary/Cargo.toml + cargo msrv verify --manifest-path coordinator/cosign/Cargo.toml cargo msrv verify --manifest-path coordinator/Cargo.toml msrv-substrate: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3adc3ac5..9f1b0a1f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -61,6 +61,7 @@ jobs: -p serai-monero-processor \ -p tendermint-machine \ -p tributary-chain \ + -p serai-cosign \ -p serai-coordinator \ -p serai-orchestrator \ -p serai-docker-tests diff --git a/Cargo.lock b/Cargo.lock index 4266d05a..8965bf2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8365,6 +8365,23 @@ dependencies = [ "zeroize", ] +[[package]] +name = "serai-cosign" +version = "0.1.0" +dependencies = [ + "blake2", + "borsh", + "ciphersuite", + "log", + "parity-scale-codec", + "schnorr-signatures", + "serai-client", + "serai-db", + "serai-task", + "sp-application-crypto", + "tokio", +] + [[package]] name = "serai-db" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 6fe5fe89..eea39f37 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -98,6 +98,7 @@ members = [ "coordinator/tributary/tendermint", "coordinator/tributary", + "coordinator/cosign", "coordinator", "substrate/primitives", @@ -208,6 +209,7 @@ pasta_curves = { git = "https://github.com/kayabaNerve/pasta_curves", rev = "a46 [workspace.lints.clippy] unwrap_or_default = "allow" +map_unwrap_or = "allow" borrow_as_ptr = "deny" cast_lossless = "deny" cast_possible_truncation = "deny" @@ -235,7 +237,6 @@ manual_instant_elapsed = "deny" manual_let_else = "deny" manual_ok_or = "deny" manual_string_new = "deny" -map_unwrap_or = "deny" match_bool = "deny" match_same_arms = "deny" missing_fields_in_debug = "deny" diff --git a/coordinator/cosign/Cargo.toml b/coordinator/cosign/Cargo.toml new file mode 100644 index 00000000..b6177121 --- /dev/null +++ b/coordinator/cosign/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "serai-cosign" +version = "0.1.0" +description = "Evaluator of cosigns for the Serai network" +license = "AGPL-3.0-only" +repository = "https://github.com/serai-dex/serai/tree/develop/coordinator/cosign" +authors = ["Luke Parker "] +keywords = [] +edition = "2021" +publish = false +rust-version = "1.81" + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] + +[lints] +workspace = true + +[dependencies] +scale = { package = "parity-scale-codec", version = "3", default-features = false, features = ["std", "derive"] } +borsh = { version = "1", default-features = false, features = ["std", "derive", "de_strict_order"] } + +blake2 = { version = "0.10", default-features = false, features = ["std"] } +ciphersuite = { path = "../../crypto/ciphersuite", default-features = false, features = ["std"] } +schnorr = { package = "schnorr-signatures", path = "../../crypto/schnorr", default-features = false, features = ["std"] } + +sp-application-crypto = { git = "https://github.com/serai-dex/substrate", default-features = false, features = ["std"] } +serai-client = { path = "../../substrate/client", default-features = false, features = ["serai", "borsh"] } + +log = { version = "0.4", default-features = false, features = ["std"] } + +tokio = { version = "1", default-features = false, features = [] } + +serai-db = { path = "../../common/db" } +serai-task = { path = "../../common/task" } diff --git a/coordinator/cosign/LICENSE b/coordinator/cosign/LICENSE new file mode 100644 index 00000000..26d57cbb --- /dev/null +++ b/coordinator/cosign/LICENSE @@ -0,0 +1,15 @@ +AGPL-3.0-only license + +Copyright (c) 2023-2024 Luke Parker + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License Version 3 as +published by the Free Software Foundation. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . diff --git a/coordinator/cosign/README.md b/coordinator/cosign/README.md new file mode 100644 index 00000000..96fe0f92 --- /dev/null +++ b/coordinator/cosign/README.md @@ -0,0 +1,121 @@ +# Serai Cosign + +The Serai blockchain is controlled by a set of validators referred to as the +Serai validators. These validators could attempt to double-spend, even if every +node on the network is a full node, via equivocating. + +Posit: + - The Serai validators control X SRI + - The Serai validators produce block A swapping X SRI to Y XYZ + - The Serai validators produce block B swapping X SRI to Z ABC + - The Serai validators finalize block A and send to the validators for XYZ + - The Serai validators finalize block B and send to the validators for ABC + +This is solved via the cosigning protocol. The validators for XYZ and the +validators for ABC each sign their view of the Serai blockchain, communicating +amongst each other to ensure consistency. + +The security of the cosigning protocol is not formally proven, and there are no +claims it achieves Byzantine Fault Tolerance. This protocol is meant to be +practical and make such attacks infeasible, when they could already be argued +difficult to perform. + +### Definitions + +- Cosign: A signature from a non-Serai validator set for a Serai block +- Cosign Commit: A collection of cosigns which achieve the necessary weight + +### Methodology + +Finalized blocks from the Serai network are intended to be cosigned if they +contain burn events. Only once cosigned should non-Serai validators process +them. + +Cosigning occurs by a non-Serai validator set, using their threshold keys +declared on the Serai blockchain. Once 83% of non-Serai validator sets, by +weight, cosign a block, a cosign commit is formed. A cosign commit for a block +is considered to also cosign for all blocks preceding it. + +### Bounds Under Asynchrony + +Assuming an asynchronous environment fully controlled by the adversary, 34% of +a validator set may cause an equivocation. Control of 67% of non-Serai +validator sets, by weight, is sufficient to produce two distinct cosign commits +at the same position. This is due to the honest stake, 33%, being split across +the two candidates (67% + 16.5% = 83.5%, just over the threshold). This means +the cosigning protocol may produce multiple cosign commits if 34% of 67%, just +22.78%, of the non-Serai validator sets, is malicious. This would be in +conjunction with 34% of the Serai validator set (assumed 20% of total stake), +for a total stake requirement of 34% of 20% + 22.78% of 80% (25.024%). This is +an increase from the 6.8% required without the cosigning protocol. + +### Bounds Under Synchrony + +Assuming the honest stake within the non-Serai validator sets detect the +malicious stake within their set prior to assisting in producing a cosign for +their set, for which there is a multi-second window, 67% of 67% of non-Serai +validator sets is required to produce cosigns for those sets. This raises the +total stake requirement to 42.712% (past the usual 34% threshold). + +### Behavior Reliant on Synchrony + +If the Serai blockchain node detects an equivocation, it will stop responding +to all RPC requests and stop participating in finalizing further blocks. This +lets the node communicate the equivocating commits to other nodes (causing them +to exhibit the same behavior), yet prevents interaction with it. + +If cosigns representing 17% of the non-Serai validators sets by weight are +detected for distinct blocks at the same position, the protocol halts. An +explicit latency period of five seconds is enacted after receiving a cosign +commit for the detection of such an equivocation. This is largely redundant +given how the Serai blockchain node will presumably have halted itself by this +time. + +### Equivocation-Detection Avoidance + +Malicious Serai validators could avoid detection of their equivocating if they +produced two distinct blockchains, A and B, with different keys declared for +the same non-Serai validator set. While the validators following A may detect +the cosigns for distinct blocks by validators following B, the cosigns would be +assumed invalid due to their signatures being verified against distinct keys. + +This is prevented by requiring cosigns on the blocks which declare new keys, +ensuring all validators have a consistent view of the keys used within the +cosigning protocol (per the bounds of the cosigning protocol). These blocks are +exempt from the general policy of cosign commits cosigning all prior blocks, +preventing the newly declared keys (which aren't yet cosigned) from being used +to cosign themselves. These cosigns are flagged as "notable", are permanently +archived, and must be synced before a validator will move forward. + +Cosigning the block which declares new keys also ensures agreement on the +preceding block which declared the new set, with an exact specification of the +participants and their weight, before it impacts the cosigning protocol. + +### Denial of Service Concerns + +Any historical Serai validator set may trigger a chain halt by producing an +equivocation after their retiry. This requires 67% to be malicious. 34% of the +active Serai validator set may also trigger a chain halt. + +17% of non-Serai validator sets equivocating causing a halt means 5.67% of +non-Serai validator sets' stake may cause a halt (in an asynchronous +environment fully controlled by the adversary). In a synchronous environment +where the honest stake cannot be split across two candidates, 11.33% of +non-Serai validator sets' stake is required. + +The more practical attack is for one to obtain 5.67% of non-Serai validator +sets' stake, under any network conditions, and simply go offline. This will +take 17% of validator sets offline with it, preventing any cosign commits +from being performed. A fallback protocol where validators individually produce +cosigns, removing the network's horizontal scalability but ensuring liveness, +prevents this, restoring the additional requirements for control of an +asynchronous network or 11.33% of non-Serai validator sets' stake. + +### TODO + +The Serai node no longer responding to RPC requests upon detecting any +equivocation, and the fallback protocol where validators individually produce +signatures, are not implemented at this time. The former means the detection of +equivocating cosigns not redundant and the latter makes 5.67% of non-Serai +validator sets' stake the DoS threshold, even without control of an +asynchronous network. diff --git a/coordinator/cosign/src/evaluator.rs b/coordinator/cosign/src/evaluator.rs new file mode 100644 index 00000000..db77281a --- /dev/null +++ b/coordinator/cosign/src/evaluator.rs @@ -0,0 +1,188 @@ +use core::future::Future; + +use serai_client::{primitives::Amount, Serai}; + +use serai_db::*; +use serai_task::ContinuallyRan; + +use crate::{*, intend::BlockHasEvents}; + +create_db!( + SubstrateCosignEvaluator { + LatestCosignedBlockNumber: () -> u64, + } +); + +/// A task to determine if a block has been cosigned and we should handle it. +pub(crate) struct CosignEvaluatorTask { + pub(crate) db: D, + pub(crate) serai: Serai, + pub(crate) request: R, +} + +// TODO: Add a cache for the stake values + +impl ContinuallyRan for CosignEvaluatorTask { + fn run_iteration(&mut self) -> impl Send + Future> { + async move { + let latest_cosigned_block_number = LatestCosignedBlockNumber::get(&self.db).unwrap_or(0); + + let mut known_cosign = None; + let mut made_progress = false; + loop { + let mut txn = self.db.txn(); + let Some((block_number, has_events)) = BlockHasEvents::try_recv(&mut txn) else { break }; + // Make sure these two feeds haven't desynchronized somehow + // We could remove our `LatestCosignedBlockNumber`, making the latest cosigned block number + // the next message in the channel's block number minus one, but that'd only work when the + // channel isn't empty + assert_eq!(block_number, latest_cosigned_block_number + 1); + + let cosigns_for_block = Cosigns::get(&txn, block_number).unwrap_or(vec![]); + + match has_events { + // Because this had notable events, we require an explicit cosign for this block by a + // supermajority of the prior block's validator sets + HasEvents::Notable => { + let mut weight_cosigned = 0; + let mut total_weight = 0; + let (_block, sets) = cosigning_sets_for_block(&self.serai, block_number).await?; + let global_session = GlobalSession::new(sets.clone()).id(); + let (_, global_session_start_block) = GlobalSessions::get(&txn, global_session).expect( + "checking if intended cosign was satisfied within an unrecognized global session", + ); + for set in sets { + // Fetch the weight for this set, as of the start of the global session + // This simplifies the logic around which set of stakes to use when evaluating + // cosigns, even if it's lossy as it isn't accurate to how stake may fluctuate within + // a session + let stake = self + .serai + .as_of(global_session_start_block) + .validator_sets() + .total_allocated_stake(set.network) + .await + .map_err(|e| format!("{e:?}"))? + .unwrap_or(Amount(0)) + .0; + total_weight += stake; + + // Check if we have the cosign from this set + if cosigns_for_block + .iter() + .any(|cosign| cosign.cosigner == Cosigner::ValidatorSet(set.network)) + { + // Since have this cosign, add the set's weight to the weight which has cosigned + weight_cosigned += stake; + } + } + // Check if the sum weight doesn't cross the required threshold + if weight_cosigned < (((total_weight * 83) / 100) + 1) { + // Request the necessary cosigns over the network + // TODO: Add a timer to ensure this isn't called too often + self + .request + .request_notable_cosigns(global_session) + .await + .map_err(|e| format!("{e:?}"))?; + // We return an error so the delay before this task is run again increases + return Err(format!( + "notable block (#{block_number}) wasn't yet cosigned. this should resolve shortly", + )); + } + } + // Since this block didn't have any notable events, we simply require a cosign for this + // block or a greater block by the current validator sets + HasEvents::NonNotable => { + // Check if this was satisfied by a cached result which wasn't calculated incrementally + let known_cosigned = if let Some(known_cosign) = known_cosign { + known_cosign >= block_number + } else { + // Clear `known_cosign` which is no longer helpful + known_cosign = None; + false + }; + + // If it isn't already known to be cosigned, evaluate the latest cosigns + if !known_cosigned { + /* + LatestCosign is populated with the latest cosigns for each network which don't + exceed the latest global session we've evaluated the start of. This current block + is during the latest global session we've evaluated the start of. + */ + + // Get the global session for this block + let (_block, sets) = cosigning_sets_for_block(&self.serai, block_number).await?; + let global_session = GlobalSession::new(sets.clone()).id(); + let (_, global_session_start_block) = GlobalSessions::get(&txn, global_session) + .expect( + "checking if intended cosign was satisfied within an unrecognized global session", + ); + + let mut weight_cosigned = 0; + let mut total_weight = 0; + let mut lowest_common_block: Option = None; + for set in sets { + let stake = self + .serai + .as_of(global_session_start_block) + .validator_sets() + .total_allocated_stake(set.network) + .await + .map_err(|e| format!("{e:?}"))? + .unwrap_or(Amount(0)) + .0; + // Increment total_weight with this set's stake + total_weight += stake; + + // Check if this set cosigned this block or not + let Some(cosign) = NetworksLatestCosignedBlock::get(&txn, set.network) else { + continue; + }; + if cosign.block_number >= block_number { + weight_cosigned += total_weight + } + + // Update the lowest block common to all of these cosigns + lowest_common_block = lowest_common_block + .map(|existing| existing.min(cosign.block_number)) + .or(Some(cosign.block_number)); + } + + // Check if the sum weight doesn't cross the required threshold + if weight_cosigned < (((total_weight * 83) / 100) + 1) { + // Request the superseding notable cosigns over the network + // If this session hasn't yet produced notable cosigns, then we presume we'll see + // the desired non-notable cosigns as part of normal operations, without needing to + // explicitly request them + self + .request + .request_notable_cosigns(global_session) + .await + .map_err(|e| format!("{e:?}"))?; + // We return an error so the delay before this task is run again increases + return Err(format!( + "block (#{block_number}) wasn't yet cosigned. this should resolve shortly", + )); + } + + // Update the cached result for the block we know is cosigned + known_cosign = lowest_common_block; + } + } + // If this block has no events necessitating cosigning, we can immediately consider the + // block cosigned (making this block a NOP) + HasEvents::No => {} + } + + // Since we checked we had the necessary cosigns, increment the latest cosigned block + LatestCosignedBlockNumber::set(&mut txn, &block_number); + txn.commit(); + + made_progress = true; + } + + Ok(made_progress) + } + } +} diff --git a/coordinator/cosign/src/intend.rs b/coordinator/cosign/src/intend.rs new file mode 100644 index 00000000..76863925 --- /dev/null +++ b/coordinator/cosign/src/intend.rs @@ -0,0 +1,116 @@ +use core::future::Future; + +use serai_client::{SeraiError, Serai, validator_sets::primitives::ValidatorSet}; + +use serai_db::*; +use serai_task::ContinuallyRan; + +use crate::*; + +create_db!( + CosignIntend { + ScanCosignFrom: () -> u64, + } +); + +db_channel! { + CosignIntendChannels { + BlockHasEvents: () -> (u64, HasEvents), + IntendedCosigns: (set: ValidatorSet) -> CosignIntent, + } +} + +async fn block_has_events_justifying_a_cosign( + serai: &Serai, + block: u64, +) -> Result { + let serai = serai.as_of( + serai + .finalized_block_by_number(block) + .await? + .expect("couldn't get block which should've been finalized") + .hash(), + ); + + if !serai.validator_sets().key_gen_events().await?.is_empty() { + return Ok(HasEvents::Notable); + } + + if !serai.coins().burn_with_instruction_events().await?.is_empty() { + return Ok(HasEvents::NonNotable); + } + + Ok(HasEvents::No) +} + +/// A task to determine which blocks we should intend to cosign. +pub(crate) struct CosignIntendTask { + pub(crate) db: D, + pub(crate) serai: Serai, +} + +impl ContinuallyRan for CosignIntendTask { + fn run_iteration(&mut self) -> impl Send + Future> { + async move { + let start_block_number = ScanCosignFrom::get(&self.db).unwrap_or(1); + let latest_block_number = + self.serai.latest_finalized_block().await.map_err(|e| format!("{e:?}"))?.number(); + + for block_number in start_block_number ..= latest_block_number { + let mut txn = self.db.txn(); + + let mut has_events = block_has_events_justifying_a_cosign(&self.serai, block_number) + .await + .map_err(|e| format!("{e:?}"))?; + + match has_events { + HasEvents::Notable | HasEvents::NonNotable => { + let (block, sets) = cosigning_sets_for_block(&self.serai, block_number).await?; + + // If this is notable, it creates a new global session, which we index into the + // database now + if has_events == HasEvents::Notable { + let sets = cosigning_sets(&self.serai.as_of(block.hash())).await?; + GlobalSessions::set( + &mut txn, + GlobalSession::new(sets).id(), + &(block.number(), block.hash()), + ); + } + + // If this block doesn't have any cosigners, meaning it'll never be cosigned, we flag it + // as not having any events requiring cosigning so we don't attempt to sign/require a + // cosign for it + if sets.is_empty() { + has_events = HasEvents::No; + } else { + let global_session = GlobalSession::new(sets.clone()).id(); + // Tell each set of their expectation to cosign this block + for set in sets { + log::debug!("{:?} will be cosigning block #{block_number}", set); + IntendedCosigns::send( + &mut txn, + set, + &CosignIntent { + global_session, + block_number, + block_hash: block.hash(), + notable: has_events == HasEvents::Notable, + }, + ); + } + } + } + HasEvents::No => {} + } + // Populate a singular feed with every block's status for the evluator to work off of + BlockHasEvents::send(&mut txn, &(block_number, has_events)); + // Mark this block as handled, meaning we should scan from the next block moving on + ScanCosignFrom::set(&mut txn, &(block_number + 1)); + txn.commit(); + } + + Ok(start_block_number <= latest_block_number) + } + } +} diff --git a/coordinator/cosign/src/lib.rs b/coordinator/cosign/src/lib.rs new file mode 100644 index 00000000..345334a4 --- /dev/null +++ b/coordinator/cosign/src/lib.rs @@ -0,0 +1,416 @@ +#![cfg_attr(docsrs, feature(doc_auto_cfg))] +#![doc = include_str!("../README.md")] +#![deny(missing_docs)] + +use core::{fmt::Debug, future::Future}; + +use blake2::{Digest, Blake2s256}; + +use borsh::{BorshSerialize, BorshDeserialize}; + +use serai_client::{ + primitives::{Amount, NetworkId, SeraiAddress}, + validator_sets::primitives::{Session, ValidatorSet, KeyPair}, + Block, Serai, TemporalSerai, +}; + +use serai_db::*; +use serai_task::*; + +/// The cosigns which are intended to be performed. +mod intend; +/// The evaluator of the cosigns. +mod evaluator; +use evaluator::LatestCosignedBlockNumber; + +/// A 'global session', defined as all validator sets used for cosigning at a given moment. +/// +/// We evaluate cosign faults within a global session. This ensures even if cosigners cosign +/// distinct blocks at distinct positions within a global session, we still identify the faults. +/* + There is the attack where a validator set is given an alternate blockchain with a key generation + event at block #n, while most validator sets are given a blockchain with a key generation event + at block number #(n+1). This prevents whoever has the alternate blockchain from verifying the + cosigns on the primary blockchain, and detecting the faults, if they use the keys as of the block + prior to the block being cosigned. + + We solve this by binding cosigns to a global session ID, which has a specific start block, and + reading the keys from the start block. This means that so long as all validator sets agree on the + start of a global session, they can verify all cosigns produced by that session, regardless of + how it advances. Since agreeing on the start of a global session is mandated, there's no way to + have validator sets follow two distinct global sessions without breaking the bounds of the + cosigning protocol. +*/ +#[derive(Debug, BorshSerialize, BorshDeserialize)] +struct GlobalSession { + cosigners: Vec, +} +impl GlobalSession { + fn new(mut cosigners: Vec) -> Self { + cosigners.sort_by_key(|a| borsh::to_vec(a).unwrap()); + Self { cosigners } + } + fn id(&self) -> [u8; 32] { + Blake2s256::digest(borsh::to_vec(self).unwrap()).into() + } +} + +create_db! { + Cosign { + // A mapping from a global session's ID to its start block (number, hash). + GlobalSessions: (global_session: [u8; 32]) -> (u64, [u8; 32]), + // An archive of all cosigns ever received. + // + // This will only be populated with cosigns predating or during the most recent global session + // to have its start cosigned. + Cosigns: (block_number: u64) -> Vec, + // The latest cosigned block for each network. + // + // This will only be populated with cosigns predating or during the most recent global session + // to have its start cosigned. + NetworksLatestCosignedBlock: (network: NetworkId) -> Cosign, + // Cosigns received for blocks not locally recognized as finalized. + Faults: (global_session: [u8; 32]) -> Vec, + // The global session which faulted. + FaultedSession: () -> [u8; 32], + } +} + +/// If the block has events. +#[derive(Clone, Copy, PartialEq, Eq, Debug, BorshSerialize, BorshDeserialize)] +enum HasEvents { + /// The block had a notable event. + /// + /// This is a special case as blocks with key gen events change the keys used for cosigning, and + /// accordingly must be cosigned before we advance past them. + Notable, + /// The block had an non-notable event justifying a cosign. + NonNotable, + /// The block didn't have an event justifying a cosign. + No, +} + +/// An intended cosign. +#[derive(Clone, Copy, PartialEq, Eq, Debug, BorshSerialize, BorshDeserialize)] +struct CosignIntent { + /// The global session this cosign is being performed under. + global_session: [u8; 32], + /// The number of the block to cosign. + block_number: u64, + /// The hash of the block to cosign. + block_hash: [u8; 32], + /// If this cosign must be handled before further cosigns are. + notable: bool, +} + +/// The identification of a cosigner. +#[derive(Clone, Copy, PartialEq, Eq, Debug, BorshSerialize, BorshDeserialize)] +enum Cosigner { + /// The network which produced this cosign. + ValidatorSet(NetworkId), + /// The individual validator which produced this cosign. + Validator(SeraiAddress), +} + +/// A cosign. +#[derive(Clone, Copy, PartialEq, Eq, Debug, BorshSerialize, BorshDeserialize)] +struct Cosign { + /// The global session this cosign is being performed under. + global_session: [u8; 32], + /// The number of the block to cosign. + block_number: u64, + /// The hash of the block to cosign. + block_hash: [u8; 32], + /// The actual cosigner. + cosigner: Cosigner, +} + +/// Construct a `TemporalSerai` bound to the time used for cosigning this block. +async fn temporal_serai_used_for_cosigning( + serai: &Serai, + block_number: u64, +) -> Result<(Block, TemporalSerai<'_>), String> { + let block = serai + .finalized_block_by_number(block_number) + .await + .map_err(|e| format!("{e:?}"))? + .ok_or("block wasn't finalized".to_string())?; + + // If we're cosigning block `n`, it's cosigned by the sets as of block `n-1` + // (as block `n` may update the sets declared but that update shouldn't take effect here + // until it's cosigned) + let serai = serai.as_of(block.header.parent_hash.into()); + + Ok((block, serai)) +} + +/// Fetch the keys used for cosigning by a specific network. +async fn keys_for_network( + serai: &TemporalSerai<'_>, + network: NetworkId, +) -> Result, String> { + let Some(latest_session) = + serai.validator_sets().session(network).await.map_err(|e| format!("{e:?}"))? + else { + // If this network hasn't had a session declared, move on + return Ok(None); + }; + + // Get the keys for the latest session + if let Some(keys) = serai + .validator_sets() + .keys(ValidatorSet { network, session: latest_session }) + .await + .map_err(|e| format!("{e:?}"))? + { + return Ok(Some((latest_session, keys))); + } + + // If the latest session has yet to set keys, use the prior session + if let Some(prior_session) = latest_session.0.checked_sub(1).map(Session) { + if let Some(keys) = serai + .validator_sets() + .keys(ValidatorSet { network, session: prior_session }) + .await + .map_err(|e| format!("{e:?}"))? + { + return Ok(Some((prior_session, keys))); + } + } + + Ok(None) +} + +/// Fetch the `ValidatorSet`s used for cosigning as of this block. +async fn cosigning_sets(serai: &TemporalSerai<'_>) -> Result, String> { + let mut sets = Vec::with_capacity(serai_client::primitives::NETWORKS.len()); + for network in serai_client::primitives::NETWORKS { + let Some((session, _)) = keys_for_network(serai, network).await? else { + // If this network doesn't have usable keys, move on + continue; + }; + + sets.push(ValidatorSet { network, session }); + } + Ok(sets) +} + +/// Fetch the `ValidatorSet`s used for cosigning this block. +async fn cosigning_sets_for_block( + serai: &Serai, + block_number: u64, +) -> Result<(Block, Vec), String> { + let (block, serai) = temporal_serai_used_for_cosigning(serai, block_number).await?; + cosigning_sets(&serai).await.map(|sets| (block, sets)) +} + +/// An object usable to request notable cosigns for a block. +pub trait RequestNotableCosigns: 'static + Send { + /// The error type which may be encountered when requesting notable cosigns. + type Error: Debug; + + /// Request the notable cosigns for this global session. + fn request_notable_cosigns( + &self, + global_session: [u8; 32], + ) -> impl Send + Future>; +} + +/// An error used to indicate the cosigning protocol has faulted. +pub struct Faulted; + +/// The interface to manage cosigning with. +pub struct Cosigning { + db: D, + serai: Serai, +} +impl Cosigning { + /// Spawn the tasks to intend and evaluate cosigns. + /// + /// The database specified must only be used with a singular instance of the Serai network, and + /// only used once at any given time. + pub fn spawn( + db: D, + serai: Serai, + request: R, + tasks_to_run_upon_cosigning: Vec, + ) -> Self { + let (intend_task, _intend_task_handle) = Task::new(); + let (evaluator_task, evaluator_task_handle) = Task::new(); + tokio::spawn( + (intend::CosignIntendTask { db: db.clone(), serai: serai.clone() }) + .continually_run(intend_task, vec![evaluator_task_handle]), + ); + tokio::spawn( + (evaluator::CosignEvaluatorTask { db: db.clone(), serai: serai.clone(), request }) + .continually_run(evaluator_task, tasks_to_run_upon_cosigning), + ); + Self { db, serai } + } + + /// The latest cosigned block number. + pub fn latest_cosigned_block_number(&self) -> Result { + if FaultedSession::get(&self.db).is_some() { + Err(Faulted)?; + } + + Ok(LatestCosignedBlockNumber::get(&self.db).unwrap_or(0)) + } + + /// Fetch the notable cosigns for a global session in order to respond to requests. + pub fn notable_cosigns(&self, global_session: [u8; 32]) -> Vec { + todo!("TODO") + } + + /// The cosigns to rebroadcast ever so often. + /// + /// This will be the most recent cosigns, in case the initial broadcast failed, or the faulty + /// cosigns, in case of a fault, to induce identification of the fault by others. + pub fn cosigns_to_rebroadcast(&self) -> Vec { + if let Some(faulted) = FaultedSession::get(&self.db) { + let mut cosigns = Faults::get(&self.db, faulted).unwrap(); + // Also include all of our recognized-as-honest cosigns in an attempt to induce fault + // identification in those who see the faulty cosigns as honest + for network in serai_client::primitives::NETWORKS { + if let Some(cosign) = NetworksLatestCosignedBlock::get(&self.db, network) { + if cosign.global_session == faulted { + cosigns.push(cosign); + } + } + } + cosigns + } else { + let mut cosigns = Vec::with_capacity(serai_client::primitives::NETWORKS.len()); + for network in serai_client::primitives::NETWORKS { + if let Some(cosign) = NetworksLatestCosignedBlock::get(&self.db, network) { + cosigns.push(cosign); + } + } + cosigns + } + } + + /// Intake a cosign from the Serai network. + /// + /// - Returns Err(_) if there was an error trying to validate the cosign and it should be retired + /// later. + /// - Returns Ok(true) if the cosign was successfully handled or could not be handled at this + /// time. + /// - Returns Ok(false) if the cosign was invalid. + // + // We collapse a cosign which shouldn't be handled yet into a valid cosign (`Ok(true)`) as we + // assume we'll either explicitly request it if we need it or we'll naturally see it (or a later, + // more relevant, cosign) again. + // + // Takes `&mut self` as this should only be called once at any given moment. + pub async fn intake_cosign(&mut self, cosign: Cosign) -> Result { + // Check if we've prior handled this cosign + let mut txn = self.db.txn(); + let mut cosigns_for_this_block_position = + Cosigns::get(&txn, cosign.block_number).unwrap_or(vec![]); + if cosigns_for_this_block_position.iter().any(|existing| *existing == cosign) { + return Ok(true); + } + + // Check we can verify this cosign's signature + let Some((global_session_start_block_number, global_session_start_block_hash)) = + GlobalSessions::get(&txn, cosign.global_session) + else { + // Unrecognized global session + return Ok(true); + }; + + // Check the cosign's signature + let network = match cosign.cosigner { + Cosigner::ValidatorSet(network) => { + let Some((_session, keys)) = + keys_for_network(&self.serai.as_of(global_session_start_block_hash), network).await? + else { + return Ok(false); + }; + + todo!("TODO"); + + network + } + Cosigner::Validator(_) => return Ok(false), + }; + + // Check our finalized blockchain exceeds this block number + if self.serai.latest_finalized_block().await.map_err(|e| format!("{e:?}"))?.number() < + cosign.block_number + { + // Unrecognized block number + return Ok(true); + } + + // Since we verified this cosign's signature, and have a chain sufficiently long, handle the + // cosign + + // Save the cosign to the database + cosigns_for_this_block_position.push(cosign); + Cosigns::set(&mut txn, cosign.block_number, &cosigns_for_this_block_position); + + let our_block_hash = self + .serai + .block_hash(cosign.block_number) + .await + .map_err(|e| format!("{e:?}"))? + .expect("requested hash of a finalized block yet received None"); + if our_block_hash == cosign.block_hash { + // If this is for a future global session, we don't acknowledge this cosign at this time + if global_session_start_block_number > LatestCosignedBlockNumber::get(&self.db).unwrap_or(0) { + drop(txn); + return Ok(true); + } + + if NetworksLatestCosignedBlock::get(&txn, network) + .map(|cosign| cosign.block_number) + .unwrap_or(0) < + cosign.block_number + { + NetworksLatestCosignedBlock::set(&mut txn, network, &cosign); + } + } else { + let mut faults = Faults::get(&txn, cosign.global_session).unwrap_or(vec![]); + // Only handle this as a fault if this set wasn't prior faulty + if !faults.iter().any(|cosign| cosign.cosigner == Cosigner::ValidatorSet(network)) { + faults.push(cosign); + Faults::set(&mut txn, cosign.global_session, &faults); + + let mut weight_cosigned = 0; + let mut total_weight = 0; + for set in cosigning_sets(&self.serai.as_of(global_session_start_block_hash)).await? { + let stake = self + .serai + .as_of(global_session_start_block_hash) + .validator_sets() + .total_allocated_stake(set.network) + .await + .map_err(|e| format!("{e:?}"))? + .unwrap_or(Amount(0)) + .0; + // Increment total_weight with this set's stake + total_weight += stake; + + // Check if this set cosigned this block or not + if faults.iter().any(|cosign| cosign.cosigner == Cosigner::ValidatorSet(set.network)) { + weight_cosigned += total_weight + } + } + + // Check if the sum weight means a fault has occurred + assert!( + total_weight != 0, + "evaluating valid cosign when no stake was present in the system" + ); + if weight_cosigned >= ((total_weight * 17) / 100) { + FaultedSession::set(&mut txn, &cosign.global_session); + } + } + } + + txn.commit(); + Ok(true) + } +} diff --git a/coordinator/src/cosign_evaluator.rs b/coordinator/src/cosign_evaluator.rs deleted file mode 100644 index 29d9cc4b..00000000 --- a/coordinator/src/cosign_evaluator.rs +++ /dev/null @@ -1,336 +0,0 @@ -use core::time::Duration; -use std::{ - sync::Arc, - collections::{HashSet, HashMap}, -}; - -use tokio::{ - sync::{mpsc, Mutex, RwLock}, - time::sleep, -}; - -use borsh::BorshSerialize; -use sp_application_crypto::RuntimePublic; -use serai_client::{ - primitives::{NETWORKS, NetworkId, Signature}, - validator_sets::primitives::{Session, ValidatorSet}, - SeraiError, TemporalSerai, Serai, -}; - -use serai_db::{Get, DbTxn, Db, create_db}; - -use processor_messages::coordinator::cosign_block_msg; - -use crate::{ - p2p::{CosignedBlock, GossipMessageKind, P2p}, - substrate::LatestCosignedBlock, -}; - -create_db! { - CosignDb { - ReceivedCosign: (set: ValidatorSet, block: [u8; 32]) -> CosignedBlock, - LatestCosign: (network: NetworkId) -> CosignedBlock, - DistinctChain: (set: ValidatorSet) -> (), - } -} - -pub struct CosignEvaluator { - db: Mutex, - serai: Arc, - stakes: RwLock>>, - latest_cosigns: RwLock>, -} - -impl CosignEvaluator { - async fn update_latest_cosign(&self) { - let stakes_lock = self.stakes.read().await; - // If we haven't gotten the stake data yet, return - let Some(stakes) = stakes_lock.as_ref() else { return }; - - let total_stake = stakes.values().copied().sum::(); - - let latest_cosigns = self.latest_cosigns.read().await; - let mut highest_block = 0; - for cosign in latest_cosigns.values() { - let mut networks = HashSet::new(); - for (network, sub_cosign) in &*latest_cosigns { - if sub_cosign.block_number >= cosign.block_number { - networks.insert(network); - } - } - let sum_stake = - networks.into_iter().map(|network| stakes.get(network).unwrap_or(&0)).sum::(); - let needed_stake = ((total_stake * 2) / 3) + 1; - if (total_stake == 0) || (sum_stake > needed_stake) { - highest_block = highest_block.max(cosign.block_number); - } - } - - let mut db_lock = self.db.lock().await; - let mut txn = db_lock.txn(); - if highest_block > LatestCosignedBlock::latest_cosigned_block(&txn) { - log::info!("setting latest cosigned block to {}", highest_block); - LatestCosignedBlock::set(&mut txn, &highest_block); - } - txn.commit(); - } - - async fn update_stakes(&self) -> Result<(), SeraiError> { - let serai = self.serai.as_of_latest_finalized_block().await?; - - let mut stakes = HashMap::new(); - for network in NETWORKS { - // Use if this network has published a Batch for a short-circuit of if they've ever set a key - let set_key = serai.in_instructions().last_batch_for_network(network).await?.is_some(); - if set_key { - stakes.insert( - network, - serai - .validator_sets() - .total_allocated_stake(network) - .await? - .expect("network which published a batch didn't have a stake set") - .0, - ); - } - } - - // Since we've successfully built stakes, set it - *self.stakes.write().await = Some(stakes); - - self.update_latest_cosign().await; - - Ok(()) - } - - // Uses Err to signify a message should be retried - async fn handle_new_cosign(&self, cosign: CosignedBlock) -> Result<(), SeraiError> { - // If we already have this cosign or a newer cosign, return - if let Some(latest) = self.latest_cosigns.read().await.get(&cosign.network) { - if latest.block_number >= cosign.block_number { - return Ok(()); - } - } - - // If this an old cosign (older than a day), drop it - let latest_block = self.serai.latest_finalized_block().await?; - if (cosign.block_number + (24 * 60 * 60 / 6)) < latest_block.number() { - log::debug!("received old cosign supposedly signed by {:?}", cosign.network); - return Ok(()); - } - - let Some(block) = self.serai.finalized_block_by_number(cosign.block_number).await? else { - log::warn!("received cosign with a block number which doesn't map to a block"); - return Ok(()); - }; - - async fn set_with_keys_fn( - serai: &TemporalSerai<'_>, - network: NetworkId, - ) -> Result, SeraiError> { - let Some(latest_session) = serai.validator_sets().session(network).await? else { - log::warn!("received cosign from {:?}, which doesn't yet have a session", network); - return Ok(None); - }; - let prior_session = Session(latest_session.0.saturating_sub(1)); - Ok(Some( - if serai - .validator_sets() - .keys(ValidatorSet { network, session: prior_session }) - .await? - .is_some() - { - ValidatorSet { network, session: prior_session } - } else { - ValidatorSet { network, session: latest_session } - }, - )) - } - - // Get the key for this network as of the prior block - // If we have two chains, this value may be different across chains depending on if one chain - // included the set_keys and one didn't - // Because set_keys will force a cosign, it will force detection of distinct blocks - // re: set_keys using keys prior to set_keys (assumed amenable to all) - let serai = self.serai.as_of(block.header.parent_hash.into()); - - let Some(set_with_keys) = set_with_keys_fn(&serai, cosign.network).await? else { - return Ok(()); - }; - let Some(keys) = serai.validator_sets().keys(set_with_keys).await? else { - log::warn!("received cosign for a block we didn't have keys for"); - return Ok(()); - }; - - if !keys - .0 - .verify(&cosign_block_msg(cosign.block_number, cosign.block), &Signature(cosign.signature)) - { - log::warn!("received cosigned block with an invalid signature"); - return Ok(()); - } - - log::info!( - "received cosign for block {} ({}) by {:?}", - block.number(), - hex::encode(cosign.block), - cosign.network - ); - - // Save this cosign to the DB - { - let mut db = self.db.lock().await; - let mut txn = db.txn(); - ReceivedCosign::set(&mut txn, set_with_keys, cosign.block, &cosign); - LatestCosign::set(&mut txn, set_with_keys.network, &(cosign)); - txn.commit(); - } - - if cosign.block != block.hash() { - log::error!( - "received cosign for a distinct block at {}. we have {}. cosign had {}", - cosign.block_number, - hex::encode(block.hash()), - hex::encode(cosign.block) - ); - - let serai = self.serai.as_of(latest_block.hash()); - - let mut db = self.db.lock().await; - // Save this set as being on a different chain - let mut txn = db.txn(); - DistinctChain::set(&mut txn, set_with_keys, &()); - txn.commit(); - - let mut total_stake = 0; - let mut total_on_distinct_chain = 0; - for network in NETWORKS { - if network == NetworkId::Serai { - continue; - } - - // Get the current set for this network - let set_with_keys = { - let mut res; - while { - res = set_with_keys_fn(&serai, cosign.network).await; - res.is_err() - } { - log::error!( - "couldn't get the set with keys when checking for a distinct chain: {:?}", - res - ); - tokio::time::sleep(core::time::Duration::from_secs(3)).await; - } - res.unwrap() - }; - - // Get its stake - // Doesn't use the stakes inside self to prevent deadlocks re: multi-lock acquisition - if let Some(set_with_keys) = set_with_keys { - let stake = { - let mut res; - while { - res = serai.validator_sets().total_allocated_stake(set_with_keys.network).await; - res.is_err() - } { - log::error!( - "couldn't get total allocated stake when checking for a distinct chain: {:?}", - res - ); - tokio::time::sleep(core::time::Duration::from_secs(3)).await; - } - res.unwrap() - }; - - if let Some(stake) = stake { - total_stake += stake.0; - - if DistinctChain::get(&*db, set_with_keys).is_some() { - total_on_distinct_chain += stake.0; - } - } - } - } - - // See https://github.com/serai-dex/serai/issues/339 for the reasoning on 17% - if (total_stake * 17 / 100) <= total_on_distinct_chain { - panic!("17% of validator sets (by stake) have co-signed a distinct chain"); - } - } else { - { - let mut latest_cosigns = self.latest_cosigns.write().await; - latest_cosigns.insert(cosign.network, cosign); - } - self.update_latest_cosign().await; - } - - Ok(()) - } - - #[allow(clippy::new_ret_no_self)] - pub fn new(db: D, p2p: P, serai: Arc) -> mpsc::UnboundedSender { - let mut latest_cosigns = HashMap::new(); - for network in NETWORKS { - if let Some(cosign) = LatestCosign::get(&db, network) { - latest_cosigns.insert(network, cosign); - } - } - - let evaluator = Arc::new(Self { - db: Mutex::new(db), - serai, - stakes: RwLock::new(None), - latest_cosigns: RwLock::new(latest_cosigns), - }); - - // Spawn a task to update stakes regularly - tokio::spawn({ - let evaluator = evaluator.clone(); - async move { - loop { - // Run this until it passes - while evaluator.update_stakes().await.is_err() { - log::warn!("couldn't update stakes in the cosign evaluator"); - // Try again in 10 seconds - sleep(Duration::from_secs(10)).await; - } - // Run it every 10 minutes as we don't need the exact stake data for this to be valid - sleep(Duration::from_secs(10 * 60)).await; - } - } - }); - - // Spawn a task to receive cosigns and handle them - let (send, mut recv) = mpsc::unbounded_channel(); - tokio::spawn({ - let evaluator = evaluator.clone(); - async move { - while let Some(msg) = recv.recv().await { - while evaluator.handle_new_cosign(msg).await.is_err() { - // Try again in 10 seconds - sleep(Duration::from_secs(10)).await; - } - } - } - }); - - // Spawn a task to rebroadcast the most recent cosigns - tokio::spawn({ - async move { - loop { - let cosigns = evaluator.latest_cosigns.read().await.values().copied().collect::>(); - for cosign in cosigns { - let mut buf = vec![]; - cosign.serialize(&mut buf).unwrap(); - P2p::broadcast(&p2p, GossipMessageKind::CosignedBlock, buf).await; - } - sleep(Duration::from_secs(60)).await; - } - } - }); - - // Return the channel to send cosigns - send - } -} diff --git a/coordinator/src/substrate/cosign.rs b/coordinator/src/substrate/cosign.rs deleted file mode 100644 index 00560763..00000000 --- a/coordinator/src/substrate/cosign.rs +++ /dev/null @@ -1,332 +0,0 @@ -/* - If: - A) This block has events and it's been at least X blocks since the last cosign or - B) This block doesn't have events but it's been X blocks since a skipped block which did - have events or - C) This block key gens (which changes who the cosigners are) - cosign this block. - - This creates both a minimum and maximum delay of X blocks before a block's cosigning begins, - barring key gens which are exceptional. The minimum delay is there to ensure we don't constantly - spawn new protocols every 6 seconds, overwriting the old ones. The maximum delay is there to - ensure any block needing cosigned is consigned within a reasonable amount of time. -*/ - -use zeroize::Zeroizing; - -use ciphersuite::{Ciphersuite, Ristretto}; - -use borsh::{BorshSerialize, BorshDeserialize}; - -use serai_client::{ - SeraiError, Serai, - primitives::NetworkId, - validator_sets::primitives::{Session, ValidatorSet}, -}; - -use serai_db::*; - -use crate::{Db, substrate::in_set, tributary::SeraiBlockNumber}; - -// 5 minutes, expressed in blocks -// TODO: Pull a constant for block time -const COSIGN_DISTANCE: u64 = 5 * 60 / 6; - -#[derive(Clone, Copy, PartialEq, Eq, Debug, BorshSerialize, BorshDeserialize)] -enum HasEvents { - KeyGen, - Yes, - No, -} - -create_db!( - SubstrateCosignDb { - ScanCosignFrom: () -> u64, - IntendedCosign: () -> (u64, Option), - BlockHasEventsCache: (block: u64) -> HasEvents, - LatestCosignedBlock: () -> u64, - } -); - -impl IntendedCosign { - // Sets the intended to cosign block, clearing the prior value entirely. - pub fn set_intended_cosign(txn: &mut impl DbTxn, intended: u64) { - Self::set(txn, &(intended, None::)); - } - - // Sets the cosign skipped since the last intended to cosign block. - pub fn set_skipped_cosign(txn: &mut impl DbTxn, skipped: u64) { - let (intended, prior_skipped) = Self::get(txn).unwrap(); - assert!(prior_skipped.is_none()); - Self::set(txn, &(intended, Some(skipped))); - } -} - -impl LatestCosignedBlock { - pub fn latest_cosigned_block(getter: &impl Get) -> u64 { - Self::get(getter).unwrap_or_default().max(1) - } -} - -db_channel! { - SubstrateDbChannels { - CosignTransactions: (network: NetworkId) -> (Session, u64, [u8; 32]), - } -} - -impl CosignTransactions { - // Append a cosign transaction. - pub fn append_cosign(txn: &mut impl DbTxn, set: ValidatorSet, number: u64, hash: [u8; 32]) { - CosignTransactions::send(txn, set.network, &(set.session, number, hash)) - } -} - -async fn block_has_events( - txn: &mut impl DbTxn, - serai: &Serai, - block: u64, -) -> Result { - let cached = BlockHasEventsCache::get(txn, block); - match cached { - None => { - let serai = serai.as_of( - serai - .finalized_block_by_number(block) - .await? - .expect("couldn't get block which should've been finalized") - .hash(), - ); - - if !serai.validator_sets().key_gen_events().await?.is_empty() { - return Ok(HasEvents::KeyGen); - } - - let has_no_events = serai.coins().burn_with_instruction_events().await?.is_empty() && - serai.in_instructions().batch_events().await?.is_empty() && - serai.validator_sets().new_set_events().await?.is_empty() && - serai.validator_sets().set_retired_events().await?.is_empty(); - - let has_events = if has_no_events { HasEvents::No } else { HasEvents::Yes }; - - BlockHasEventsCache::set(txn, block, &has_events); - Ok(has_events) - } - Some(code) => Ok(code), - } -} - -async fn potentially_cosign_block( - txn: &mut impl DbTxn, - serai: &Serai, - block: u64, - skipped_block: Option, - window_end_exclusive: u64, -) -> Result { - // The following code regarding marking cosigned if prior block is cosigned expects this block to - // not be zero - // While we could perform this check there, there's no reason not to optimize the entire function - // as such - if block == 0 { - return Ok(false); - } - - let block_has_events = block_has_events(txn, serai, block).await?; - - // If this block had no events and immediately follows a cosigned block, mark it as cosigned - if (block_has_events == HasEvents::No) && - (LatestCosignedBlock::latest_cosigned_block(txn) == (block - 1)) - { - log::debug!("automatically co-signing next block ({block}) since it has no events"); - LatestCosignedBlock::set(txn, &block); - } - - // If we skipped a block, we're supposed to sign it plus the COSIGN_DISTANCE if no other blocks - // trigger a cosigning protocol covering it - // This means there will be the maximum delay allowed from a block needing cosigning occurring - // and a cosign for it triggering - let maximally_latent_cosign_block = - skipped_block.map(|skipped_block| skipped_block + COSIGN_DISTANCE); - - // If this block is within the window, - if block < window_end_exclusive { - // and set a key, cosign it - if block_has_events == HasEvents::KeyGen { - IntendedCosign::set_intended_cosign(txn, block); - // Carry skipped if it isn't included by cosigning this block - if let Some(skipped) = skipped_block { - if skipped > block { - IntendedCosign::set_skipped_cosign(txn, block); - } - } - return Ok(true); - } - } else if (Some(block) == maximally_latent_cosign_block) || (block_has_events != HasEvents::No) { - // Since this block was outside the window and had events/was maximally latent, cosign it - IntendedCosign::set_intended_cosign(txn, block); - return Ok(true); - } - Ok(false) -} - -/* - Advances the cosign protocol as should be done per the latest block. - - A block is considered cosigned if: - A) It was cosigned - B) It's the parent of a cosigned block - C) It immediately follows a cosigned block and has no events requiring cosigning - - This only actually performs advancement within a limited bound (generally until it finds a block - which should be cosigned). Accordingly, it is necessary to call multiple times even if - `latest_number` doesn't change. -*/ -async fn advance_cosign_protocol_inner( - db: &mut impl Db, - key: &Zeroizing<::F>, - serai: &Serai, - latest_number: u64, -) -> Result<(), SeraiError> { - let mut txn = db.txn(); - - const INITIAL_INTENDED_COSIGN: u64 = 1; - let (last_intended_to_cosign_block, mut skipped_block) = { - let intended_cosign = IntendedCosign::get(&txn); - // If we haven't prior intended to cosign a block, set the intended cosign to 1 - if let Some(intended_cosign) = intended_cosign { - intended_cosign - } else { - IntendedCosign::set_intended_cosign(&mut txn, INITIAL_INTENDED_COSIGN); - IntendedCosign::get(&txn).unwrap() - } - }; - - // "windows" refers to the window of blocks where even if there's a block which should be - // cosigned, it won't be due to proximity due to the prior cosign - let mut window_end_exclusive = last_intended_to_cosign_block + COSIGN_DISTANCE; - // If we've never triggered a cosign, don't skip any cosigns based on proximity - if last_intended_to_cosign_block == INITIAL_INTENDED_COSIGN { - window_end_exclusive = 1; - } - - // The consensus rules for this are `last_intended_to_cosign_block + 1` - let scan_start_block = last_intended_to_cosign_block + 1; - // As a practical optimization, we don't re-scan old blocks since old blocks are independent to - // new state - let scan_start_block = scan_start_block.max(ScanCosignFrom::get(&txn).unwrap_or(1)); - - // Check all blocks within the window to see if they should be cosigned - // If so, we're skipping them and need to flag them as skipped so that once the window closes, we - // do cosign them - // We only perform this check if we haven't already marked a block as skipped since the cosign - // the skipped block will cause will cosign all other blocks within this window - if skipped_block.is_none() { - let window_end_inclusive = window_end_exclusive - 1; - for b in scan_start_block ..= window_end_inclusive.min(latest_number) { - if block_has_events(&mut txn, serai, b).await? == HasEvents::Yes { - skipped_block = Some(b); - log::debug!("skipping cosigning {b} due to proximity to prior cosign"); - IntendedCosign::set_skipped_cosign(&mut txn, b); - break; - } - } - } - - // A block which should be cosigned - let mut to_cosign = None; - // A list of sets which are cosigning, along with a boolean of if we're in the set - let mut cosigning = vec![]; - - for block in scan_start_block ..= latest_number { - let actual_block = serai - .finalized_block_by_number(block) - .await? - .expect("couldn't get block which should've been finalized"); - - // Save the block number for this block, as needed by the cosigner to perform cosigning - SeraiBlockNumber::set(&mut txn, actual_block.hash(), &block); - - if potentially_cosign_block(&mut txn, serai, block, skipped_block, window_end_exclusive).await? - { - to_cosign = Some((block, actual_block.hash())); - - // Get the keys as of the prior block - // If this key sets new keys, the coordinator won't acknowledge so until we process this - // block - // We won't process this block until its co-signed - // Using the keys of the prior block ensures this deadlock isn't reached - let serai = serai.as_of(actual_block.header.parent_hash.into()); - - for network in serai_client::primitives::NETWORKS { - // Get the latest session to have set keys - let set_with_keys = { - let Some(latest_session) = serai.validator_sets().session(network).await? else { - continue; - }; - let prior_session = Session(latest_session.0.saturating_sub(1)); - if serai - .validator_sets() - .keys(ValidatorSet { network, session: prior_session }) - .await? - .is_some() - { - ValidatorSet { network, session: prior_session } - } else { - let set = ValidatorSet { network, session: latest_session }; - if serai.validator_sets().keys(set).await?.is_none() { - continue; - } - set - } - }; - - log::debug!("{:?} will be cosigning {block}", set_with_keys.network); - cosigning.push((set_with_keys, in_set(key, &serai, set_with_keys).await?.unwrap())); - } - - break; - } - - // If this TX is committed, always start future scanning from the next block - ScanCosignFrom::set(&mut txn, &(block + 1)); - // Since we're scanning *from* the next block, tidy the cache - BlockHasEventsCache::del(&mut txn, block); - } - - if let Some((number, hash)) = to_cosign { - // If this block doesn't have cosigners, yet does have events, automatically mark it as - // cosigned - if cosigning.is_empty() { - log::debug!("{} had no cosigners available, marking as cosigned", number); - LatestCosignedBlock::set(&mut txn, &number); - } else { - for (set, in_set) in cosigning { - if in_set { - log::debug!("cosigning {number} with {:?} {:?}", set.network, set.session); - CosignTransactions::append_cosign(&mut txn, set, number, hash); - } - } - } - } - txn.commit(); - - Ok(()) -} - -pub async fn advance_cosign_protocol( - db: &mut impl Db, - key: &Zeroizing<::F>, - serai: &Serai, - latest_number: u64, -) -> Result<(), SeraiError> { - loop { - let scan_from = ScanCosignFrom::get(db).unwrap_or(1); - // Only scan 1000 blocks at a time to limit a massive txn from forming - let scan_to = latest_number.min(scan_from + 1000); - advance_cosign_protocol_inner(db, key, serai, scan_to).await?; - // If we didn't limit the scan_to, break - if scan_to == latest_number { - break; - } - } - Ok(()) -} diff --git a/deny.toml b/deny.toml index 66bd4bc6..cc45984a 100644 --- a/deny.toml +++ b/deny.toml @@ -73,6 +73,7 @@ exceptions = [ { allow = ["AGPL-3.0"], name = "serai-monero-processor" }, { allow = ["AGPL-3.0"], name = "tributary-chain" }, + { allow = ["AGPL-3.0"], name = "serai-cosign" }, { allow = ["AGPL-3.0"], name = "serai-coordinator" }, { allow = ["AGPL-3.0"], name = "serai-coins-pallet" },