Update the git tags

Does no actual migration work. This allows establishing the difference in
dependencies between substrate and polkadot-sdk/substrate.

.github/nightly-version

@@ -1 +1 @@
-nightly-2024-07-01
+nightly-2025-02-01

.github/workflows/monero-tests.yaml

@@ -39,9 +39,6 @@ jobs:
           GITHUB_CI=true RUST_BACKTRACE=1 cargo test --package monero-simple-request-rpc --lib
           GITHUB_CI=true RUST_BACKTRACE=1 cargo test --package monero-address --lib
           GITHUB_CI=true RUST_BACKTRACE=1 cargo test --package monero-wallet --lib
-          GITHUB_CI=true RUST_BACKTRACE=1 cargo test --package monero-seed --lib
-          GITHUB_CI=true RUST_BACKTRACE=1 cargo test --package polyseed --lib
-          GITHUB_CI=true RUST_BACKTRACE=1 cargo test --package monero-wallet-util --lib
 
       # Doesn't run unit tests with features as the tests workflow will
 
@@ -65,7 +62,6 @@ jobs:
           GITHUB_CI=true RUST_BACKTRACE=1 cargo test --package monero-serai --test '*'
           GITHUB_CI=true RUST_BACKTRACE=1 cargo test --package monero-simple-request-rpc --test '*'
           GITHUB_CI=true RUST_BACKTRACE=1 cargo test --package monero-wallet --test '*'
-          GITHUB_CI=true RUST_BACKTRACE=1 cargo test --package monero-wallet-util --test '*'
 
       - name: Run Integration Tests
         # Don't run if the the tests workflow also will
@@ -74,4 +70,3 @@ jobs:
           GITHUB_CI=true RUST_BACKTRACE=1 cargo test --package monero-serai --all-features --test '*'
           GITHUB_CI=true RUST_BACKTRACE=1 cargo test --package monero-simple-request-rpc --test '*'
           GITHUB_CI=true RUST_BACKTRACE=1 cargo test --package monero-wallet --all-features --test '*'
-          GITHUB_CI=true RUST_BACKTRACE=1 cargo test --package monero-wallet-util --all-features --test '*'

.github/workflows/networks-tests.yml

@@ -46,7 +46,4 @@ jobs:
             -p monero-simple-request-rpc \
             -p monero-address \
             -p monero-wallet \
-            -p monero-seed \
-            -p polyseed \
-            -p monero-wallet-util \
             -p monero-serai-verify-chain

.github/workflows/pages.yml

@@ -1,6 +1,7 @@
 # MIT License
 #
 # Copyright (c) 2022 just-the-docs
+# Copyright (c) 2022-2024 Luke Parker
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -20,31 +21,21 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-# This workflow uses actions that are not certified by GitHub.
-# They are provided by a third-party and are governed by
-# separate terms of service, privacy policy, and support
-# documentation.
-
-# Sample workflow for building and deploying a Jekyll site to GitHub Pages
-name: Deploy Jekyll site to Pages
+name: Deploy Rust docs and Jekyll site to Pages
 
 on:
   push:
     branches:
       - "develop"
-    paths:
-      - "docs/**"
 
-  # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
-# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
 permissions:
   contents: read
   pages: write
   id-token: write
 
-# Allow one concurrent deployment
+# Only allow one concurrent deployment
 concurrency:
   group: "pages"
   cancel-in-progress: true
@@ -53,9 +44,6 @@ jobs:
   # Build job
   build:
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: docs
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -69,11 +57,24 @@ jobs:
         id: pages
         uses: actions/configure-pages@v3
       - name: Build with Jekyll
-        run: bundle exec jekyll build --baseurl "${{ steps.pages.outputs.base_path }}"
+        run: cd ${{ github.workspace }}/docs && bundle exec jekyll build --baseurl "${{ steps.pages.outputs.base_path }}"
         env:
           JEKYLL_ENV: production
+
+      - name: Get nightly version to use
+        id: nightly
+        shell: bash
+        run: echo "version=$(cat .github/nightly-version)" >> $GITHUB_OUTPUT
+      - name: Build Dependencies
+        uses: ./.github/actions/build-dependencies
+      - name: Buld Rust docs
+        run: |
+          rustup toolchain install ${{ steps.nightly.outputs.version }} --profile minimal -t wasm32-unknown-unknown -c rust-docs
+          RUSTDOCFLAGS="--cfg docsrs" cargo +${{ steps.nightly.outputs.version }} doc --workspace --all-features
+          mv target/doc docs/_site/rust
+
       - name: Upload artifact
-        uses: actions/upload-pages-artifact@v1
+        uses: actions/upload-pages-artifact@v3
         with:
           path: "docs/_site/"
 
@@ -87,4 +88,4 @@ jobs:
     steps:
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v2
+        uses: actions/deploy-pages@v4

Cargo.toml

@@ -64,9 +64,6 @@ members = [
   "networks/monero/rpc/simple-request",
   "networks/monero/wallet/address",
   "networks/monero/wallet",
-  "networks/monero/wallet/seed",
-  "networks/monero/wallet/polyseed",
-  "networks/monero/wallet/util",
   "networks/monero/verify-chain",
 
   "message-queue",
@@ -144,9 +141,9 @@ members = [
 
   "tests/docker",
   "tests/message-queue",
-  "tests/processor",
-  "tests/coordinator",
-  "tests/full-stack",
+  # TODO "tests/processor",
+  # TODO "tests/coordinator",
+  # TODO "tests/full-stack",
   "tests/reproducible-runtime",
 ]
 
@@ -208,12 +205,10 @@ matches = { path = "patches/matches" }
 option-ext = { path = "patches/option-ext" }
 directories-next = { path = "patches/directories-next" }
 
-# The official pasta_curves repo doesn't support Zeroize
-pasta_curves = { git = "https://github.com/kayabaNerve/pasta_curves", rev = "a46b5be95cacbff54d06aad8d3bbcba42e05d616" }
-
 [workspace.lints.clippy]
 unwrap_or_default = "allow"
 map_unwrap_or = "allow"
+needless_continue = "allow"
 borrow_as_ptr = "deny"
 cast_lossless = "deny"
 cast_possible_truncation = "deny"
@@ -244,7 +239,6 @@ manual_string_new = "deny"
 match_bool = "deny"
 match_same_arms = "deny"
 missing_fields_in_debug = "deny"
-needless_continue = "deny"
 needless_pass_by_value = "deny"
 ptr_cast_constness = "deny"
 range_minus_one = "deny"

LICENSE

@@ -5,4 +5,4 @@ a full copy of the AGPL-3.0 License is included in the root of this repository
 as a reference text. This copy should be provided with any distribution of a
 crate licensed under the AGPL-3.0, as per its terms.
 
-The GitHub actions (`.github/actions`) are licensed under the MIT license.
+The GitHub actions/workflows (`.github`) are licensed under the MIT license.

common/request/src/response.rs

@@ -11,7 +11,7 @@ use crate::{Client, Error};
 #[allow(dead_code)]
 #[derive(Debug)]
 pub struct Response<'a>(pub(crate) hyper::Response<Incoming>, pub(crate) &'a Client);
-impl<'a> Response<'a> {
+impl Response<'_> {
   pub fn status(&self) -> StatusCode {
     self.0.status()
   }

coordinator/Cargo.toml

@@ -25,12 +25,13 @@ rand_core = { version = "0.6", default-features = false, features = ["std"] }
 blake2 = { version = "0.10", default-features = false, features = ["std"] }
 schnorrkel = { version = "0.11", default-features = false, features = ["std"] }
 
-ciphersuite = { path = "../crypto/ciphersuite", default-features = false, features = ["std"] }
-schnorr = { package = "schnorr-signatures", path = "../crypto/schnorr", default-features = false, features = ["std"] }
-frost = { package = "modular-frost", path = "../crypto/frost" }
+ciphersuite = { path = "../crypto/ciphersuite", default-features = false, features = ["std", "ristretto"] }
+dkg = { path = "../crypto/dkg", default-features = false, features = ["std"] }
 frost-schnorrkel = { path = "../crypto/schnorrkel" }
 
+hex = { version = "0.4", default-features = false, features = ["std"] }
 scale = { package = "parity-scale-codec", version = "3", default-features = false, features = ["std", "derive", "bit-vec"] }
+borsh = { version = "1", default-features = false, features = ["std", "derive", "de_strict_order"] }
 
 zalloc = { path = "../common/zalloc" }
 serai-db = { path = "../common/db" }
@@ -43,9 +44,6 @@ tributary-sdk = { path = "./tributary-sdk" }
 
 serai-client = { path = "../substrate/client", default-features = false, features = ["serai", "borsh"] }
 
-hex = { version = "0.4", default-features = false, features = ["std"] }
-borsh = { version = "1", default-features = false, features = ["std", "derive", "de_strict_order"] }
-
 log = { version = "0.4", default-features = false, features = ["std"] }
 env_logger = { version = "0.10", default-features = false, features = ["humantime"] }
 

coordinator/cosign/src/evaluator.rs

@@ -1,5 +1,5 @@
 use core::future::Future;
-use std::time::{Duration, SystemTime};
+use std::time::{Duration, Instant, SystemTime};
 
 use serai_db::*;
 use serai_task::ContinuallyRan;
@@ -77,12 +77,22 @@ pub(crate) fn currently_evaluated_global_session(getter: &impl Get) -> Option<[u
 pub(crate) struct CosignEvaluatorTask<D: Db, R: RequestNotableCosigns> {
   pub(crate) db: D,
   pub(crate) request: R,
+  pub(crate) last_request_for_cosigns: Instant,
 }
 
 impl<D: Db, R: RequestNotableCosigns> ContinuallyRan for CosignEvaluatorTask<D, R> {
   type Error = String;
 
   fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, Self::Error>> {
+    let should_request_cosigns = |last_request_for_cosigns: &mut Instant| {
+      const REQUEST_COSIGNS_SPACING: Duration = Duration::from_secs(60);
+      if Instant::now() < (*last_request_for_cosigns + REQUEST_COSIGNS_SPACING) {
+        return false;
+      }
+      *last_request_for_cosigns = Instant::now();
+      true
+    };
+
     async move {
       let mut known_cosign = None;
       let mut made_progress = false;
@@ -118,12 +128,13 @@ impl<D: Db, R: RequestNotableCosigns> ContinuallyRan for CosignEvaluatorTask<D,
             // Check if the sum weight doesn't cross the required threshold
             if weight_cosigned < (((global_session_info.total_stake * 83) / 100) + 1) {
               // Request the necessary cosigns over the network
-              // TODO: Add a timer to ensure this isn't called too often
-              self
-                .request
-                .request_notable_cosigns(global_session)
-                .await
-                .map_err(|e| format!("{e:?}"))?;
+              if should_request_cosigns(&mut self.last_request_for_cosigns) {
+                self
+                  .request
+                  .request_notable_cosigns(global_session)
+                  .await
+                  .map_err(|e| format!("{e:?}"))?;
+              }
               // We return an error so the delay before this task is run again increases
               return Err(format!(
                 "notable block (#{block_number}) wasn't yet cosigned. this should resolve shortly",
@@ -180,11 +191,13 @@ impl<D: Db, R: RequestNotableCosigns> ContinuallyRan for CosignEvaluatorTask<D,
                 // If this session hasn't yet produced notable cosigns, then we presume we'll see
                 // the desired non-notable cosigns as part of normal operations, without needing to
                 // explicitly request them
-                self
-                  .request
-                  .request_notable_cosigns(global_session)
-                  .await
-                  .map_err(|e| format!("{e:?}"))?;
+                if should_request_cosigns(&mut self.last_request_for_cosigns) {
+                  self
+                    .request
+                    .request_notable_cosigns(global_session)
+                    .await
+                    .map_err(|e| format!("{e:?}"))?;
+                }
                 // We return an error so the delay before this task is run again increases
                 return Err(format!(
                   "block (#{block_number}) wasn't yet cosigned. this should resolve shortly",

coordinator/cosign/src/intend.rs

@@ -3,7 +3,7 @@ use std::{sync::Arc, collections::HashMap};
 
 use serai_client::{
   primitives::{SeraiAddress, Amount},
-  validator_sets::primitives::ValidatorSet,
+  validator_sets::primitives::ExternalValidatorSet,
   Serai,
 };
 
@@ -28,7 +28,7 @@ db_channel! {
   CosignIntendChannels {
     GlobalSessionsChannel: () -> ([u8; 32], GlobalSession),
     BlockEvents: () -> BlockEventData,
-    IntendedCosigns: (set: ValidatorSet) -> CosignIntent,
+    IntendedCosigns: (set: ExternalValidatorSet) -> CosignIntent,
   }
 }
 
@@ -110,7 +110,7 @@ impl<D: Db> ContinuallyRan for CosignIntendTask<D> {
             keys.insert(set.network, SeraiAddress::from(*key));
             let stake = serai
               .validator_sets()
-              .total_allocated_stake(set.network)
+              .total_allocated_stake(set.network.into())
               .await
               .map_err(|e| format!("{e:?}"))?
               .unwrap_or(Amount(0))

coordinator/cosign/src/lib.rs

@@ -3,7 +3,7 @@
 #![deny(missing_docs)]
 
 use core::{fmt::Debug, future::Future};
-use std::{sync::Arc, collections::HashMap};
+use std::{sync::Arc, collections::HashMap, time::Instant};
 
 use blake2::{Digest, Blake2s256};
 
@@ -11,8 +11,8 @@ use scale::{Encode, Decode};
 use borsh::{BorshSerialize, BorshDeserialize};
 
 use serai_client::{
-  primitives::{NetworkId, SeraiAddress},
-  validator_sets::primitives::{Session, ValidatorSet, KeyPair},
+  primitives::{ExternalNetworkId, SeraiAddress},
+  validator_sets::primitives::{Session, ExternalValidatorSet, KeyPair},
   Public, Block, Serai, TemporalSerai,
 };
 
@@ -52,13 +52,13 @@ pub const COSIGN_CONTEXT: &[u8] = b"/serai/coordinator/cosign";
 #[derive(Debug, BorshSerialize, BorshDeserialize)]
 pub(crate) struct GlobalSession {
   pub(crate) start_block_number: u64,
-  pub(crate) sets: Vec<ValidatorSet>,
-  pub(crate) keys: HashMap<NetworkId, SeraiAddress>,
-  pub(crate) stakes: HashMap<NetworkId, u64>,
+  pub(crate) sets: Vec<ExternalValidatorSet>,
+  pub(crate) keys: HashMap<ExternalNetworkId, SeraiAddress>,
+  pub(crate) stakes: HashMap<ExternalNetworkId, u64>,
   pub(crate) total_stake: u64,
 }
 impl GlobalSession {
-  fn id(mut cosigners: Vec<ValidatorSet>) -> [u8; 32] {
+  fn id(mut cosigners: Vec<ExternalValidatorSet>) -> [u8; 32] {
     cosigners.sort_by_key(|a| borsh::to_vec(a).unwrap());
     Blake2s256::digest(borsh::to_vec(&cosigners).unwrap()).into()
   }
@@ -101,7 +101,25 @@ pub struct Cosign {
   /// The hash of the block to cosign.
   pub block_hash: [u8; 32],
   /// The actual cosigner.
-  pub cosigner: NetworkId,
+  pub cosigner: ExternalNetworkId,
+}
+
+impl CosignIntent {
+  /// Convert this into a `Cosign`.
+  pub fn into_cosign(self, cosigner: ExternalNetworkId) -> Cosign {
+    let CosignIntent { global_session, block_number, block_hash, notable: _ } = self;
+    Cosign { global_session, block_number, block_hash, cosigner }
+  }
+}
+
+impl Cosign {
+  /// The message to sign to sign this cosign.
+  ///
+  /// This must be signed with schnorrkel, the context set to `COSIGN_CONTEXT`.
+  pub fn signature_message(&self) -> Vec<u8> {
+    // We use a schnorrkel context to domain-separate this
+    self.encode()
+  }
 }
 
 /// A signed cosign.
@@ -118,7 +136,7 @@ impl SignedCosign {
     let Ok(signer) = schnorrkel::PublicKey::from_bytes(&signer.0) else { return false };
     let Ok(signature) = schnorrkel::Signature::from_bytes(&self.signature) else { return false };
 
-    signer.verify_simple(COSIGN_CONTEXT, &self.cosign.encode(), &signature).is_ok()
+    signer.verify_simple(COSIGN_CONTEXT, &self.cosign.signature_message(), &signature).is_ok()
   }
 }
 
@@ -148,7 +166,10 @@ create_db! {
     // one notable block. All validator sets will explicitly produce a cosign for their notable
     // block, causing the latest cosigned block for a global session to either be the global
     // session's notable cosigns or the network's latest cosigns.
-    NetworksLatestCosignedBlock: (global_session: [u8; 32], network: NetworkId) -> SignedCosign,
+    NetworksLatestCosignedBlock: (
+      global_session: [u8; 32],
+      network: ExternalNetworkId
+    ) -> SignedCosign,
     // Cosigns received for blocks not locally recognized as finalized.
     Faults: (global_session: [u8; 32]) -> Vec<SignedCosign>,
     // The global session which faulted.
@@ -159,15 +180,10 @@ create_db! {
 /// Fetch the keys used for cosigning by a specific network.
 async fn keys_for_network(
   serai: &TemporalSerai<'_>,
-  network: NetworkId,
+  network: ExternalNetworkId,
 ) -> Result<Option<(Session, KeyPair)>, String> {
-  // The Serai network never cosigns so it has no keys for cosigning
-  if network == NetworkId::Serai {
-    return Ok(None);
-  }
-
   let Some(latest_session) =
-    serai.validator_sets().session(network).await.map_err(|e| format!("{e:?}"))?
+    serai.validator_sets().session(network.into()).await.map_err(|e| format!("{e:?}"))?
   else {
     // If this network hasn't had a session declared, move on
     return Ok(None);
@@ -176,7 +192,7 @@ async fn keys_for_network(
   // Get the keys for the latest session
   if let Some(keys) = serai
     .validator_sets()
-    .keys(ValidatorSet { network, session: latest_session })
+    .keys(ExternalValidatorSet { network, session: latest_session })
     .await
     .map_err(|e| format!("{e:?}"))?
   {
@@ -187,7 +203,7 @@ async fn keys_for_network(
   if let Some(prior_session) = latest_session.0.checked_sub(1).map(Session) {
     if let Some(keys) = serai
       .validator_sets()
-      .keys(ValidatorSet { network, session: prior_session })
+      .keys(ExternalValidatorSet { network, session: prior_session })
       .await
       .map_err(|e| format!("{e:?}"))?
     {
@@ -198,16 +214,19 @@ async fn keys_for_network(
   Ok(None)
 }
 
-/// Fetch the `ValidatorSet`s, and their associated keys, used for cosigning as of this block.
-async fn cosigning_sets(serai: &TemporalSerai<'_>) -> Result<Vec<(ValidatorSet, Public)>, String> {
-  let mut sets = Vec::with_capacity(serai_client::primitives::NETWORKS.len());
-  for network in serai_client::primitives::NETWORKS {
+/// Fetch the `ExternalValidatorSet`s, and their associated keys, used for cosigning as of this
+/// block.
+async fn cosigning_sets(
+  serai: &TemporalSerai<'_>,
+) -> Result<Vec<(ExternalValidatorSet, Public)>, String> {
+  let mut sets = Vec::with_capacity(serai_client::primitives::EXTERNAL_NETWORKS.len());
+  for network in serai_client::primitives::EXTERNAL_NETWORKS {
     let Some((session, keys)) = keys_for_network(serai, network).await? else {
       // If this network doesn't have usable keys, move on
       continue;
     };
 
-    sets.push((ValidatorSet { network, session }, keys.0));
+    sets.push((ExternalValidatorSet { network, session }, keys.0));
   }
   Ok(sets)
 }
@@ -288,8 +307,12 @@ impl<D: Db> Cosigning<D> {
         .continually_run(intend_task, vec![evaluator_task_handle]),
     );
     tokio::spawn(
-      (evaluator::CosignEvaluatorTask { db: db.clone(), request })
-        .continually_run(evaluator_task, vec![delay_task_handle]),
+      (evaluator::CosignEvaluatorTask {
+        db: db.clone(),
+        request,
+        last_request_for_cosigns: Instant::now(),
+      })
+      .continually_run(evaluator_task, vec![delay_task_handle]),
     );
     tokio::spawn(
       (delay::CosignDelayTask { db: db.clone() })
@@ -323,8 +346,8 @@ impl<D: Db> Cosigning<D> {
   /// If this global session hasn't produced any notable cosigns, this will return the latest
   /// cosigns for this session.
   pub fn notable_cosigns(getter: &impl Get, global_session: [u8; 32]) -> Vec<SignedCosign> {
-    let mut cosigns = Vec::with_capacity(serai_client::primitives::NETWORKS.len());
-    for network in serai_client::primitives::NETWORKS {
+    let mut cosigns = Vec::with_capacity(serai_client::primitives::EXTERNAL_NETWORKS.len());
+    for network in serai_client::primitives::EXTERNAL_NETWORKS {
       if let Some(cosign) = NetworksLatestCosignedBlock::get(getter, global_session, network) {
         cosigns.push(cosign);
       }
@@ -341,7 +364,7 @@ impl<D: Db> Cosigning<D> {
       let mut cosigns = Faults::get(&self.db, faulted).expect("faulted with no faults");
       // Also include all of our recognized-as-honest cosigns in an attempt to induce fault
       // identification in those who see the faulty cosigns as honest
-      for network in serai_client::primitives::NETWORKS {
+      for network in serai_client::primitives::EXTERNAL_NETWORKS {
         if let Some(cosign) = NetworksLatestCosignedBlock::get(&self.db, faulted, network) {
           if cosign.cosign.global_session == faulted {
             cosigns.push(cosign);
@@ -353,8 +376,8 @@ impl<D: Db> Cosigning<D> {
       let Some(global_session) = evaluator::currently_evaluated_global_session(&self.db) else {
         return vec![];
       };
-      let mut cosigns = Vec::with_capacity(serai_client::primitives::NETWORKS.len());
-      for network in serai_client::primitives::NETWORKS {
+      let mut cosigns = Vec::with_capacity(serai_client::primitives::EXTERNAL_NETWORKS.len());
+      for network in serai_client::primitives::EXTERNAL_NETWORKS {
         if let Some(cosign) = NetworksLatestCosignedBlock::get(&self.db, global_session, network) {
           cosigns.push(cosign);
         }
@@ -465,12 +488,12 @@ impl<D: Db> Cosigning<D> {
     Ok(())
   }
 
-  /// Receive intended cosigns to produce for this ValidatorSet.
+  /// Receive intended cosigns to produce for this ExternalValidatorSet.
   ///
   /// All cosigns intended, up to and including the next notable cosign, are returned.
   ///
   /// This will drain the internal channel and not re-yield these intentions again.
-  pub fn intended_cosigns(txn: &mut impl DbTxn, set: ValidatorSet) -> Vec<CosignIntent> {
+  pub fn intended_cosigns(txn: &mut impl DbTxn, set: ExternalValidatorSet) -> Vec<CosignIntent> {
     let mut res: Vec<CosignIntent> = vec![];
     // While we have yet to find a notable cosign...
     while !res.last().map(|cosign| cosign.notable).unwrap_or(false) {

coordinator/p2p/libp2p/Cargo.toml

@@ -35,7 +35,7 @@ tributary-sdk = { path = "../../tributary-sdk" }
 
 futures-util = { version = "0.3", default-features = false, features = ["std"] }
 tokio = { version = "1", default-features = false, features = ["sync"] }
-libp2p = { version = "0.52", default-features = false, features = ["tokio", "tcp", "noise", "yamux", "ping", "request-response", "gossipsub", "macros"] }
+libp2p = { version = "0.54", default-features = false, features = ["tokio", "tcp", "noise", "yamux", "ping", "request-response", "gossipsub", "macros"] }
 
 log = { version = "0.4", default-features = false, features = ["std"] }
 serai-task = { path = "../../../common/task", version = "0.1" }

coordinator/p2p/libp2p/src/authenticate.rs

@@ -11,8 +11,7 @@ use serai_client::primitives::PublicKey as Public;
 
 use futures_util::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
 use libp2p::{
-  core::UpgradeInfo,
-  InboundUpgrade, OutboundUpgrade,
+  core::upgrade::{UpgradeInfo, InboundConnectionUpgrade, OutboundConnectionUpgrade},
   identity::{self, PeerId},
   noise,
 };
@@ -119,12 +118,18 @@ impl UpgradeInfo for OnlyValidators {
   }
 }
 
-impl<S: 'static + Send + Unpin + AsyncRead + AsyncWrite> InboundUpgrade<S> for OnlyValidators {
+impl<S: 'static + Send + Unpin + AsyncRead + AsyncWrite> InboundConnectionUpgrade<S>
+  for OnlyValidators
+{
   type Output = (PeerId, noise::Output<S>);
   type Error = io::Error;
   type Future = Pin<Box<dyn Send + Future<Output = Result<Self::Output, Self::Error>>>>;
 
-  fn upgrade_inbound(self, socket: S, info: Self::Info) -> Self::Future {
+  fn upgrade_inbound(
+    self,
+    socket: S,
+    info: <Self as UpgradeInfo>::Info,
+  ) -> <Self as InboundConnectionUpgrade<S>>::Future {
     Box::pin(async move {
       let (dialer_noise_peer_id, mut socket) = noise::Config::new(&self.noise_keypair)
         .unwrap()
@@ -147,12 +152,18 @@ impl<S: 'static + Send + Unpin + AsyncRead + AsyncWrite> InboundUpgrade<S> for O
   }
 }
 
-impl<S: 'static + Send + Unpin + AsyncRead + AsyncWrite> OutboundUpgrade<S> for OnlyValidators {
+impl<S: 'static + Send + Unpin + AsyncRead + AsyncWrite> OutboundConnectionUpgrade<S>
+  for OnlyValidators
+{
   type Output = (PeerId, noise::Output<S>);
   type Error = io::Error;
   type Future = Pin<Box<dyn Send + Future<Output = Result<Self::Output, Self::Error>>>>;
 
-  fn upgrade_outbound(self, socket: S, info: Self::Info) -> Self::Future {
+  fn upgrade_outbound(
+    self,
+    socket: S,
+    info: <Self as UpgradeInfo>::Info,
+  ) -> <Self as OutboundConnectionUpgrade<S>>::Future {
     Box::pin(async move {
       let (listener_noise_peer_id, mut socket) = noise::Config::new(&self.noise_keypair)
         .unwrap()

coordinator/p2p/libp2p/src/lib.rs

@@ -14,8 +14,8 @@ use zeroize::Zeroizing;
 use schnorrkel::Keypair;
 
 use serai_client::{
-  primitives::{NetworkId, PublicKey},
-  validator_sets::primitives::ValidatorSet,
+  primitives::{ExternalNetworkId, PublicKey},
+  validator_sets::primitives::ExternalValidatorSet,
   Serai,
 };
 
@@ -50,7 +50,7 @@ mod ping;
 
 /// The request-response messages and behavior
 mod reqres;
-use reqres::{RequestId, Request, Response};
+use reqres::{InboundRequestId, Request, Response};
 
 /// The gossip messages and behavior
 mod gossip;
@@ -66,14 +66,6 @@ use dial::DialTask;
 
 const PORT: u16 = 30563; // 5132 ^ (('c' << 8) | 'o')
 
-// usize::max, manually implemented, as max isn't a const fn
-const MAX_LIBP2P_MESSAGE_SIZE: usize =
-  if gossip::MAX_LIBP2P_GOSSIP_MESSAGE_SIZE > reqres::MAX_LIBP2P_REQRES_MESSAGE_SIZE {
-    gossip::MAX_LIBP2P_GOSSIP_MESSAGE_SIZE
-  } else {
-    reqres::MAX_LIBP2P_REQRES_MESSAGE_SIZE
-  };
-
 fn peer_id_from_public(public: PublicKey) -> PeerId {
   // 0 represents the identity Multihash, that no hash was performed
   // It's an internal constant so we can't refer to the constant inside libp2p
@@ -112,7 +104,7 @@ impl serai_coordinator_p2p::Peer<'_> for Peer<'_> {
 
 #[derive(Clone)]
 struct Peers {
-  peers: Arc<RwLock<HashMap<NetworkId, HashSet<PeerId>>>>,
+  peers: Arc<RwLock<HashMap<ExternalNetworkId, HashSet<PeerId>>>>,
 }
 
 // Consider adding identify/kad/autonat/rendevous/(relay + dcutr). While we currently use the Serai
@@ -143,9 +135,10 @@ struct Libp2pInner {
   signed_cosigns: Mutex<mpsc::UnboundedReceiver<SignedCosign>>,
   signed_cosigns_send: mpsc::UnboundedSender<SignedCosign>,
 
-  heartbeat_requests: Mutex<mpsc::UnboundedReceiver<(RequestId, ValidatorSet, [u8; 32])>>,
-  notable_cosign_requests: Mutex<mpsc::UnboundedReceiver<(RequestId, [u8; 32])>>,
-  inbound_request_responses: mpsc::UnboundedSender<(RequestId, Response)>,
+  heartbeat_requests:
+    Mutex<mpsc::UnboundedReceiver<(InboundRequestId, ExternalValidatorSet, [u8; 32])>>,
+  notable_cosign_requests: Mutex<mpsc::UnboundedReceiver<(InboundRequestId, [u8; 32])>>,
+  inbound_request_responses: mpsc::UnboundedSender<(InboundRequestId, Response)>,
 }
 
 /// The libp2p-backed P2P implementation.
@@ -176,19 +169,9 @@ impl Libp2p {
         Ok(OnlyValidators { serai_key: serai_key.clone(), noise_keypair: noise_keypair.clone() })
       };
 
-      let new_yamux = || {
-        let mut config = yamux::Config::default();
-        // 1 MiB default + max message size
-        config.set_max_buffer_size((1024 * 1024) + MAX_LIBP2P_MESSAGE_SIZE);
-        // 256 KiB default + max message size
-        config
-          .set_receive_window_size(((256 * 1024) + MAX_LIBP2P_MESSAGE_SIZE).try_into().unwrap());
-        config
-      };
-
       let mut swarm = SwarmBuilder::with_existing_identity(identity::Keypair::generate_ed25519())
         .with_tokio()
-        .with_tcp(TcpConfig::default().nodelay(true), new_only_validators, new_yamux)
+        .with_tcp(TcpConfig::default().nodelay(true), new_only_validators, yamux::Config::default)
         .unwrap()
         .with_behaviour(|_| Behavior {
           allow_list: allow_block_list::Behaviour::default(),
@@ -330,7 +313,7 @@ impl serai_cosign::RequestNotableCosigns for Libp2p {
 impl serai_coordinator_p2p::P2p for Libp2p {
   type Peer<'a> = Peer<'a>;
 
-  fn peers(&self, network: NetworkId) -> impl Send + Future<Output = Vec<Self::Peer<'_>>> {
+  fn peers(&self, network: ExternalNetworkId) -> impl Send + Future<Output = Vec<Self::Peer<'_>>> {
     async move {
       let Some(peer_ids) = self.0.peers.peers.read().await.get(&network).cloned() else {
         return vec![];

coordinator/p2p/libp2p/src/reqres.rs

@@ -10,7 +10,7 @@ use futures_util::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
 use libp2p::request_response::{
   self, Codec as CodecTrait, Event as GenericEvent, Config, Behaviour, ProtocolSupport,
 };
-pub use request_response::{RequestId, Message};
+pub use request_response::{InboundRequestId, Message};
 
 use serai_cosign::SignedCosign;
 
@@ -129,7 +129,6 @@ pub(crate) type Event = GenericEvent<Request, Response>;
 
 pub(crate) type Behavior = Behaviour<Codec>;
 pub(crate) fn new_behavior() -> Behavior {
-  let mut config = Config::default();
-  config.set_request_timeout(Duration::from_secs(5));
+  let config = Config::default().with_request_timeout(Duration::from_secs(5));
   Behavior::new([(PROTOCOL, ProtocolSupport::Full)], config)
 }

coordinator/p2p/libp2p/src/swarm.rs

@@ -6,7 +6,7 @@ use std::{
 
 use borsh::BorshDeserialize;
 
-use serai_client::validator_sets::primitives::ValidatorSet;
+use serai_client::validator_sets::primitives::ExternalValidatorSet;
 
 use tokio::sync::{mpsc, oneshot, RwLock};
 
@@ -17,7 +17,7 @@ use serai_cosign::SignedCosign;
 use futures_util::StreamExt;
 use libp2p::{
   identity::PeerId,
-  request_response::{RequestId, ResponseChannel},
+  request_response::{InboundRequestId, OutboundRequestId, ResponseChannel},
   swarm::{dial_opts::DialOpts, SwarmEvent, Swarm},
 };
 
@@ -65,12 +65,12 @@ pub(crate) struct SwarmTask {
   tributary_gossip: mpsc::UnboundedSender<([u8; 32], Vec<u8>)>,
 
   outbound_requests: mpsc::UnboundedReceiver<(PeerId, Request, oneshot::Sender<Response>)>,
-  outbound_request_responses: HashMap<RequestId, oneshot::Sender<Response>>,
+  outbound_request_responses: HashMap<OutboundRequestId, oneshot::Sender<Response>>,
 
-  inbound_request_response_channels: HashMap<RequestId, ResponseChannel<Response>>,
-  heartbeat_requests: mpsc::UnboundedSender<(RequestId, ValidatorSet, [u8; 32])>,
-  notable_cosign_requests: mpsc::UnboundedSender<(RequestId, [u8; 32])>,
-  inbound_request_responses: mpsc::UnboundedReceiver<(RequestId, Response)>,
+  inbound_request_response_channels: HashMap<InboundRequestId, ResponseChannel<Response>>,
+  heartbeat_requests: mpsc::UnboundedSender<(InboundRequestId, ExternalValidatorSet, [u8; 32])>,
+  notable_cosign_requests: mpsc::UnboundedSender<(InboundRequestId, [u8; 32])>,
+  inbound_request_responses: mpsc::UnboundedReceiver<(InboundRequestId, Response)>,
 }
 
 impl SwarmTask {
@@ -222,25 +222,21 @@ impl SwarmTask {
               }
             }
 
-            SwarmEvent::Behaviour(
-              BehaviorEvent::AllowList(event) | BehaviorEvent::ConnectionLimits(event)
-            ) => {
-              // This *is* an exhaustive match as these events are empty enums
-              match event {}
-            }
-            SwarmEvent::Behaviour(
-              BehaviorEvent::Ping(ping::Event { peer: _, connection, result, })
-            ) => {
-              if result.is_err() {
-                self.swarm.close_connection(connection);
+            SwarmEvent::Behaviour(event) => {
+              match event {
+                BehaviorEvent::AllowList(event) | BehaviorEvent::ConnectionLimits(event) => {
+                  // This *is* an exhaustive match as these events are empty enums
+                  match event {}
+                }
+                BehaviorEvent::Ping(ping::Event { peer: _, connection, result, }) => {
+                  if result.is_err() {
+                    self.swarm.close_connection(connection);
+                  }
+                }
+                BehaviorEvent::Reqres(event) => self.handle_reqres(event),
+                BehaviorEvent::Gossip(event) => self.handle_gossip(event),
               }
             }
-            SwarmEvent::Behaviour(BehaviorEvent::Reqres(event)) => {
-              self.handle_reqres(event)
-            }
-            SwarmEvent::Behaviour(BehaviorEvent::Gossip(event)) => {
-              self.handle_gossip(event)
-            }
 
             // We don't handle any of these
             SwarmEvent::IncomingConnection { .. } |
@@ -250,7 +246,14 @@ impl SwarmTask {
             SwarmEvent::ExpiredListenAddr { .. } |
             SwarmEvent::ListenerClosed { .. } |
             SwarmEvent::ListenerError { .. } |
-            SwarmEvent::Dialing { .. } => {}
+            SwarmEvent::Dialing { .. } |
+            SwarmEvent::NewExternalAddrCandidate { .. } |
+            SwarmEvent::ExternalAddrConfirmed { .. } |
+            SwarmEvent::ExternalAddrExpired { .. } |
+            SwarmEvent::NewExternalAddrOfPeer { .. } => {}
+
+            // Requires as SwarmEvent is non-exhaustive
+            _ => log::warn!("unhandled SwarmEvent: {event:?}"),
           }
         }
 
@@ -321,9 +324,9 @@ impl SwarmTask {
 
     outbound_requests: mpsc::UnboundedReceiver<(PeerId, Request, oneshot::Sender<Response>)>,
 
-    heartbeat_requests: mpsc::UnboundedSender<(RequestId, ValidatorSet, [u8; 32])>,
-    notable_cosign_requests: mpsc::UnboundedSender<(RequestId, [u8; 32])>,
-    inbound_request_responses: mpsc::UnboundedReceiver<(RequestId, Response)>,
+    heartbeat_requests: mpsc::UnboundedSender<(InboundRequestId, ExternalValidatorSet, [u8; 32])>,
+    notable_cosign_requests: mpsc::UnboundedSender<(InboundRequestId, [u8; 32])>,
+    inbound_request_responses: mpsc::UnboundedReceiver<(InboundRequestId, Response)>,
   ) {
     tokio::spawn(
       SwarmTask {

coordinator/p2p/libp2p/src/validators.rs

@@ -4,7 +4,9 @@ use std::{
   collections::{HashSet, HashMap},
 };
 
-use serai_client::{primitives::NetworkId, validator_sets::primitives::Session, SeraiError, Serai};
+use serai_client::{
+  primitives::ExternalNetworkId, validator_sets::primitives::Session, SeraiError, Serai,
+};
 
 use serai_task::{Task, ContinuallyRan};
 
@@ -24,11 +26,11 @@ pub(crate) struct Validators {
   serai: Arc<Serai>,
 
   // A cache for which session we're populated with the validators of
-  sessions: HashMap<NetworkId, Session>,
+  sessions: HashMap<ExternalNetworkId, Session>,
   // The validators by network
-  by_network: HashMap<NetworkId, HashSet<PeerId>>,
+  by_network: HashMap<ExternalNetworkId, HashSet<PeerId>>,
   // The validators and their networks
-  validators: HashMap<PeerId, HashSet<NetworkId>>,
+  validators: HashMap<PeerId, HashSet<ExternalNetworkId>>,
 
   // The channel to send the changes down
   changes: mpsc::UnboundedSender<Changes>,
@@ -49,8 +51,16 @@ impl Validators {
 
   async fn session_changes(
     serai: impl Borrow<Serai>,
-    sessions: impl Borrow<HashMap<NetworkId, Session>>,
-  ) -> Result<Vec<(NetworkId, Session, HashSet<PeerId>)>, SeraiError> {
+    sessions: impl Borrow<HashMap<ExternalNetworkId, Session>>,
+  ) -> Result<Vec<(ExternalNetworkId, Session, HashSet<PeerId>)>, SeraiError> {
+    /*
+      This uses the latest finalized block, not the latest cosigned block, which should be fine as
+      in the worst case, we'd connect to unexpected validators. They still shouldn't be able to
+      bypass the cosign protocol unless a historical global session was malicious, in which case
+      the cosign protocol already breaks.
+
+      Besides, we can't connect to historical validators, only the current validators.
+    */
     let temporal_serai = serai.borrow().as_of_latest_finalized_block().await?;
     let temporal_serai = temporal_serai.validator_sets();
 
@@ -59,13 +69,10 @@ impl Validators {
       // FuturesUnordered can be bad practice as it'll cause timeouts if infrequently polled, but
       // we poll it till it yields all futures with the most minimal processing possible
       let mut futures = FuturesUnordered::new();
-      for network in serai_client::primitives::NETWORKS {
-        if network == NetworkId::Serai {
-          continue;
-        }
+      for network in serai_client::primitives::EXTERNAL_NETWORKS {
         let sessions = sessions.borrow();
         futures.push(async move {
-          let session = match temporal_serai.session(network).await {
+          let session = match temporal_serai.session(network.into()).await {
             Ok(Some(session)) => session,
             Ok(None) => return Ok(None),
             Err(e) => return Err(e),
@@ -74,7 +81,7 @@ impl Validators {
           if sessions.get(&network) == Some(&session) {
             Ok(None)
           } else {
-            match temporal_serai.active_network_validators(network).await {
+            match temporal_serai.active_network_validators(network.into()).await {
               Ok(validators) => Ok(Some((
                 network,
                 session,
@@ -97,7 +104,7 @@ impl Validators {
 
   fn incorporate_session_changes(
     &mut self,
-    session_changes: Vec<(NetworkId, Session, HashSet<PeerId>)>,
+    session_changes: Vec<(ExternalNetworkId, Session, HashSet<PeerId>)>,
   ) {
     let mut removed = HashSet::new();
     let mut added = HashSet::new();
@@ -152,11 +159,11 @@ impl Validators {
     Ok(())
   }
 
-  pub(crate) fn by_network(&self) -> &HashMap<NetworkId, HashSet<PeerId>> {
+  pub(crate) fn by_network(&self) -> &HashMap<ExternalNetworkId, HashSet<PeerId>> {
     &self.by_network
   }
 
-  pub(crate) fn networks(&self, peer_id: &PeerId) -> Option<&HashSet<NetworkId>> {
+  pub(crate) fn networks(&self, peer_id: &PeerId) -> Option<&HashSet<ExternalNetworkId>> {
     self.validators.get(peer_id)
   }
 }

coordinator/p2p/src/heartbeat.rs

@@ -1,7 +1,7 @@
 use core::future::Future;
 use std::time::{Duration, SystemTime};
 
-use serai_client::validator_sets::primitives::{MAX_KEY_SHARES_PER_SET, ValidatorSet};
+use serai_client::validator_sets::primitives::{MAX_KEY_SHARES_PER_SET, ExternalValidatorSet};
 
 use futures_lite::FutureExt;
 
@@ -38,7 +38,7 @@ pub const BATCH_SIZE_LIMIT: usize = MIN_BLOCKS_PER_BATCH *
 /// If the other validator has more blocks then we do, they're expected to inform us. This forms
 /// the sync protocol for our Tributaries.
 pub(crate) struct HeartbeatTask<TD: Db, Tx: TransactionTrait, P: P2p> {
-  pub(crate) set: ValidatorSet,
+  pub(crate) set: ExternalValidatorSet,
   pub(crate) tributary: Tributary<TD, Tx, P>,
   pub(crate) reader: TributaryReader<TD, Tx>,
   pub(crate) p2p: P,

coordinator/p2p/src/lib.rs

@@ -7,7 +7,7 @@ use std::collections::HashMap;
 
 use borsh::{BorshSerialize, BorshDeserialize};
 
-use serai_client::{primitives::NetworkId, validator_sets::primitives::ValidatorSet};
+use serai_client::{primitives::ExternalNetworkId, validator_sets::primitives::ExternalValidatorSet};
 
 use serai_db::Db;
 use tributary_sdk::{ReadWrite, TransactionTrait, Tributary, TributaryReader};
@@ -25,7 +25,7 @@ use crate::heartbeat::HeartbeatTask;
 #[derive(Clone, Copy, BorshSerialize, BorshDeserialize, Debug)]
 pub struct Heartbeat {
   /// The Tributary this is the heartbeat of.
-  pub set: ValidatorSet,
+  pub set: ExternalValidatorSet,
   /// The hash of the latest block added to the Tributary.
   pub latest_block_hash: [u8; 32],
 }
@@ -56,7 +56,7 @@ pub trait P2p:
   type Peer<'a>: Peer<'a>;
 
   /// Fetch the peers for this network.
-  fn peers(&self, network: NetworkId) -> impl Send + Future<Output = Vec<Self::Peer<'_>>>;
+  fn peers(&self, network: ExternalNetworkId) -> impl Send + Future<Output = Vec<Self::Peer<'_>>>;
 
   /// Broadcast a cosign.
   fn publish_cosign(&self, cosign: SignedCosign) -> impl Send + Future<Output = ()>;
@@ -131,13 +131,13 @@ fn handle_heartbeat<D: Db, T: TransactionTrait>(
 pub async fn run<TD: Db, Tx: TransactionTrait, P: P2p>(
   db: impl Db,
   p2p: P,
-  mut add_tributary: mpsc::UnboundedReceiver<(ValidatorSet, Tributary<TD, Tx, P>)>,
-  mut retire_tributary: mpsc::UnboundedReceiver<ValidatorSet>,
+  mut add_tributary: mpsc::UnboundedReceiver<(ExternalValidatorSet, Tributary<TD, Tx, P>)>,
+  mut retire_tributary: mpsc::UnboundedReceiver<ExternalValidatorSet>,
   send_cosigns: mpsc::UnboundedSender<SignedCosign>,
 ) {
-  let mut readers = HashMap::<ValidatorSet, TributaryReader<TD, Tx>>::new();
+  let mut readers = HashMap::<ExternalValidatorSet, TributaryReader<TD, Tx>>::new();
   let mut tributaries = HashMap::<[u8; 32], mpsc::UnboundedSender<Vec<u8>>>::new();
-  let mut heartbeat_tasks = HashMap::<ValidatorSet, _>::new();
+  let mut heartbeat_tasks = HashMap::<ExternalValidatorSet, _>::new();
 
   loop {
     tokio::select! {

coordinator/src/db.rs

@@ -3,9 +3,11 @@ use std::{path::Path, fs};
 pub(crate) use serai_db::{Get, DbTxn, Db as DbTrait};
 use serai_db::{create_db, db_channel};
 
+use dkg::Participant;
+
 use serai_client::{
-  primitives::NetworkId,
-  validator_sets::primitives::{Session, ValidatorSet},
+  primitives::ExternalNetworkId,
+  validator_sets::primitives::{Session, ExternalValidatorSet, KeyPair},
 };
 
 use serai_cosign::SignedCosign;
@@ -13,7 +15,7 @@ use serai_coordinator_substrate::NewSetInformation;
 use serai_coordinator_tributary::Transaction;
 
 #[cfg(all(feature = "parity-db", not(feature = "rocksdb")))]
-pub(crate) type Db = serai_db::ParityDb;
+pub(crate) type Db = std::sync::Arc<serai_db::ParityDb>;
 #[cfg(feature = "rocksdb")]
 pub(crate) type Db = serai_db::RocksDB;
 
@@ -41,22 +43,21 @@ pub(crate) fn coordinator_db() -> Db {
   db(&format!("{root_path}/coordinator/db"))
 }
 
-fn tributary_db_folder(set: ValidatorSet) -> String {
+fn tributary_db_folder(set: ExternalValidatorSet) -> String {
   let root_path = serai_env::var("DB_PATH").expect("path to DB wasn't specified");
   let network = match set.network {
-    NetworkId::Serai => panic!("creating Tributary for the Serai network"),
-    NetworkId::Bitcoin => "Bitcoin",
-    NetworkId::Ethereum => "Ethereum",
-    NetworkId::Monero => "Monero",
+    ExternalNetworkId::Bitcoin => "Bitcoin",
+    ExternalNetworkId::Ethereum => "Ethereum",
+    ExternalNetworkId::Monero => "Monero",
   };
   format!("{root_path}/tributary-{network}-{}", set.session.0)
 }
 
-pub(crate) fn tributary_db(set: ValidatorSet) -> Db {
+pub(crate) fn tributary_db(set: ExternalValidatorSet) -> Db {
   db(&format!("{}/db", tributary_db_folder(set)))
 }
 
-pub(crate) fn prune_tributary_db(set: ValidatorSet) {
+pub(crate) fn prune_tributary_db(set: ExternalValidatorSet) {
   log::info!("pruning data directory for tributary {set:?}");
   let db = tributary_db_folder(set);
   if fs::exists(&db).expect("couldn't check if tributary DB exists") {
@@ -71,11 +72,15 @@ create_db! {
     // The latest Tributary to have been retired for a network
     // Since Tributaries are retired sequentially, this is informative to if any Tributary has been
     // retired
-    RetiredTributary: (network: NetworkId) -> Session,
+    RetiredTributary: (network: ExternalNetworkId) -> Session,
     // The last handled message from a Processor
-    LastProcessorMessage: (network: NetworkId) -> u64,
+    LastProcessorMessage: (network: ExternalNetworkId) -> u64,
     // Cosigns we produced and tried to intake yet incurred an error while doing so
     ErroneousCosigns: () -> Vec<SignedCosign>,
+    // The keys to confirm and set on the Serai network
+    KeysToConfirm: (set: ExternalValidatorSet) -> KeyPair,
+    // The key was set on the Serai network
+    KeySet: (set: ExternalValidatorSet) -> (),
   }
 }
 
@@ -84,7 +89,7 @@ db_channel! {
     // Cosigns we produced
     SignedCosigns: () -> SignedCosign,
     // Tributaries to clean up upon reboot
-    TributaryCleanup: () -> ValidatorSet,
+    TributaryCleanup: () -> ExternalValidatorSet,
   }
 }
 
@@ -93,21 +98,51 @@ mod _internal_db {
 
   db_channel! {
     Coordinator {
-      // Tributary transactions to publish
-      TributaryTransactions: (set: ValidatorSet) -> Transaction,
+      // Tributary transactions to publish from the Processor messages
+      TributaryTransactionsFromProcessorMessages: (set: ExternalValidatorSet) -> Transaction,
+      // Tributary transactions to publish from the DKG confirmation task
+      TributaryTransactionsFromDkgConfirmation: (set: ExternalValidatorSet) -> Transaction,
+      // Participants to remove
+      RemoveParticipant: (set: ExternalValidatorSet) -> Participant,
     }
   }
 }
 
-pub(crate) struct TributaryTransactions;
-impl TributaryTransactions {
-  pub(crate) fn send(txn: &mut impl DbTxn, set: ValidatorSet, tx: &Transaction) {
+pub(crate) struct TributaryTransactionsFromProcessorMessages;
+impl TributaryTransactionsFromProcessorMessages {
+  pub(crate) fn send(txn: &mut impl DbTxn, set: ExternalValidatorSet, tx: &Transaction) {
     // If this set has yet to be retired, send this transaction
     if RetiredTributary::get(txn, set.network).map(|session| session.0) < Some(set.session.0) {
-      _internal_db::TributaryTransactions::send(txn, set, tx);
+      _internal_db::TributaryTransactionsFromProcessorMessages::send(txn, set, tx);
     }
   }
-  pub(crate) fn try_recv(txn: &mut impl DbTxn, set: ValidatorSet) -> Option<Transaction> {
-    _internal_db::TributaryTransactions::try_recv(txn, set)
+  pub(crate) fn try_recv(txn: &mut impl DbTxn, set: ExternalValidatorSet) -> Option<Transaction> {
+    _internal_db::TributaryTransactionsFromProcessorMessages::try_recv(txn, set)
+  }
+}
+
+pub(crate) struct TributaryTransactionsFromDkgConfirmation;
+impl TributaryTransactionsFromDkgConfirmation {
+  pub(crate) fn send(txn: &mut impl DbTxn, set: ExternalValidatorSet, tx: &Transaction) {
+    // If this set has yet to be retired, send this transaction
+    if RetiredTributary::get(txn, set.network).map(|session| session.0) < Some(set.session.0) {
+      _internal_db::TributaryTransactionsFromDkgConfirmation::send(txn, set, tx);
+    }
+  }
+  pub(crate) fn try_recv(txn: &mut impl DbTxn, set: ExternalValidatorSet) -> Option<Transaction> {
+    _internal_db::TributaryTransactionsFromDkgConfirmation::try_recv(txn, set)
+  }
+}
+
+pub(crate) struct RemoveParticipant;
+impl RemoveParticipant {
+  pub(crate) fn send(txn: &mut impl DbTxn, set: ExternalValidatorSet, participant: Participant) {
+    // If this set has yet to be retired, send this transaction
+    if RetiredTributary::get(txn, set.network).map(|session| session.0) < Some(set.session.0) {
+      _internal_db::RemoveParticipant::send(txn, set, &participant);
+    }
+  }
+  pub(crate) fn try_recv(txn: &mut impl DbTxn, set: ExternalValidatorSet) -> Option<Participant> {
+    _internal_db::RemoveParticipant::try_recv(txn, set)
   }
 }

coordinator/src/dkg_confirmation.rs

@@ -0,0 +1,437 @@
+use core::{ops::Deref, future::Future};
+use std::{boxed::Box, collections::HashMap};
+
+use zeroize::Zeroizing;
+use rand_core::OsRng;
+use ciphersuite::{group::GroupEncoding, Ciphersuite, Ristretto};
+use frost_schnorrkel::{
+  frost::{
+    dkg::{Participant, musig::musig},
+    FrostError,
+    sign::*,
+  },
+  Schnorrkel,
+};
+
+use serai_db::{DbTxn, Db as DbTrait};
+
+use serai_client::{
+  primitives::SeraiAddress,
+  validator_sets::primitives::{ExternalValidatorSet, musig_context, set_keys_message},
+};
+
+use serai_task::{DoesNotError, ContinuallyRan};
+
+use serai_coordinator_substrate::{NewSetInformation, Keys};
+use serai_coordinator_tributary::{Transaction, DkgConfirmationMessages};
+
+use crate::{KeysToConfirm, KeySet, TributaryTransactionsFromDkgConfirmation};
+
+fn schnorrkel() -> Schnorrkel {
+  Schnorrkel::new(b"substrate") // TODO: Pull the constant for this
+}
+
+fn our_i(
+  set: &NewSetInformation,
+  key: &Zeroizing<<Ristretto as Ciphersuite>::F>,
+  data: &HashMap<Participant, Vec<u8>>,
+) -> Participant {
+  let public = SeraiAddress((Ristretto::generator() * key.deref()).to_bytes());
+
+  let mut our_i = None;
+  for participant in data.keys() {
+    let validator_index = usize::from(u16::from(*participant) - 1);
+    let (validator, _weight) = set.validators[validator_index];
+    if validator == public {
+      our_i = Some(*participant);
+    }
+  }
+  our_i.unwrap()
+}
+
+// Take a HashMap of participations with non-contiguous Participants and convert them to a
+// contiguous sequence.
+//
+// The input data is expected to not include our own data, which also won't be in the output data.
+//
+// Returns the mapping from the contiguous Participants to the original Participants.
+fn make_contiguous<T>(
+  our_i: Participant,
+  mut data: HashMap<Participant, Vec<u8>>,
+  transform: impl Fn(Vec<u8>) -> std::io::Result<T>,
+) -> Result<HashMap<Participant, T>, Participant> {
+  assert!(!data.contains_key(&our_i));
+
+  let mut ordered_participants = data.keys().copied().collect::<Vec<_>>();
+  ordered_participants.sort_by_key(|participant| u16::from(*participant));
+
+  let mut our_i = Some(our_i);
+  let mut contiguous = HashMap::new();
+  let mut i = 1;
+  for participant in ordered_participants {
+    // If this is the first participant after our own index, increment to account for our index
+    if let Some(our_i_value) = our_i {
+      if u16::from(participant) > u16::from(our_i_value) {
+        i += 1;
+        our_i = None;
+      }
+    }
+
+    let contiguous_index = Participant::new(i).unwrap();
+    let data = match transform(data.remove(&participant).unwrap()) {
+      Ok(data) => data,
+      Err(_) => Err(participant)?,
+    };
+    contiguous.insert(contiguous_index, data);
+    i += 1;
+  }
+  Ok(contiguous)
+}
+
+fn handle_frost_error<T>(result: Result<T, FrostError>) -> Result<T, Participant> {
+  match &result {
+    Ok(_) => Ok(result.unwrap()),
+    Err(FrostError::InvalidPreprocess(participant) | FrostError::InvalidShare(participant)) => {
+      Err(*participant)
+    }
+    // All of these should be unreachable
+    Err(
+      FrostError::InternalError(_) |
+      FrostError::InvalidParticipant(_, _) |
+      FrostError::InvalidSigningSet(_) |
+      FrostError::InvalidParticipantQuantity(_, _) |
+      FrostError::DuplicatedParticipant(_) |
+      FrostError::MissingParticipant(_),
+    ) => {
+      result.unwrap();
+      unreachable!("continued execution after unwrapping Result::Err");
+    }
+  }
+}
+
+#[rustfmt::skip]
+enum Signer {
+  Preprocess { attempt: u32, seed: CachedPreprocess, preprocess: [u8; 64] },
+  Share {
+    attempt: u32,
+    musig_validators: Vec<SeraiAddress>,
+    share: [u8; 32],
+    machine: Box<AlgorithmSignatureMachine<Ristretto, Schnorrkel>>,
+  },
+}
+
+/// Performs the DKG Confirmation protocol.
+pub(crate) struct ConfirmDkgTask<CD: DbTrait, TD: DbTrait> {
+  db: CD,
+
+  set: NewSetInformation,
+  tributary_db: TD,
+
+  key: Zeroizing<<Ristretto as Ciphersuite>::F>,
+  signer: Option<Signer>,
+}
+
+impl<CD: DbTrait, TD: DbTrait> ConfirmDkgTask<CD, TD> {
+  pub(crate) fn new(
+    db: CD,
+    set: NewSetInformation,
+    tributary_db: TD,
+    key: Zeroizing<<Ristretto as Ciphersuite>::F>,
+  ) -> Self {
+    Self { db, set, tributary_db, key, signer: None }
+  }
+
+  fn slash(db: &mut CD, set: ExternalValidatorSet, validator: SeraiAddress) {
+    let mut txn = db.txn();
+    TributaryTransactionsFromDkgConfirmation::send(
+      &mut txn,
+      set,
+      &Transaction::RemoveParticipant { participant: validator, signed: Default::default() },
+    );
+    txn.commit();
+  }
+
+  fn preprocess(
+    db: &mut CD,
+    set: ExternalValidatorSet,
+    attempt: u32,
+    key: &Zeroizing<<Ristretto as Ciphersuite>::F>,
+    signer: &mut Option<Signer>,
+  ) {
+    // Perform the preprocess
+    let (machine, preprocess) = AlgorithmMachine::new(
+      schnorrkel(),
+      // We use a 1-of-1 Musig here as we don't know who will actually be in this Musig yet
+      musig(&musig_context(set.into()), key, &[Ristretto::generator() * key.deref()])
+        .unwrap()
+        .into(),
+    )
+    .preprocess(&mut OsRng);
+    // We take the preprocess so we can use it in a distinct machine with the actual Musig
+    // parameters
+    let seed = machine.cache();
+
+    let mut preprocess_bytes = [0u8; 64];
+    preprocess_bytes.copy_from_slice(&preprocess.serialize());
+    let preprocess = preprocess_bytes;
+
+    let mut txn = db.txn();
+    // If this attempt has already been preprocessed for, the Tributary will de-duplicate it
+    // This may mean the Tributary preprocess is distinct from ours, but we check for that later
+    TributaryTransactionsFromDkgConfirmation::send(
+      &mut txn,
+      set,
+      &Transaction::DkgConfirmationPreprocess { attempt, preprocess, signed: Default::default() },
+    );
+    txn.commit();
+
+    *signer = Some(Signer::Preprocess { attempt, seed, preprocess });
+  }
+}
+
+impl<CD: DbTrait, TD: DbTrait> ContinuallyRan for ConfirmDkgTask<CD, TD> {
+  type Error = DoesNotError;
+
+  fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, Self::Error>> {
+    async move {
+      let mut made_progress = false;
+
+      // If we were sent a key to set, create the signer for it
+      if self.signer.is_none() && KeysToConfirm::get(&self.db, self.set.set).is_some() {
+        // Create and publish the initial preprocess
+        Self::preprocess(&mut self.db, self.set.set, 0, &self.key, &mut self.signer);
+
+        made_progress = true;
+      }
+
+      // If we have keys to confirm, handle all messages from the tributary
+      if let Some(key_pair) = KeysToConfirm::get(&self.db, self.set.set) {
+        // Handle all messages from the Tributary
+        loop {
+          let mut tributary_txn = self.tributary_db.txn();
+          let Some(msg) = DkgConfirmationMessages::try_recv(&mut tributary_txn, self.set.set)
+          else {
+            break;
+          };
+
+          match msg {
+            messages::sign::CoordinatorMessage::Reattempt {
+              id: messages::sign::SignId { attempt, .. },
+            } => {
+              // Create and publish the preprocess for the specified attempt
+              Self::preprocess(&mut self.db, self.set.set, attempt, &self.key, &mut self.signer);
+            }
+            messages::sign::CoordinatorMessage::Preprocesses {
+              id: messages::sign::SignId { attempt, .. },
+              mut preprocesses,
+            } => {
+              // Confirm the preprocess we're expected to sign with is the one we locally have
+              // It may be different if we rebooted and made a second preprocess for this attempt
+              let Some(Signer::Preprocess { attempt: our_attempt, seed, preprocess }) =
+                self.signer.take()
+              else {
+                // If this message is not expected, commit the txn to drop it and move on
+                // At some point, we'll get a Reattempt and reset
+                tributary_txn.commit();
+                break;
+              };
+
+              // Determine the MuSig key signed with
+              let musig_validators = {
+                let mut ordered_participants = preprocesses.keys().copied().collect::<Vec<_>>();
+                ordered_participants.sort_by_key(|participant| u16::from(*participant));
+
+                let mut res = vec![];
+                for participant in ordered_participants {
+                  let (validator, _weight) =
+                    self.set.validators[usize::from(u16::from(participant) - 1)];
+                  res.push(validator);
+                }
+                res
+              };
+
+              let musig_public_keys = musig_validators
+                .iter()
+                .map(|key| {
+                  Ristretto::read_G(&mut key.0.as_slice())
+                    .expect("Serai validator had invalid public key")
+                })
+                .collect::<Vec<_>>();
+
+              let keys = musig(&musig_context(self.set.set.into()), &self.key, &musig_public_keys)
+                .unwrap()
+                .into();
+
+              // Rebuild the machine
+              let (machine, preprocess_from_cache) =
+                AlgorithmSignMachine::from_cache(schnorrkel(), keys, seed);
+              assert_eq!(preprocess.as_slice(), preprocess_from_cache.serialize().as_slice());
+
+              // Ensure this is a consistent signing session
+              let our_i = our_i(&self.set, &self.key, &preprocesses);
+              let consistent = (attempt == our_attempt) &&
+                (preprocesses.remove(&our_i).unwrap().as_slice() == preprocess.as_slice());
+              if !consistent {
+                tributary_txn.commit();
+                break;
+              }
+
+              // Reformat the preprocesses into the expected format for Musig
+              let preprocesses = match make_contiguous(our_i, preprocesses, |preprocess| {
+                machine.read_preprocess(&mut preprocess.as_slice())
+              }) {
+                Ok(preprocesses) => preprocesses,
+                // This yields the *original participant index*
+                Err(participant) => {
+                  Self::slash(
+                    &mut self.db,
+                    self.set.set,
+                    self.set.validators[usize::from(u16::from(participant) - 1)].0,
+                  );
+                  tributary_txn.commit();
+                  break;
+                }
+              };
+
+              // Calculate our share
+              let (machine, share) = match handle_frost_error(
+                machine.sign(preprocesses, &set_keys_message(&self.set.set, &key_pair)),
+              ) {
+                Ok((machine, share)) => (machine, share),
+                // This yields the *musig participant index*
+                Err(participant) => {
+                  Self::slash(
+                    &mut self.db,
+                    self.set.set,
+                    musig_validators[usize::from(u16::from(participant) - 1)],
+                  );
+                  tributary_txn.commit();
+                  break;
+                }
+              };
+
+              // Send our share
+              let share = <[u8; 32]>::try_from(share.serialize()).unwrap();
+              let mut txn = self.db.txn();
+              TributaryTransactionsFromDkgConfirmation::send(
+                &mut txn,
+                self.set.set,
+                &Transaction::DkgConfirmationShare { attempt, share, signed: Default::default() },
+              );
+              txn.commit();
+
+              self.signer = Some(Signer::Share {
+                attempt,
+                musig_validators,
+                share,
+                machine: Box::new(machine),
+              });
+            }
+            messages::sign::CoordinatorMessage::Shares {
+              id: messages::sign::SignId { attempt, .. },
+              mut shares,
+            } => {
+              let Some(Signer::Share { attempt: our_attempt, musig_validators, share, machine }) =
+                self.signer.take()
+              else {
+                tributary_txn.commit();
+                break;
+              };
+
+              // Ensure this is a consistent signing session
+              let our_i = our_i(&self.set, &self.key, &shares);
+              let consistent = (attempt == our_attempt) &&
+                (shares.remove(&our_i).unwrap().as_slice() == share.as_slice());
+              if !consistent {
+                tributary_txn.commit();
+                break;
+              }
+
+              // Reformat the shares into the expected format for Musig
+              let shares = match make_contiguous(our_i, shares, |share| {
+                machine.read_share(&mut share.as_slice())
+              }) {
+                Ok(shares) => shares,
+                // This yields the *original participant index*
+                Err(participant) => {
+                  Self::slash(
+                    &mut self.db,
+                    self.set.set,
+                    self.set.validators[usize::from(u16::from(participant) - 1)].0,
+                  );
+                  tributary_txn.commit();
+                  break;
+                }
+              };
+
+              match handle_frost_error(machine.complete(shares)) {
+                Ok(signature) => {
+                  // Create the bitvec of the participants
+                  let mut signature_participants;
+                  {
+                    use bitvec::prelude::*;
+                    signature_participants = bitvec![u8, Lsb0; 0; 0];
+                    let mut i = 0;
+                    for (validator, _) in &self.set.validators {
+                      if Some(validator) == musig_validators.get(i) {
+                        signature_participants.push(true);
+                        i += 1;
+                      } else {
+                        signature_participants.push(false);
+                      }
+                    }
+                  }
+
+                  // This is safe to call multiple times as it'll just change which *valid*
+                  // signature to publish
+                  let mut txn = self.db.txn();
+                  Keys::set(
+                    &mut txn,
+                    self.set.set,
+                    key_pair.clone(),
+                    signature_participants,
+                    signature.into(),
+                  );
+                  txn.commit();
+                }
+                // This yields the *musig participant index*
+                Err(participant) => {
+                  Self::slash(
+                    &mut self.db,
+                    self.set.set,
+                    musig_validators[usize::from(u16::from(participant) - 1)],
+                  );
+                  tributary_txn.commit();
+                  break;
+                }
+              }
+            }
+          }
+
+          // Because we successfully handled this message, note we made proress
+          made_progress = true;
+          tributary_txn.commit();
+        }
+      }
+
+      // Check if the key has been set on Serai
+      if KeysToConfirm::get(&self.db, self.set.set).is_some() &&
+        KeySet::get(&self.db, self.set.set).is_some()
+      {
+        // Take the keys to confirm so we never instantiate the signer again
+        let mut txn = self.db.txn();
+        KeysToConfirm::take(&mut txn, self.set.set);
+        KeySet::take(&mut txn, self.set.set);
+        txn.commit();
+
+        // Drop our own signer
+        // The task won't die until the Tributary does, but now it'll never do anything again
+        self.signer = None;
+
+        made_progress = true;
+      }
+
+      Ok(made_progress)
+    }
+  }
+}

coordinator/src/main.rs

@@ -14,8 +14,8 @@ use borsh::BorshDeserialize;
 use tokio::sync::mpsc;
 
 use serai_client::{
-  primitives::{NetworkId, PublicKey},
-  validator_sets::primitives::ValidatorSet,
+  primitives::{ExternalNetworkId, PublicKey, SeraiAddress, Signature},
+  validator_sets::primitives::{ExternalValidatorSet, KeyPair},
   Serai,
 };
 use message_queue::{Service, client::MessageQueue};
@@ -23,13 +23,17 @@ use message_queue::{Service, client::MessageQueue};
 use serai_task::{Task, TaskHandle, ContinuallyRan};
 
 use serai_cosign::{Faulted, SignedCosign, Cosigning};
-use serai_coordinator_substrate::{CanonicalEventStream, EphemeralEventStream, SignSlashReport};
-use serai_coordinator_tributary::{Signed, Transaction, SubstrateBlockPlans};
+use serai_coordinator_substrate::{
+  CanonicalEventStream, EphemeralEventStream, SignSlashReport, SetKeysTask, SignedBatches,
+  PublishBatchTask, SlashReports, PublishSlashReportTask,
+};
+use serai_coordinator_tributary::{SigningProtocolRound, Signed, Transaction, SubstrateBlockPlans};
 
 mod db;
 use db::*;
 
 mod tributary;
+mod dkg_confirmation;
 
 mod substrate;
 use substrate::SubstrateTask;
@@ -145,11 +149,24 @@ fn spawn_cosigning<D: serai_db::Db>(
   });
 }
 
-async fn handle_processor_messages(
+async fn handle_network(
   mut db: impl serai_db::Db,
   message_queue: Arc<MessageQueue>,
-  network: NetworkId,
+  serai: Arc<Serai>,
+  network: ExternalNetworkId,
 ) {
+  // Spawn the task to publish batches for this network
+  {
+    let (publish_batch_task_def, publish_batch_task) = Task::new();
+    tokio::spawn(
+      PublishBatchTask::new(db.clone(), serai.clone(), network)
+        .continually_run(publish_batch_task_def, vec![]),
+    );
+    // Forget its handle so it always runs in the background
+    core::mem::forget(publish_batch_task);
+  }
+
+  // Handle Processor messages
   loop {
     let (msg_id, msg) = {
       let msg = message_queue.next(Service::Processor(network)).await;
@@ -179,8 +196,8 @@ async fn handle_processor_messages(
     match msg {
       messages::ProcessorMessage::KeyGen(msg) => match msg {
         messages::key_gen::ProcessorMessage::Participation { session, participation } => {
-          let set = ValidatorSet { network, session };
-          TributaryTransactions::send(
+          let set = ExternalValidatorSet { network, session };
+          TributaryTransactionsFromProcessorMessages::send(
             &mut txn,
             set,
             &Transaction::DkgParticipation { participation, signed: Signed::default() },
@@ -190,45 +207,84 @@ async fn handle_processor_messages(
           session,
           substrate_key,
           network_key,
-        } => todo!("TODO Transaction::DkgConfirmationPreprocess"),
-        messages::key_gen::ProcessorMessage::Blame { session, participant } => {
-          let set = ValidatorSet { network, session };
-          TributaryTransactions::send(
+        } => {
+          KeysToConfirm::set(
             &mut txn,
-            set,
-            &Transaction::RemoveParticipant {
-              participant: todo!("TODO"),
-              signed: Signed::default(),
-            },
+            ExternalValidatorSet { network, session },
+            &KeyPair(
+              PublicKey::from_raw(substrate_key),
+              network_key
+                .try_into()
+                .expect("generated a network key which exceeds the maximum key length"),
+            ),
           );
         }
+        messages::key_gen::ProcessorMessage::Blame { session, participant } => {
+          RemoveParticipant::send(&mut txn, ExternalValidatorSet { network, session }, participant);
+        }
       },
       messages::ProcessorMessage::Sign(msg) => match msg {
         messages::sign::ProcessorMessage::InvalidParticipant { session, participant } => {
-          let set = ValidatorSet { network, session };
-          TributaryTransactions::send(
+          RemoveParticipant::send(&mut txn, ExternalValidatorSet { network, session }, participant);
+        }
+        messages::sign::ProcessorMessage::Preprocesses { id, preprocesses } => {
+          let set = ExternalValidatorSet { network, session: id.session };
+          if id.attempt == 0 {
+            // Batches are declared by their intent to be signed
+            if let messages::sign::VariantSignId::Batch(hash) = id.id {
+              TributaryTransactionsFromProcessorMessages::send(
+                &mut txn,
+                set,
+                &Transaction::Batch { hash },
+              );
+            }
+          }
+
+          TributaryTransactionsFromProcessorMessages::send(
             &mut txn,
             set,
-            &Transaction::RemoveParticipant {
-              participant: todo!("TODO"),
+            &Transaction::Sign {
+              id: id.id,
+              attempt: id.attempt,
+              round: SigningProtocolRound::Preprocess,
+              data: preprocesses,
               signed: Signed::default(),
             },
           );
         }
-        messages::sign::ProcessorMessage::Preprocesses { id, preprocesses } => {
-          todo!("TODO Transaction::Batch + Transaction::Sign")
+        messages::sign::ProcessorMessage::Shares { id, shares } => {
+          let set = ExternalValidatorSet { network, session: id.session };
+          TributaryTransactionsFromProcessorMessages::send(
+            &mut txn,
+            set,
+            &Transaction::Sign {
+              id: id.id,
+              attempt: id.attempt,
+              round: SigningProtocolRound::Share,
+              data: shares,
+              signed: Signed::default(),
+            },
+          );
         }
-        messages::sign::ProcessorMessage::Shares { id, shares } => todo!("TODO Transaction::Sign"),
       },
       messages::ProcessorMessage::Coordinator(msg) => match msg {
         messages::coordinator::ProcessorMessage::CosignedBlock { cosign } => {
           SignedCosigns::send(&mut txn, &cosign);
         }
         messages::coordinator::ProcessorMessage::SignedBatch { batch } => {
-          todo!("TODO PublishBatchTask")
+          SignedBatches::send(&mut txn, &batch);
         }
-        messages::coordinator::ProcessorMessage::SignedSlashReport { session, signature } => {
-          todo!("TODO PublishSlashReportTask")
+        messages::coordinator::ProcessorMessage::SignedSlashReport {
+          session,
+          slash_report,
+          signature,
+        } => {
+          SlashReports::set(
+            &mut txn,
+            ExternalValidatorSet { network, session },
+            slash_report,
+            Signature(signature),
+          );
         }
       },
       messages::ProcessorMessage::Substrate(msg) => match msg {
@@ -241,9 +297,9 @@ async fn handle_processor_messages(
               .push(plan.transaction_plan_id);
           }
           for (session, plans) in by_session {
-            let set = ValidatorSet { network, session };
+            let set = ExternalValidatorSet { network, session };
             SubstrateBlockPlans::set(&mut txn, set, block, &plans);
-            TributaryTransactions::send(
+            TributaryTransactionsFromProcessorMessages::send(
               &mut txn,
               set,
               &Transaction::SubstrateBlock { hash: block },
@@ -309,10 +365,16 @@ async fn main() {
     // Cleanup all historic Tributaries
     while let Some(to_cleanup) = TributaryCleanup::try_recv(&mut txn) {
       prune_tributary_db(to_cleanup);
+      // Remove the keys to confirm for this network
+      KeysToConfirm::take(&mut txn, to_cleanup);
+      KeySet::take(&mut txn, to_cleanup);
       // Drain the cosign intents created for this set
       while !Cosigning::<Db>::intended_cosigns(&mut txn, to_cleanup).is_empty() {}
       // Drain the transactions to publish for this set
-      while TributaryTransactions::try_recv(&mut txn, to_cleanup).is_some() {}
+      while TributaryTransactionsFromProcessorMessages::try_recv(&mut txn, to_cleanup).is_some() {}
+      while TributaryTransactionsFromDkgConfirmation::try_recv(&mut txn, to_cleanup).is_some() {}
+      // Drain the participants to remove for this set
+      while RemoveParticipant::try_recv(&mut txn, to_cleanup).is_some() {}
       // Remove the SignSlashReport notification
       SignSlashReport::try_recv(&mut txn, to_cleanup);
     }
@@ -376,7 +438,7 @@ async fn main() {
     EphemeralEventStream::new(
       db.clone(),
       serai.clone(),
-      PublicKey::from_raw((<Ristretto as Ciphersuite>::generator() * serai_key.deref()).to_bytes()),
+      SeraiAddress((<Ristretto as Ciphersuite>::generator() * serai_key.deref()).to_bytes()),
     )
     .continually_run(substrate_ephemeral_task_def, vec![substrate_task]),
   );
@@ -417,12 +479,29 @@ async fn main() {
     .continually_run(substrate_task_def, vec![]),
   );
 
-  // Handle all of the Processors' messages
-  for network in serai_client::primitives::NETWORKS {
-    if network == NetworkId::Serai {
-      continue;
-    }
-    tokio::spawn(handle_processor_messages(db.clone(), message_queue.clone(), network));
+  // Handle each of the networks
+  for network in serai_client::primitives::EXTERNAL_NETWORKS {
+    tokio::spawn(handle_network(db.clone(), message_queue.clone(), serai.clone(), network));
+  }
+
+  // Spawn the task to set keys
+  {
+    let (set_keys_task_def, set_keys_task) = Task::new();
+    tokio::spawn(
+      SetKeysTask::new(db.clone(), serai.clone()).continually_run(set_keys_task_def, vec![]),
+    );
+    // Forget its handle so it always runs in the background
+    core::mem::forget(set_keys_task);
+  }
+
+  // Spawn the task to publish slash reports
+  {
+    let (publish_slash_report_task_def, publish_slash_report_task) = Task::new();
+    tokio::spawn(
+      PublishSlashReportTask::new(db, serai).continually_run(publish_slash_report_task_def, vec![]),
+    );
+    // Always have this run in the background
+    core::mem::forget(publish_slash_report_task);
   }
 
   // Run the spawned tasks ad-infinitum

coordinator/src/substrate.rs

@@ -9,7 +9,7 @@ use tokio::sync::mpsc;
 
 use serai_db::{DbTxn, Db as DbTrait};
 
-use serai_client::validator_sets::primitives::{Session, ValidatorSet};
+use serai_client::validator_sets::primitives::{Session, ExternalValidatorSet};
 use message_queue::{Service, Metadata, client::MessageQueue};
 
 use tributary_sdk::Tributary;
@@ -19,7 +19,7 @@ use serai_task::ContinuallyRan;
 use serai_coordinator_tributary::Transaction;
 use serai_coordinator_p2p::P2p;
 
-use crate::Db;
+use crate::{Db, KeySet};
 
 pub(crate) struct SubstrateTask<P: P2p> {
   pub(crate) serai_key: Zeroizing<<Ristretto as Ciphersuite>::F>,
@@ -27,8 +27,8 @@ pub(crate) struct SubstrateTask<P: P2p> {
   pub(crate) message_queue: Arc<MessageQueue>,
   pub(crate) p2p: P,
   pub(crate) p2p_add_tributary:
-    mpsc::UnboundedSender<(ValidatorSet, Tributary<Db, Transaction, P>)>,
-  pub(crate) p2p_retire_tributary: mpsc::UnboundedSender<ValidatorSet>,
+    mpsc::UnboundedSender<(ExternalValidatorSet, Tributary<Db, Transaction, P>)>,
+  pub(crate) p2p_retire_tributary: mpsc::UnboundedSender<ExternalValidatorSet>,
 }
 
 impl<P: P2p> ContinuallyRan for SubstrateTask<P> {
@@ -38,7 +38,7 @@ impl<P: P2p> ContinuallyRan for SubstrateTask<P> {
       let mut made_progress = false;
 
       // Handle the Canonical events
-      for network in serai_client::primitives::NETWORKS {
+      for network in serai_client::primitives::EXTERNAL_NETWORKS {
         loop {
           let mut txn = self.db.txn();
           let Some(msg) = serai_coordinator_substrate::Canonical::try_recv(&mut txn, network)
@@ -47,8 +47,9 @@ impl<P: P2p> ContinuallyRan for SubstrateTask<P> {
           };
 
           match msg {
-            // TODO: Stop trying to confirm the DKG
-            messages::substrate::CoordinatorMessage::SetKeys { .. } => todo!("TODO"),
+            messages::substrate::CoordinatorMessage::SetKeys { session, .. } => {
+              KeySet::set(&mut txn, ExternalValidatorSet { network, session }, &());
+            }
             messages::substrate::CoordinatorMessage::SlashesReported { session } => {
               let prior_retired = crate::db::RetiredTributary::get(&txn, network);
               let next_to_be_retired =
@@ -57,7 +58,7 @@ impl<P: P2p> ContinuallyRan for SubstrateTask<P> {
               crate::db::RetiredTributary::set(&mut txn, network, &session);
               self
                 .p2p_retire_tributary
-                .send(ValidatorSet { network, session })
+                .send(ExternalValidatorSet { network, session })
                 .expect("p2p retire_tributary channel dropped?");
             }
             messages::substrate::CoordinatorMessage::Block { .. } => {}
@@ -107,7 +108,10 @@ impl<P: P2p> ContinuallyRan for SubstrateTask<P> {
           */
           crate::db::TributaryCleanup::send(
             &mut txn,
-            &ValidatorSet { network: new_set.set.network, session: Session(historic_session) },
+            &ExternalValidatorSet {
+              network: new_set.set.network,
+              session: Session(historic_session),
+            },
           );
         }
 

coordinator/src/tributary.rs

@@ -11,7 +11,7 @@ use tokio::sync::mpsc;
 use serai_db::{Get, DbTxn, Db as DbTrait, create_db, db_channel};
 
 use scale::Encode;
-use serai_client::validator_sets::primitives::ValidatorSet;
+use serai_client::validator_sets::primitives::ExternalValidatorSet;
 
 use tributary_sdk::{TransactionKind, TransactionError, ProvidedError, TransactionTrait, Tributary};
 
@@ -21,14 +21,25 @@ use message_queue::{Service, Metadata, client::MessageQueue};
 
 use serai_cosign::{Faulted, CosignIntent, Cosigning};
 use serai_coordinator_substrate::{NewSetInformation, SignSlashReport};
-use serai_coordinator_tributary::{Transaction, ProcessorMessages, CosignIntents, ScanTributaryTask};
+use serai_coordinator_tributary::{
+  Topic, Transaction, ProcessorMessages, CosignIntents, RecognizedTopics, ScanTributaryTask,
+};
 use serai_coordinator_p2p::P2p;
 
-use crate::{Db, TributaryTransactions};
+use crate::{
+  Db, TributaryTransactionsFromProcessorMessages, TributaryTransactionsFromDkgConfirmation,
+  RemoveParticipant, dkg_confirmation::ConfirmDkgTask,
+};
+
+create_db! {
+  Coordinator {
+     PublishOnRecognition: (set: ExternalValidatorSet, topic: Topic) -> Transaction,
+  }
+}
 
 db_channel! {
   Coordinator {
-    PendingCosigns: (set: ValidatorSet) -> CosignIntent,
+    PendingCosigns: (set: ExternalValidatorSet) -> CosignIntent,
   }
 }
 
@@ -37,7 +48,7 @@ db_channel! {
 /// This is not a well-designed function. This is specific to the context in which its called,
 /// within this file. It should only be considered an internal helper for this domain alone.
 async fn provide_transaction<TD: DbTrait, P: P2p>(
-  set: ValidatorSet,
+  set: ExternalValidatorSet,
   tributary: &Tributary<TD, Transaction, P>,
   tx: Transaction,
 ) {
@@ -147,12 +158,101 @@ impl<CD: DbTrait, TD: DbTrait, P: P2p> ContinuallyRan
   }
 }
 
-/// Adds all of the transactions sent via `TributaryTransactions`.
+#[must_use]
+async fn add_signed_unsigned_transaction<TD: DbTrait, P: P2p>(
+  tributary: &Tributary<TD, Transaction, P>,
+  key: &Zeroizing<<Ristretto as Ciphersuite>::F>,
+  mut tx: Transaction,
+) -> bool {
+  // If this is a signed transaction, sign it
+  if matches!(tx.kind(), TransactionKind::Signed(_, _)) {
+    tx.sign(&mut OsRng, tributary.genesis(), key);
+  }
+
+  let res = tributary.add_transaction(tx.clone()).await;
+  match &res {
+    // Fresh publication, already published
+    Ok(true | false) => {}
+    Err(
+      TransactionError::TooLargeTransaction |
+      TransactionError::InvalidSigner |
+      TransactionError::InvalidSignature |
+      TransactionError::InvalidContent,
+    ) => {
+      panic!("created an invalid transaction, tx: {tx:?}, err: {res:?}");
+    }
+    // InvalidNonce may be out-of-order TXs, not invalid ones, but we only create nonce #n+1 after
+    // on-chain inclusion of the TX with nonce #n, so it is invalid within our context unless the
+    // issue is this transaction was already included on-chain
+    Err(TransactionError::InvalidNonce) => {
+      let TransactionKind::Signed(order, signed) = tx.kind() else {
+        panic!("non-Signed transaction had InvalidNonce");
+      };
+      let next_nonce = tributary
+        .next_nonce(&signed.signer, &order)
+        .await
+        .expect("signer who is a present validator didn't have a nonce");
+      assert!(next_nonce != signed.nonce);
+      // We're publishing an old transaction
+      if next_nonce > signed.nonce {
+        return true;
+      }
+      panic!("nonce in transaction wasn't contiguous with nonce on-chain");
+    }
+    // We've published too many transactions recently
+    Err(TransactionError::TooManyInMempool) => {
+      return false;
+    }
+    // This isn't a Provided transaction so this should never be hit
+    Err(TransactionError::ProvidedAddedToMempool) => unreachable!(),
+  }
+
+  true
+}
+
+async fn add_with_recognition_check<TD: DbTrait, P: P2p>(
+  set: ExternalValidatorSet,
+  tributary_db: &mut TD,
+  tributary: &Tributary<TD, Transaction, P>,
+  key: &Zeroizing<<Ristretto as Ciphersuite>::F>,
+  tx: Transaction,
+) -> bool {
+  let kind = tx.kind();
+  match kind {
+    TransactionKind::Provided(_) => provide_transaction(set, tributary, tx).await,
+    TransactionKind::Unsigned | TransactionKind::Signed(_, _) => {
+      // If this is a transaction with signing data, check the topic is recognized before
+      // publishing
+      let topic = tx.topic();
+      let still_requires_recognition = if let Some(topic) = topic {
+        (topic.requires_recognition() && (!RecognizedTopics::recognized(tributary_db, set, topic)))
+          .then_some(topic)
+      } else {
+        None
+      };
+      if let Some(topic) = still_requires_recognition {
+        // Queue the transaction until the topic is recognized
+        // We use the Tributary DB for this so it's cleaned up when the Tributary DB is
+        let mut tributary_txn = tributary_db.txn();
+        PublishOnRecognition::set(&mut tributary_txn, set, topic, &tx);
+        tributary_txn.commit();
+      } else {
+        // Actually add the transaction
+        if !add_signed_unsigned_transaction(tributary, key, tx).await {
+          return false;
+        }
+      }
+    }
+  }
+  true
+}
+
+/// Adds all of the transactions sent via `TributaryTransactionsFromProcessorMessages`.
 pub(crate) struct AddTributaryTransactionsTask<CD: DbTrait, TD: DbTrait, P: P2p> {
   db: CD,
   tributary_db: TD,
   tributary: Tributary<TD, Transaction, P>,
-  set: ValidatorSet,
+  set: NewSetInformation,
   key: Zeroizing<<Ristretto as Ciphersuite>::F>,
 }
 impl<CD: DbTrait, TD: DbTrait, P: P2p> ContinuallyRan for AddTributaryTransactionsTask<CD, TD, P> {
@@ -161,49 +261,87 @@ impl<CD: DbTrait, TD: DbTrait, P: P2p> ContinuallyRan for AddTributaryTransactio
   fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, Self::Error>> {
     async move {
       let mut made_progress = false;
+
+      // Provide/add all transactions sent our way
       loop {
         let mut txn = self.db.txn();
-        let Some(mut tx) = TributaryTransactions::try_recv(&mut txn, self.set) else { break };
+        let Some(tx) = TributaryTransactionsFromDkgConfirmation::try_recv(&mut txn, self.set.set)
+        else {
+          break;
+        };
 
-        let kind = tx.kind();
-        match kind {
-          TransactionKind::Provided(_) => provide_transaction(self.set, &self.tributary, tx).await,
-          TransactionKind::Unsigned | TransactionKind::Signed(_, _) => {
-            // If this is a signed transaction, sign it
-            if matches!(kind, TransactionKind::Signed(_, _)) {
-              tx.sign(&mut OsRng, self.tributary.genesis(), &self.key);
-            }
-
-            // Actually add the transaction
-            // TODO: If this is a preprocess, make sure the topic has been recognized
-            let res = self.tributary.add_transaction(tx.clone()).await;
-            match &res {
-              // Fresh publication, already published
-              Ok(true | false) => {}
-              Err(
-                TransactionError::TooLargeTransaction |
-                TransactionError::InvalidSigner |
-                TransactionError::InvalidNonce |
-                TransactionError::InvalidSignature |
-                TransactionError::InvalidContent,
-              ) => {
-                panic!("created an invalid transaction, tx: {tx:?}, err: {res:?}");
-              }
-              // We've published too many transactions recently
-              // Drop this txn to try to publish it again later on a future iteration
-              Err(TransactionError::TooManyInMempool) => {
-                drop(txn);
-                break;
-              }
-              // This isn't a Provided transaction so this should never be hit
-              Err(TransactionError::ProvidedAddedToMempool) => unreachable!(),
-            }
-          }
+        if !add_with_recognition_check(
+          self.set.set,
+          &mut self.tributary_db,
+          &self.tributary,
+          &self.key,
+          tx,
+        )
+        .await
+        {
+          break;
         }
 
         made_progress = true;
         txn.commit();
       }
+
+      loop {
+        let mut txn = self.db.txn();
+        let Some(tx) = TributaryTransactionsFromProcessorMessages::try_recv(&mut txn, self.set.set)
+        else {
+          break;
+        };
+
+        if !add_with_recognition_check(
+          self.set.set,
+          &mut self.tributary_db,
+          &self.tributary,
+          &self.key,
+          tx,
+        )
+        .await
+        {
+          break;
+        }
+
+        made_progress = true;
+        txn.commit();
+      }
+
+      // Provide/add all transactions due to newly recognized topics
+      loop {
+        let mut tributary_txn = self.tributary_db.txn();
+        let Some(topic) =
+          RecognizedTopics::try_recv_topic_requiring_recognition(&mut tributary_txn, self.set.set)
+        else {
+          break;
+        };
+        if let Some(tx) = PublishOnRecognition::take(&mut tributary_txn, self.set.set, topic) {
+          if !add_signed_unsigned_transaction(&self.tributary, &self.key, tx).await {
+            break;
+          }
+        }
+
+        made_progress = true;
+        tributary_txn.commit();
+      }
+
+      // Publish any participant removals
+      loop {
+        let mut txn = self.db.txn();
+        let Some(participant) = RemoveParticipant::try_recv(&mut txn, self.set.set) else { break };
+        let tx = Transaction::RemoveParticipant {
+          participant: self.set.participant_indexes_reverse_lookup[&participant],
+          signed: Default::default(),
+        };
+        if !add_signed_unsigned_transaction(&self.tributary, &self.key, tx).await {
+          break;
+        }
+        made_progress = true;
+        txn.commit();
+      }
+
       Ok(made_progress)
     }
   }
@@ -212,7 +350,7 @@ impl<CD: DbTrait, TD: DbTrait, P: P2p> ContinuallyRan for AddTributaryTransactio
 /// Takes the messages from ScanTributaryTask and publishes them to the message-queue.
 pub(crate) struct TributaryProcessorMessagesTask<TD: DbTrait> {
   tributary_db: TD,
-  set: ValidatorSet,
+  set: ExternalValidatorSet,
   message_queue: Arc<MessageQueue>,
 }
 impl<TD: DbTrait> ContinuallyRan for TributaryProcessorMessagesTask<TD> {
@@ -292,7 +430,7 @@ impl<CD: DbTrait, TD: DbTrait, P: P2p> ContinuallyRan for SignSlashReportTask<CD
 /// Run the scan task whenever the Tributary adds a new block.
 async fn scan_on_new_block<CD: DbTrait, TD: DbTrait, P: P2p>(
   db: CD,
-  set: ValidatorSet,
+  set: ExternalValidatorSet,
   tributary: Tributary<TD, Transaction, P>,
   scan_tributary_task: TaskHandle,
   tasks_to_keep_alive: Vec<TaskHandle>,
@@ -323,13 +461,15 @@ async fn scan_on_new_block<CD: DbTrait, TD: DbTrait, P: P2p>(
 /// - Spawn the ScanTributaryTask
 /// - Spawn the ProvideCosignCosignedTransactionsTask
 /// - Spawn the TributaryProcessorMessagesTask
+/// - Spawn the AddTributaryTransactionsTask
+/// - Spawn the ConfirmDkgTask
 /// - Spawn the SignSlashReportTask
 /// - Iterate the scan task whenever a new block occurs (not just on the standard interval)
 pub(crate) async fn spawn_tributary<P: P2p>(
   db: Db,
   message_queue: Arc<MessageQueue>,
   p2p: P,
-  p2p_add_tributary: &mpsc::UnboundedSender<(ValidatorSet, Tributary<Db, Transaction, P>)>,
+  p2p_add_tributary: &mpsc::UnboundedSender<(ExternalValidatorSet, Tributary<Db, Transaction, P>)>,
   set: NewSetInformation,
   serai_key: Zeroizing<<Ristretto as Ciphersuite>::F>,
 ) {
@@ -403,38 +543,45 @@ pub(crate) async fn spawn_tributary<P: P2p>(
   // Spawn the scan task
   let (scan_tributary_task_def, scan_tributary_task) = Task::new();
   tokio::spawn(
-    ScanTributaryTask::<_, P>::new(tributary_db.clone(), &set, reader)
+    ScanTributaryTask::<_, P>::new(tributary_db.clone(), set.clone(), reader)
       // This is the only handle for this TributaryProcessorMessagesTask, so when this task is
       // dropped, it will be too
       .continually_run(scan_tributary_task_def, vec![scan_tributary_messages_task]),
   );
 
-  // Spawn the sign slash report task
-  let (sign_slash_report_task_def, sign_slash_report_task) = Task::new();
-  tokio::spawn(
-    (SignSlashReportTask {
-      db: db.clone(),
-      tributary_db: tributary_db.clone(),
-      tributary: tributary.clone(),
-      set: set.clone(),
-      key: serai_key.clone(),
-    })
-    .continually_run(sign_slash_report_task_def, vec![]),
-  );
-
   // Spawn the add transactions task
   let (add_tributary_transactions_task_def, add_tributary_transactions_task) = Task::new();
   tokio::spawn(
     (AddTributaryTransactionsTask {
       db: db.clone(),
-      tributary_db,
+      tributary_db: tributary_db.clone(),
       tributary: tributary.clone(),
-      set: set.set,
-      key: serai_key,
+      set: set.clone(),
+      key: serai_key.clone(),
     })
     .continually_run(add_tributary_transactions_task_def, vec![]),
   );
 
+  // Spawn the task to confirm the DKG result
+  let (confirm_dkg_task_def, confirm_dkg_task) = Task::new();
+  tokio::spawn(
+    ConfirmDkgTask::new(db.clone(), set.clone(), tributary_db.clone(), serai_key.clone())
+      .continually_run(confirm_dkg_task_def, vec![add_tributary_transactions_task]),
+  );
+
+  // Spawn the sign slash report task
+  let (sign_slash_report_task_def, sign_slash_report_task) = Task::new();
+  tokio::spawn(
+    (SignSlashReportTask {
+      db: db.clone(),
+      tributary_db,
+      tributary: tributary.clone(),
+      set: set.clone(),
+      key: serai_key,
+    })
+    .continually_run(sign_slash_report_task_def, vec![]),
+  );
+
   // Whenever a new block occurs, immediately run the scan task
   // This function also preserves the ProvideCosignCosignedTransactionsTask handle until the
   // Tributary is retired, ensuring it isn't dropped prematurely and that the task don't run ad
@@ -444,10 +591,6 @@ pub(crate) async fn spawn_tributary<P: P2p>(
     set.set,
     tributary,
     scan_tributary_task,
-    vec![
-      provide_cosign_cosigned_transactions_task,
-      sign_slash_report_task,
-      add_tributary_transactions_task,
-    ],
+    vec![provide_cosign_cosigned_transactions_task, confirm_dkg_task, sign_slash_report_task],
   ));
 }

coordinator/substrate/Cargo.toml

@@ -22,6 +22,9 @@ bitvec = { version = "1", default-features = false, features = ["std"] }
 
 scale = { package = "parity-scale-codec", version = "3", default-features = false, features = ["std", "derive", "bit-vec"] }
 borsh = { version = "1", default-features = false, features = ["std", "derive", "de_strict_order"] }
+
+dkg = { path = "../../crypto/dkg", default-features = false, features = ["std"] }
+
 serai-client = { path = "../../substrate/client", version = "0.1", default-features = false, features = ["serai", "borsh"] }
 
 log = { version = "0.4", default-features = false, features = ["std"] }

coordinator/substrate/src/canonical.rs

@@ -3,7 +3,7 @@ use std::sync::Arc;
 
 use futures::stream::{StreamExt, FuturesOrdered};
 
-use serai_client::Serai;
+use serai_client::{validator_sets::primitives::ExternalValidatorSet, Serai};
 
 use messages::substrate::{InInstructionResult, ExecutedBatch, CoordinatorMessage};
 
@@ -152,6 +152,7 @@ impl<D: Db> ContinuallyRan for CanonicalEventStream<D> {
           else {
             panic!("SetRetired event wasn't a SetRetired event: {set_retired:?}");
           };
+          let Ok(set) = ExternalValidatorSet::try_from(*set) else { continue };
           crate::Canonical::send(
             &mut txn,
             set.network,
@@ -159,7 +160,7 @@ impl<D: Db> ContinuallyRan for CanonicalEventStream<D> {
           );
         }
 
-        for network in serai_client::primitives::NETWORKS {
+        for network in serai_client::primitives::EXTERNAL_NETWORKS {
           let mut batch = None;
           for this_batch in &block.batch_events {
             let serai_client::in_instructions::InInstructionsEvent::Batch {
@@ -180,7 +181,7 @@ impl<D: Db> ContinuallyRan for CanonicalEventStream<D> {
               batch = Some(ExecutedBatch {
                 id: *id,
                 publisher: *publishing_session,
-                external_network_block_hash: *external_network_block_hash,
+                external_network_block_hash: external_network_block_hash.0,
                 in_instructions_hash: *in_instructions_hash,
                 in_instruction_results: in_instruction_results
                   .iter()
@@ -201,7 +202,7 @@ impl<D: Db> ContinuallyRan for CanonicalEventStream<D> {
             let serai_client::coins::CoinsEvent::BurnWithInstruction { from: _, instruction } =
               &burn
             else {
-              panic!("Burn event wasn't a Burn.in event: {burn:?}");
+              panic!("BurnWithInstruction event wasn't a BurnWithInstruction event: {burn:?}");
             };
             if instruction.balance.coin.network() == network {
               burns.push(instruction.clone());

coordinator/substrate/src/ephemeral.rs

@@ -4,8 +4,8 @@ use std::sync::Arc;
 use futures::stream::{StreamExt, FuturesOrdered};
 
 use serai_client::{
-  primitives::{PublicKey, NetworkId, EmbeddedEllipticCurve},
-  validator_sets::primitives::MAX_KEY_SHARES_PER_SET,
+  primitives::{SeraiAddress, EmbeddedEllipticCurve},
+  validator_sets::primitives::{MAX_KEY_SHARES_PER_SET, ExternalValidatorSet},
   Serai,
 };
 
@@ -26,14 +26,14 @@ create_db!(
 pub struct EphemeralEventStream<D: Db> {
   db: D,
   serai: Arc<Serai>,
-  validator: PublicKey,
+  validator: SeraiAddress,
 }
 
 impl<D: Db> EphemeralEventStream<D> {
   /// Create a new ephemeral event stream.
   ///
   /// Only one of these may exist over the provided database.
-  pub fn new(db: D, serai: Arc<Serai>, validator: PublicKey) -> Self {
+  pub fn new(db: D, serai: Arc<Serai>, validator: SeraiAddress) -> Self {
     Self { db, serai, validator }
   }
 }
@@ -130,21 +130,22 @@ impl<D: Db> ContinuallyRan for EphemeralEventStream<D> {
           let serai_client::validator_sets::ValidatorSetsEvent::NewSet { set } = &new_set else {
             panic!("NewSet event wasn't a NewSet event: {new_set:?}");
           };
-
           // We only coordinate over external networks
-          if set.network == NetworkId::Serai {
-            continue;
-          }
+          let Ok(set) = ExternalValidatorSet::try_from(*set) else { continue };
 
           let serai = self.serai.as_of(block.block_hash);
           let serai = serai.validator_sets();
           let Some(validators) =
-            serai.participants(set.network).await.map_err(|e| format!("{e:?}"))?
+            serai.participants(set.network.into()).await.map_err(|e| format!("{e:?}"))?
           else {
             Err(format!(
               "block #{block_number} declared a new set but didn't have the participants"
             ))?
           };
+          let validators = validators
+            .into_iter()
+            .map(|(validator, weight)| (SeraiAddress::from(validator), weight))
+            .collect::<Vec<_>>();
           let in_set = validators.iter().any(|(validator, _)| *validator == self.validator);
           if in_set {
             if u16::try_from(validators.len()).is_err() {
@@ -177,14 +178,16 @@ impl<D: Db> ContinuallyRan for EphemeralEventStream<D> {
               embedded_elliptic_curve_keys.push_back(async move {
                 tokio::try_join!(
                   // One future to fetch the substrate embedded key
-                  serai
-                    .embedded_elliptic_curve_key(validator, EmbeddedEllipticCurve::Embedwards25519),
+                  serai.embedded_elliptic_curve_key(
+                    validator.into(),
+                    EmbeddedEllipticCurve::Embedwards25519
+                  ),
                   // One future to fetch the external embedded key, if there is a distinct curve
                   async {
                     // `embedded_elliptic_curves` is documented to have the second entry be the
                     // network-specific curve (if it exists and is distinct from Embedwards25519)
                     if let Some(curve) = set.network.embedded_elliptic_curves().get(1) {
-                      serai.embedded_elliptic_curve_key(validator, *curve).await.map(Some)
+                      serai.embedded_elliptic_curve_key(validator.into(), *curve).await.map(Some)
                     } else {
                       Ok(None)
                     }
@@ -215,19 +218,22 @@ impl<D: Db> ContinuallyRan for EphemeralEventStream<D> {
               }
             }
 
-            crate::NewSet::send(
-              &mut txn,
-              &NewSetInformation {
-                set: *set,
-                serai_block: block.block_hash,
-                declaration_time: block.time,
-                // TODO: Why do we have this as an explicit field here?
-                // Shouldn't thiis be inlined into the Processor's key gen code, where it's used?
-                threshold: ((total_weight * 2) / 3) + 1,
-                validators,
-                evrf_public_keys,
-              },
-            );
+            let mut new_set = NewSetInformation {
+              set,
+              serai_block: block.block_hash,
+              declaration_time: block.time,
+              // TODO: This should be inlined into the Processor's key gen code
+              // It's legacy from when we removed participants from the key gen
+              threshold: ((total_weight * 2) / 3) + 1,
+              validators,
+              evrf_public_keys,
+              participant_indexes: Default::default(),
+              participant_indexes_reverse_lookup: Default::default(),
+            };
+            // These aren't serialized, and we immediately serialize and drop this, so this isn't
+            // necessary. It's just good practice not have this be dirty
+            new_set.init_participant_indexes();
+            crate::NewSet::send(&mut txn, &new_set);
           }
         }
 
@@ -237,7 +243,8 @@ impl<D: Db> ContinuallyRan for EphemeralEventStream<D> {
           else {
             panic!("AcceptedHandover event wasn't a AcceptedHandover event: {accepted_handover:?}");
           };
-          crate::SignSlashReport::send(&mut txn, *set);
+          let Ok(set) = ExternalValidatorSet::try_from(*set) else { continue };
+          crate::SignSlashReport::send(&mut txn, set);
         }
 
         txn.commit();

coordinator/substrate/src/lib.rs

@@ -2,12 +2,16 @@
 #![doc = include_str!("../README.md")]
 #![deny(missing_docs)]
 
+use std::collections::HashMap;
+
 use scale::{Encode, Decode};
-use borsh::{io, BorshSerialize, BorshDeserialize};
+use borsh::{BorshSerialize, BorshDeserialize};
+
+use dkg::Participant;
 
 use serai_client::{
-  primitives::{NetworkId, PublicKey, Signature, SeraiAddress},
-  validator_sets::primitives::{Session, ValidatorSet, KeyPair},
+  primitives::{ExternalNetworkId, SeraiAddress, Signature},
+  validator_sets::primitives::{Session, ExternalValidatorSet, KeyPair, SlashReport},
   in_instructions::primitives::SignedBatch,
   Transaction,
 };
@@ -26,25 +30,12 @@ pub use publish_batch::PublishBatchTask;
 mod publish_slash_report;
 pub use publish_slash_report::PublishSlashReportTask;
 
-fn borsh_serialize_validators<W: io::Write>(
-  validators: &Vec<(PublicKey, u16)>,
-  writer: &mut W,
-) -> Result<(), io::Error> {
-  // This doesn't use `encode_to` as `encode_to` panics if the writer returns an error
-  writer.write_all(&validators.encode())
-}
-
-fn borsh_deserialize_validators<R: io::Read>(
-  reader: &mut R,
-) -> Result<Vec<(PublicKey, u16)>, io::Error> {
-  Decode::decode(&mut scale::IoReader(reader)).map_err(io::Error::other)
-}
-
 /// The information for a new set.
 #[derive(Clone, Debug, BorshSerialize, BorshDeserialize)]
+#[borsh(init = init_participant_indexes)]
 pub struct NewSetInformation {
   /// The set.
-  pub set: ValidatorSet,
+  pub set: ExternalValidatorSet,
   /// The Serai block which declared it.
   pub serai_block: [u8; 32],
   /// The time of the block which declared it, in seconds.
@@ -52,13 +43,37 @@ pub struct NewSetInformation {
   /// The threshold to use.
   pub threshold: u16,
   /// The validators, with the amount of key shares they have.
-  #[borsh(
-    serialize_with = "borsh_serialize_validators",
-    deserialize_with = "borsh_deserialize_validators"
-  )]
-  pub validators: Vec<(PublicKey, u16)>,
+  pub validators: Vec<(SeraiAddress, u16)>,
   /// The eVRF public keys.
+  ///
+  /// This will have the necessary copies of the keys proper for each validator's weight,
+  /// accordingly syncing up with `participant_indexes`.
   pub evrf_public_keys: Vec<([u8; 32], Vec<u8>)>,
+  /// The participant indexes, indexed by their validator.
+  #[borsh(skip)]
+  pub participant_indexes: HashMap<SeraiAddress, Vec<Participant>>,
+  /// The validators, indexed by their participant indexes.
+  #[borsh(skip)]
+  pub participant_indexes_reverse_lookup: HashMap<Participant, SeraiAddress>,
+}
+
+impl NewSetInformation {
+  fn init_participant_indexes(&mut self) {
+    let mut next_i = 1;
+    self.participant_indexes = HashMap::with_capacity(self.validators.len());
+    self.participant_indexes_reverse_lookup = HashMap::with_capacity(self.validators.len());
+    for (validator, weight) in &self.validators {
+      let mut these_is = Vec::with_capacity((*weight).into());
+      for _ in 0 .. *weight {
+        let this_i = Participant::new(next_i).unwrap();
+        next_i += 1;
+
+        these_is.push(this_i);
+        self.participant_indexes_reverse_lookup.insert(this_i, *validator);
+      }
+      self.participant_indexes.insert(*validator, these_is);
+    }
+  }
 }
 
 mod _public_db {
@@ -67,24 +82,24 @@ mod _public_db {
   db_channel!(
     CoordinatorSubstrate {
       // Canonical messages to send to the processor
-      Canonical: (network: NetworkId) -> messages::substrate::CoordinatorMessage,
+      Canonical: (network: ExternalNetworkId) -> messages::substrate::CoordinatorMessage,
 
       // Relevant new set, from an ephemeral event stream
       NewSet: () -> NewSetInformation,
       // Potentially relevant sign slash report, from an ephemeral event stream
-      SignSlashReport: (set: ValidatorSet) -> (),
+      SignSlashReport: (set: ExternalValidatorSet) -> (),
 
       // Signed batches to publish onto the Serai network
-      SignedBatches: (network: NetworkId) -> SignedBatch,
+      SignedBatches: (network: ExternalNetworkId) -> SignedBatch,
     }
   );
 
   create_db!(
     CoordinatorSubstrate {
       // Keys to set on the Serai network
-      Keys: (network: NetworkId) -> (Session, Vec<u8>),
+      Keys: (network: ExternalNetworkId) -> (Session, Vec<u8>),
       // Slash reports to publish onto the Serai network
-      SlashReports: (network: NetworkId) -> (Session, Vec<u8>),
+      SlashReports: (network: ExternalNetworkId) -> (Session, Vec<u8>),
     }
   );
 }
@@ -94,7 +109,7 @@ pub struct Canonical;
 impl Canonical {
   pub(crate) fn send(
     txn: &mut impl DbTxn,
-    network: NetworkId,
+    network: ExternalNetworkId,
     msg: &messages::substrate::CoordinatorMessage,
   ) {
     _public_db::Canonical::send(txn, network, msg);
@@ -102,7 +117,7 @@ impl Canonical {
   /// Try to receive a canonical event, returning `None` if there is none to receive.
   pub fn try_recv(
     txn: &mut impl DbTxn,
-    network: NetworkId,
+    network: ExternalNetworkId,
   ) -> Option<messages::substrate::CoordinatorMessage> {
     _public_db::Canonical::try_recv(txn, network)
   }
@@ -126,12 +141,12 @@ impl NewSet {
 /// notifications for all relevant validator sets will be included.
 pub struct SignSlashReport;
 impl SignSlashReport {
-  pub(crate) fn send(txn: &mut impl DbTxn, set: ValidatorSet) {
+  pub(crate) fn send(txn: &mut impl DbTxn, set: ExternalValidatorSet) {
     _public_db::SignSlashReport::send(txn, set, &());
   }
   /// Try to receive a notification to sign a slash report, returning `None` if there is none to
   /// receive.
-  pub fn try_recv(txn: &mut impl DbTxn, set: ValidatorSet) -> Option<()> {
+  pub fn try_recv(txn: &mut impl DbTxn, set: ExternalValidatorSet) -> Option<()> {
     _public_db::SignSlashReport::try_recv(txn, set)
   }
 }
@@ -145,7 +160,7 @@ impl Keys {
   /// reported at once.
   pub fn set(
     txn: &mut impl DbTxn,
-    set: ValidatorSet,
+    set: ExternalValidatorSet,
     key_pair: KeyPair,
     signature_participants: bitvec::vec::BitVec<u8, bitvec::order::Lsb0>,
     signature: Signature,
@@ -165,7 +180,10 @@ impl Keys {
     );
     _public_db::Keys::set(txn, set.network, &(set.session, tx.encode()));
   }
-  pub(crate) fn take(txn: &mut impl DbTxn, network: NetworkId) -> Option<(Session, Transaction)> {
+  pub(crate) fn take(
+    txn: &mut impl DbTxn,
+    network: ExternalNetworkId,
+  ) -> Option<(Session, Transaction)> {
     let (session, tx) = _public_db::Keys::take(txn, network)?;
     Some((session, <_>::decode(&mut tx.as_slice()).unwrap()))
   }
@@ -175,20 +193,14 @@ impl Keys {
 pub struct SignedBatches;
 impl SignedBatches {
   /// Send a `SignedBatch` to publish onto Serai.
-  ///
-  /// These will be published sequentially. Out-of-order sending risks hanging the task.
   pub fn send(txn: &mut impl DbTxn, batch: &SignedBatch) {
     _public_db::SignedBatches::send(txn, batch.batch.network, batch);
   }
-  pub(crate) fn try_recv(txn: &mut impl DbTxn, network: NetworkId) -> Option<SignedBatch> {
+  pub(crate) fn try_recv(txn: &mut impl DbTxn, network: ExternalNetworkId) -> Option<SignedBatch> {
     _public_db::SignedBatches::try_recv(txn, network)
   }
 }
 
-/// The slash report was invalid.
-#[derive(Debug)]
-pub struct InvalidSlashReport;
-
 /// The slash reports to publish onto Serai.
 pub struct SlashReports;
 impl SlashReports {
@@ -196,32 +208,30 @@ impl SlashReports {
   ///
   /// This only saves the most recent slashes as only a single session is eligible to have its
   /// slashes reported at once.
-  ///
-  /// Returns Err if the slashes are invalid. Returns Ok if the slashes weren't detected as
-  /// invalid. Slashes may be considered invalid by the Serai blockchain later even if not detected
-  /// as invalid here.
   pub fn set(
     txn: &mut impl DbTxn,
-    set: ValidatorSet,
-    slashes: Vec<(SeraiAddress, u32)>,
+    set: ExternalValidatorSet,
+    slash_report: SlashReport,
     signature: Signature,
-  ) -> Result<(), InvalidSlashReport> {
+  ) {
     // If we have a more recent slash report, don't write this historic one
     if let Some((existing_session, _)) = _public_db::SlashReports::get(txn, set.network) {
       if existing_session.0 >= set.session.0 {
-        return Ok(());
+        return;
       }
     }
 
     let tx = serai_client::validator_sets::SeraiValidatorSets::report_slashes(
       set.network,
-      slashes.try_into().map_err(|_| InvalidSlashReport)?,
+      slash_report,
       signature,
     );
     _public_db::SlashReports::set(txn, set.network, &(set.session, tx.encode()));
-    Ok(())
   }
-  pub(crate) fn take(txn: &mut impl DbTxn, network: NetworkId) -> Option<(Session, Transaction)> {
+  pub(crate) fn take(
+    txn: &mut impl DbTxn,
+    network: ExternalNetworkId,
+  ) -> Option<(Session, Transaction)> {
     let (session, tx) = _public_db::SlashReports::take(txn, network)?;
     Some((session, <_>::decode(&mut tx.as_slice()).unwrap()))
   }

coordinator/substrate/src/publish_batch.rs

@@ -1,31 +1,32 @@
 use core::future::Future;
 use std::sync::Arc;
 
-use serai_db::{DbTxn, Db};
-
-use serai_client::{primitives::NetworkId, SeraiError, Serai};
+#[rustfmt::skip]
+use serai_client::{primitives::ExternalNetworkId, in_instructions::primitives::SignedBatch, SeraiError, Serai};
 
+use serai_db::{Get, DbTxn, Db, create_db};
 use serai_task::ContinuallyRan;
 
 use crate::SignedBatches;
 
+create_db!(
+  CoordinatorSubstrate {
+    LastPublishedBatch: (network: ExternalNetworkId) -> u32,
+    BatchesToPublish: (network: ExternalNetworkId, batch: u32) -> SignedBatch,
+  }
+);
+
 /// Publish `SignedBatch`s from `SignedBatches` onto Serai.
 pub struct PublishBatchTask<D: Db> {
   db: D,
   serai: Arc<Serai>,
-  network: NetworkId,
+  network: ExternalNetworkId,
 }
 
 impl<D: Db> PublishBatchTask<D> {
   /// Create a task to publish `SignedBatch`s onto Serai.
-  ///
-  /// Returns None if `network == NetworkId::Serai`.
-  // TODO: ExternalNetworkId
-  pub fn new(db: D, serai: Arc<Serai>, network: NetworkId) -> Option<Self> {
-    if network == NetworkId::Serai {
-      None?
-    };
-    Some(Self { db, serai, network })
+  pub fn new(db: D, serai: Arc<Serai>, network: ExternalNetworkId) -> Self {
+    Self { db, serai, network }
   }
 }
 
@@ -34,32 +35,52 @@ impl<D: Db> ContinuallyRan for PublishBatchTask<D> {
 
   fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, Self::Error>> {
     async move {
-      let mut made_progress = false;
-
+      // Read from SignedBatches, which is sequential, into our own mapping
       loop {
         let mut txn = self.db.txn();
         let Some(batch) = SignedBatches::try_recv(&mut txn, self.network) else {
-          // No batch to publish at this time
           break;
         };
 
-        // Publish this Batch if it hasn't already been published
+        // If this is a Batch not yet published, save it into our unordered mapping
+        if LastPublishedBatch::get(&txn, self.network) < Some(batch.batch.id) {
+          BatchesToPublish::set(&mut txn, self.network, batch.batch.id, &batch);
+        }
+
+        txn.commit();
+      }
+
+      // Synchronize our last published batch with the Serai network's
+      let next_to_publish = {
+        // This uses the latest finalized block, not the latest cosigned block, which should be
+        // fine as in the worst case, the only impact is no longer attempting TX publication
         let serai = self.serai.as_of_latest_finalized_block().await?;
         let last_batch = serai.in_instructions().last_batch_for_network(self.network).await?;
-        if last_batch < Some(batch.batch.id) {
-          // This stream of Batches *should* be sequential within the larger context of the Serai
-          // coordinator. In this library, we use a more relaxed definition and don't assert
-          // sequence. This does risk hanging the task, if Batch #n+1 is sent before Batch #n, but
-          // that is a documented fault of the `SignedBatches` API.
+
+        let mut txn = self.db.txn();
+        let mut our_last_batch = LastPublishedBatch::get(&txn, self.network);
+        while our_last_batch < last_batch {
+          let next_batch = our_last_batch.map(|batch| batch + 1).unwrap_or(0);
+          // Clean up the Batch to publish since it's already been published
+          BatchesToPublish::take(&mut txn, self.network, next_batch);
+          our_last_batch = Some(next_batch);
+        }
+        if let Some(last_batch) = our_last_batch {
+          LastPublishedBatch::set(&mut txn, self.network, &last_batch);
+        }
+        last_batch.map(|batch| batch + 1).unwrap_or(0)
+      };
+
+      let made_progress =
+        if let Some(batch) = BatchesToPublish::get(&self.db, self.network, next_to_publish) {
           self
             .serai
             .publish(&serai_client::in_instructions::SeraiInInstructions::execute_batch(batch))
             .await?;
-        }
-
-        txn.commit();
-        made_progress = true;
-      }
+          true
+        } else {
+          false
+        };
       Ok(made_progress)
     }
   }

coordinator/substrate/src/publish_slash_report.rs

@@ -3,7 +3,7 @@ use std::sync::Arc;
 
 use serai_db::{DbTxn, Db};
 
-use serai_client::{primitives::NetworkId, validator_sets::primitives::Session, Serai};
+use serai_client::{primitives::ExternalNetworkId, validator_sets::primitives::Session, Serai};
 
 use serai_task::ContinuallyRan;
 
@@ -22,66 +22,78 @@ impl<D: Db> PublishSlashReportTask<D> {
   }
 }
 
+impl<D: Db> PublishSlashReportTask<D> {
+  // Returns if a slash report was successfully published
+  async fn publish(&mut self, network: ExternalNetworkId) -> Result<bool, String> {
+    let mut txn = self.db.txn();
+    let Some((session, slash_report)) = SlashReports::take(&mut txn, network) else {
+      // No slash report to publish
+      return Ok(false);
+    };
+
+    // This uses the latest finalized block, not the latest cosigned block, which should be
+    // fine as in the worst case, the only impact is no longer attempting TX publication
+    let serai = self.serai.as_of_latest_finalized_block().await.map_err(|e| format!("{e:?}"))?;
+    let serai = serai.validator_sets();
+    let session_after_slash_report = Session(session.0 + 1);
+    let current_session = serai.session(network.into()).await.map_err(|e| format!("{e:?}"))?;
+    let current_session = current_session.map(|session| session.0);
+    // Only attempt to publish the slash report for session #n while session #n+1 is still
+    // active
+    let session_after_slash_report_retired = current_session > Some(session_after_slash_report.0);
+    if session_after_slash_report_retired {
+      // Commit the txn to drain this slash report from the database and not try it again later
+      txn.commit();
+      return Ok(false);
+    }
+
+    if Some(session_after_slash_report.0) != current_session {
+      // We already checked the current session wasn't greater, and they're not equal
+      assert!(current_session < Some(session_after_slash_report.0));
+      // This would mean the Serai node is resyncing and is behind where it prior was
+      Err("have a slash report for a session Serai has yet to retire".to_string())?;
+    }
+
+    // If this session which should publish a slash report already has, move on
+    let key_pending_slash_report =
+      serai.key_pending_slash_report(network).await.map_err(|e| format!("{e:?}"))?;
+    if key_pending_slash_report.is_none() {
+      txn.commit();
+      return Ok(false);
+    };
+
+    match self.serai.publish(&slash_report).await {
+      Ok(()) => {
+        txn.commit();
+        Ok(true)
+      }
+      // This could be specific to this TX (such as an already in mempool error) and it may be
+      // worthwhile to continue iteration with the other pending slash reports. We assume this
+      // error ephemeral and that the latency incurred for this ephemeral error to resolve is
+      // miniscule compared to the window available to publish the slash report. That makes
+      // this a non-issue.
+      Err(e) => Err(format!("couldn't publish slash report transaction: {e:?}")),
+    }
+  }
+}
+
 impl<D: Db> ContinuallyRan for PublishSlashReportTask<D> {
   type Error = String;
 
   fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, Self::Error>> {
     async move {
       let mut made_progress = false;
-      for network in serai_client::primitives::NETWORKS {
-        if network == NetworkId::Serai {
-          continue;
-        };
-
-        let mut txn = self.db.txn();
-        let Some((session, slash_report)) = SlashReports::take(&mut txn, network) else {
-          // No slash report to publish
-          continue;
-        };
-
-        let serai =
-          self.serai.as_of_latest_finalized_block().await.map_err(|e| format!("{e:?}"))?;
-        let serai = serai.validator_sets();
-        let session_after_slash_report = Session(session.0 + 1);
-        let current_session = serai.session(network).await.map_err(|e| format!("{e:?}"))?;
-        let current_session = current_session.map(|session| session.0);
-        // Only attempt to publish the slash report for session #n while session #n+1 is still
-        // active
-        let session_after_slash_report_retired =
-          current_session > Some(session_after_slash_report.0);
-        if session_after_slash_report_retired {
-          // Commit the txn to drain this slash report from the database and not try it again later
-          txn.commit();
-          continue;
-        }
-
-        if Some(session_after_slash_report.0) != current_session {
-          // We already checked the current session wasn't greater, and they're not equal
-          assert!(current_session < Some(session_after_slash_report.0));
-          // This would mean the Serai node is resyncing and is behind where it prior was
-          Err("have a slash report for a session Serai has yet to retire".to_string())?;
-        }
-
-        // If this session which should publish a slash report already has, move on
-        let key_pending_slash_report =
-          serai.key_pending_slash_report(network).await.map_err(|e| format!("{e:?}"))?;
-        if key_pending_slash_report.is_none() {
-          txn.commit();
-          continue;
-        };
-
-        match self.serai.publish(&slash_report).await {
-          Ok(()) => {
-            txn.commit();
-            made_progress = true;
-          }
-          // This could be specific to this TX (such as an already in mempool error) and it may be
-          // worthwhile to continue iteration with the other pending slash reports. We assume this
-          // error ephemeral and that the latency incurred for this ephemeral error to resolve is
-          // miniscule compared to the window available to publish the slash report. That makes
-          // this a non-issue.
-          Err(e) => Err(format!("couldn't publish slash report transaction: {e:?}"))?,
-        }
+      let mut error = None;
+      for network in serai_client::primitives::EXTERNAL_NETWORKS {
+        let network_res = self.publish(network).await;
+        // We made progress if any network successfully published their slash report
+        made_progress |= network_res == Ok(true);
+        // We want to yield the first error *after* attempting for every network
+        error = error.or(network_res.err());
+      }
+      // Yield the error
+      if let Some(error) = error {
+        Err(error)?
       }
       Ok(made_progress)
     }

coordinator/substrate/src/set_keys.rs

@@ -3,7 +3,7 @@ use std::sync::Arc;
 
 use serai_db::{DbTxn, Db};
 
-use serai_client::{primitives::NetworkId, validator_sets::primitives::ValidatorSet, Serai};
+use serai_client::{validator_sets::primitives::ExternalValidatorSet, Serai};
 
 use serai_task::ContinuallyRan;
 
@@ -28,21 +28,19 @@ impl<D: Db> ContinuallyRan for SetKeysTask<D> {
   fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, Self::Error>> {
     async move {
       let mut made_progress = false;
-      for network in serai_client::primitives::NETWORKS {
-        if network == NetworkId::Serai {
-          continue;
-        };
-
+      for network in serai_client::primitives::EXTERNAL_NETWORKS {
         let mut txn = self.db.txn();
         let Some((session, keys)) = Keys::take(&mut txn, network) else {
           // No keys to set
           continue;
         };
 
+        // This uses the latest finalized block, not the latest cosigned block, which should be
+        // fine as in the worst case, the only impact is no longer attempting TX publication
         let serai =
           self.serai.as_of_latest_finalized_block().await.map_err(|e| format!("{e:?}"))?;
         let serai = serai.validator_sets();
-        let current_session = serai.session(network).await.map_err(|e| format!("{e:?}"))?;
+        let current_session = serai.session(network.into()).await.map_err(|e| format!("{e:?}"))?;
         let current_session = current_session.map(|session| session.0);
         // Only attempt to set these keys if this isn't a retired session
         if Some(session.0) < current_session {
@@ -60,7 +58,7 @@ impl<D: Db> ContinuallyRan for SetKeysTask<D> {
 
         // If this session already has had its keys set, move on
         if serai
-          .keys(ValidatorSet { network, session })
+          .keys(ExternalValidatorSet { network, session })
           .await
           .map_err(|e| format!("{e:?}"))?
           .is_some()

coordinator/tributary-sdk/src/merkle.rs

@@ -9,7 +9,7 @@ pub(crate) fn merkle(hash_args: &[[u8; 32]]) -> [u8; 32] {
   let zero = [0; 32];
   let mut interim;
   while hashes.len() > 1 {
-    interim = Vec::with_capacity((hashes.len() + 1) / 2);
+    interim = Vec::with_capacity(hashes.len().div_ceil(2));
 
     let mut i = 0;
     while i < hashes.len() {

coordinator/tributary/Cargo.toml

@@ -21,13 +21,14 @@ workspace = true
 zeroize = { version = "^1.5", default-features = false, features = ["std"] }
 rand_core = { version = "0.6", default-features = false, features = ["std"] }
 
-blake2 = { version = "0.10", default-features = false, features = ["std"] }
-ciphersuite = { path = "../../crypto/ciphersuite", default-features = false, features = ["std"] }
-schnorr = { package = "schnorr-signatures", path = "../../crypto/schnorr", default-features = false, features = ["std"] }
-
 scale = { package = "parity-scale-codec", version = "3", default-features = false, features = ["std", "derive"] }
 borsh = { version = "1", default-features = false, features = ["std", "derive", "de_strict_order"] }
 
+blake2 = { version = "0.10", default-features = false, features = ["std"] }
+ciphersuite = { path = "../../crypto/ciphersuite", default-features = false, features = ["std"] }
+dkg = { path = "../../crypto/dkg", default-features = false, features = ["std"] }
+schnorr = { package = "schnorr-signatures", path = "../../crypto/schnorr", default-features = false, features = ["std"] }
+
 serai-client = { path = "../../substrate/client", default-features = false, features = ["serai", "borsh"] }
 
 serai-db = { path = "../../common/db" }

coordinator/tributary/src/db.rs

@@ -3,7 +3,7 @@ use std::collections::HashMap;
 use scale::Encode;
 use borsh::{BorshSerialize, BorshDeserialize};
 
-use serai_client::{primitives::SeraiAddress, validator_sets::primitives::ValidatorSet};
+use serai_client::{primitives::SeraiAddress, validator_sets::primitives::ExternalValidatorSet};
 
 use messages::sign::{VariantSignId, SignId};
 
@@ -15,20 +15,35 @@ use crate::transaction::SigningProtocolRound;
 
 /// A topic within the database which the group participates in
 #[derive(Clone, Copy, PartialEq, Eq, Debug, Encode, BorshSerialize, BorshDeserialize)]
-pub(crate) enum Topic {
+pub enum Topic {
   /// Vote to remove a participant
-  RemoveParticipant { participant: SeraiAddress },
+  RemoveParticipant {
+    /// The participant to remove
+    participant: SeraiAddress,
+  },
 
   // DkgParticipation isn't represented here as participations are immediately sent to the
   // processor, not accumulated within this databse
   /// Participation in the signing protocol to confirm the DKG results on Substrate
-  DkgConfirmation { attempt: u32, round: SigningProtocolRound },
+  DkgConfirmation {
+    /// The attempt number this is for
+    attempt: u32,
+    /// The round of the signing protocol
+    round: SigningProtocolRound,
+  },
 
   /// The local view of the SlashReport, to be aggregated into the final SlashReport
   SlashReport,
 
   /// Participation in a signing protocol
-  Sign { id: VariantSignId, attempt: u32, round: SigningProtocolRound },
+  Sign {
+    /// The ID of the signing protocol
+    id: VariantSignId,
+    /// The attempt number this is for
+    attempt: u32,
+    /// The round of the signing protocol
+    round: SigningProtocolRound,
+  },
 }
 
 enum Participating {
@@ -46,7 +61,7 @@ impl Topic {
         attempt: attempt + 1,
         round: SigningProtocolRound::Preprocess,
       }),
-      Topic::SlashReport { .. } => None,
+      Topic::SlashReport => None,
       Topic::Sign { id, attempt, round: _ } => {
         Some(Topic::Sign { id, attempt: attempt + 1, round: SigningProtocolRound::Preprocess })
       }
@@ -68,7 +83,7 @@ impl Topic {
         }
         SigningProtocolRound::Share => None,
       },
-      Topic::SlashReport { .. } => None,
+      Topic::SlashReport => None,
       Topic::Sign { id, attempt, round } => match round {
         SigningProtocolRound::Preprocess => {
           let attempt = attempt + 1;
@@ -79,19 +94,46 @@ impl Topic {
     }
   }
 
-  // The SignId for this topic
-  //
-  // Returns None if Topic isn't Topic::Sign
-  pub(crate) fn sign_id(self, set: ValidatorSet) -> Option<messages::sign::SignId> {
+  /// The SignId for this topic
+  ///
+  /// Returns None if Topic isn't Topic::Sign
+  pub(crate) fn sign_id(self, set: ExternalValidatorSet) -> Option<messages::sign::SignId> {
     #[allow(clippy::match_same_arms)]
     match self {
       Topic::RemoveParticipant { .. } => None,
       Topic::DkgConfirmation { .. } => None,
-      Topic::SlashReport { .. } => None,
+      Topic::SlashReport => None,
       Topic::Sign { id, attempt, round: _ } => Some(SignId { session: set.session, id, attempt }),
     }
   }
 
+  /// The SignId for this DKG Confirmation.
+  ///
+  /// This is undefined except for being consistent to the DKG Confirmation signing protocol and
+  /// unique across sets.
+  ///
+  /// Returns None if Topic isn't Topic::DkgConfirmation.
+  pub(crate) fn dkg_confirmation_sign_id(
+    self,
+    set: ExternalValidatorSet,
+  ) -> Option<messages::sign::SignId> {
+    #[allow(clippy::match_same_arms)]
+    match self {
+      Topic::RemoveParticipant { .. } => None,
+      Topic::DkgConfirmation { attempt, round: _ } => Some({
+        let id = {
+          let mut id = [0; 32];
+          let encoded_set = set.encode();
+          id[.. encoded_set.len()].copy_from_slice(&encoded_set);
+          VariantSignId::Batch(id)
+        };
+        SignId { session: set.session, id, attempt }
+      }),
+      Topic::SlashReport => None,
+      Topic::Sign { .. } => None,
+    }
+  }
+
   /// The topic which precedes this topic as a prerequisite
   ///
   /// The preceding topic must define this topic as succeeding
@@ -105,7 +147,7 @@ impl Topic {
           Some(Topic::DkgConfirmation { attempt, round: SigningProtocolRound::Preprocess })
         }
       },
-      Topic::SlashReport { .. } => None,
+      Topic::SlashReport => None,
       Topic::Sign { id, attempt, round } => match round {
         SigningProtocolRound::Preprocess => None,
         SigningProtocolRound::Share => {
@@ -128,7 +170,7 @@ impl Topic {
         }
         SigningProtocolRound::Share => None,
       },
-      Topic::SlashReport { .. } => None,
+      Topic::SlashReport => None,
       Topic::Sign { id, attempt, round } => match round {
         SigningProtocolRound::Preprocess => {
           Some(Topic::Sign { id, attempt, round: SigningProtocolRound::Share })
@@ -138,21 +180,22 @@ impl Topic {
     }
   }
 
-  fn requires_whitelisting(&self) -> bool {
+  /// If this topic requires recognition before entries are permitted for it.
+  pub fn requires_recognition(&self) -> bool {
     #[allow(clippy::match_same_arms)]
     match self {
-      // We don't require whitelisting to remove a participant
+      // We don't require recognition to remove a participant
       Topic::RemoveParticipant { .. } => false,
-      // We don't require whitelisting for the first attempt, solely the re-attempts
+      // We don't require recognition for the first attempt, solely the re-attempts
       Topic::DkgConfirmation { attempt, .. } => *attempt != 0,
-      // We don't require whitelisting for the slash report
-      Topic::SlashReport { .. } => false,
-      // We do require whitelisting for every sign protocol
+      // We don't require recognition for the slash report
+      Topic::SlashReport => false,
+      // We do require recognition for every sign protocol
       Topic::Sign { .. } => true,
     }
   }
 
-  fn required_participation(&self, n: u64) -> u64 {
+  fn required_participation(&self, n: u16) -> u16 {
     let _ = self;
     // All of our topics require 2/3rds participation
     ((2 * n) / 3) + 1
@@ -163,7 +206,7 @@ impl Topic {
     match self {
       Topic::RemoveParticipant { .. } => Participating::Everyone,
       Topic::DkgConfirmation { .. } => Participating::Participated,
-      Topic::SlashReport { .. } => Participating::Everyone,
+      Topic::SlashReport => Participating::Everyone,
       Topic::Sign { .. } => Participating::Participated,
     }
   }
@@ -184,36 +227,48 @@ pub(crate) enum DataSet<D: Borshy> {
 create_db!(
   CoordinatorTributary {
     // The last handled tributary block's (number, hash)
-    LastHandledTributaryBlock: (set: ValidatorSet) -> (u64, [u8; 32]),
+    LastHandledTributaryBlock: (set: ExternalValidatorSet) -> (u64, [u8; 32]),
 
     // The slash points a validator has accrued, with u32::MAX representing a fatal slash.
-    SlashPoints: (set: ValidatorSet, validator: SeraiAddress) -> u32,
+    SlashPoints: (set: ExternalValidatorSet, validator: SeraiAddress) -> u32,
 
     // The cosign intent for a Substrate block
-    CosignIntents: (set: ValidatorSet, substrate_block_hash: [u8; 32]) -> CosignIntent,
+    CosignIntents: (set: ExternalValidatorSet, substrate_block_hash: [u8; 32]) -> CosignIntent,
     // The latest Substrate block to cosign.
-    LatestSubstrateBlockToCosign: (set: ValidatorSet) -> [u8; 32],
+    LatestSubstrateBlockToCosign: (set: ExternalValidatorSet) -> [u8; 32],
     // The hash of the block we're actively cosigning.
-    ActivelyCosigning: (set: ValidatorSet) -> [u8; 32],
+    ActivelyCosigning: (set: ExternalValidatorSet) -> [u8; 32],
     // If this block has already been cosigned.
-    Cosigned: (set: ValidatorSet, substrate_block_hash: [u8; 32]) -> (),
+    Cosigned: (set: ExternalValidatorSet, substrate_block_hash: [u8; 32]) -> (),
 
-    // The plans to whitelist upon a `Transaction::SubstrateBlock` being included on-chain.
-    SubstrateBlockPlans: (set: ValidatorSet, substrate_block_hash: [u8; 32]) -> Vec<[u8; 32]>,
+    // The plans to recognize upon a `Transaction::SubstrateBlock` being included on-chain.
+    SubstrateBlockPlans: (
+      set: ExternalValidatorSet,
+      substrate_block_hash: [u8; 32]
+    ) -> Vec<[u8; 32]>,
 
     // The weight accumulated for a topic.
-    AccumulatedWeight: (set: ValidatorSet, topic: Topic) -> u64,
+    AccumulatedWeight: (set: ExternalValidatorSet, topic: Topic) -> u16,
     // The entries accumulated for a topic, by validator.
-    Accumulated: <D: Borshy>(set: ValidatorSet, topic: Topic, validator: SeraiAddress) -> D,
+    Accumulated: <D: Borshy>(
+      set: ExternalValidatorSet,
+      topic: Topic,
+      validator: SeraiAddress
+    ) -> D,
 
     // Topics to be recognized as of a certain block number due to the reattempt protocol.
-    Reattempt: (set: ValidatorSet, block_number: u64) -> Vec<Topic>,
+    Reattempt: (set: ExternalValidatorSet, block_number: u64) -> Vec<Topic>,
   }
 );
 
 db_channel!(
   CoordinatorTributary {
-    ProcessorMessages: (set: ValidatorSet) -> messages::CoordinatorMessage,
+    // Messages to send to the processor
+    ProcessorMessages: (set: ExternalValidatorSet) -> messages::CoordinatorMessage,
+    // Messages for the DKG confirmation
+    DkgConfirmationMessages: (set: ExternalValidatorSet) -> messages::sign::CoordinatorMessage,
+    // Topics which have been explicitly recognized
+    RecognizedTopics: (set: ExternalValidatorSet) -> Topic,
   }
 );
 
@@ -221,13 +276,13 @@ pub(crate) struct TributaryDb;
 impl TributaryDb {
   pub(crate) fn last_handled_tributary_block(
     getter: &impl Get,
-    set: ValidatorSet,
+    set: ExternalValidatorSet,
   ) -> Option<(u64, [u8; 32])> {
     LastHandledTributaryBlock::get(getter, set)
   }
   pub(crate) fn set_last_handled_tributary_block(
     txn: &mut impl DbTxn,
-    set: ValidatorSet,
+    set: ExternalValidatorSet,
     block_number: u64,
     block_hash: [u8; 32],
   ) {
@@ -236,23 +291,26 @@ impl TributaryDb {
 
   pub(crate) fn latest_substrate_block_to_cosign(
     getter: &impl Get,
-    set: ValidatorSet,
+    set: ExternalValidatorSet,
   ) -> Option<[u8; 32]> {
     LatestSubstrateBlockToCosign::get(getter, set)
   }
   pub(crate) fn set_latest_substrate_block_to_cosign(
     txn: &mut impl DbTxn,
-    set: ValidatorSet,
+    set: ExternalValidatorSet,
     substrate_block_hash: [u8; 32],
   ) {
     LatestSubstrateBlockToCosign::set(txn, set, &substrate_block_hash);
   }
-  pub(crate) fn actively_cosigning(txn: &mut impl DbTxn, set: ValidatorSet) -> Option<[u8; 32]> {
+  pub(crate) fn actively_cosigning(
+    txn: &mut impl DbTxn,
+    set: ExternalValidatorSet,
+  ) -> Option<[u8; 32]> {
     ActivelyCosigning::get(txn, set)
   }
   pub(crate) fn start_cosigning(
     txn: &mut impl DbTxn,
-    set: ValidatorSet,
+    set: ExternalValidatorSet,
     substrate_block_hash: [u8; 32],
     substrate_block_number: u64,
   ) {
@@ -262,7 +320,7 @@ impl TributaryDb {
     );
     ActivelyCosigning::set(txn, set, &substrate_block_hash);
 
-    TributaryDb::recognize_topic(
+    Self::recognize_topic(
       txn,
       set,
       Topic::Sign {
@@ -272,29 +330,33 @@ impl TributaryDb {
       },
     );
   }
-  pub(crate) fn finish_cosigning(txn: &mut impl DbTxn, set: ValidatorSet) {
+  pub(crate) fn finish_cosigning(txn: &mut impl DbTxn, set: ExternalValidatorSet) {
     assert!(ActivelyCosigning::take(txn, set).is_some(), "finished cosigning but not cosigning");
   }
   pub(crate) fn mark_cosigned(
     txn: &mut impl DbTxn,
-    set: ValidatorSet,
+    set: ExternalValidatorSet,
     substrate_block_hash: [u8; 32],
   ) {
     Cosigned::set(txn, set, substrate_block_hash, &());
   }
   pub(crate) fn cosigned(
     txn: &mut impl DbTxn,
-    set: ValidatorSet,
+    set: ExternalValidatorSet,
     substrate_block_hash: [u8; 32],
   ) -> bool {
     Cosigned::get(txn, set, substrate_block_hash).is_some()
   }
 
-  pub(crate) fn recognize_topic(txn: &mut impl DbTxn, set: ValidatorSet, topic: Topic) {
+  pub(crate) fn recognize_topic(txn: &mut impl DbTxn, set: ExternalValidatorSet, topic: Topic) {
     AccumulatedWeight::set(txn, set, topic, &0);
+    RecognizedTopics::send(txn, set, &topic);
+  }
+  pub(crate) fn recognized(getter: &impl Get, set: ExternalValidatorSet, topic: Topic) -> bool {
+    AccumulatedWeight::get(getter, set, topic).is_some()
   }
 
-  pub(crate) fn start_of_block(txn: &mut impl DbTxn, set: ValidatorSet, block_number: u64) {
+  pub(crate) fn start_of_block(txn: &mut impl DbTxn, set: ExternalValidatorSet, block_number: u64) {
     for topic in Reattempt::take(txn, set, block_number).unwrap_or(vec![]) {
       /*
         TODO: Slash all people who preprocessed but didn't share, and add a delay to their
@@ -312,13 +374,19 @@ impl TributaryDb {
       Self::recognize_topic(txn, set, topic);
       if let Some(id) = topic.sign_id(set) {
         Self::send_message(txn, set, messages::sign::CoordinatorMessage::Reattempt { id });
+      } else if let Some(id) = topic.dkg_confirmation_sign_id(set) {
+        DkgConfirmationMessages::send(
+          txn,
+          set,
+          &messages::sign::CoordinatorMessage::Reattempt { id },
+        );
       }
     }
   }
 
   pub(crate) fn fatal_slash(
     txn: &mut impl DbTxn,
-    set: ValidatorSet,
+    set: ExternalValidatorSet,
     validator: SeraiAddress,
     reason: &str,
   ) {
@@ -328,7 +396,7 @@ impl TributaryDb {
 
   pub(crate) fn is_fatally_slashed(
     getter: &impl Get,
-    set: ValidatorSet,
+    set: ExternalValidatorSet,
     validator: SeraiAddress,
   ) -> bool {
     SlashPoints::get(getter, set, validator).unwrap_or(0) == u32::MAX
@@ -337,21 +405,26 @@ impl TributaryDb {
   #[allow(clippy::too_many_arguments)]
   pub(crate) fn accumulate<D: Borshy>(
     txn: &mut impl DbTxn,
-    set: ValidatorSet,
+    set: ExternalValidatorSet,
     validators: &[SeraiAddress],
-    total_weight: u64,
+    total_weight: u16,
     block_number: u64,
     topic: Topic,
     validator: SeraiAddress,
-    validator_weight: u64,
+    validator_weight: u16,
     data: &D,
   ) -> DataSet<D> {
     // This function will only be called once for a (validator, topic) tuple due to how we handle
     // nonces on transactions (deterministically to the topic)
 
     let accumulated_weight = AccumulatedWeight::get(txn, set, topic);
-    if topic.requires_whitelisting() && accumulated_weight.is_none() {
-      Self::fatal_slash(txn, set, validator, "participated in unrecognized topic");
+    if topic.requires_recognition() && accumulated_weight.is_none() {
+      Self::fatal_slash(
+        txn,
+        set,
+        validator,
+        "participated in unrecognized topic which requires recognition",
+      );
       return DataSet::None;
     }
     let mut accumulated_weight = accumulated_weight.unwrap_or(0);
@@ -448,7 +521,7 @@ impl TributaryDb {
 
   pub(crate) fn send_message(
     txn: &mut impl DbTxn,
-    set: ValidatorSet,
+    set: ExternalValidatorSet,
     message: impl Into<messages::CoordinatorMessage>,
   ) {
     ProcessorMessages::send(txn, set, &message.into());

coordinator/tributary/src/lib.rs

@@ -6,10 +6,11 @@ use core::{marker::PhantomData, future::Future};
 use std::collections::HashMap;
 
 use ciphersuite::group::GroupEncoding;
+use dkg::Participant;
 
 use serai_client::{
   primitives::SeraiAddress,
-  validator_sets::primitives::{ValidatorSet, Slash},
+  validator_sets::primitives::{ExternalValidatorSet, Slash},
 };
 
 use serai_db::*;
@@ -27,59 +28,99 @@ use tributary_sdk::{
 use serai_cosign::CosignIntent;
 use serai_coordinator_substrate::NewSetInformation;
 
-use messages::sign::VariantSignId;
+use messages::sign::{VariantSignId, SignId};
 
 mod transaction;
 pub use transaction::{SigningProtocolRound, Signed, Transaction};
 
 mod db;
 use db::*;
+pub use db::Topic;
 
 /// Messages to send to the Processors.
 pub struct ProcessorMessages;
 impl ProcessorMessages {
   /// Try to receive a message to send to a Processor.
-  pub fn try_recv(txn: &mut impl DbTxn, set: ValidatorSet) -> Option<messages::CoordinatorMessage> {
+  pub fn try_recv(
+    txn: &mut impl DbTxn,
+    set: ExternalValidatorSet,
+  ) -> Option<messages::CoordinatorMessage> {
     db::ProcessorMessages::try_recv(txn, set)
   }
 }
 
+/// Messages for the DKG confirmation.
+pub struct DkgConfirmationMessages;
+impl DkgConfirmationMessages {
+  /// Receive a message for the DKG confirmation.
+  ///
+  /// These messages use the ProcessorMessage API as that's what existing flows are designed
+  /// around, enabling their reuse. The ProcessorMessage includes a VariantSignId which isn't
+  /// applicable to the DKG confirmation (as there's no such variant of the VariantSignId). The
+  /// actual ID is undefined other than it will be consistent to the signing protocol and unique
+  /// across validator sets, with no guarantees of uniqueness across contexts.
+  pub fn try_recv(
+    txn: &mut impl DbTxn,
+    set: ExternalValidatorSet,
+  ) -> Option<messages::sign::CoordinatorMessage> {
+    db::DkgConfirmationMessages::try_recv(txn, set)
+  }
+}
+
 /// The cosign intents.
 pub struct CosignIntents;
 impl CosignIntents {
   /// Provide a CosignIntent for this Tributary.
   ///
   /// This must be done before the associated `Transaction::Cosign` is provided.
-  pub fn provide(txn: &mut impl DbTxn, set: ValidatorSet, intent: &CosignIntent) {
+  pub fn provide(txn: &mut impl DbTxn, set: ExternalValidatorSet, intent: &CosignIntent) {
     db::CosignIntents::set(txn, set, intent.block_hash, intent);
   }
   fn take(
     txn: &mut impl DbTxn,
-    set: ValidatorSet,
+    set: ExternalValidatorSet,
     substrate_block_hash: [u8; 32],
   ) -> Option<CosignIntent> {
     db::CosignIntents::take(txn, set, substrate_block_hash)
   }
 }
 
-/// The plans to whitelist upon a `Transaction::SubstrateBlock` being included on-chain.
+/// An interface to the topics recognized on this Tributary.
+pub struct RecognizedTopics;
+impl RecognizedTopics {
+  /// If this topic has been recognized by this Tributary.
+  ///
+  /// This will either be by explicit recognition or participation.
+  pub fn recognized(getter: &impl Get, set: ExternalValidatorSet, topic: Topic) -> bool {
+    TributaryDb::recognized(getter, set, topic)
+  }
+  /// The next topic requiring recognition which has been recognized by this Tributary.
+  pub fn try_recv_topic_requiring_recognition(
+    txn: &mut impl DbTxn,
+    set: ExternalValidatorSet,
+  ) -> Option<Topic> {
+    db::RecognizedTopics::try_recv(txn, set)
+  }
+}
+
+/// The plans to recognize upon a `Transaction::SubstrateBlock` being included on-chain.
 pub struct SubstrateBlockPlans;
 impl SubstrateBlockPlans {
-  /// Set the plans to whitelist upon the associated `Transaction::SubstrateBlock` being included
+  /// Set the plans to recognize upon the associated `Transaction::SubstrateBlock` being included
   /// on-chain.
   ///
   /// This must be done before the associated `Transaction::Cosign` is provided.
   pub fn set(
     txn: &mut impl DbTxn,
-    set: ValidatorSet,
+    set: ExternalValidatorSet,
     substrate_block_hash: [u8; 32],
     plans: &Vec<[u8; 32]>,
   ) {
-    db::SubstrateBlockPlans::set(txn, set, substrate_block_hash, &plans);
+    db::SubstrateBlockPlans::set(txn, set, substrate_block_hash, plans);
   }
   fn take(
     txn: &mut impl DbTxn,
-    set: ValidatorSet,
+    set: ExternalValidatorSet,
     substrate_block_hash: [u8; 32],
   ) -> Option<Vec<[u8; 32]>> {
     db::SubstrateBlockPlans::take(txn, set, substrate_block_hash)
@@ -90,32 +131,32 @@ struct ScanBlock<'a, TD: Db, TDT: DbTxn, P: P2p> {
   _td: PhantomData<TD>,
   _p2p: PhantomData<P>,
   tributary_txn: &'a mut TDT,
-  set: ValidatorSet,
+  set: &'a NewSetInformation,
   validators: &'a [SeraiAddress],
-  total_weight: u64,
-  validator_weights: &'a HashMap<SeraiAddress, u64>,
+  total_weight: u16,
+  validator_weights: &'a HashMap<SeraiAddress, u16>,
 }
-impl<'a, TD: Db, TDT: DbTxn, P: P2p> ScanBlock<'a, TD, TDT, P> {
+impl<TD: Db, TDT: DbTxn, P: P2p> ScanBlock<'_, TD, TDT, P> {
   fn potentially_start_cosign(&mut self) {
     // Don't start a new cosigning instance if we're actively running one
-    if TributaryDb::actively_cosigning(self.tributary_txn, self.set).is_some() {
+    if TributaryDb::actively_cosigning(self.tributary_txn, self.set.set).is_some() {
       return;
     }
 
     // Fetch the latest intended-to-be-cosigned block
     let Some(latest_substrate_block_to_cosign) =
-      TributaryDb::latest_substrate_block_to_cosign(self.tributary_txn, self.set)
+      TributaryDb::latest_substrate_block_to_cosign(self.tributary_txn, self.set.set)
     else {
       return;
     };
 
     // If it was already cosigned, return
-    if TributaryDb::cosigned(self.tributary_txn, self.set, latest_substrate_block_to_cosign) {
+    if TributaryDb::cosigned(self.tributary_txn, self.set.set, latest_substrate_block_to_cosign) {
       return;
     }
 
     let intent =
-      CosignIntents::take(self.tributary_txn, self.set, latest_substrate_block_to_cosign)
+      CosignIntents::take(self.tributary_txn, self.set.set, latest_substrate_block_to_cosign)
         .expect("Transaction::Cosign locally provided but CosignIntents wasn't populated");
     assert_eq!(
       intent.block_hash, latest_substrate_block_to_cosign,
@@ -125,20 +166,71 @@ impl<'a, TD: Db, TDT: DbTxn, P: P2p> ScanBlock<'a, TD, TDT, P> {
     // Mark us as actively cosigning
     TributaryDb::start_cosigning(
       self.tributary_txn,
-      self.set,
+      self.set.set,
       latest_substrate_block_to_cosign,
       intent.block_number,
     );
     // Send the message for the processor to start signing
     TributaryDb::send_message(
       self.tributary_txn,
-      self.set,
+      self.set.set,
       messages::coordinator::CoordinatorMessage::CosignSubstrateBlock {
-        session: self.set.session,
-        intent,
+        session: self.set.set.session,
+        cosign: intent.into_cosign(self.set.set.network),
       },
     );
   }
+
+  fn accumulate_dkg_confirmation<D: AsRef<[u8]> + Borshy>(
+    &mut self,
+    block_number: u64,
+    topic: Topic,
+    data: &D,
+    signer: SeraiAddress,
+  ) -> Option<(SignId, HashMap<Participant, Vec<u8>>)> {
+    match TributaryDb::accumulate::<D>(
+      self.tributary_txn,
+      self.set.set,
+      self.validators,
+      self.total_weight,
+      block_number,
+      topic,
+      signer,
+      self.validator_weights[&signer],
+      data,
+    ) {
+      DataSet::None => None,
+      DataSet::Participating(data_set) => {
+        let id = topic.dkg_confirmation_sign_id(self.set.set).unwrap();
+
+        // This will be used in a MuSig protocol, so the Participant indexes are the validator's
+        // position in the list regardless of their weight
+        let flatten_data_set = |data_set: HashMap<_, D>| {
+          let mut entries = HashMap::with_capacity(usize::from(self.total_weight));
+          for (validator, participation) in data_set {
+            let (index, (_validator, _weight)) = &self
+              .set
+              .validators
+              .iter()
+              .enumerate()
+              .find(|(_i, (validator_i, _weight))| validator == *validator_i)
+              .unwrap();
+            // The index is zero-indexed yet participants are one-indexed
+            let index = index + 1;
+
+            entries.insert(
+              Participant::new(u16::try_from(index).unwrap()).unwrap(),
+              participation.as_ref().to_vec(),
+            );
+          }
+          entries
+        };
+        let data_set = flatten_data_set(data_set);
+        Some((id, data_set))
+      }
+    }
+  }
+
   fn handle_application_tx(&mut self, block_number: u64, tx: Transaction) {
     let signer = |signed: Signed| SeraiAddress(signed.signer().to_bytes());
 
@@ -147,13 +239,14 @@ impl<'a, TD: Db, TDT: DbTxn, P: P2p> ScanBlock<'a, TD, TDT, P> {
       // TODO: The fact they can publish these TXs makes this a notable spam vector
       if TributaryDb::is_fatally_slashed(
         self.tributary_txn,
-        self.set,
+        self.set.set,
         SeraiAddress(signer.to_bytes()),
       ) {
         return;
       }
     }
 
+    let topic = tx.topic();
     match tx {
       // Accumulate this vote and fatally slash the participant if past the threshold
       Transaction::RemoveParticipant { participant, signed } => {
@@ -163,7 +256,7 @@ impl<'a, TD: Db, TDT: DbTxn, P: P2p> ScanBlock<'a, TD, TDT, P> {
         if !self.validators.iter().any(|validator| *validator == participant) {
           TributaryDb::fatal_slash(
             self.tributary_txn,
-            self.set,
+            self.set.set,
             signer,
             "voted to remove non-existent participant",
           );
@@ -172,18 +265,23 @@ impl<'a, TD: Db, TDT: DbTxn, P: P2p> ScanBlock<'a, TD, TDT, P> {
 
         match TributaryDb::accumulate(
           self.tributary_txn,
-          self.set,
+          self.set.set,
           self.validators,
           self.total_weight,
           block_number,
-          Topic::RemoveParticipant { participant },
+          topic.unwrap(),
           signer,
           self.validator_weights[&signer],
           &(),
         ) {
           DataSet::None => {}
           DataSet::Participating(_) => {
-            TributaryDb::fatal_slash(self.tributary_txn, self.set, participant, "voted to remove");
+            TributaryDb::fatal_slash(
+              self.tributary_txn,
+              self.set.set,
+              participant,
+              "voted to remove",
+            );
           }
         };
       }
@@ -192,28 +290,52 @@ impl<'a, TD: Db, TDT: DbTxn, P: P2p> ScanBlock<'a, TD, TDT, P> {
       Transaction::DkgParticipation { participation, signed } => {
         TributaryDb::send_message(
           self.tributary_txn,
-          self.set,
+          self.set.set,
           messages::key_gen::CoordinatorMessage::Participation {
-            session: self.set.session,
-            participant: todo!("TODO"),
+            session: self.set.set.session,
+            participant: self.set.participant_indexes[&signer(signed)][0],
             participation,
           },
         );
       }
-      Transaction::DkgConfirmationPreprocess { attempt, preprocess, signed } => {
-        // Accumulate the preprocesses into our own FROST attempt manager
-        todo!("TODO")
+      Transaction::DkgConfirmationPreprocess { attempt: _, preprocess, signed } => {
+        let topic = topic.unwrap();
+        let signer = signer(signed);
+
+        let Some((id, data_set)) =
+          self.accumulate_dkg_confirmation(block_number, topic, &preprocess, signer)
+        else {
+          return;
+        };
+
+        db::DkgConfirmationMessages::send(
+          self.tributary_txn,
+          self.set.set,
+          &messages::sign::CoordinatorMessage::Preprocesses { id, preprocesses: data_set },
+        );
       }
-      Transaction::DkgConfirmationShare { attempt, share, signed } => {
-        // Accumulate the shares into our own FROST attempt manager
-        todo!("TODO: SetKeysTask")
+      Transaction::DkgConfirmationShare { attempt: _, share, signed } => {
+        let topic = topic.unwrap();
+        let signer = signer(signed);
+
+        let Some((id, data_set)) =
+          self.accumulate_dkg_confirmation(block_number, topic, &share, signer)
+        else {
+          return;
+        };
+
+        db::DkgConfirmationMessages::send(
+          self.tributary_txn,
+          self.set.set,
+          &messages::sign::CoordinatorMessage::Shares { id, shares: data_set },
+        );
       }
 
       Transaction::Cosign { substrate_block_hash } => {
         // Update the latest intended-to-be-cosigned Substrate block
         TributaryDb::set_latest_substrate_block_to_cosign(
           self.tributary_txn,
-          self.set,
+          self.set.set,
           substrate_block_hash,
         );
         // Start a new cosign if we aren't already working on one
@@ -226,32 +348,32 @@ impl<'a, TD: Db, TDT: DbTxn, P: P2p> ScanBlock<'a, TD, TDT, P> {
           not-yet-Cosigned cosigns, we flag all cosigned blocks as cosigned. Then, when we choose
           the next block to work on, we won't if it's already been cosigned.
         */
-        TributaryDb::mark_cosigned(self.tributary_txn, self.set, substrate_block_hash);
+        TributaryDb::mark_cosigned(self.tributary_txn, self.set.set, substrate_block_hash);
 
         // If we aren't actively cosigning this block, return
         // This occurs when we have Cosign TXs A, B, C, we received Cosigned for A and start on C,
         // and then receive Cosigned for B
-        if TributaryDb::actively_cosigning(self.tributary_txn, self.set) !=
+        if TributaryDb::actively_cosigning(self.tributary_txn, self.set.set) !=
           Some(substrate_block_hash)
         {
           return;
         }
 
         // Since this is the block we were cosigning, mark us as having finished cosigning
-        TributaryDb::finish_cosigning(self.tributary_txn, self.set);
+        TributaryDb::finish_cosigning(self.tributary_txn, self.set.set);
 
         // Start working on the next cosign
         self.potentially_start_cosign();
       }
       Transaction::SubstrateBlock { hash } => {
-        // Whitelist all of the IDs this Substrate block causes to be signed
-        let plans = SubstrateBlockPlans::take(self.tributary_txn, self.set, hash).expect(
+        // Recognize all of the IDs this Substrate block causes to be signed
+        let plans = SubstrateBlockPlans::take(self.tributary_txn, self.set.set, hash).expect(
           "Transaction::SubstrateBlock locally provided but SubstrateBlockPlans wasn't populated",
         );
         for plan in plans {
           TributaryDb::recognize_topic(
             self.tributary_txn,
-            self.set,
+            self.set.set,
             Topic::Sign {
               id: VariantSignId::Transaction(plan),
               attempt: 0,
@@ -261,10 +383,10 @@ impl<'a, TD: Db, TDT: DbTxn, P: P2p> ScanBlock<'a, TD, TDT, P> {
         }
       }
       Transaction::Batch { hash } => {
-        // Whitelist the signing of this batch
+        // Recognize the signing of this batch
         TributaryDb::recognize_topic(
           self.tributary_txn,
-          self.set,
+          self.set.set,
           Topic::Sign {
             id: VariantSignId::Batch(hash),
             attempt: 0,
@@ -279,7 +401,7 @@ impl<'a, TD: Db, TDT: DbTxn, P: P2p> ScanBlock<'a, TD, TDT, P> {
         if slash_points.len() != self.validators.len() {
           TributaryDb::fatal_slash(
             self.tributary_txn,
-            self.set,
+            self.set.set,
             signer,
             "slash report was for a distinct amount of signers",
           );
@@ -289,11 +411,11 @@ impl<'a, TD: Db, TDT: DbTxn, P: P2p> ScanBlock<'a, TD, TDT, P> {
         // Accumulate, and if past the threshold, calculate *the* slash report and start signing it
         match TributaryDb::accumulate(
           self.tributary_txn,
-          self.set,
+          self.set.set,
           self.validators,
           self.total_weight,
           block_number,
-          Topic::SlashReport,
+          topic.unwrap(),
           signer,
           self.validator_weights[&signer],
           &slash_points,
@@ -307,10 +429,6 @@ impl<'a, TD: Db, TDT: DbTxn, P: P2p> ScanBlock<'a, TD, TDT, P> {
               have a supermajority agree the slash should be fatal. If there isn't a supermajority,
               but the median believe the slash should be fatal, we need to fallback to a large
               constant.
-
-              Also, TODO, each slash point should probably be considered as
-              `MAX_KEY_SHARES_PER_SET * BLOCK_TIME` seconds of downtime. As this time crosses
-              various thresholds (1 day, 3 days, etc), a multiplier should be attached.
             */
             let mut median_slash_report = Vec::with_capacity(self.validators.len());
             for i in 0 .. self.validators.len() {
@@ -351,7 +469,7 @@ impl<'a, TD: Db, TDT: DbTxn, P: P2p> ScanBlock<'a, TD, TDT, P> {
 
             // Create the resulting slash report
             let mut slash_report = vec![];
-            for (validator, points) in self.validators.iter().copied().zip(amortized_slash_report) {
+            for points in amortized_slash_report {
               // TODO: Natively store this as a `Slash`
               if points == u32::MAX {
                 slash_report.push(Slash::Fatal);
@@ -364,7 +482,7 @@ impl<'a, TD: Db, TDT: DbTxn, P: P2p> ScanBlock<'a, TD, TDT, P> {
             // Recognize the topic for signing the slash report
             TributaryDb::recognize_topic(
               self.tributary_txn,
-              self.set,
+              self.set.set,
               Topic::Sign {
                 id: VariantSignId::SlashReport,
                 attempt: 0,
@@ -374,24 +492,24 @@ impl<'a, TD: Db, TDT: DbTxn, P: P2p> ScanBlock<'a, TD, TDT, P> {
             // Send the message for the processor to start signing
             TributaryDb::send_message(
               self.tributary_txn,
-              self.set,
+              self.set.set,
               messages::coordinator::CoordinatorMessage::SignSlashReport {
-                session: self.set.session,
-                report: slash_report,
+                session: self.set.set.session,
+                slash_report: slash_report.try_into().unwrap(),
               },
             );
           }
         };
       }
 
-      Transaction::Sign { id, attempt, round, data, signed } => {
-        let topic = Topic::Sign { id, attempt, round };
+      Transaction::Sign { id: _, attempt: _, round, data, signed } => {
+        let topic = topic.unwrap();
         let signer = signer(signed);
 
-        if u64::try_from(data.len()).unwrap() != self.validator_weights[&signer] {
+        if data.len() != usize::from(self.validator_weights[&signer]) {
           TributaryDb::fatal_slash(
             self.tributary_txn,
-            self.set,
+            self.set.set,
             signer,
             "signer signed with a distinct amount of key shares than they had key shares",
           );
@@ -400,7 +518,7 @@ impl<'a, TD: Db, TDT: DbTxn, P: P2p> ScanBlock<'a, TD, TDT, P> {
 
         match TributaryDb::accumulate(
           self.tributary_txn,
-          self.set,
+          self.set.set,
           self.validators,
           self.total_weight,
           block_number,
@@ -411,12 +529,22 @@ impl<'a, TD: Db, TDT: DbTxn, P: P2p> ScanBlock<'a, TD, TDT, P> {
         ) {
           DataSet::None => {}
           DataSet::Participating(data_set) => {
-            let id = topic.sign_id(self.set).expect("Topic::Sign didn't have SignId");
-            let flatten_data_set = |data_set| todo!("TODO");
+            let id = topic.sign_id(self.set.set).expect("Topic::Sign didn't have SignId");
+            let flatten_data_set = |data_set: HashMap<_, Vec<_>>| {
+              let mut entries = HashMap::with_capacity(usize::from(self.total_weight));
+              for (validator, shares) in data_set {
+                let indexes = &self.set.participant_indexes[&validator];
+                assert_eq!(indexes.len(), shares.len());
+                for (index, share) in indexes.iter().zip(shares) {
+                  entries.insert(*index, share);
+                }
+              }
+              entries
+            };
             let data_set = flatten_data_set(data_set);
             TributaryDb::send_message(
               self.tributary_txn,
-              self.set,
+              self.set.set,
               match round {
                 SigningProtocolRound::Preprocess => {
                   messages::sign::CoordinatorMessage::Preprocesses { id, preprocesses: data_set }
@@ -427,13 +555,13 @@ impl<'a, TD: Db, TDT: DbTxn, P: P2p> ScanBlock<'a, TD, TDT, P> {
               },
             )
           }
-        };
+        }
       }
     }
   }
 
   fn handle_block(mut self, block_number: u64, block: Block<Transaction>) {
-    TributaryDb::start_of_block(self.tributary_txn, self.set, block_number);
+    TributaryDb::start_of_block(self.tributary_txn, self.set.set, block_number);
 
     for tx in block.transactions {
       match tx {
@@ -460,7 +588,7 @@ impl<'a, TD: Db, TDT: DbTxn, P: P2p> ScanBlock<'a, TD, TDT, P> {
           // errors, mark the node as fatally slashed
           TributaryDb::fatal_slash(
             self.tributary_txn,
-            self.set,
+            self.set.set,
             SeraiAddress(msgs.0.msg.sender),
             &format!("invalid tendermint messages: {msgs:?}"),
           );
@@ -476,10 +604,10 @@ impl<'a, TD: Db, TDT: DbTxn, P: P2p> ScanBlock<'a, TD, TDT, P> {
 /// The task to scan the Tributary, populating `ProcessorMessages`.
 pub struct ScanTributaryTask<TD: Db, P: P2p> {
   tributary_db: TD,
-  set: ValidatorSet,
+  set: NewSetInformation,
   validators: Vec<SeraiAddress>,
-  total_weight: u64,
-  validator_weights: HashMap<SeraiAddress, u64>,
+  total_weight: u16,
+  validator_weights: HashMap<SeraiAddress, u16>,
   tributary: TributaryReader<TD, Transaction>,
   _p2p: PhantomData<P>,
 }
@@ -488,15 +616,13 @@ impl<TD: Db, P: P2p> ScanTributaryTask<TD, P> {
   /// Create a new instance of this task.
   pub fn new(
     tributary_db: TD,
-    new_set: &NewSetInformation,
+    set: NewSetInformation,
     tributary: TributaryReader<TD, Transaction>,
   ) -> Self {
-    let mut validators = Vec::with_capacity(new_set.validators.len());
+    let mut validators = Vec::with_capacity(set.validators.len());
     let mut total_weight = 0;
-    let mut validator_weights = HashMap::with_capacity(new_set.validators.len());
-    for (validator, weight) in new_set.validators.iter().copied() {
-      let validator = SeraiAddress::from(validator);
-      let weight = u64::from(weight);
+    let mut validator_weights = HashMap::with_capacity(set.validators.len());
+    for (validator, weight) in set.validators.iter().copied() {
       validators.push(validator);
       total_weight += weight;
       validator_weights.insert(validator, weight);
@@ -504,7 +630,7 @@ impl<TD: Db, P: P2p> ScanTributaryTask<TD, P> {
 
     ScanTributaryTask {
       tributary_db,
-      set: new_set.set,
+      set,
       validators,
       total_weight,
       validator_weights,
@@ -520,7 +646,7 @@ impl<TD: Db, P: P2p> ContinuallyRan for ScanTributaryTask<TD, P> {
   fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, Self::Error>> {
     async move {
       let (mut last_block_number, mut last_block_hash) =
-        TributaryDb::last_handled_tributary_block(&self.tributary_db, self.set)
+        TributaryDb::last_handled_tributary_block(&self.tributary_db, self.set.set)
           .unwrap_or((0, self.tributary.genesis()));
 
       let mut made_progress = false;
@@ -539,7 +665,7 @@ impl<TD: Db, P: P2p> ContinuallyRan for ScanTributaryTask<TD, P> {
           if !self.tributary.locally_provided_txs_in_block(&block_hash, order) {
             return Err(format!(
               "didn't have the provided Transactions on-chain for set (ephemeral error): {:?}",
-              self.set
+              self.set.set
             ));
           }
         }
@@ -549,7 +675,7 @@ impl<TD: Db, P: P2p> ContinuallyRan for ScanTributaryTask<TD, P> {
           _td: PhantomData::<TD>,
           _p2p: PhantomData::<P>,
           tributary_txn: &mut tributary_txn,
-          set: self.set,
+          set: &self.set,
           validators: &self.validators,
           total_weight: self.total_weight,
           validator_weights: &self.validator_weights,
@@ -557,7 +683,7 @@ impl<TD: Db, P: P2p> ContinuallyRan for ScanTributaryTask<TD, P> {
         .handle_block(block_number, block);
         TributaryDb::set_last_handled_tributary_block(
           &mut tributary_txn,
-          self.set,
+          self.set.set,
           block_number,
           block_hash,
         );
@@ -577,7 +703,6 @@ impl<TD: Db, P: P2p> ContinuallyRan for ScanTributaryTask<TD, P> {
 pub fn slash_report_transaction(getter: &impl Get, set: &NewSetInformation) -> Transaction {
   let mut slash_points = Vec::with_capacity(set.validators.len());
   for (validator, _weight) in set.validators.iter().copied() {
-    let validator = SeraiAddress::from(validator);
     slash_points.push(SlashPoints::get(getter, set.set, validator).unwrap_or(0));
   }
   Transaction::SlashReport { slash_points, signed: Signed::default() }

coordinator/tributary/src/transaction.rs

@@ -25,6 +25,8 @@ use tributary_sdk::{
   },
 };
 
+use crate::db::Topic;
+
 /// The round this data is for, within a signing protocol.
 #[derive(Clone, Copy, PartialEq, Eq, Debug, Encode, BorshSerialize, BorshDeserialize)]
 pub enum SigningProtocolRound {
@@ -180,7 +182,7 @@ pub enum Transaction {
   ///
   /// This is provided after the block has been cosigned.
   ///
-  /// With the acknowledgement of a Substrate block, we can whitelist all the `VariantSignId`s
+  /// With the acknowledgement of a Substrate block, we can recognize all the `VariantSignId`s
   /// resulting from its handling.
   SubstrateBlock {
     /// The hash of the Substrate block
@@ -257,9 +259,7 @@ impl TransactionTrait for Transaction {
 
       Transaction::Cosign { .. } => TransactionKind::Provided("Cosign"),
       Transaction::Cosigned { .. } => TransactionKind::Provided("Cosigned"),
-      // TODO: Provide this
       Transaction::SubstrateBlock { .. } => TransactionKind::Provided("SubstrateBlock"),
-      // TODO: Provide this
       Transaction::Batch { .. } => TransactionKind::Provided("Batch"),
 
       Transaction::Sign { id, attempt, round, signed, .. } => TransactionKind::Signed(
@@ -318,6 +318,36 @@ impl TransactionTrait for Transaction {
 }
 
 impl Transaction {
+  /// The topic in the database for this transaction.
+  pub fn topic(&self) -> Option<Topic> {
+    #[allow(clippy::match_same_arms)] // This doesn't make semantic sense here
+    match self {
+      Transaction::RemoveParticipant { participant, .. } => {
+        Some(Topic::RemoveParticipant { participant: *participant })
+      }
+
+      Transaction::DkgParticipation { .. } => None,
+      Transaction::DkgConfirmationPreprocess { attempt, .. } => {
+        Some(Topic::DkgConfirmation { attempt: *attempt, round: SigningProtocolRound::Preprocess })
+      }
+      Transaction::DkgConfirmationShare { attempt, .. } => {
+        Some(Topic::DkgConfirmation { attempt: *attempt, round: SigningProtocolRound::Share })
+      }
+
+      // Provided TXs
+      Transaction::Cosign { .. } |
+      Transaction::Cosigned { .. } |
+      Transaction::SubstrateBlock { .. } |
+      Transaction::Batch { .. } => None,
+
+      Transaction::Sign { id, attempt, round, .. } => {
+        Some(Topic::Sign { id: *id, attempt: *attempt, round: *round })
+      }
+
+      Transaction::SlashReport { .. } => Some(Topic::SlashReport),
+    }
+  }
+
   /// Sign a transaction.
   ///
   /// Panics if signing a transaction whose type isn't `TransactionKind::Signed`.
@@ -335,10 +365,12 @@ impl Transaction {
         Transaction::DkgConfirmationPreprocess { ref mut signed, .. } => signed,
         Transaction::DkgConfirmationShare { ref mut signed, .. } => signed,
 
-        Transaction::Cosign { .. } => panic!("signing CosignSubstrateBlock"),
-        Transaction::Cosigned { .. } => panic!("signing Cosigned"),
-        Transaction::SubstrateBlock { .. } => panic!("signing SubstrateBlock"),
-        Transaction::Batch { .. } => panic!("signing Batch"),
+        Transaction::Cosign { .. } => panic!("signing Cosign transaction (provided)"),
+        Transaction::Cosigned { .. } => panic!("signing Cosigned transaction (provided)"),
+        Transaction::SubstrateBlock { .. } => {
+          panic!("signing SubstrateBlock transaction (provided)")
+        }
+        Transaction::Batch { .. } => panic!("signing Batch transaction (provided)"),
 
         Transaction::Sign { ref mut signed, .. } => signed,
 

crypto/ciphersuite/src/dalek.rs

@@ -28,6 +28,12 @@ macro_rules! dalek_curve {
         $Point::generator()
       }
 
+      fn reduce_512(mut scalar: [u8; 64]) -> Self::F {
+        let res = Scalar::from_bytes_mod_order_wide(&scalar);
+        scalar.zeroize();
+        res
+      }
+
       fn hash_to_F(dst: &[u8], data: &[u8]) -> Self::F {
         Scalar::from_hash(Sha512::new_with_prefix(&[dst, data].concat()))
       }

crypto/ciphersuite/src/ed448.rs

@@ -66,6 +66,12 @@ impl Ciphersuite for Ed448 {
     Point::generator()
   }
 
+  fn reduce_512(mut scalar: [u8; 64]) -> Self::F {
+    let res = Self::hash_to_F(b"Ciphersuite-reduce_512", &scalar);
+    scalar.zeroize();
+    res
+  }
+
   fn hash_to_F(dst: &[u8], data: &[u8]) -> Self::F {
     Scalar::wide_reduce(Self::H::digest([dst, data].concat()).as_ref().try_into().unwrap())
   }

crypto/ciphersuite/src/kp256.rs

@@ -6,7 +6,7 @@ use group::ff::PrimeField;
 
 use elliptic_curve::{
   generic_array::GenericArray,
-  bigint::{NonZero, CheckedAdd, Encoding, U384},
+  bigint::{NonZero, CheckedAdd, Encoding, U384, U512},
   hash2curve::{Expander, ExpandMsg, ExpandMsgXmd},
 };
 
@@ -31,6 +31,22 @@ macro_rules! kp_curve {
         $lib::ProjectivePoint::GENERATOR
       }
 
+      fn reduce_512(scalar: [u8; 64]) -> Self::F {
+        let mut modulus = [0; 64];
+        modulus[32 ..].copy_from_slice(&(Self::F::ZERO - Self::F::ONE).to_bytes());
+        let modulus = U512::from_be_slice(&modulus).checked_add(&U512::ONE).unwrap();
+
+        let mut wide =
+          U512::from_be_bytes(scalar).rem(&NonZero::new(modulus).unwrap()).to_be_bytes();
+
+        let mut array = *GenericArray::from_slice(&wide[32 ..]);
+        let res = $lib::Scalar::from_repr(array).unwrap();
+
+        wide.zeroize();
+        array.zeroize();
+        res
+      }
+
       fn hash_to_F(dst: &[u8], msg: &[u8]) -> Self::F {
         // While one of these two libraries does support directly hashing to the Scalar field, the
         // other doesn't. While that's probably an oversight, this is a universally working method

crypto/ciphersuite/src/lib.rs

@@ -62,6 +62,12 @@ pub trait Ciphersuite:
   // While group does provide this in its API, privacy coins may want to use a custom basepoint
   fn generator() -> Self::G;
 
+  /// Reduce 512 bits into a uniform scalar.
+  ///
+  /// If 512 bits is insufficient to perform a reduction into a uniform scalar, the ciphersuite
+  /// will perform a hash to sample the necessary bits.
+  fn reduce_512(scalar: [u8; 64]) -> Self::F;
+
   /// Hash the provided domain-separation tag and message to a scalar. Ciphersuites MAY naively
   /// prefix the tag to the message, enabling transpotion between the two. Accordingly, this
   /// function should NOT be used in any scheme where one tag is a valid substring of another
@@ -99,6 +105,9 @@ pub trait Ciphersuite:
   }
 
   /// Read a canonical point from something implementing std::io::Read.
+  ///
+  /// The provided implementation is safe so long as `GroupEncoding::to_bytes` always returns a
+  /// canonical serialization.
   #[cfg(any(feature = "alloc", feature = "std"))]
   #[allow(non_snake_case)]
   fn read_G<R: Read>(reader: &mut R) -> io::Result<Self::G> {

crypto/dalek-ff-group/src/field.rs

@@ -92,7 +92,7 @@ impl Neg for FieldElement {
   }
 }
 
-impl<'a> Neg for &'a FieldElement {
+impl Neg for &FieldElement {
   type Output = FieldElement;
   fn neg(self) -> Self::Output {
     (*self).neg()
@@ -244,7 +244,16 @@ impl FieldElement {
             res *= res;
           }
         }
-        res *= table[usize::from(bits)];
+
+        let mut scale_by = FieldElement::ONE;
+        #[allow(clippy::needless_range_loop)]
+        for i in 0 .. 16 {
+          #[allow(clippy::cast_possible_truncation)] // Safe since 0 .. 16
+          {
+            scale_by = <_>::conditional_select(&scale_by, &table[i], bits.ct_eq(&(i as u8)));
+          }
+        }
+        res *= scale_by;
         bits = 0;
       }
     }

crypto/dalek-ff-group/src/lib.rs

@@ -208,7 +208,16 @@ impl Scalar {
             res *= res;
           }
         }
-        res *= table[usize::from(bits)];
+
+        let mut scale_by = Scalar::ONE;
+        #[allow(clippy::needless_range_loop)]
+        for i in 0 .. 16 {
+          #[allow(clippy::cast_possible_truncation)] // Safe since 0 .. 16
+          {
+            scale_by = <_>::conditional_select(&scale_by, &table[i], bits.ct_eq(&(i as u8)));
+          }
+        }
+        res *= scale_by;
         bits = 0;
       }
     }

crypto/dkg/Cargo.toml

@@ -54,7 +54,7 @@ rand = { version = "0.8", default-features = false, features = ["std"] }
 ciphersuite = { path = "../ciphersuite", default-features = false, features = ["ristretto"] }
 generalized-bulletproofs = { path = "../evrf/generalized-bulletproofs", features = ["tests"] }
 ec-divisors = { path = "../evrf/divisors", features = ["pasta"] }
-pasta_curves = "0.5"
+pasta_curves = { git = "https://github.com/kayabaNerve/pasta_curves", rev = "a46b5be95cacbff54d06aad8d3bbcba42e05d616" }
 
 [features]
 std = [

crypto/dkg/src/evrf/mod.rs

@@ -85,7 +85,7 @@ use ciphersuite::{
 };
 use multiexp::multiexp_vartime;
 
-use generalized_bulletproofs::arithmetic_circuit_proof::*;
+use generalized_bulletproofs::{Generators, arithmetic_circuit_proof::*};
 use ec_divisors::DivisorCurve;
 
 use crate::{Participant, ThresholdParams, Interpolation, ThresholdCore, ThresholdKeys};
@@ -277,6 +277,7 @@ impl<C: EvrfCurve> EvrfDkg<C> {
     if evrf_public_keys.iter().any(|key| bool::from(key.is_identity())) {
       Err(EvrfError::PublicKeyWasIdentity)?;
     };
+    // This also checks the private key is not 0
     let evrf_public_key = <C::EmbeddedCurve as Ciphersuite>::generator() * evrf_private_key.deref();
     if !evrf_public_keys.iter().any(|key| *key == evrf_public_key) {
       Err(EvrfError::NotAParticipant)?;
@@ -359,7 +360,7 @@ impl<C: EvrfCurve> EvrfDkg<C> {
 
     let transcript = Self::initial_transcript(context, evrf_public_keys, t);
 
-    let mut evrf_verifier = generators.0.batch_verifier();
+    let mut evrf_verifier = Generators::batch_verifier();
     for (i, participation) in participations {
       let evrf_public_key = evrf_public_keys[usize::from(u16::from(*i)) - 1];
 
@@ -395,7 +396,7 @@ impl<C: EvrfCurve> EvrfDkg<C> {
         if faulty.contains(i) {
           continue;
         }
-        let mut evrf_verifier = generators.0.batch_verifier();
+        let mut evrf_verifier = Generators::batch_verifier();
         Evrf::<C>::verify(
           rng,
           &generators.0,

crypto/dkg/src/evrf/proof.rs

@@ -129,15 +129,11 @@ impl<C: EvrfCurve> Evrf<C> {
   /// Read a Variable from a theoretical vector commitment tape
   fn read_one_from_tape(generators_to_use: usize, start: &mut usize) -> Variable {
     // Each commitment has twice as many variables as generators in use
-    let commitment = *start / (2 * generators_to_use);
+    let commitment = *start / generators_to_use;
     // The index will be less than the amount of generators in use, as half are left and half are
     // right
     let index = *start % generators_to_use;
-    let res = if (*start / generators_to_use) % 2 == 0 {
-      Variable::CG { commitment, index }
-    } else {
-      Variable::CH { commitment, index }
-    };
+    let res = Variable::CG { commitment, index };
     *start += 1;
     res
   }
@@ -202,8 +198,8 @@ impl<C: EvrfCurve> Evrf<C> {
         padded_pow_of_2 <<= 1;
       }
       // This may as small as 16, which would create an excessive amount of vector commitments
-      // We set a floor of 1024 rows for bandwidth reasons
-      padded_pow_of_2.max(1024)
+      // We set a floor of 2048 rows for bandwidth reasons
+      padded_pow_of_2.max(2048)
     };
     (expected_muls, generators_to_use)
   }
@@ -213,7 +209,7 @@ impl<C: EvrfCurve> Evrf<C> {
     evrf_public_key: (C::F, C::F),
     coefficients: usize,
     ecdh_commitments: &[[(C::F, C::F); 2]],
-    generator_tables: &[GeneratorTable<C::F, C::EmbeddedCurveParameters>],
+    generator_tables: &[&GeneratorTable<C::F, C::EmbeddedCurveParameters>],
     circuit: &mut Circuit<C>,
     transcript: &mut impl Transcript,
   ) {
@@ -376,8 +372,10 @@ impl<C: EvrfCurve> Evrf<C> {
     let evrf_public_key;
     let mut actual_coefficients = Vec::with_capacity(coefficients);
     {
+      // This is checked at a higher level
       let dlog =
-        ScalarDecomposition::<<C::EmbeddedCurve as Ciphersuite>::F>::new(**evrf_private_key);
+        ScalarDecomposition::<<C::EmbeddedCurve as Ciphersuite>::F>::new(**evrf_private_key)
+          .expect("eVRF private key was zero");
       let points = Self::transcript_to_points(transcript, coefficients);
 
       // Start by pushing the discrete logarithm onto the tape
@@ -431,7 +429,8 @@ impl<C: EvrfCurve> Evrf<C> {
           }
         }
         let dlog =
-          ScalarDecomposition::<<C::EmbeddedCurve as Ciphersuite>::F>::new(ecdh_private_key);
+          ScalarDecomposition::<<C::EmbeddedCurve as Ciphersuite>::F>::new(ecdh_private_key)
+            .expect("ECDH private key was zero");
         let ecdh_commitment = <C::EmbeddedCurve as Ciphersuite>::generator() * ecdh_private_key;
         ecdh_commitments.push(ecdh_commitment);
         ecdh_commitments_xy.last_mut().unwrap()[j] =
@@ -471,15 +470,10 @@ impl<C: EvrfCurve> Evrf<C> {
       Self::muls_and_generators_to_use(coefficients, ecdh_public_keys.len());
 
     let mut vector_commitments =
-      Vec::with_capacity(vector_commitment_tape.len().div_ceil(2 * generators_to_use));
-    for chunk in vector_commitment_tape.chunks(2 * generators_to_use) {
+      Vec::with_capacity(vector_commitment_tape.len().div_ceil(generators_to_use));
+    for chunk in vector_commitment_tape.chunks(generators_to_use) {
       let g_values = chunk[.. generators_to_use.min(chunk.len())].to_vec().into();
-      let h_values = chunk[generators_to_use.min(chunk.len()) ..].to_vec().into();
-      vector_commitments.push(PedersenVectorCommitment {
-        g_values,
-        h_values,
-        mask: C::F::random(&mut *rng),
-      });
+      vector_commitments.push(PedersenVectorCommitment { g_values, mask: C::F::random(&mut *rng) });
     }
 
     vector_commitment_tape.zeroize();
@@ -499,7 +493,7 @@ impl<C: EvrfCurve> Evrf<C> {
         .iter()
         .map(|commitment| {
           commitment
-            .commit(generators.g_bold_slice(), generators.h_bold_slice(), generators.h())
+            .commit(generators.g_bold_slice(), generators.h())
             .ok_or(AcError::NotEnoughGenerators)
         })
         .collect::<Result<_, _>>()?,
@@ -518,7 +512,7 @@ impl<C: EvrfCurve> Evrf<C> {
       evrf_public_key,
       coefficients,
       &ecdh_commitments_xy,
-      &generator_tables,
+      &generator_tables.iter().collect::<Vec<_>>(),
       &mut circuit,
       &mut transcript,
     );
@@ -543,7 +537,7 @@ impl<C: EvrfCurve> Evrf<C> {
     let mut agg_weights = Vec::with_capacity(commitments.len());
     agg_weights.push(C::F::ONE);
     while agg_weights.len() < commitments.len() {
-      agg_weights.push(transcript.challenge::<C::F>());
+      agg_weights.push(transcript.challenge::<C>());
     }
     let mut x = commitments
       .iter()
@@ -554,7 +548,7 @@ impl<C: EvrfCurve> Evrf<C> {
     // Do a Schnorr PoK for the randomness of the aggregated Pedersen commitment
     let mut r = C::F::random(&mut *rng);
     transcript.push_point(generators.h() * r);
-    let c = transcript.challenge::<C::F>();
+    let c = transcript.challenge::<C>();
     transcript.push_scalar(r + (c * x));
     r.zeroize();
     x.zeroize();
@@ -615,7 +609,7 @@ impl<C: EvrfCurve> Evrf<C> {
     let coeffs_vc_variables = dlog_len + ((1 + (2 * coefficients)) * dlog_proof_len);
     let ecdhs_vc_variables = ((2 * ecdh_public_keys.len()) * dlog_len) +
       ((2 * 2 * ecdh_public_keys.len()) * dlog_proof_len);
-    let vcs = (coeffs_vc_variables + ecdhs_vc_variables).div_ceil(2 * generators_to_use);
+    let vcs = (coeffs_vc_variables + ecdhs_vc_variables).div_ceil(generators_to_use);
 
     let all_commitments =
       transcript.read_commitments(vcs, coefficients + ecdh_public_keys.len()).map_err(|_| ())?;
@@ -642,7 +636,7 @@ impl<C: EvrfCurve> Evrf<C> {
       <C::EmbeddedCurve as Ciphersuite>::G::to_xy(evrf_public_key).ok_or(())?,
       coefficients,
       &ecdh_keys_xy,
-      &generator_tables,
+      &generator_tables.iter().collect::<Vec<_>>(),
       &mut circuit,
       &mut transcript,
     );
@@ -665,7 +659,7 @@ impl<C: EvrfCurve> Evrf<C> {
     let mut agg_weights = Vec::with_capacity(commitments.len());
     agg_weights.push(C::F::ONE);
     while agg_weights.len() < commitments.len() {
-      agg_weights.push(transcript.challenge::<C::F>());
+      agg_weights.push(transcript.challenge::<C>());
     }
 
     let sum_points =
@@ -677,7 +671,7 @@ impl<C: EvrfCurve> Evrf<C> {
 
     #[allow(non_snake_case)]
     let R = transcript.read_point::<C>().map_err(|_| ())?;
-    let c = transcript.challenge::<C::F>();
+    let c = transcript.challenge::<C>();
     let s = transcript.read_scalar::<C>().map_err(|_| ())?;
 
     // Doesn't batch verify this as we can't access the internals of the GBP batch verifier

crypto/dkg/src/tests/evrf/proof.rs

@@ -15,7 +15,7 @@ use ciphersuite::{
 };
 use pasta_curves::{Ep, Eq, Fp, Fq};
 
-use generalized_bulletproofs::tests::generators;
+use generalized_bulletproofs::{Generators, tests::generators};
 use generalized_bulletproofs_ec_gadgets::DiscreteLogParameters;
 
 use crate::evrf::proof::*;
@@ -35,6 +35,9 @@ impl Ciphersuite for Pallas {
     // This is solely test code so it's fine
     Self::F::from_uniform_bytes(&Self::H::digest([dst, msg].concat()).into())
   }
+  fn reduce_512(scalar: [u8; 64]) -> Self::F {
+    Self::F::from_uniform_bytes(&scalar)
+  }
 }
 
 #[derive(Clone, Copy, PartialEq, Eq, Debug, Zeroize)]
@@ -52,6 +55,9 @@ impl Ciphersuite for Vesta {
     // This is solely test code so it's fine
     Self::F::from_uniform_bytes(&Self::H::digest([dst, msg].concat()).into())
   }
+  fn reduce_512(scalar: [u8; 64]) -> Self::F {
+    Self::F::from_uniform_bytes(&scalar)
+  }
 }
 
 pub struct VestaParams;
@@ -68,7 +74,7 @@ impl EvrfCurve for Pallas {
 }
 
 fn evrf_proof_test<C: EvrfCurve>() {
-  let generators = generators(1024);
+  let generators = generators(2048);
   let vesta_private_key = Zeroizing::new(<C::EmbeddedCurve as Ciphersuite>::F::random(&mut OsRng));
   let ecdh_public_keys = [
     <C::EmbeddedCurve as Ciphersuite>::G::random(&mut OsRng),
@@ -81,7 +87,7 @@ fn evrf_proof_test<C: EvrfCurve>() {
   println!("Proving time: {:?}", time.elapsed());
 
   let time = Instant::now();
-  let mut verifier = generators.batch_verifier();
+  let mut verifier = Generators::batch_verifier();
   Evrf::<C>::verify(
     &mut OsRng,
     &generators,

crypto/dkg/src/tests/promote.rs

@@ -28,6 +28,10 @@ impl<C: Ciphersuite> Ciphersuite for AltGenerator<C> {
     C::G::generator() * <C as Ciphersuite>::hash_to_F(b"DKG Promotion Test", b"generator")
   }
 
+  fn reduce_512(scalar: [u8; 64]) -> Self::F {
+    <C as Ciphersuite>::reduce_512(scalar)
+  }
+
   fn hash_to_F(dst: &[u8], data: &[u8]) -> Self::F {
     <C as Ciphersuite>::hash_to_F(dst, data)
   }

crypto/dleq/src/lib.rs

@@ -37,11 +37,11 @@ pub(crate) fn challenge<T: Transcript, F: PrimeField>(transcript: &mut T) -> F {
   // Get a wide amount of bytes to safely reduce without bias
   // In most cases, <=1.5x bytes is enough. 2x is still standard and there's some theoretical
   // groups which may technically require more than 1.5x bytes for this to work as intended
-  let target_bytes = ((usize::try_from(F::NUM_BITS).unwrap() + 7) / 8) * 2;
+  let target_bytes = usize::try_from(F::NUM_BITS).unwrap().div_ceil(8) * 2;
   let mut challenge_bytes = transcript.challenge(b"challenge");
   let challenge_bytes_len = challenge_bytes.as_ref().len();
   // If the challenge is 32 bytes, and we need 64, we need two challenges
-  let needed_challenges = (target_bytes + (challenge_bytes_len - 1)) / challenge_bytes_len;
+  let needed_challenges = target_bytes.div_ceil(challenge_bytes_len);
 
   // The following algorithm should be equivalent to a wide reduction of the challenges,
   // interpreted as concatenated, big-endian byte string

crypto/ed448/src/backend.rs

@@ -161,7 +161,16 @@ macro_rules! field {
                 res *= res;
               }
             }
-            res *= table[usize::from(bits)];
+
+            let mut scale_by = $FieldName(Residue::ONE);
+            #[allow(clippy::needless_range_loop)]
+            for i in 0 .. 16 {
+              #[allow(clippy::cast_possible_truncation)] // Safe since 0 .. 16
+              {
+                scale_by = <_>::conditional_select(&scale_by, &table[i], bits.ct_eq(&(i as u8)));
+              }
+            }
+            res *= scale_by;
             bits = 0;
           }
         }

crypto/ed448/src/point.rs

@@ -242,7 +242,16 @@ impl Mul<Scalar> for Point {
             res = res.double();
           }
         }
-        res += table[usize::from(bits)];
+
+        let mut add_by = Point::identity();
+        #[allow(clippy::needless_range_loop)]
+        for i in 0 .. 16 {
+          #[allow(clippy::cast_possible_truncation)] // Safe since 0 .. 16
+          {
+            add_by = <_>::conditional_select(&add_by, &table[i], bits.ct_eq(&(i as u8)));
+          }
+        }
+        res += add_by;
         bits = 0;
       }
     }

crypto/evrf/circuit-abstraction/Cargo.toml

@@ -3,19 +3,25 @@ name = "generalized-bulletproofs-circuit-abstraction"
 version = "0.1.0"
 description = "An abstraction for arithmetic circuits over Generalized Bulletproofs"
 license = "MIT"
-repository = "https://github.com/serai-dex/serai/tree/develop/crypto/evrf/circuit-abstraction"
+repository = "https://github.com/serai-dex/serai/tree/develop/crypto/fcmps/circuit-abstraction"
 authors = ["Luke Parker <lukeparker5132@gmail.com>"]
 keywords = ["bulletproofs", "circuit"]
 edition = "2021"
-rust-version = "1.80"
+rust-version = "1.69"
 
 [package.metadata.docs.rs]
 all-features = true
 rustdoc-args = ["--cfg", "docsrs"]
 
 [dependencies]
+std-shims = { path = "../../../common/std-shims", version = "^0.1.1", default-features = false }
+
 zeroize = { version = "^1.5", default-features = false, features = ["zeroize_derive"] }
 
-ciphersuite = { path = "../../ciphersuite", version = "0.4", default-features = false, features = ["std"] }
+ciphersuite = { path = "../../ciphersuite", version = "0.4", default-features = false }
 
-generalized-bulletproofs = { path = "../generalized-bulletproofs" }
+generalized-bulletproofs = { path = "../generalized-bulletproofs", default-features = false }
+
+[features]
+std = ["std-shims/std", "zeroize/std", "ciphersuite/std", "generalized-bulletproofs/std"]
+default = ["std"]

crypto/evrf/circuit-abstraction/src/lib.rs

@@ -1,14 +1,14 @@
 #![cfg_attr(docsrs, feature(doc_auto_cfg))]
 #![doc = include_str!("../README.md")]
+#![cfg_attr(not(feature = "std"), no_std)]
 #![deny(missing_docs)]
 #![allow(non_snake_case)]
 
+use std_shims::{vec, vec::Vec};
+
 use zeroize::{Zeroize, ZeroizeOnDrop};
 
-use ciphersuite::{
-  group::ff::{Field, PrimeField},
-  Ciphersuite,
-};
+use ciphersuite::{group::ff::Field, Ciphersuite};
 
 use generalized_bulletproofs::{
   ScalarVector, PedersenCommitment, PedersenVectorCommitment, ProofGenerators,
@@ -26,16 +26,28 @@ pub trait Transcript {
   ///
   /// It is the caller's responsibility to have properly transcripted all variables prior to
   /// sampling this challenge.
-  fn challenge<F: PrimeField>(&mut self) -> F;
+  fn challenge<C: Ciphersuite>(&mut self) -> C::F;
+
+  /// Sample a challenge as a byte array.
+  ///
+  /// It is the caller's responsibility to have properly transcripted all variables prior to
+  /// sampling this challenge.
+  fn challenge_bytes(&mut self) -> [u8; 64];
 }
 impl Transcript for ProverTranscript {
-  fn challenge<F: PrimeField>(&mut self) -> F {
-    self.challenge()
+  fn challenge<C: Ciphersuite>(&mut self) -> C::F {
+    self.challenge::<C>()
+  }
+  fn challenge_bytes(&mut self) -> [u8; 64] {
+    self.challenge_bytes()
   }
 }
 impl Transcript for VerifierTranscript<'_> {
-  fn challenge<F: PrimeField>(&mut self) -> F {
-    self.challenge()
+  fn challenge<C: Ciphersuite>(&mut self) -> C::F {
+    self.challenge::<C>()
+  }
+  fn challenge_bytes(&mut self) -> [u8; 64] {
+    self.challenge_bytes()
   }
 }
 
@@ -64,7 +76,6 @@ impl<C: Ciphersuite> Circuit<C> {
   }
 
   /// Create an instance to prove satisfaction of a circuit with.
-  // TODO: Take the transcript here
   #[allow(clippy::type_complexity)]
   pub fn prove(
     vector_commitments: Vec<PedersenVectorCommitment<C>>,
@@ -78,14 +89,13 @@ impl<C: Ciphersuite> Circuit<C> {
   }
 
   /// Create an instance to verify a proof with.
-  // TODO: Take the transcript here
   pub fn verify() -> Self {
     Self { muls: 0, constraints: vec![], prover: None }
   }
 
   /// Evaluate a linear combination.
   ///
-  /// Yields WL aL + WR aR + WO aO + WCG CG + WCH CH + WV V + c.
+  /// Yields WL aL + WR aR + WO aO + WCG CG + WV V + c.
   ///
   /// May panic if the linear combination references non-existent terms.
   ///
@@ -107,11 +117,6 @@ impl<C: Ciphersuite> Circuit<C> {
           res += C.g_values[*j] * weight;
         }
       }
-      for (WCH, C) in lincomb.WCH().iter().zip(&prover.C) {
-        for (j, weight) in WCH {
-          res += C.h_values[*j] * weight;
-        }
-      }
       for (index, weight) in lincomb.WV() {
         res += prover.V[*index].value * weight;
       }
@@ -176,13 +181,13 @@ impl<C: Ciphersuite> Circuit<C> {
         // We can't deconstruct the witness as it implements Drop (per ZeroizeOnDrop)
         // Accordingly, we take the values within it and move forward with those
         let mut aL = vec![];
-        std::mem::swap(&mut prover.aL, &mut aL);
+        core::mem::swap(&mut prover.aL, &mut aL);
         let mut aR = vec![];
-        std::mem::swap(&mut prover.aR, &mut aR);
+        core::mem::swap(&mut prover.aR, &mut aR);
         let mut C = vec![];
-        std::mem::swap(&mut prover.C, &mut C);
+        core::mem::swap(&mut prover.C, &mut C);
         let mut V = vec![];
-        std::mem::swap(&mut prover.V, &mut V);
+        core::mem::swap(&mut prover.V, &mut V);
         ArithmeticCircuitWitness::new(ScalarVector::from(aL), ScalarVector::from(aR), C, V)
       })
       .transpose()?;

crypto/evrf/divisors/Cargo.toml

@@ -3,35 +3,39 @@ name = "ec-divisors"
 version = "0.1.0"
 description = "A library for calculating elliptic curve divisors"
 license = "MIT"
-repository = "https://github.com/serai-dex/serai/tree/develop/crypto/evrf/divisors"
+repository = "https://github.com/serai-dex/serai/tree/develop/crypto/divisors"
 authors = ["Luke Parker <lukeparker5132@gmail.com>"]
 keywords = ["ciphersuite", "ff", "group"]
 edition = "2021"
-rust-version = "1.71"
+rust-version = "1.69"
 
 [package.metadata.docs.rs]
 all-features = true
 rustdoc-args = ["--cfg", "docsrs"]
 
 [dependencies]
-rand_core = { version = "0.6", default-features = false }
-zeroize = { version = "^1.5", default-features = false, features = ["std", "zeroize_derive"] }
+std-shims = { path = "../../../common/std-shims", version = "^0.1.1", default-features = false }
 
-subtle = { version = "2", default-features = false, features = ["std"] }
-ff = { version = "0.13", default-features = false, features = ["std", "bits"] }
+rand_core = { version = "0.6", default-features = false }
+zeroize = { version = "^1.5", default-features = false, features = ["zeroize_derive"] }
+
+subtle = { version = "2", default-features = false }
+ff = { version = "0.13", default-features = false, features = ["bits"] }
 group = { version = "0.13", default-features = false }
 
-hex = { version = "0.4", optional = true }
-dalek-ff-group = { path = "../../dalek-ff-group", features = ["std"], optional = true }
-pasta_curves = { version = "0.5", default-features = false, features = ["bits", "alloc"], optional = true }
+hex = { version = "0.4", default-features = false, optional = true }
+dalek-ff-group = { path = "../../dalek-ff-group", default-features = false, optional = true }
+pasta_curves = { version = "0.5", git = "https://github.com/kayabaNerve/pasta_curves.git", rev = "a46b5be95cacbff54d06aad8d3bbcba42e05d616", default-features = false, features = ["bits", "alloc"], optional = true }
 
 [dev-dependencies]
 rand_core = { version = "0.6", features = ["getrandom"] }
 
 hex = "0.4"
 dalek-ff-group = { path = "../../dalek-ff-group", features = ["std"] }
-pasta_curves = { version = "0.5", default-features = false, features = ["bits", "alloc"] }
+pasta_curves = { version = "0.5", git = "https://github.com/kayabaNerve/pasta_curves.git", rev = "a46b5be95cacbff54d06aad8d3bbcba42e05d616", default-features = false, features = ["bits", "alloc"] }
 
 [features]
-ed25519 = ["hex", "dalek-ff-group"]
+std = ["std-shims/std", "zeroize/std", "subtle/std", "ff/std", "dalek-ff-group?/std"]
+ed25519 = ["hex/alloc", "dalek-ff-group"]
 pasta = ["pasta_curves"]
+default = ["std"]

crypto/evrf/divisors/src/lib.rs

@@ -1,8 +1,11 @@
 #![cfg_attr(docsrs, feature(doc_auto_cfg))]
 #![doc = include_str!("../README.md")]
+#![cfg_attr(not(feature = "std"), no_std)]
 #![deny(missing_docs)]
 #![allow(non_snake_case)]
 
+use std_shims::{vec, vec::Vec};
+
 use subtle::{Choice, ConstantTimeEq, ConstantTimeGreater, ConditionallySelectable};
 use zeroize::{Zeroize, ZeroizeOnDrop};
 
@@ -18,7 +21,7 @@ pub use poly::Poly;
 mod tests;
 
 /// A curve usable with this library.
-pub trait DivisorCurve: Group + ConstantTimeEq + ConditionallySelectable {
+pub trait DivisorCurve: Group + ConstantTimeEq + ConditionallySelectable + Zeroize {
   /// An element of the field this curve is defined over.
   type FieldElement: Zeroize + PrimeField + ConditionallySelectable;
 
@@ -54,6 +57,8 @@ pub trait DivisorCurve: Group + ConstantTimeEq + ConditionallySelectable {
   /// Convert a point to its x and y coordinates.
   ///
   /// Returns None if passed the point at infinity.
+  ///
+  /// This function may run in time variable to if the point is the identity.
   fn to_xy(point: Self) -> Option<(Self::FieldElement, Self::FieldElement)>;
 }
 
@@ -271,8 +276,16 @@ pub struct ScalarDecomposition<F: Zeroize + PrimeFieldBits> {
 }
 
 impl<F: Zeroize + PrimeFieldBits> ScalarDecomposition<F> {
-  /// Decompose a scalar.
-  pub fn new(scalar: F) -> Self {
+  /// Decompose a non-zero scalar.
+  ///
+  /// Returns `None` if the scalar is zero.
+  ///
+  /// This function is constant time if the scalar is non-zero.
+  pub fn new(scalar: F) -> Option<Self> {
+    if bool::from(scalar.is_zero()) {
+      None?;
+    }
+
     /*
       We need the sum of the coefficients to equal F::NUM_BITS. The scalar's bits will be less than
       F::NUM_BITS. Accordingly, we need to increment the sum of the coefficients without
@@ -400,7 +413,12 @@ impl<F: Zeroize + PrimeFieldBits> ScalarDecomposition<F> {
     }
     debug_assert!(bool::from(decomposition.iter().sum::<u64>().ct_eq(&num_bits)));
 
-    ScalarDecomposition { scalar, decomposition }
+    Some(ScalarDecomposition { scalar, decomposition })
+  }
+
+  /// The scalar.
+  pub fn scalar(&self) -> &F {
+    &self.scalar
   }
 
   /// The decomposition of the scalar.
@@ -414,7 +432,7 @@ impl<F: Zeroize + PrimeFieldBits> ScalarDecomposition<F> {
   ///
   /// This function executes in constant time with regards to the scalar.
   ///
-  /// This function MAY panic if this scalar is zero.
+  /// This function MAY panic if the generator is the point at infinity.
   pub fn scalar_mul_divisor<C: Zeroize + DivisorCurve<Scalar = F>>(
     &self,
     mut generator: C,
@@ -430,37 +448,19 @@ impl<F: Zeroize + PrimeFieldBits> ScalarDecomposition<F> {
     divisor_points[0] = -generator * self.scalar;
 
     // Write the decomposition
-    let mut write_to: u32 = 1;
+    let mut write_above: u64 = 0;
     for coefficient in &self.decomposition {
-      let mut coefficient = *coefficient;
-      // Iterate over the maximum amount of iters for this value to be constant time regardless of
-      // any branch prediction algorithms
-      for _ in 0 .. <C::Scalar as PrimeField>::NUM_BITS {
-        // Write the generator to the slot we're supposed to
-        /*
-          Without this loop, we'd increment this dependent on the distribution within the
-          decomposition. If the distribution is bottom-heavy, we won't access the tail of
-          `divisor_points` for a while, risking it being ejected out of the cache (causing a cache
-          miss which may not occur with a top-heavy distribution which quickly moves to the tail).
-
-          This is O(log2(NUM_BITS) ** 3) though, as this the third loop, which is horrific.
-        */
-        for i in 1 ..= <C::Scalar as PrimeField>::NUM_BITS {
-          divisor_points[i as usize] =
-            <_>::conditional_select(&divisor_points[i as usize], &generator, i.ct_eq(&write_to));
-        }
-        // If the coefficient isn't zero, increment write_to (so we don't overwrite this generator
-        // when it should be there)
-        let coefficient_not_zero = !coefficient.ct_eq(&0);
-        write_to = <_>::conditional_select(&write_to, &(write_to + 1), coefficient_not_zero);
-        // Subtract one from the coefficient, if it's not zero and won't underflow
-        coefficient =
-          <_>::conditional_select(&coefficient, &coefficient.wrapping_sub(1), coefficient_not_zero);
+      // Write the generator to every slot except the slots we have already written to.
+      for i in 1 ..= (<C::Scalar as PrimeField>::NUM_BITS as u64) {
+        divisor_points[i as usize].conditional_assign(&generator, i.ct_gt(&write_above));
       }
+
+      // Increase the next write start by the coefficient.
+      write_above += coefficient;
       generator = generator.double();
     }
 
-    // Create a divisor out of all points except the last point which is solely scratch
+    // Create a divisor out of the points
     let res = new_divisor(&divisor_points).unwrap();
     divisor_points.zeroize();
     res
@@ -511,6 +511,7 @@ mod pasta {
 
 #[cfg(any(test, feature = "ed25519"))]
 mod ed25519 {
+  use subtle::{Choice, ConditionallySelectable};
   use group::{
     ff::{Field, PrimeField},
     Group, GroupEncoding,
@@ -558,9 +559,13 @@ mod ed25519 {
         ((D * edwards_y_sq) + Self::FieldElement::ONE).invert().unwrap())
       .sqrt()
       .unwrap();
-      if u8::from(bool::from(edwards_x.is_odd())) != x_is_odd {
-        edwards_x = -edwards_x;
-      }
+
+      // Negate the x coordinate if the sign doesn't match
+      edwards_x = <_>::conditional_select(
+        &edwards_x,
+        &-edwards_x,
+        edwards_x.is_odd() ^ Choice::from(x_is_odd),
+      );
 
       // Calculate the x and y coordinates for Wei25519
       let edwards_y_plus_one = Self::FieldElement::ONE + edwards_y;

crypto/evrf/divisors/src/poly.rs

@@ -1,4 +1,5 @@
 use core::ops::{Add, Neg, Sub, Mul, Rem};
+use std_shims::{vec, vec::Vec};
 
 use subtle::{Choice, ConstantTimeEq, ConstantTimeGreater, ConditionallySelectable};
 use zeroize::{Zeroize, ZeroizeOnDrop};
@@ -257,7 +258,7 @@ impl<F: From<u64> + Zeroize + PrimeField> Poly<F> {
     self.zero_coefficient = F::ZERO;
 
     // Move the x coefficients
-    std::mem::swap(&mut self.yx_coefficients[power_of_y - 1], &mut self.x_coefficients);
+    core::mem::swap(&mut self.yx_coefficients[power_of_y - 1], &mut self.x_coefficients);
     self.x_coefficients = vec![];
 
     self
@@ -564,7 +565,7 @@ impl<F: From<u64> + Zeroize + PrimeField> Poly<F> {
     quotient = conditional_select_poly(
       quotient,
       // If the dividing coefficient was for y**0 x**0, we return the poly scaled by its inverse
-      self.clone() * denominator_dividing_coefficient_inv,
+      self * denominator_dividing_coefficient_inv,
       denominator_dividing_coefficient.ct_eq(&CoefficientIndex { y_pow: 0, x_pow: 0 }),
     );
     remainder = conditional_select_poly(

crypto/evrf/ec-gadgets/Cargo.toml

@@ -3,19 +3,25 @@ name = "generalized-bulletproofs-ec-gadgets"
 version = "0.1.0"
 description = "Gadgets for working with an embedded Elliptic Curve in a Generalized Bulletproofs circuit"
 license = "MIT"
-repository = "https://github.com/serai-dex/serai/tree/develop/crypto/evrf/ec-gadgets"
+repository = "https://github.com/serai-dex/serai/tree/develop/crypto/fcmps/ec-gadgets"
 authors = ["Luke Parker <lukeparker5132@gmail.com>"]
 keywords = ["bulletproofs", "circuit", "divisors"]
 edition = "2021"
-rust-version = "1.80"
+rust-version = "1.69"
 
 [package.metadata.docs.rs]
 all-features = true
 rustdoc-args = ["--cfg", "docsrs"]
 
 [dependencies]
+std-shims = { path = "../../../common/std-shims", version = "^0.1.1", default-features = false }
+
 generic-array = { version = "1", default-features = false, features = ["alloc"] }
 
-ciphersuite = { path = "../../ciphersuite", version = "0.4", default-features = false, features = ["std"] }
+ciphersuite = { path = "../../ciphersuite", version = "0.4", default-features = false }
 
-generalized-bulletproofs-circuit-abstraction = { path = "../circuit-abstraction" }
+generalized-bulletproofs-circuit-abstraction = { path = "../circuit-abstraction", default-features = false }
+
+[features]
+std = ["std-shims/std", "ciphersuite/std", "generalized-bulletproofs-circuit-abstraction/std"]
+default = ["std"]

crypto/evrf/ec-gadgets/src/dlog.rs

@@ -1,4 +1,5 @@
 use core::fmt;
+use std_shims::{vec, vec::Vec};
 
 use ciphersuite::{
   group::ff::{Field, PrimeField, BatchInverter},
@@ -10,11 +11,6 @@ use generalized_bulletproofs_circuit_abstraction::*;
 use crate::*;
 
 /// Parameters for a discrete logarithm proof.
-///
-/// This isn't required to be implemented by the Field/Group/Ciphersuite, solely a struct, to
-/// enable parameterization of discrete log proofs to the bitlength of the discrete logarithm.
-/// While that may be F::NUM_BITS, a discrete log proof a for a full scalar, it could also be 64,
-/// a discrete log proof for a u64 (such as if opening a Pedersen commitment in-circuit).
 pub trait DiscreteLogParameters {
   /// The amount of bits used to represent a scalar.
   type ScalarBits: ArrayLength;
@@ -30,8 +26,8 @@ pub trait DiscreteLogParameters {
 
   /// The amount of y x**i coefficients in a divisor.
   ///
-  /// This is the amount of points in a divisor (the amount of bits in a scalar, plus one) plus
-  /// one, divided by two, minus two.
+  /// This is the amount of points in a divisor (the amount of bits in a scalar, plus one) divided
+  /// by two, minus two.
   type YxCoefficients: ArrayLength;
 }
 
@@ -106,8 +102,6 @@ pub struct Divisor<Parameters: DiscreteLogParameters> {
   /// exceeding trivial complexity.
   pub y: Variable,
   /// The coefficients for the `y**1 x**i` terms of the polynomial.
-  // This subtraction enforces the divisor to have at least 4 points which is acceptable.
-  // TODO: Double check these constants
   pub yx: GenericArray<Variable, Parameters::YxCoefficients>,
   /// The coefficients for the `x**i` terms of the polynomial, skipping x**1.
   ///
@@ -324,7 +318,7 @@ pub trait EcDlogGadgets<C: Ciphersuite> {
     &self,
     transcript: &mut T,
     curve: &CurveSpec<C::F>,
-    generators: &[GeneratorTable<C::F, Parameters>],
+    generators: &[&GeneratorTable<C::F, Parameters>],
   ) -> (DiscreteLogChallenge<C::F, Parameters>, Vec<ChallengedGenerator<C::F, Parameters>>);
 
   /// Prove this point has the specified discrete logarithm over the specified generator.
@@ -355,12 +349,14 @@ impl<C: Ciphersuite> EcDlogGadgets<C> for Circuit<C> {
     &self,
     transcript: &mut T,
     curve: &CurveSpec<C::F>,
-    generators: &[GeneratorTable<C::F, Parameters>],
+    generators: &[&GeneratorTable<C::F, Parameters>],
   ) -> (DiscreteLogChallenge<C::F, Parameters>, Vec<ChallengedGenerator<C::F, Parameters>>) {
     // Get the challenge points
-    // TODO: Implement a proper hash to curve
+    let sign_of_points = transcript.challenge_bytes();
+    let sign_of_point_0 = (sign_of_points[0] & 1) == 1;
+    let sign_of_point_1 = ((sign_of_points[0] >> 1) & 1) == 1;
     let (c0_x, c0_y) = loop {
-      let c0_x: C::F = transcript.challenge();
+      let c0_x = transcript.challenge::<C>();
       let Some(c0_y) =
         Option::<C::F>::from(((c0_x.square() * c0_x) + (curve.a * c0_x) + curve.b).sqrt())
       else {
@@ -368,17 +364,16 @@ impl<C: Ciphersuite> EcDlogGadgets<C> for Circuit<C> {
       };
       // Takes the even y coordinate as to not be dependent on whatever root the above sqrt
       // happens to returns
-      // TODO: Randomly select which to take
-      break (c0_x, if bool::from(c0_y.is_odd()) { -c0_y } else { c0_y });
+      break (c0_x, if bool::from(c0_y.is_odd()) != sign_of_point_0 { -c0_y } else { c0_y });
     };
     let (c1_x, c1_y) = loop {
-      let c1_x: C::F = transcript.challenge();
+      let c1_x = transcript.challenge::<C>();
       let Some(c1_y) =
         Option::<C::F>::from(((c1_x.square() * c1_x) + (curve.a * c1_x) + curve.b).sqrt())
       else {
         continue;
       };
-      break (c1_x, if bool::from(c1_y.is_odd()) { -c1_y } else { c1_y });
+      break (c1_x, if bool::from(c1_y.is_odd()) != sign_of_point_1 { -c1_y } else { c1_y });
     };
 
     // mmadd-1998-cmo
@@ -483,7 +478,7 @@ impl<C: Ciphersuite> EcDlogGadgets<C> for Circuit<C> {
     let arg_iter = arg_iter.chain(dlog.iter());
     for variable in arg_iter {
       debug_assert!(
-        matches!(variable, Variable::CG { .. } | Variable::CH { .. } | Variable::V(_)),
+        matches!(variable, Variable::CG { .. } | Variable::V(_)),
         "discrete log proofs requires all arguments belong to commitments",
       );
     }

crypto/evrf/ec-gadgets/src/lib.rs

@@ -1,5 +1,6 @@
 #![cfg_attr(docsrs, feature(doc_auto_cfg))]
 #![doc = include_str!("../README.md")]
+#![cfg_attr(not(feature = "std"), no_std)]
 #![deny(missing_docs)]
 #![allow(non_snake_case)]
 

crypto/evrf/embedwards25519/Cargo.toml

@@ -17,20 +17,22 @@ rustdoc-args = ["--cfg", "docsrs"]
 rustversion = "1"
 hex-literal = { version = "0.4", default-features = false }
 
-rand_core = { version = "0.6", default-features = false, features = ["std"] }
+std-shims = { version = "0.1", path = "../../../common/std-shims", default-features = false, optional = true }
 
-zeroize = { version = "^1.5", default-features = false, features = ["std", "zeroize_derive"] }
-subtle = { version = "^2.4", default-features = false, features = ["std"] }
+rand_core = { version = "0.6", default-features = false }
+
+zeroize = { version = "^1.5", default-features = false, features = ["zeroize_derive"] }
+subtle = { version = "^2.4", default-features = false }
 
 generic-array = { version = "1", default-features = false }
 crypto-bigint = { version = "0.5", default-features = false, features = ["zeroize"] }
 
 dalek-ff-group = { path = "../../dalek-ff-group", version = "0.4", default-features = false }
 
-blake2 = { version = "0.10", default-features = false, features = ["std"] }
-ciphersuite = { path = "../../ciphersuite", version = "0.4", default-features = false, features = ["std"] }
-ec-divisors = { path = "../divisors" }
-generalized-bulletproofs-ec-gadgets = { path = "../ec-gadgets" }
+blake2 = { version = "0.10", default-features = false }
+ciphersuite = { path = "../../ciphersuite", version = "0.4", default-features = false }
+ec-divisors = { path = "../divisors", default-features = false }
+generalized-bulletproofs-ec-gadgets = { path = "../ec-gadgets", default-features = false }
 
 [dev-dependencies]
 hex = "0.4"
@@ -38,3 +40,8 @@ hex = "0.4"
 rand_core = { version = "0.6", features = ["std"] }
 
 ff-group-tests = { path = "../../ff-group-tests" }
+
+[features]
+alloc = ["std-shims", "zeroize/alloc", "ciphersuite/alloc"]
+std = ["std-shims/std", "rand_core/std", "zeroize/std", "subtle/std", "blake2/std", "ciphersuite/std", "ec-divisors/std", "generalized-bulletproofs-ec-gadgets/std"]
+default = ["std"]

crypto/evrf/embedwards25519/README.md

@@ -7,7 +7,7 @@ This curve was found via
 for finding curves (specifically, curve cycles), modified to search for curves
 whose field is the Ed25519 scalar field (not the Ed25519 field).
 
-```
+```ignore
 p = 0x1000000000000000000000000000000014def9dea2f79cd65812631a5cf5d3ed
 q = 0x0fffffffffffffffffffffffffffffffe53f4debb78ff96877063f0306eef96b
 D = -420435

crypto/evrf/embedwards25519/src/lib.rs

@@ -1,5 +1,9 @@
 #![cfg_attr(docsrs, feature(doc_auto_cfg))]
 #![doc = include_str!("../README.md")]
+#![cfg_attr(not(feature = "std"), no_std)]
+
+#[cfg(any(feature = "alloc", feature = "std"))]
+use std_shims::io::{self, Read};
 
 use generic_array::typenum::{Sum, Diff, Quot, U, U1, U2};
 use ciphersuite::group::{ff::PrimeField, Group};
@@ -33,10 +37,29 @@ impl ciphersuite::Ciphersuite for Embedwards25519 {
     Point::generator()
   }
 
+  fn reduce_512(scalar: [u8; 64]) -> Self::F {
+    Scalar::wide_reduce(scalar)
+  }
+
   fn hash_to_F(dst: &[u8], data: &[u8]) -> Self::F {
     use blake2::Digest;
     Scalar::wide_reduce(Self::H::digest([dst, data].concat()).as_slice().try_into().unwrap())
   }
+
+  // We override the provided impl, which compares against the reserialization, because
+  // we already require canonicity
+  #[cfg(any(feature = "alloc", feature = "std"))]
+  #[allow(non_snake_case)]
+  fn read_G<R: Read>(reader: &mut R) -> io::Result<Self::G> {
+    use ciphersuite::group::GroupEncoding;
+
+    let mut encoding = <Self::G as GroupEncoding>::Repr::default();
+    reader.read_exact(encoding.as_mut())?;
+
+    let point = Option::<Self::G>::from(Self::G::from_bytes(&encoding))
+      .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "invalid point"))?;
+    Ok(point)
+  }
 }
 
 impl generalized_bulletproofs_ec_gadgets::DiscreteLogParameters for Embedwards25519 {

crypto/evrf/embedwards25519/src/point.rs

@@ -46,7 +46,8 @@ impl ConstantTimeEq for Point {
     let y1 = self.y * other.z;
     let y2 = other.y * self.z;
 
-    (self.x.is_zero() & other.x.is_zero()) | (x1.ct_eq(&x2) & y1.ct_eq(&y2))
+    // Both identity or equivalent over their denominators
+    (self.z.is_zero() & other.z.is_zero()) | (x1.ct_eq(&x2) & y1.ct_eq(&y2))
   }
 }
 
@@ -198,6 +199,7 @@ impl Group for Point {
     Point { x: FieldElement::ZERO, y: FieldElement::ONE, z: FieldElement::ZERO }
   }
   fn generator() -> Self {
+    // Point with the lowest valid x-coordinate
     Point {
       x: FieldElement::from_repr(hex_literal::hex!(
         "0100000000000000000000000000000000000000000000000000000000000000"
@@ -335,8 +337,10 @@ impl GroupEncoding for Point {
       // If this the identity, set y to 1
       let y =
         CtOption::conditional_select(&y, &CtOption::new(FieldElement::ONE, 1.into()), is_identity);
+      // If this the identity, set y to 1 and z to 0 (instead of 1)
+      let z = <_>::conditional_select(&FieldElement::ONE, &FieldElement::ZERO, is_identity);
       // Create the point if we have a y solution
-      let point = y.map(|y| Point { x, y, z: FieldElement::ONE });
+      let point = y.map(|y| Point { x, y, z });
 
       let not_negative_zero = !(is_identity & sign);
       // Only return the point if it isn't -0

crypto/evrf/generalized-bulletproofs/Cargo.toml

@@ -3,25 +3,27 @@ name = "generalized-bulletproofs"
 version = "0.1.0"
 description = "Generalized Bulletproofs"
 license = "MIT"
-repository = "https://github.com/serai-dex/serai/tree/develop/crypto/evrf/generalized-bulletproofs"
+repository = "https://github.com/serai-dex/serai/tree/develop/crypto/generalized-bulletproofs"
 authors = ["Luke Parker <lukeparker5132@gmail.com>"]
 keywords = ["ciphersuite", "ff", "group"]
 edition = "2021"
-rust-version = "1.80"
+rust-version = "1.69"
 
 [package.metadata.docs.rs]
 all-features = true
 rustdoc-args = ["--cfg", "docsrs"]
 
 [dependencies]
-rand_core = { version = "0.6", default-features = false, features = ["std"] }
+std-shims = { path = "../../../common/std-shims", version = "^0.1.1", default-features = false }
 
-zeroize = { version = "^1.5", default-features = false, features = ["std", "zeroize_derive"] }
+rand_core = { version = "0.6", default-features = false }
 
-blake2 = { version = "0.10", default-features = false, features = ["std"] }
+zeroize = { version = "^1.5", default-features = false, features = ["zeroize_derive"] }
 
-multiexp = { path = "../../multiexp", version = "0.4", default-features = false, features = ["std", "batch"] }
-ciphersuite = { path = "../../ciphersuite", version = "0.4", default-features = false, features = ["std"] }
+blake2 = { version = "0.10", default-features = false }
+
+multiexp = { path = "../../multiexp", version = "0.4", default-features = false, features = ["batch"] }
+ciphersuite = { path = "../../ciphersuite", version = "0.4", default-features = false }
 
 [dev-dependencies]
 rand_core = { version = "0.6", features = ["getrandom"] }
@@ -31,4 +33,6 @@ transcript = { package = "flexible-transcript", path = "../../transcript", featu
 ciphersuite = { path = "../../ciphersuite", features = ["ristretto"] }
 
 [features]
-tests = []
+std = ["std-shims/std", "rand_core/std", "zeroize/std", "blake2/std", "multiexp/std", "ciphersuite/std"]
+tests = ["std"]
+default = ["std"]

crypto/evrf/generalized-bulletproofs/src/arithmetic_circuit_proof.rs

@@ -1,3 +1,5 @@
+use std_shims::{vec, vec::Vec};
+
 use rand_core::{RngCore, CryptoRng};
 
 use zeroize::{Zeroize, ZeroizeOnDrop};
@@ -20,10 +22,10 @@ pub use crate::lincomb::{Variable, LinComb};
 ///  `aL * aR = aO, WL * aL + WR * aR + WO * aO = WV * V + c`.
 ///
 /// Generalized Bulletproofs modifies this to
-/// `aL * aR = aO, WL * aL + WR * aR + WO * aO + WCG * C_G + WCH * C_H = WV * V + c`.
+/// `aL * aR = aO, WL * aL + WR * aR + WO * aO + WCG * C_G = WV * V + c`.
 ///
 /// We implement the latter, yet represented (for simplicity) as
-/// `aL * aR = aO, WL * aL + WR * aR + WO * aO + WCG * C_G + WCH * C_H + WV * V + c = 0`.
+/// `aL * aR = aO, WL * aL + WR * aR + WO * aO + WCG * C_G + WV * V + c = 0`.
 #[derive(Clone, Debug)]
 pub struct ArithmeticCircuitStatement<'a, C: Ciphersuite> {
   generators: ProofGenerators<'a, C>,
@@ -33,7 +35,7 @@ pub struct ArithmeticCircuitStatement<'a, C: Ciphersuite> {
   V: PointVector<C>,
 }
 
-impl<'a, C: Ciphersuite> Zeroize for ArithmeticCircuitStatement<'a, C> {
+impl<C: Ciphersuite> Zeroize for ArithmeticCircuitStatement<'_, C> {
   fn zeroize(&mut self) {
     self.constraints.zeroize();
     self.C.zeroize();
@@ -202,16 +204,10 @@ impl<'a, C: Ciphersuite> ArithmeticCircuitStatement<'a, C> {
       if c.g_values.len() > n {
         Err(AcError::NotEnoughGenerators)?;
       }
-      if c.h_values.len() > n {
-        Err(AcError::NotEnoughGenerators)?;
-      }
       // The Pedersen vector commitments internally have n terms
       while c.g_values.len() < n {
         c.g_values.0.push(C::F::ZERO);
       }
-      while c.h_values.len() < n {
-        c.h_values.0.push(C::F::ZERO);
-      }
     }
 
     // Check the witness's consistency with the statement
@@ -227,12 +223,7 @@ impl<'a, C: Ciphersuite> ArithmeticCircuitStatement<'a, C> {
         }
       }
       for (commitment, opening) in self.C.0.iter().zip(witness.c.iter()) {
-        if Some(*commitment) !=
-          opening.commit(
-            self.generators.g_bold_slice(),
-            self.generators.h_bold_slice(),
-            self.generators.h(),
-          )
+        if Some(*commitment) != opening.commit(self.generators.g_bold_slice(), self.generators.h())
         {
           Err(AcError::InconsistentWitness)?;
         }
@@ -250,11 +241,6 @@ impl<'a, C: Ciphersuite> ArithmeticCircuitStatement<'a, C> {
                 weights.iter().map(|(j, weight)| *weight * c.g_values[*j])
               }),
             )
-            .chain(
-              constraint.WCH.iter().zip(&witness.c).flat_map(|(weights, c)| {
-                weights.iter().map(|(j, weight)| *weight * c.h_values[*j])
-              }),
-            )
             .chain(constraint.WV.iter().map(|(i, weight)| *weight * witness.v[*i].value))
             .chain(core::iter::once(constraint.c))
             .sum::<C::F>();
@@ -306,8 +292,8 @@ impl<'a, C: Ciphersuite> ArithmeticCircuitStatement<'a, C> {
     transcript.push_point(AI);
     transcript.push_point(AO);
     transcript.push_point(S);
-    let y = transcript.challenge();
-    let z = transcript.challenge();
+    let y = transcript.challenge::<C>();
+    let z = transcript.challenge::<C>();
     let YzChallenges { y_inv, z } = self.yz_challenges(y, z);
     let y = ScalarVector::powers(y, n);
 
@@ -318,7 +304,7 @@ impl<'a, C: Ciphersuite> ArithmeticCircuitStatement<'a, C> {
     // polynomial).
 
     // ni = n'
-    let ni = 2 * (c + 1);
+    let ni = 2 + (2 * (c / 2));
     // These indexes are from the Generalized Bulletproofs paper
     #[rustfmt::skip]
     let ilr = ni / 2; // 1 if c = 0
@@ -379,32 +365,25 @@ impl<'a, C: Ciphersuite> ArithmeticCircuitStatement<'a, C> {
     // r decreasing from n' (skipping jlr)
 
     let mut cg_weights = Vec::with_capacity(witness.c.len());
-    let mut ch_weights = Vec::with_capacity(witness.c.len());
     for i in 0 .. witness.c.len() {
       let mut cg = ScalarVector::new(n);
-      let mut ch = ScalarVector::new(n);
       for (constraint, z) in self.constraints.iter().zip(&z.0) {
         if let Some(WCG) = constraint.WCG.get(i) {
           accumulate_vector(&mut cg, WCG, *z);
         }
-        if let Some(WCH) = constraint.WCH.get(i) {
-          accumulate_vector(&mut ch, WCH, *z);
-        }
       }
       cg_weights.push(cg);
-      ch_weights.push(ch);
     }
 
-    for (i, (c, (cg_weights, ch_weights))) in
-      witness.c.iter().zip(cg_weights.into_iter().zip(ch_weights)).enumerate()
-    {
-      let i = i + 1;
+    for (mut i, (c, cg_weights)) in witness.c.iter().zip(cg_weights).enumerate() {
+      if i >= ilr {
+        i += 1;
+      }
+      // Because i has skipped ilr, j will skip jlr
       let j = ni - i;
 
       l[i] = c.g_values.clone();
-      l[j] = ch_weights * &y_inv;
       r[j] = cg_weights;
-      r[i] = (c.h_values.clone() * &y) + &r[i];
     }
 
     // Multiply them to obtain t
@@ -437,7 +416,7 @@ impl<'a, C: Ciphersuite> ArithmeticCircuitStatement<'a, C> {
       transcript.push_point(multiexp(&[(*t, self.generators.g()), (*tau, self.generators.h())]));
     }
 
-    let x: ScalarVector<C::F> = ScalarVector::powers(transcript.challenge(), t.len());
+    let x: ScalarVector<C::F> = ScalarVector::powers(transcript.challenge::<C>(), t.len());
 
     let poly_eval = |poly: &[ScalarVector<C::F>], x: &ScalarVector<_>| -> ScalarVector<_> {
       let mut res = ScalarVector::<C::F>::new(poly[0].0.len());
@@ -477,8 +456,11 @@ impl<'a, C: Ciphersuite> ArithmeticCircuitStatement<'a, C> {
       let mut u = (alpha * x[ilr]) + (beta * x[io]) + (rho * x[is]);
 
       // Incorporate the commitment masks multiplied by the associated power of x
-      for (i, commitment) in witness.c.iter().enumerate() {
-        let i = i + 1;
+      for (mut i, commitment) in witness.c.iter().enumerate() {
+        // If this index is ni / 2, skip it
+        if i >= (ni / 2) {
+          i += 1;
+        }
         u += x[i] * commitment.mask;
       }
       u
@@ -498,7 +480,7 @@ impl<'a, C: Ciphersuite> ArithmeticCircuitStatement<'a, C> {
     transcript.push_scalar(tau_x);
     transcript.push_scalar(u);
     transcript.push_scalar(t_caret);
-    let ip_x = transcript.challenge();
+    let ip_x = transcript.challenge::<C>();
     P_terms.push((ip_x * t_caret, self.generators.g()));
     IpStatement::new(
       self.generators,
@@ -513,16 +495,27 @@ impl<'a, C: Ciphersuite> ArithmeticCircuitStatement<'a, C> {
   }
 
   /// Verify a proof for this statement.
+  ///
+  /// This solely queues the statement for batch verification. The resulting BatchVerifier MUST
+  /// still be verified.
+  ///
+  /// If this proof returns an error, the BatchVerifier MUST be assumed corrupted and discarded.
   pub fn verify<R: RngCore + CryptoRng>(
     self,
     rng: &mut R,
     verifier: &mut BatchVerifier<C>,
     transcript: &mut VerifierTranscript,
   ) -> Result<(), AcError> {
+    if verifier.g_bold.len() < self.generators.len() {
+      verifier.g_bold.resize(self.generators.len(), C::F::ZERO);
+      verifier.h_bold.resize(self.generators.len(), C::F::ZERO);
+      verifier.h_sum.resize(self.generators.len(), C::F::ZERO);
+    }
+
     let n = self.n();
     let c = self.c();
 
-    let ni = 2 * (c + 1);
+    let ni = 2 + (2 * (c / 2));
 
     let ilr = ni / 2;
     let io = ni;
@@ -535,8 +528,8 @@ impl<'a, C: Ciphersuite> ArithmeticCircuitStatement<'a, C> {
     let AI = transcript.read_point::<C>().map_err(|_| AcError::IncompleteProof)?;
     let AO = transcript.read_point::<C>().map_err(|_| AcError::IncompleteProof)?;
     let S = transcript.read_point::<C>().map_err(|_| AcError::IncompleteProof)?;
-    let y = transcript.challenge();
-    let z = transcript.challenge();
+    let y = transcript.challenge::<C>();
+    let z = transcript.challenge::<C>();
     let YzChallenges { y_inv, z } = self.yz_challenges(y, z);
 
     let mut l_weights = ScalarVector::new(n);
@@ -559,7 +552,7 @@ impl<'a, C: Ciphersuite> ArithmeticCircuitStatement<'a, C> {
     for _ in 0 .. (t_poly_len - ni - 1) {
       T_after_ni.push(transcript.read_point::<C>().map_err(|_| AcError::IncompleteProof)?);
     }
-    let x: ScalarVector<C::F> = ScalarVector::powers(transcript.challenge(), t_poly_len);
+    let x: ScalarVector<C::F> = ScalarVector::powers(transcript.challenge::<C>(), t_poly_len);
 
     let tau_x = transcript.read_scalar::<C>().map_err(|_| AcError::IncompleteProof)?;
     let u = transcript.read_scalar::<C>().map_err(|_| AcError::IncompleteProof)?;
@@ -624,34 +617,25 @@ impl<'a, C: Ciphersuite> ArithmeticCircuitStatement<'a, C> {
       h_bold_scalars = h_bold_scalars + &(o_weights * verifier_weight);
 
       let mut cg_weights = Vec::with_capacity(self.C.len());
-      let mut ch_weights = Vec::with_capacity(self.C.len());
       for i in 0 .. self.C.len() {
         let mut cg = ScalarVector::new(n);
-        let mut ch = ScalarVector::new(n);
         for (constraint, z) in self.constraints.iter().zip(&z.0) {
           if let Some(WCG) = constraint.WCG.get(i) {
             accumulate_vector(&mut cg, WCG, *z);
           }
-          if let Some(WCH) = constraint.WCH.get(i) {
-            accumulate_vector(&mut ch, WCH, *z);
-          }
         }
         cg_weights.push(cg);
-        ch_weights.push(ch);
       }
 
       // Push the terms for C, which increment from 0, and the terms for WC, which decrement from
       // n'
-      for (i, (C, (WCG, WCH))) in
-        self.C.0.into_iter().zip(cg_weights.into_iter().zip(ch_weights)).enumerate()
-      {
-        let i = i + 1;
+      for (mut i, (C, WCG)) in self.C.0.into_iter().zip(cg_weights).enumerate() {
+        if i >= (ni / 2) {
+          i += 1;
+        }
         let j = ni - i;
         verifier.additional.push((x[i], C));
         h_bold_scalars = h_bold_scalars + &(WCG * x[j]);
-        for (i, scalar) in (WCH * &y_inv * x[j]).0.into_iter().enumerate() {
-          verifier.g_bold[i] += scalar;
-        }
       }
 
       // All terms for h_bold here have actually been for h_bold', h_bold * y_inv
@@ -666,7 +650,7 @@ impl<'a, C: Ciphersuite> ArithmeticCircuitStatement<'a, C> {
 
     // Prove for lines 88, 92 with an Inner-Product statement
     // This inlines Protocol 1, as our IpStatement implements Protocol 2
-    let ip_x = transcript.challenge();
+    let ip_x = transcript.challenge::<C>();
     // P is amended with this additional term
     verifier.g += verifier_weight * ip_x * t_caret;
     IpStatement::new(self.generators, y_inv, ip_x, P::Verifier { verifier_weight })

crypto/evrf/generalized-bulletproofs/src/inner_product.rs

@@ -1,3 +1,5 @@
+use std_shims::{vec, vec::Vec};
+
 use multiexp::multiexp_vartime;
 use ciphersuite::{group::ff::Field, Ciphersuite};
 
@@ -186,7 +188,7 @@ impl<'a, C: Ciphersuite> IpStatement<'a, C> {
       // Now that we've calculate L, R, transcript them to receive x (26-27)
       transcript.push_point(L);
       transcript.push_point(R);
-      let x: C::F = transcript.challenge();
+      let x: C::F = transcript.challenge::<C>();
       let x_inv = x.invert().unwrap();
 
       // The prover and verifier now calculate the following (28-31)
@@ -269,11 +271,19 @@ impl<'a, C: Ciphersuite> IpStatement<'a, C> {
   /// This will return Err if there is an error. This will return Ok if the proof was successfully
   /// queued for batch verification. The caller is required to verify the batch in order to ensure
   /// the proof is actually correct.
+  ///
+  /// If this proof returns an error, the BatchVerifier MUST be assumed corrupted and discarded.
   pub(crate) fn verify(
     self,
     verifier: &mut BatchVerifier<C>,
     transcript: &mut VerifierTranscript,
   ) -> Result<(), IpError> {
+    if verifier.g_bold.len() < self.generators.len() {
+      verifier.g_bold.resize(self.generators.len(), C::F::ZERO);
+      verifier.h_bold.resize(self.generators.len(), C::F::ZERO);
+      verifier.h_sum.resize(self.generators.len(), C::F::ZERO);
+    }
+
     let IpStatement { generators, h_bold_weights, u, P } = self;
 
     // Calculate the discrete log w.r.t. 2 for the amount of generators present
@@ -296,7 +306,7 @@ impl<'a, C: Ciphersuite> IpStatement<'a, C> {
     for _ in 0 .. lr_len {
       L.push(transcript.read_point::<C>().map_err(|_| IpError::IncompleteProof)?);
       R.push(transcript.read_point::<C>().map_err(|_| IpError::IncompleteProof)?);
-      xs.push(transcript.challenge());
+      xs.push(transcript.challenge::<C>());
     }
 
     // We calculate their inverse in batch

crypto/evrf/generalized-bulletproofs/src/lib.rs

@@ -1,10 +1,11 @@
 #![cfg_attr(docsrs, feature(doc_auto_cfg))]
 #![doc = include_str!("../README.md")]
+#![cfg_attr(not(feature = "std"), no_std)]
 #![deny(missing_docs)]
 #![allow(non_snake_case)]
 
 use core::fmt;
-use std::collections::HashSet;
+use std_shims::{vec, vec::Vec, collections::HashSet};
 
 use zeroize::Zeroize;
 
@@ -70,14 +71,26 @@ pub struct Generators<C: Ciphersuite> {
 #[must_use]
 #[derive(Clone)]
 pub struct BatchVerifier<C: Ciphersuite> {
-  g: C::F,
-  h: C::F,
+  /// The summed scalar for the G generator.
+  pub g: C::F,
+  /// The summed scalar for the G generator.
+  pub h: C::F,
 
-  g_bold: Vec<C::F>,
-  h_bold: Vec<C::F>,
-  h_sum: Vec<C::F>,
+  /// The summed scalars for the G_bold generators.
+  pub g_bold: Vec<C::F>,
+  /// The summed scalars for the H_bold generators.
+  pub h_bold: Vec<C::F>,
+  /// The summed scalars for the sums of all H generators prior to the index.
+  ///
+  /// This is not populated with the full set of summed H generators. This is only populated with
+  /// the powers of 2. Accordingly, an index i specifies a scalar for the sum of all H generators
+  /// from H**2**0 ..= H**2**i.
+  pub h_sum: Vec<C::F>,
 
-  additional: Vec<(C::F, C::G)>,
+  /// Additional (non-fixed) points to include in the multiexp.
+  ///
+  /// This is used for proof-specific elements.
+  pub additional: Vec<(C::F, C::G)>,
 }
 
 impl<C: Ciphersuite> fmt::Debug for Generators<C> {
@@ -171,15 +184,15 @@ impl<C: Ciphersuite> Generators<C> {
     Ok(Generators { g, h, g_bold, h_bold, h_sum })
   }
 
-  /// Create a BatchVerifier for proofs which use these generators.
-  pub fn batch_verifier(&self) -> BatchVerifier<C> {
+  /// Create a BatchVerifier for proofs which use a consistent set of generators.
+  pub fn batch_verifier() -> BatchVerifier<C> {
     BatchVerifier {
       g: C::F::ZERO,
       h: C::F::ZERO,
 
-      g_bold: vec![C::F::ZERO; self.g_bold.len()],
-      h_bold: vec![C::F::ZERO; self.h_bold.len()],
-      h_sum: vec![C::F::ZERO; self.h_sum.len()],
+      g_bold: vec![],
+      h_bold: vec![],
+      h_sum: vec![],
 
       additional: Vec::with_capacity(128),
     }
@@ -247,7 +260,7 @@ impl<C: Ciphersuite> Generators<C> {
   }
 }
 
-impl<'a, C: Ciphersuite> ProofGenerators<'a, C> {
+impl<C: Ciphersuite> ProofGenerators<'_, C> {
   pub(crate) fn len(&self) -> usize {
     self.g_bold.len()
   }
@@ -298,8 +311,6 @@ impl<C: Ciphersuite> PedersenCommitment<C> {
 pub struct PedersenVectorCommitment<C: Ciphersuite> {
   /// The values committed to across the `g` (bold) generators.
   pub g_values: ScalarVector<C::F>,
-  /// The values committed to across the `h` (bold) generators.
-  pub h_values: ScalarVector<C::F>,
   /// The mask blinding the values committed to.
   pub mask: C::F,
 }
@@ -309,8 +320,8 @@ impl<C: Ciphersuite> PedersenVectorCommitment<C> {
   ///
   /// This function returns None if the amount of generators is less than the amount of values
   /// within the relevant vector.
-  pub fn commit(&self, g_bold: &[C::G], h_bold: &[C::G], h: C::G) -> Option<C::G> {
-    if (g_bold.len() < self.g_values.len()) || (h_bold.len() < self.h_values.len()) {
+  pub fn commit(&self, g_bold: &[C::G], h: C::G) -> Option<C::G> {
+    if g_bold.len() < self.g_values.len() {
       None?;
     };
 
@@ -318,9 +329,6 @@ impl<C: Ciphersuite> PedersenVectorCommitment<C> {
     for pair in self.g_values.0.iter().cloned().zip(g_bold.iter().cloned()) {
       terms.push(pair);
     }
-    for pair in self.h_values.0.iter().cloned().zip(h_bold.iter().cloned()) {
-      terms.push(pair);
-    }
     let res = multiexp(&terms);
     terms.zeroize();
     Some(res)

crypto/evrf/generalized-bulletproofs/src/lincomb.rs

@@ -1,4 +1,5 @@
 use core::ops::{Add, Sub, Mul};
+use std_shims::{vec, vec::Vec};
 
 use zeroize::Zeroize;
 
@@ -23,13 +24,6 @@ pub enum Variable {
     /// The index of the variable.
     index: usize,
   },
-  /// A variable within a Pedersen vector commitment, committed to with a generator from `h` (bold).
-  CH {
-    /// The commitment being indexed.
-    commitment: usize,
-    /// The index of the variable.
-    index: usize,
-  },
   /// A variable within a Pedersen commitment.
   V(usize),
 }
@@ -41,7 +35,7 @@ impl Zeroize for Variable {
 
 /// A linear combination.
 ///
-/// Specifically, `WL aL + WR aR + WO aO + WCG C_G + WCH C_H + WV V + c`.
+/// Specifically, `WL aL + WR aR + WO aO + WCG C_G + WV V + c`.
 #[derive(Clone, PartialEq, Eq, Debug, Zeroize)]
 #[must_use]
 pub struct LinComb<F: PrimeField> {
@@ -55,7 +49,6 @@ pub struct LinComb<F: PrimeField> {
   pub(crate) WO: Vec<(usize, F)>,
   // Sparse representation once within a commitment
   pub(crate) WCG: Vec<Vec<(usize, F)>>,
-  pub(crate) WCH: Vec<Vec<(usize, F)>>,
   // Sparse representation of WV
   pub(crate) WV: Vec<(usize, F)>,
   pub(crate) c: F,
@@ -81,15 +74,9 @@ impl<F: PrimeField> Add<&LinComb<F>> for LinComb<F> {
     while self.WCG.len() < constraint.WCG.len() {
       self.WCG.push(vec![]);
     }
-    while self.WCH.len() < constraint.WCH.len() {
-      self.WCH.push(vec![]);
-    }
     for (sWC, cWC) in self.WCG.iter_mut().zip(&constraint.WCG) {
       sWC.extend(cWC);
     }
-    for (sWC, cWC) in self.WCH.iter_mut().zip(&constraint.WCH) {
-      sWC.extend(cWC);
-    }
     self.WV.extend(&constraint.WV);
     self.c += constraint.c;
     self
@@ -110,15 +97,9 @@ impl<F: PrimeField> Sub<&LinComb<F>> for LinComb<F> {
     while self.WCG.len() < constraint.WCG.len() {
       self.WCG.push(vec![]);
     }
-    while self.WCH.len() < constraint.WCH.len() {
-      self.WCH.push(vec![]);
-    }
     for (sWC, cWC) in self.WCG.iter_mut().zip(&constraint.WCG) {
       sWC.extend(cWC.iter().map(|(i, weight)| (*i, -*weight)));
     }
-    for (sWC, cWC) in self.WCH.iter_mut().zip(&constraint.WCH) {
-      sWC.extend(cWC.iter().map(|(i, weight)| (*i, -*weight)));
-    }
     self.WV.extend(constraint.WV.iter().map(|(i, weight)| (*i, -*weight)));
     self.c -= constraint.c;
     self
@@ -143,11 +124,6 @@ impl<F: PrimeField> Mul<F> for LinComb<F> {
         *weight *= scalar;
       }
     }
-    for WC in self.WCH.iter_mut() {
-      for (_, weight) in WC {
-        *weight *= scalar;
-      }
-    }
     for (_, weight) in self.WV.iter_mut() {
       *weight *= scalar;
     }
@@ -167,7 +143,6 @@ impl<F: PrimeField> LinComb<F> {
       WR: vec![],
       WO: vec![],
       WCG: vec![],
-      WCH: vec![],
       WV: vec![],
       c: F::ZERO,
     }
@@ -196,14 +171,6 @@ impl<F: PrimeField> LinComb<F> {
         }
         self.WCG[i].push((j, scalar))
       }
-      Variable::CH { commitment: i, index: j } => {
-        self.highest_c_index = self.highest_c_index.max(Some(i));
-        self.highest_a_index = self.highest_a_index.max(Some(j));
-        while self.WCH.len() <= i {
-          self.WCH.push(vec![]);
-        }
-        self.WCH[i].push((j, scalar))
-      }
       Variable::V(i) => {
         self.highest_v_index = self.highest_v_index.max(Some(i));
         self.WV.push((i, scalar));
@@ -238,11 +205,6 @@ impl<F: PrimeField> LinComb<F> {
     &self.WCG
   }
 
-  /// View the current weights for CH.
-  pub fn WCH(&self) -> &[Vec<(usize, F)>] {
-    &self.WCH
-  }
-
   /// View the current weights for V.
   pub fn WV(&self) -> &[(usize, F)] {
     &self.WV

crypto/evrf/generalized-bulletproofs/src/point_vector.rs

@@ -1,4 +1,5 @@
 use core::ops::{Index, IndexMut};
+use std_shims::vec::Vec;
 
 use zeroize::Zeroize;
 

crypto/evrf/generalized-bulletproofs/src/scalar_vector.rs

@@ -1,4 +1,5 @@
 use core::ops::{Index, IndexMut, Add, Sub, Mul};
+use std_shims::{vec, vec::Vec};
 
 use zeroize::Zeroize;
 

crypto/evrf/generalized-bulletproofs/src/tests/arithmetic_circuit_proof.rs

@@ -3,7 +3,7 @@ use rand_core::{RngCore, OsRng};
 use ciphersuite::{group::ff::Field, Ciphersuite, Ristretto};
 
 use crate::{
-  ScalarVector, PedersenCommitment, PedersenVectorCommitment,
+  ScalarVector, PedersenCommitment, PedersenVectorCommitment, Generators,
   transcript::*,
   arithmetic_circuit_proof::{
     Variable, LinComb, ArithmeticCircuitStatement, ArithmeticCircuitWitness,
@@ -43,7 +43,7 @@ fn test_zero_arithmetic_circuit() {
     statement.clone().prove(&mut OsRng, &mut transcript, witness).unwrap();
     transcript.complete()
   };
-  let mut verifier = generators.batch_verifier();
+  let mut verifier = Generators::batch_verifier();
 
   let mut transcript = VerifierTranscript::new([0; 32], &proof);
   let verifier_commmitments = transcript.read_commitments(0, 1);
@@ -59,14 +59,8 @@ fn test_vector_commitment_arithmetic_circuit() {
 
   let v1 = <Ristretto as Ciphersuite>::F::random(&mut OsRng);
   let v2 = <Ristretto as Ciphersuite>::F::random(&mut OsRng);
-  let v3 = <Ristretto as Ciphersuite>::F::random(&mut OsRng);
-  let v4 = <Ristretto as Ciphersuite>::F::random(&mut OsRng);
   let gamma = <Ristretto as Ciphersuite>::F::random(&mut OsRng);
-  let commitment = (reduced.g_bold(0) * v1) +
-    (reduced.g_bold(1) * v2) +
-    (reduced.h_bold(0) * v3) +
-    (reduced.h_bold(1) * v4) +
-    (generators.h() * gamma);
+  let commitment = (reduced.g_bold(0) * v1) + (reduced.g_bold(1) * v2) + (generators.h() * gamma);
   let V = vec![];
   let C = vec![commitment];
 
@@ -83,20 +77,14 @@ fn test_vector_commitment_arithmetic_circuit() {
     vec![LinComb::empty()
       .term(<Ristretto as Ciphersuite>::F::ONE, Variable::CG { commitment: 0, index: 0 })
       .term(<Ristretto as Ciphersuite>::F::from(2u64), Variable::CG { commitment: 0, index: 1 })
-      .term(<Ristretto as Ciphersuite>::F::from(3u64), Variable::CH { commitment: 0, index: 0 })
-      .term(<Ristretto as Ciphersuite>::F::from(4u64), Variable::CH { commitment: 0, index: 1 })
-      .constant(-(v1 + (v2 + v2) + (v3 + v3 + v3) + (v4 + v4 + v4 + v4)))],
+      .constant(-(v1 + (v2 + v2)))],
     commitments.clone(),
   )
   .unwrap();
   let witness = ArithmeticCircuitWitness::<Ristretto>::new(
     aL,
     aR,
-    vec![PedersenVectorCommitment {
-      g_values: ScalarVector(vec![v1, v2]),
-      h_values: ScalarVector(vec![v3, v4]),
-      mask: gamma,
-    }],
+    vec![PedersenVectorCommitment { g_values: ScalarVector(vec![v1, v2]), mask: gamma }],
     vec![],
   )
   .unwrap();
@@ -105,7 +93,7 @@ fn test_vector_commitment_arithmetic_circuit() {
     statement.clone().prove(&mut OsRng, &mut transcript, witness).unwrap();
     transcript.complete()
   };
-  let mut verifier = generators.batch_verifier();
+  let mut verifier = Generators::batch_verifier();
 
   let mut transcript = VerifierTranscript::new([0; 32], &proof);
   let verifier_commmitments = transcript.read_commitments(1, 0);
@@ -139,13 +127,8 @@ fn fuzz_test_arithmetic_circuit() {
       while g_values.0.len() < ((OsRng.next_u64() % 8) + 1).try_into().unwrap() {
         g_values.0.push(<Ristretto as Ciphersuite>::F::random(&mut OsRng));
       }
-      let mut h_values = ScalarVector(vec![]);
-      while h_values.0.len() < ((OsRng.next_u64() % 8) + 1).try_into().unwrap() {
-        h_values.0.push(<Ristretto as Ciphersuite>::F::random(&mut OsRng));
-      }
       C.push(PedersenVectorCommitment {
         g_values,
-        h_values,
         mask: <Ristretto as Ciphersuite>::F::random(&mut OsRng),
       });
     }
@@ -193,13 +176,6 @@ fn fuzz_test_arithmetic_circuit() {
           constraint = constraint.term(weight, Variable::CG { commitment, index });
           eval += weight * C.g_values[index];
         }
-
-        for _ in 0 .. (OsRng.next_u64() % 4) {
-          let index = usize::try_from(OsRng.next_u64()).unwrap() % C.h_values.len();
-          let weight = <Ristretto as Ciphersuite>::F::random(&mut OsRng);
-          constraint = constraint.term(weight, Variable::CH { commitment, index });
-          eval += weight * C.h_values[index];
-        }
       }
 
       if !V.is_empty() {
@@ -218,11 +194,7 @@ fn fuzz_test_arithmetic_circuit() {
 
     let mut transcript = Transcript::new([0; 32]);
     let commitments = transcript.write_commitments(
-      C.iter()
-        .map(|C| {
-          C.commit(generators.g_bold_slice(), generators.h_bold_slice(), generators.h()).unwrap()
-        })
-        .collect(),
+      C.iter().map(|C| C.commit(generators.g_bold_slice(), generators.h()).unwrap()).collect(),
       V.iter().map(|V| V.commit(generators.g(), generators.h())).collect(),
     );
 
@@ -239,7 +211,7 @@ fn fuzz_test_arithmetic_circuit() {
       statement.clone().prove(&mut OsRng, &mut transcript, witness).unwrap();
       transcript.complete()
     };
-    let mut verifier = generators.batch_verifier();
+    let mut verifier = Generators::batch_verifier();
 
     let mut transcript = VerifierTranscript::new([0; 32], &proof);
     let verifier_commmitments = transcript.read_commitments(C.len(), V.len());

crypto/evrf/generalized-bulletproofs/src/tests/inner_product.rs

@@ -8,7 +8,7 @@ use ciphersuite::{
 };
 
 use crate::{
-  ScalarVector, PointVector,
+  ScalarVector, PointVector, Generators,
   transcript::*,
   inner_product::{P, IpStatement, IpWitness},
   tests::generators,
@@ -41,7 +41,7 @@ fn test_zero_inner_product() {
     transcript.complete()
   };
 
-  let mut verifier = generators.batch_verifier();
+  let mut verifier = Generators::batch_verifier();
   IpStatement::<Ristretto>::new(
     reduced,
     ScalarVector(vec![<Ristretto as Ciphersuite>::F::ONE; 1]),
@@ -58,7 +58,7 @@ fn test_zero_inner_product() {
 fn test_inner_product() {
   // P = sum(g_bold * a, h_bold * b)
   let generators = generators::<Ristretto>(32);
-  let mut verifier = generators.batch_verifier();
+  let mut verifier = Generators::batch_verifier();
   for i in [1, 2, 4, 8, 16, 32] {
     let generators = generators.reduce(i).unwrap();
     let g = generators.g();

crypto/evrf/generalized-bulletproofs/src/transcript.rs

@@ -1,9 +1,12 @@
-use std::io;
+use std_shims::{vec::Vec, io};
 
 use blake2::{Digest, Blake2b512};
 
 use ciphersuite::{
-  group::{ff::PrimeField, GroupEncoding},
+  group::{
+    ff::{Field, PrimeField},
+    GroupEncoding,
+  },
   Ciphersuite,
 };
 
@@ -13,27 +16,11 @@ const SCALAR: u8 = 0;
 const POINT: u8 = 1;
 const CHALLENGE: u8 = 2;
 
-fn challenge<F: PrimeField>(digest: &mut Blake2b512) -> F {
-  // Panic if this is such a wide field, we won't successfully perform a reduction into an unbiased
-  // scalar
-  debug_assert!((F::NUM_BITS + 128) < 512);
-
+fn challenge<C: Ciphersuite>(digest: &mut Blake2b512) -> C::F {
   digest.update([CHALLENGE]);
-  let chl = digest.clone().finalize();
+  let chl = digest.clone().finalize().into();
 
-  let mut res = F::ZERO;
-  for (i, mut byte) in chl.iter().cloned().enumerate() {
-    for j in 0 .. 8 {
-      let lsb = byte & 1;
-      let mut bit = F::from(u64::from(lsb));
-      for _ in 0 .. ((i * 8) + j) {
-        bit = bit.double();
-      }
-      res += bit;
-
-      byte >>= 1;
-    }
-  }
+  let res = C::reduce_512(chl);
 
   // Negligible probability
   if bool::from(res.is_zero()) {
@@ -83,6 +70,8 @@ impl Transcript {
   }
 
   /// Push a scalar onto the transcript.
+  ///
+  /// The order and layout of this must be constant to the context.
   pub fn push_scalar(&mut self, scalar: impl PrimeField) {
     self.digest.update([SCALAR]);
     let bytes = scalar.to_repr();
@@ -91,6 +80,8 @@ impl Transcript {
   }
 
   /// Push a point onto the transcript.
+  ///
+  /// The order and layout of this must be constant to the context.
   pub fn push_point(&mut self, point: impl GroupEncoding) {
     self.digest.update([POINT]);
     let bytes = point.to_bytes();
@@ -104,9 +95,11 @@ impl Transcript {
     C: Vec<C::G>,
     V: Vec<C::G>,
   ) -> Commitments<C> {
+    self.digest.update(u32::try_from(C.len()).unwrap().to_le_bytes());
     for C in &C {
       self.push_point(*C);
     }
+    self.digest.update(u32::try_from(V.len()).unwrap().to_le_bytes());
     for V in &V {
       self.push_point(*V);
     }
@@ -114,8 +107,14 @@ impl Transcript {
   }
 
   /// Sample a challenge.
-  pub fn challenge<F: PrimeField>(&mut self) -> F {
-    challenge(&mut self.digest)
+  pub fn challenge<C: Ciphersuite>(&mut self) -> C::F {
+    challenge::<C>(&mut self.digest)
+  }
+
+  /// Sample a challenge as a byte array.
+  pub fn challenge_bytes(&mut self) -> [u8; 64] {
+    self.digest.update([CHALLENGE]);
+    self.digest.clone().finalize().into()
   }
 
   /// Complete a transcript, yielding the fully serialized proof.
@@ -139,20 +138,36 @@ impl<'a> VerifierTranscript<'a> {
   }
 
   /// Read a scalar from the transcript.
+  ///
+  /// The order and layout of this must be constant to the context.
   pub fn read_scalar<C: Ciphersuite>(&mut self) -> io::Result<C::F> {
-    let scalar = C::read_F(&mut self.transcript)?;
+    // Read the scalar onto the transcript using the serialization present in the transcript
     self.digest.update([SCALAR]);
-    let bytes = scalar.to_repr();
-    self.digest.update(bytes);
+    let scalar_len = <C::F as PrimeField>::Repr::default().as_ref().len();
+    if self.transcript.len() < scalar_len {
+      Err(io::Error::new(io::ErrorKind::Other, "not enough bytes to read_scalar"))?;
+    }
+    self.digest.update(&self.transcript[.. scalar_len]);
+
+    // Read the actual scalar, where `read_F` ensures its canonically serialized
+    let scalar = C::read_F(&mut self.transcript)?;
     Ok(scalar)
   }
 
   /// Read a point from the transcript.
+  ///
+  /// The order and layout of this must be constant to the context.
   pub fn read_point<C: Ciphersuite>(&mut self) -> io::Result<C::G> {
-    let point = C::read_G(&mut self.transcript)?;
+    // Read the point onto the transcript using the serialization present in the transcript
     self.digest.update([POINT]);
-    let bytes = point.to_bytes();
-    self.digest.update(bytes);
+    let point_len = <C::G as GroupEncoding>::Repr::default().as_ref().len();
+    if self.transcript.len() < point_len {
+      Err(io::Error::new(io::ErrorKind::Other, "not enough bytes to read_point"))?;
+    }
+    self.digest.update(&self.transcript[.. point_len]);
+
+    // Read the actual point, where `read_G` ensures its canonically serialized
+    let point = C::read_G(&mut self.transcript)?;
     Ok(point)
   }
 
@@ -165,10 +180,12 @@ impl<'a> VerifierTranscript<'a> {
     C: usize,
     V: usize,
   ) -> io::Result<Commitments<C>> {
+    self.digest.update(u32::try_from(C).unwrap().to_le_bytes());
     let mut C_vec = Vec::with_capacity(C);
     for _ in 0 .. C {
       C_vec.push(self.read_point::<C>()?);
     }
+    self.digest.update(u32::try_from(V).unwrap().to_le_bytes());
     let mut V_vec = Vec::with_capacity(V);
     for _ in 0 .. V {
       V_vec.push(self.read_point::<C>()?);
@@ -177,11 +194,17 @@ impl<'a> VerifierTranscript<'a> {
   }
 
   /// Sample a challenge.
-  pub fn challenge<F: PrimeField>(&mut self) -> F {
-    challenge(&mut self.digest)
+  pub fn challenge<C: Ciphersuite>(&mut self) -> C::F {
+    challenge::<C>(&mut self.digest)
   }
 
-  /// Complete the transcript, returning the advanced slice.
+  /// Sample a challenge as a byte array.
+  pub fn challenge_bytes(&mut self) -> [u8; 64] {
+    self.digest.update([CHALLENGE]);
+    self.digest.clone().finalize().into()
+  }
+
+  /// Complete the transcript transcript, yielding what remains.
   pub fn complete(self) -> &'a [u8] {
     self.transcript
   }

crypto/evrf/secq256k1/Cargo.toml

@@ -17,20 +17,22 @@ rustdoc-args = ["--cfg", "docsrs"]
 rustversion = "1"
 hex-literal = { version = "0.4", default-features = false }
 
-rand_core = { version = "0.6", default-features = false, features = ["std"] }
+std-shims = { version = "0.1", path = "../../../common/std-shims", default-features = false, optional = true }
 
-zeroize = { version = "^1.5", default-features = false, features = ["std", "zeroize_derive"] }
-subtle = { version = "^2.4", default-features = false, features = ["std"] }
+rand_core = { version = "0.6", default-features = false }
+
+zeroize = { version = "^1.5", default-features = false, features = ["zeroize_derive"] }
+subtle = { version = "^2.4", default-features = false }
 
 generic-array = { version = "0.14", default-features = false }
 crypto-bigint = { version = "0.5", default-features = false, features = ["zeroize"] }
 
 k256 = { version = "0.13", default-features = false, features = ["arithmetic"] }
 
-blake2 = { version = "0.10", default-features = false, features = ["std"] }
-ciphersuite = { path = "../../ciphersuite", version = "0.4", default-features = false, features = ["std"] }
-ec-divisors = { path = "../divisors" }
-generalized-bulletproofs-ec-gadgets = { path = "../ec-gadgets" }
+blake2 = { version = "0.10", default-features = false }
+ciphersuite = { path = "../../ciphersuite", version = "0.4", default-features = false }
+ec-divisors = { path = "../divisors", default-features = false }
+generalized-bulletproofs-ec-gadgets = { path = "../ec-gadgets", default-features = false }
 
 [dev-dependencies]
 hex = "0.4"
@@ -38,3 +40,8 @@ hex = "0.4"
 rand_core = { version = "0.6", features = ["std"] }
 
 ff-group-tests = { path = "../../ff-group-tests" }
+
+[features]
+alloc = ["std-shims", "zeroize/alloc", "ciphersuite/alloc"]
+std = ["std-shims/std", "rand_core/std", "zeroize/std", "subtle/std", "blake2/std", "ciphersuite/std", "ec-divisors/std", "generalized-bulletproofs-ec-gadgets/std"]
+default = ["std"]

crypto/evrf/secq256k1/src/lib.rs

@@ -1,5 +1,9 @@
 #![cfg_attr(docsrs, feature(doc_auto_cfg))]
 #![doc = include_str!("../README.md")]
+#![cfg_attr(not(feature = "std"), no_std)]
+
+#[cfg(any(feature = "alloc", feature = "std"))]
+use std_shims::io::{self, Read};
 
 use generic_array::typenum::{Sum, Diff, Quot, U, U1, U2};
 use ciphersuite::group::{ff::PrimeField, Group};
@@ -33,10 +37,29 @@ impl ciphersuite::Ciphersuite for Secq256k1 {
     Point::generator()
   }
 
+  fn reduce_512(scalar: [u8; 64]) -> Self::F {
+    Scalar::wide_reduce(scalar)
+  }
+
   fn hash_to_F(dst: &[u8], data: &[u8]) -> Self::F {
     use blake2::Digest;
     Scalar::wide_reduce(Self::H::digest([dst, data].concat()).as_slice().try_into().unwrap())
   }
+
+  // We override the provided impl, which compares against the reserialization, because
+  // we already require canonicity
+  #[cfg(any(feature = "alloc", feature = "std"))]
+  #[allow(non_snake_case)]
+  fn read_G<R: Read>(reader: &mut R) -> io::Result<Self::G> {
+    use ciphersuite::group::GroupEncoding;
+
+    let mut encoding = <Self::G as GroupEncoding>::Repr::default();
+    reader.read_exact(encoding.as_mut())?;
+
+    let point = Option::<Self::G>::from(Self::G::from_bytes(&encoding))
+      .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "invalid point"))?;
+    Ok(point)
+  }
 }
 
 impl generalized_bulletproofs_ec_gadgets::DiscreteLogParameters for Secq256k1 {

crypto/evrf/secq256k1/src/point.rs

@@ -40,7 +40,8 @@ impl ConstantTimeEq for Point {
     let y1 = self.y * other.z;
     let y2 = other.y * self.z;
 
-    (self.x.is_zero() & other.x.is_zero()) | (x1.ct_eq(&x2) & y1.ct_eq(&y2))
+    // Identity or equivalent
+    (self.z.is_zero() & other.z.is_zero()) | (x1.ct_eq(&x2) & y1.ct_eq(&y2))
   }
 }
 
@@ -192,6 +193,7 @@ impl Group for Point {
     Point { x: FieldElement::ZERO, y: FieldElement::ONE, z: FieldElement::ZERO }
   }
   fn generator() -> Self {
+    // Point with the lowest valid x-coordinate
     Point {
       x: FieldElement::from_repr(
         hex_literal::hex!("0000000000000000000000000000000000000000000000000000000000000001")
@@ -334,8 +336,10 @@ impl GroupEncoding for Point {
       // If this the identity, set y to 1
       let y =
         CtOption::conditional_select(&y, &CtOption::new(FieldElement::ONE, 1.into()), is_identity);
+      // If this the identity, set y to 1 and z to 0 (instead of 1)
+      let z = <_>::conditional_select(&FieldElement::ONE, &FieldElement::ZERO, is_identity);
       // Create the point if we have a y solution
-      let point = y.map(|y| Point { x, y, z: FieldElement::ONE });
+      let point = y.map(|y| Point { x, y, z });
 
       let not_negative_zero = !(is_identity & sign);
       // Only return the point if it isn't -0 and the sign byte wasn't malleated

crypto/ff-group-tests/Cargo.toml

@@ -30,4 +30,4 @@ p256 = { version = "^0.13.1", default-features = false, features = ["std", "arit
 
 bls12_381 = "0.8"
 
-pasta_curves = "0.5"
+pasta_curves = { git = "https://github.com/kayabaNerve/pasta_curves", rev = "a46b5be95cacbff54d06aad8d3bbcba42e05d616" }

crypto/ff-group-tests/src/group.rs

@@ -154,18 +154,20 @@ pub fn test_group<R: RngCore, G: Group>(rng: &mut R) {
 
 /// Test encoding and decoding of group elements.
 pub fn test_encoding<G: PrimeGroup>() {
-  let test = |point: G, msg| {
+  let test = |point: G, msg| -> G {
     let bytes = point.to_bytes();
     let mut repr = G::Repr::default();
     repr.as_mut().copy_from_slice(bytes.as_ref());
-    assert_eq!(point, G::from_bytes(&repr).unwrap(), "{msg} couldn't be encoded and decoded");
+    let decoded = G::from_bytes(&repr).unwrap();
+    assert_eq!(point, decoded, "{msg} couldn't be encoded and decoded");
     assert_eq!(
       point,
       G::from_bytes_unchecked(&repr).unwrap(),
       "{msg} couldn't be encoded and decoded",
     );
+    decoded
   };
-  test(G::identity(), "identity");
+  assert!(bool::from(test(G::identity(), "identity").is_identity()));
   test(G::generator(), "generator");
   test(G::generator() + G::generator(), "(generator * 2)");
 }

crypto/frost/src/sign.rs

@@ -203,14 +203,15 @@ pub trait SignMachine<S>: Send + Sync + Sized {
   /// SignatureMachine this SignMachine turns into.
   type SignatureMachine: SignatureMachine<S, SignatureShare = Self::SignatureShare>;
 
-  /// Cache this preprocess for usage later. This cached preprocess MUST only be used once. Reuse
-  /// of it enables recovery of your private key share. Third-party recovery of a cached preprocess
-  /// also enables recovery of your private key share, so this MUST be treated with the same
-  /// security as your private key share.
+  /// Cache this preprocess for usage later.
+  ///
+  /// This cached preprocess MUST only be used once. Reuse of it enables recovery of your private
+  /// key share. Third-party recovery of a cached preprocess also enables recovery of your private
+  /// key share, so this MUST be treated with the same security as your private key share.
   fn cache(self) -> CachedPreprocess;
 
   /// Create a sign machine from a cached preprocess.
-
+  ///
   /// After this, the preprocess must be deleted so it's never reused. Any reuse will presumably
   /// cause the signer to leak their secret share.
   fn from_cache(
@@ -219,11 +220,14 @@ pub trait SignMachine<S>: Send + Sync + Sized {
     cache: CachedPreprocess,
   ) -> (Self, Self::Preprocess);
 
-  /// Read a Preprocess message. Despite taking self, this does not save the preprocess.
-  /// It must be externally cached and passed into sign.
+  /// Read a Preprocess message.
+  ///
+  /// Despite taking self, this does not save the preprocess. It must be externally cached and
+  /// passed into sign.
   fn read_preprocess<R: Read>(&self, reader: &mut R) -> io::Result<Self::Preprocess>;
 
   /// Sign a message.
+  ///
   /// Takes in the participants' preprocess messages. Returns the signature share to be broadcast
   /// to all participants, over an authenticated channel. The parties who participate here will
   /// become the signing set for this session.

crypto/multiexp/src/lib.rs

@@ -59,7 +59,7 @@ pub(crate) fn prep_bits<G: Group<Scalar: PrimeFieldBits>>(
   for pair in pairs {
     let p = groupings.len();
     let mut bits = pair.0.to_le_bits();
-    groupings.push(vec![0; (bits.len() + (w_usize - 1)) / w_usize]);
+    groupings.push(vec![0; bits.len().div_ceil(w_usize)]);
 
     for (i, mut bit) in bits.iter_mut().enumerate() {
       let mut bit = u8_from_bool(&mut bit);

crypto/schnorr/src/aggregate.rs

@@ -31,9 +31,8 @@ fn weight<D: Send + Clone + SecureDigest, F: PrimeField>(digest: &mut DigestTran
   // Derive a scalar from enough bits of entropy that bias is < 2^128
   // This can't be const due to its usage of a generic
   // Also due to the usize::try_from, yet that could be replaced with an `as`
-  // The + 7 forces it to round up
   #[allow(non_snake_case)]
-  let BYTES: usize = usize::try_from(((F::NUM_BITS + 128) + 7) / 8).unwrap();
+  let BYTES: usize = usize::try_from((F::NUM_BITS + 128).div_ceil(8)).unwrap();
 
   let mut remaining = BYTES;
 

deny.toml

@@ -121,7 +121,10 @@ license-files = [
 multiple-versions = "warn"
 wildcards = "warn"
 highlight = "all"
-deny = [ { name = "serde_derive", version = ">=1.0.172, <1.0.185" } ]
+deny = [
+  { name = "serde_derive", version = ">=1.0.172, <1.0.185" },
+  { name = "hashbrown", version = ">=0.15" },
+]
 
 [sources]
 unknown-registry = "deny"
@@ -130,7 +133,6 @@ allow-registry = ["https://github.com/rust-lang/crates.io-index"]
 allow-git = [
   "https://github.com/rust-lang-nursery/lazy-static.rs",
   "https://github.com/serai-dex/substrate-bip39",
-  "https://github.com/serai-dex/substrate",
+  "https://github.com/serai-dex/polkadot-sdk",
   "https://github.com/kayabaNerve/pasta_curves",
-  "https://github.com/alloy-rs/core",
 ]

docs/Gemfile.lock

@@ -5,20 +5,20 @@ GEM
       public_suffix (>= 2.0.2, < 7.0)
     bigdecimal (3.1.8)
     colorator (1.1.0)
-    concurrent-ruby (1.3.3)
+    concurrent-ruby (1.3.4)
     em-websocket (0.5.3)
       eventmachine (>= 0.12.9)
       http_parser.rb (~> 0)
     eventmachine (1.2.7)
     ffi (1.17.0-x86_64-linux-gnu)
     forwardable-extended (2.6.0)
-    google-protobuf (4.27.3-x86_64-linux)
+    google-protobuf (4.28.2-x86_64-linux)
       bigdecimal
       rake (>= 13)
     http_parser.rb (0.8.0)
-    i18n (1.14.5)
+    i18n (1.14.6)
       concurrent-ruby (~> 1.0)
-    jekyll (4.3.3)
+    jekyll (4.3.4)
       addressable (~> 2.4)
       colorator (~> 1.0)
       em-websocket (~> 0.5)
@@ -63,17 +63,15 @@ GEM
     rb-fsevent (0.11.2)
     rb-inotify (0.11.1)
       ffi (~> 1.0)
-    rexml (3.3.4)
-      strscan
-    rouge (4.3.0)
+    rexml (3.3.7)
+    rouge (4.4.0)
     safe_yaml (1.0.5)
-    sass-embedded (1.77.8-x86_64-linux-gnu)
-      google-protobuf (~> 4.26)
-    strscan (3.1.0)
+    sass-embedded (1.79.3-x86_64-linux-gnu)
+      google-protobuf (~> 4.27)
     terminal-table (3.0.2)
       unicode-display_width (>= 1.1.1, < 3)
-    unicode-display_width (2.5.0)
-    webrick (1.8.1)
+    unicode-display_width (2.6.0)
+    webrick (1.8.2)
 
 PLATFORMS
   x86_64-linux

message-queue/src/main.rs

@@ -6,7 +6,7 @@ pub(crate) use std::{
 pub(crate) use ciphersuite::{group::GroupEncoding, Ciphersuite, Ristretto};
 pub(crate) use schnorr_signatures::SchnorrSignature;
 
-pub(crate) use serai_primitives::NetworkId;
+pub(crate) use serai_primitives::ExternalNetworkId;
 
 pub(crate) use tokio::{
   io::{AsyncReadExt, AsyncWriteExt},
@@ -197,10 +197,7 @@ async fn main() {
     KEYS.write().unwrap().insert(service, key);
     let mut queues = QUEUES.write().unwrap();
     if service == Service::Coordinator {
-      for network in serai_primitives::NETWORKS {
-        if network == NetworkId::Serai {
-          continue;
-        }
+      for network in serai_primitives::EXTERNAL_NETWORKS {
         queues.insert(
           (service, Service::Processor(network)),
           RwLock::new(Queue(db.clone(), service, Service::Processor(network))),
@@ -214,17 +211,13 @@ async fn main() {
     }
   };
 
-  // Make queues for each NetworkId, other than Serai
-  for network in serai_primitives::NETWORKS {
-    if network == NetworkId::Serai {
-      continue;
-    }
+  // Make queues for each ExternalNetworkId
+  for network in serai_primitives::EXTERNAL_NETWORKS {
     // Use a match so we error if the list of NetworkIds changes
     let Some(key) = read_key(match network {
-      NetworkId::Serai => unreachable!(),
-      NetworkId::Bitcoin => "BITCOIN_KEY",
-      NetworkId::Ethereum => "ETHEREUM_KEY",
-      NetworkId::Monero => "MONERO_KEY",
+      ExternalNetworkId::Bitcoin => "BITCOIN_KEY",
+      ExternalNetworkId::Ethereum => "ETHEREUM_KEY",
+      ExternalNetworkId::Monero => "MONERO_KEY",
     }) else {
       continue;
     };

message-queue/src/messages.rs

@@ -3,11 +3,11 @@ use ciphersuite::{group::GroupEncoding, Ciphersuite, Ristretto};
 
 use borsh::{BorshSerialize, BorshDeserialize};
 
-use serai_primitives::NetworkId;
+use serai_primitives::ExternalNetworkId;
 
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, BorshSerialize, BorshDeserialize)]
 pub enum Service {
-  Processor(NetworkId),
+  Processor(ExternalNetworkId),
   Coordinator,
 }
 

networks/ethereum/schnorr/README.md

@@ -2,4 +2,5 @@
 
 An Ethereum contract to verify Schnorr signatures.
 
-This crate will fail to build if `solc` is not installed and available.
+This crate will fail to build if the expected version of `solc` is not
+installed and available.

networks/ethereum/schnorr/src/lib.rs

@@ -3,10 +3,6 @@
 #![deny(missing_docs)]
 #![allow(non_snake_case)]
 
-/// The initialization bytecode of the Schnorr library.
-pub const INIT_BYTECODE: &str =
-  include_str!(concat!(env!("OUT_DIR"), "/ethereum-schnorr-contract/Schnorr.bin"));
-
 mod public_key;
 pub use public_key::PublicKey;
 mod signature;

networks/ethereum/schnorr/src/tests/premise.rs

@@ -18,14 +18,10 @@ use crate::{Signature, tests::test_key};
 fn ecrecover(message: Scalar, odd_y: bool, r: Scalar, s: Scalar) -> Option<[u8; 20]> {
   let sig = ecdsa::Signature::from_scalars(r, s).ok()?;
   let message: [u8; 32] = message.to_repr().into();
-  alloy_core::primitives::Signature::from_signature_and_parity(
-    sig,
-    alloy_core::primitives::Parity::Parity(odd_y),
-  )
-  .ok()?
-  .recover_address_from_prehash(&alloy_core::primitives::B256::from(message))
-  .ok()
-  .map(Into::into)
+  alloy_core::primitives::PrimitiveSignature::from_signature_and_parity(sig, odd_y)
+    .recover_address_from_prehash(&alloy_core::primitives::B256::from(message))
+    .ok()
+    .map(Into::into)
 }
 
 // Test ecrecover behaves as expected

networks/monero/ringct/bulletproofs/src/original/mod.rs

@@ -56,7 +56,7 @@ impl AggregateRangeWitness {
   }
 }
 
-impl<'a> AggregateRangeStatement<'a> {
+impl AggregateRangeStatement<'_> {
   fn initial_transcript(&self) -> (Scalar, Vec<EdwardsPoint>) {
     let V = self.commitments.iter().map(|c| c * INV_EIGHT()).collect::<Vec<_>>();
     (keccak256_to_scalar(V.iter().flat_map(|V| V.compress().to_bytes()).collect::<Vec<_>>()), V)

networks/monero/wallet/polyseed/Cargo.toml

@@ -1,46 +0,0 @@
-[package]
-name = "polyseed"
-version = "0.1.0"
-description = "Rust implementation of Polyseed"
-license = "MIT"
-repository = "https://github.com/serai-dex/serai/tree/develop/networks/monero/wallet/polyseed"
-authors = ["Luke Parker <lukeparker5132@gmail.com>"]
-edition = "2021"
-rust-version = "1.80"
-
-[package.metadata.docs.rs]
-all-features = true
-rustdoc-args = ["--cfg", "docsrs"]
-
-[lints]
-workspace = true
-
-[dependencies]
-std-shims = { path = "../../../../common/std-shims", version = "^0.1.1", default-features = false }
-
-thiserror = { version = "2", default-features = false }
-
-subtle = { version = "^2.4", default-features = false }
-zeroize = { version = "^1.5", default-features = false, features = ["zeroize_derive"] }
-rand_core = { version = "0.6", default-features = false }
-
-sha3 = { version = "0.10", default-features = false }
-pbkdf2 = { version = "0.12", features = ["simple"], default-features = false }
-
-[dev-dependencies]
-hex = { version = "0.4", default-features = false, features = ["std"] }
-
-[features]
-std = [
-  "std-shims/std",
-
-  "thiserror/std",
-
-  "subtle/std",
-  "zeroize/std",
-  "rand_core/std",
-
-  "sha3/std",
-  "pbkdf2/std",
-]
-default = ["std"]

networks/monero/wallet/polyseed/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2022-2024 Luke Parker
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

networks/monero/wallet/polyseed/README.md

@@ -1,11 +0,0 @@
-# Polyseed
-
-Rust implementation of [Polyseed](https://github.com/tevador/polyseed).
-
-This library is usable under no-std when the `std` feature (on by default) is
-disabled.
-
-### Cargo Features
-
-- `std` (on by default): Enables `std` (and with it, more efficient internal
-  implementations).

networks/monero/wallet/polyseed/src/lib.rs

@@ -1,472 +0,0 @@
-#![cfg_attr(docsrs, feature(doc_auto_cfg))]
-#![doc = include_str!("../README.md")]
-#![deny(missing_docs)]
-#![cfg_attr(not(feature = "std"), no_std)]
-
-use core::fmt;
-use std_shims::{sync::LazyLock, string::String, collections::HashMap};
-#[cfg(feature = "std")]
-use std::time::{SystemTime, UNIX_EPOCH};
-
-use subtle::ConstantTimeEq;
-use zeroize::{Zeroize, Zeroizing, ZeroizeOnDrop};
-use rand_core::{RngCore, CryptoRng};
-
-use sha3::Sha3_256;
-use pbkdf2::pbkdf2_hmac;
-
-#[cfg(test)]
-mod tests;
-
-// Features
-const FEATURE_BITS: u8 = 5;
-#[allow(dead_code)]
-const INTERNAL_FEATURES: u8 = 2;
-const USER_FEATURES: u8 = 3;
-
-const USER_FEATURES_MASK: u8 = (1 << USER_FEATURES) - 1;
-const ENCRYPTED_MASK: u8 = 1 << 4;
-const RESERVED_FEATURES_MASK: u8 = ((1 << FEATURE_BITS) - 1) ^ ENCRYPTED_MASK;
-
-fn user_features(features: u8) -> u8 {
-  features & USER_FEATURES_MASK
-}
-
-fn polyseed_features_supported(features: u8) -> bool {
-  (features & RESERVED_FEATURES_MASK) == 0
-}
-
-// Dates
-const DATE_BITS: u8 = 10;
-const DATE_MASK: u16 = (1u16 << DATE_BITS) - 1;
-const POLYSEED_EPOCH: u64 = 1635768000; // 1st November 2021 12:00 UTC
-const TIME_STEP: u64 = 2629746; // 30.436875 days = 1/12 of the Gregorian year
-
-// After ~85 years, this will roll over.
-fn birthday_encode(time: u64) -> u16 {
-  u16::try_from((time.saturating_sub(POLYSEED_EPOCH) / TIME_STEP) & u64::from(DATE_MASK))
-    .expect("value masked by 2**10 - 1 didn't fit into a u16")
-}
-
-fn birthday_decode(birthday: u16) -> u64 {
-  POLYSEED_EPOCH + (u64::from(birthday) * TIME_STEP)
-}
-
-// Polyseed parameters
-const SECRET_BITS: usize = 150;
-
-const BITS_PER_BYTE: usize = 8;
-const SECRET_SIZE: usize = SECRET_BITS.div_ceil(BITS_PER_BYTE); // 19
-const CLEAR_BITS: usize = (SECRET_SIZE * BITS_PER_BYTE) - SECRET_BITS; // 2
-
-// Polyseed calls this CLEAR_MASK and has a very complicated formula for this fundamental
-// equivalency
-#[allow(clippy::cast_possible_truncation)]
-const LAST_BYTE_SECRET_BITS_MASK: u8 = ((1 << (BITS_PER_BYTE - CLEAR_BITS)) - 1) as u8;
-
-const SECRET_BITS_PER_WORD: usize = 10;
-
-// The amount of words in a seed.
-const POLYSEED_LENGTH: usize = 16;
-// Amount of characters each word must have if trimmed
-pub(crate) const PREFIX_LEN: usize = 4;
-
-const POLY_NUM_CHECK_DIGITS: usize = 1;
-const DATA_WORDS: usize = POLYSEED_LENGTH - POLY_NUM_CHECK_DIGITS;
-
-// Polynomial
-const GF_BITS: usize = 11;
-const POLYSEED_MUL2_TABLE: [u16; 8] = [5, 7, 1, 3, 13, 15, 9, 11];
-
-type Poly = [u16; POLYSEED_LENGTH];
-
-fn elem_mul2(x: u16) -> u16 {
-  if x < 1024 {
-    return 2 * x;
-  }
-  POLYSEED_MUL2_TABLE[usize::from(x % 8)] + (16 * ((x - 1024) / 8))
-}
-
-fn poly_eval(poly: &Poly) -> u16 {
-  // Horner's method at x = 2
-  let mut result = poly[POLYSEED_LENGTH - 1];
-  for i in (0 .. (POLYSEED_LENGTH - 1)).rev() {
-    result = elem_mul2(result) ^ poly[i];
-  }
-  result
-}
-
-// Key gen parameters
-const POLYSEED_SALT: &[u8] = b"POLYSEED key";
-const POLYSEED_KEYGEN_ITERATIONS: u32 = 10000;
-
-// Polyseed technically supports multiple coins, and the value for Monero is 0
-// See: https://github.com/tevador/polyseed/blob/dfb05d8edb682b0e8f743b1b70c9131712ff4157
-//   /include/polyseed.h#L57
-const COIN: u16 = 0;
-
-/// An error when working with a Polyseed.
-#[derive(Clone, Copy, PartialEq, Eq, Debug, thiserror::Error)]
-pub enum PolyseedError {
-  /// The seed was invalid.
-  #[error("invalid seed")]
-  InvalidSeed,
-  /// The entropy was invalid.
-  #[error("invalid entropy")]
-  InvalidEntropy,
-  /// The checksum did not match the data.
-  #[error("invalid checksum")]
-  InvalidChecksum,
-  /// Unsupported feature bits were set.
-  #[error("unsupported features")]
-  UnsupportedFeatures,
-}
-
-/// Language options for Polyseed.
-#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Zeroize)]
-pub enum Language {
-  /// English language option.
-  English,
-  /// Spanish language option.
-  Spanish,
-  /// French language option.
-  French,
-  /// Italian language option.
-  Italian,
-  /// Japanese language option.
-  Japanese,
-  /// Korean language option.
-  Korean,
-  /// Czech language option.
-  Czech,
-  /// Portuguese language option.
-  Portuguese,
-  /// Simplified Chinese language option.
-  ChineseSimplified,
-  /// Traditional Chinese language option.
-  ChineseTraditional,
-}
-
-struct WordList {
-  words: &'static [&'static str],
-  has_prefix: bool,
-  has_accent: bool,
-}
-
-impl WordList {
-  fn new(words: &'static [&'static str], has_prefix: bool, has_accent: bool) -> WordList {
-    let res = WordList { words, has_prefix, has_accent };
-    // This is needed for a later unwrap to not fails
-    assert!(words.len() < usize::from(u16::MAX));
-    res
-  }
-}
-
-static LANGUAGES: LazyLock<HashMap<Language, WordList>> = LazyLock::new(|| {
-  HashMap::from([
-    (Language::Czech, WordList::new(include!("./words/cs.rs"), true, false)),
-    (Language::French, WordList::new(include!("./words/fr.rs"), true, true)),
-    (Language::Korean, WordList::new(include!("./words/ko.rs"), false, false)),
-    (Language::English, WordList::new(include!("./words/en.rs"), true, false)),
-    (Language::Italian, WordList::new(include!("./words/it.rs"), true, false)),
-    (Language::Spanish, WordList::new(include!("./words/es.rs"), true, true)),
-    (Language::Japanese, WordList::new(include!("./words/ja.rs"), false, false)),
-    (Language::Portuguese, WordList::new(include!("./words/pt.rs"), true, false)),
-    (
-      Language::ChineseSimplified,
-      WordList::new(include!("./words/zh_simplified.rs"), false, false),
-    ),
-    (
-      Language::ChineseTraditional,
-      WordList::new(include!("./words/zh_traditional.rs"), false, false),
-    ),
-  ])
-});
-
-/// A Polyseed.
-#[derive(Clone, PartialEq, Eq, Zeroize, ZeroizeOnDrop)]
-pub struct Polyseed {
-  language: Language,
-  features: u8,
-  birthday: u16,
-  entropy: Zeroizing<[u8; 32]>,
-  checksum: u16,
-}
-
-impl fmt::Debug for Polyseed {
-  fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-    f.debug_struct("Polyseed").finish_non_exhaustive()
-  }
-}
-
-fn valid_entropy(entropy: &Zeroizing<[u8; 32]>) -> bool {
-  // Last byte of the entropy should only use certain bits
-  let mut res =
-    entropy[SECRET_SIZE - 1].ct_eq(&(entropy[SECRET_SIZE - 1] & LAST_BYTE_SECRET_BITS_MASK));
-  // Last 13 bytes of the buffer should be unused
-  for b in SECRET_SIZE .. entropy.len() {
-    res &= entropy[b].ct_eq(&0);
-  }
-  res.into()
-}
-
-impl Polyseed {
-  // TODO: Clean this
-  fn to_poly(&self) -> Poly {
-    let mut extra_bits = u32::from(FEATURE_BITS + DATE_BITS);
-    let extra_val = (u16::from(self.features) << DATE_BITS) | self.birthday;
-
-    let mut entropy_idx = 0;
-    let mut secret_bits = BITS_PER_BYTE;
-    let mut seed_rem_bits = SECRET_BITS - BITS_PER_BYTE;
-
-    let mut poly = [0; POLYSEED_LENGTH];
-    for i in 0 .. DATA_WORDS {
-      extra_bits -= 1;
-
-      let mut word_bits = 0;
-      let mut word_val = 0;
-      while word_bits < SECRET_BITS_PER_WORD {
-        if secret_bits == 0 {
-          entropy_idx += 1;
-          secret_bits = seed_rem_bits.min(BITS_PER_BYTE);
-          seed_rem_bits -= secret_bits;
-        }
-        let chunk_bits = secret_bits.min(SECRET_BITS_PER_WORD - word_bits);
-        secret_bits -= chunk_bits;
-        word_bits += chunk_bits;
-        word_val <<= chunk_bits;
-        word_val |=
-          (u16::from(self.entropy[entropy_idx]) >> secret_bits) & ((1u16 << chunk_bits) - 1);
-      }
-
-      word_val <<= 1;
-      word_val |= (extra_val >> extra_bits) & 1;
-      poly[POLY_NUM_CHECK_DIGITS + i] = word_val;
-    }
-
-    poly
-  }
-
-  fn from_internal(
-    language: Language,
-    masked_features: u8,
-    encoded_birthday: u16,
-    entropy: Zeroizing<[u8; 32]>,
-  ) -> Result<Polyseed, PolyseedError> {
-    if !polyseed_features_supported(masked_features) {
-      Err(PolyseedError::UnsupportedFeatures)?;
-    }
-
-    if !valid_entropy(&entropy) {
-      Err(PolyseedError::InvalidEntropy)?;
-    }
-
-    let mut res = Polyseed {
-      language,
-      birthday: encoded_birthday,
-      features: masked_features,
-      entropy,
-      checksum: 0,
-    };
-    res.checksum = poly_eval(&res.to_poly());
-    Ok(res)
-  }
-
-  /// Create a new `Polyseed` with specific internals.
-  ///
-  /// `birthday` is defined in seconds since the epoch.
-  pub fn from(
-    language: Language,
-    features: u8,
-    birthday: u64,
-    entropy: Zeroizing<[u8; 32]>,
-  ) -> Result<Polyseed, PolyseedError> {
-    Self::from_internal(language, user_features(features), birthday_encode(birthday), entropy)
-  }
-
-  /// Create a new `Polyseed`.
-  ///
-  /// This uses the system's time for the birthday, if available, else 0.
-  pub fn new<R: RngCore + CryptoRng>(rng: &mut R, language: Language) -> Polyseed {
-    // Get the birthday
-    #[cfg(feature = "std")]
-    let birthday =
-      SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or(core::time::Duration::ZERO).as_secs();
-    #[cfg(not(feature = "std"))]
-    let birthday = 0;
-
-    // Derive entropy
-    let mut entropy = Zeroizing::new([0; 32]);
-    rng.fill_bytes(entropy.as_mut());
-    entropy[SECRET_SIZE ..].fill(0);
-    entropy[SECRET_SIZE - 1] &= LAST_BYTE_SECRET_BITS_MASK;
-
-    Self::from(language, 0, birthday, entropy).unwrap()
-  }
-
-  /// Create a new `Polyseed` from a String.
-  #[allow(clippy::needless_pass_by_value)]
-  pub fn from_string(lang: Language, seed: Zeroizing<String>) -> Result<Polyseed, PolyseedError> {
-    // Decode the seed into its polynomial coefficients
-    let mut poly = [0; POLYSEED_LENGTH];
-
-    // Validate words are in the lang word list
-    let lang_word_list: &WordList = &LANGUAGES[&lang];
-    for (i, word) in seed.split_whitespace().enumerate() {
-      // Find the word's index
-      fn check_if_matches<S: AsRef<str>, I: Iterator<Item = S>>(
-        has_prefix: bool,
-        mut lang_words: I,
-        word: &str,
-      ) -> Option<usize> {
-        if has_prefix {
-          // Get the position of the word within the iterator
-          // Doesn't use starts_with and some words are substrs of others, leading to false
-          // positives
-          let mut get_position = || {
-            lang_words.position(|lang_word| {
-              let mut lang_word = lang_word.as_ref().chars();
-              let mut word = word.chars();
-
-              let mut res = true;
-              for _ in 0 .. PREFIX_LEN {
-                res &= lang_word.next() == word.next();
-              }
-              res
-            })
-          };
-          let res = get_position();
-          // If another word has this prefix, don't call it a match
-          if get_position().is_some() {
-            return None;
-          }
-          res
-        } else {
-          lang_words.position(|lang_word| lang_word.as_ref() == word)
-        }
-      }
-
-      let Some(coeff) = (if lang_word_list.has_accent {
-        let ascii = |word: &str| word.chars().filter(char::is_ascii).collect::<String>();
-        check_if_matches(
-          lang_word_list.has_prefix,
-          lang_word_list.words.iter().map(|lang_word| ascii(lang_word)),
-          &ascii(word),
-        )
-      } else {
-        check_if_matches(lang_word_list.has_prefix, lang_word_list.words.iter(), word)
-      }) else {
-        Err(PolyseedError::InvalidSeed)?
-      };
-
-      // WordList asserts the word list length is less than u16::MAX
-      poly[i] = u16::try_from(coeff).expect("coeff exceeded u16");
-    }
-
-    // xor out the coin
-    poly[POLY_NUM_CHECK_DIGITS] ^= COIN;
-
-    // Validate the checksum
-    if poly_eval(&poly) != 0 {
-      Err(PolyseedError::InvalidChecksum)?;
-    }
-
-    // Convert the polynomial into entropy
-    let mut entropy = Zeroizing::new([0; 32]);
-
-    let mut extra = 0;
-
-    let mut entropy_idx = 0;
-    let mut entropy_bits = 0;
-
-    let checksum = poly[0];
-    for mut word_val in poly.into_iter().skip(POLY_NUM_CHECK_DIGITS) {
-      // Parse the bottom bit, which is one of the bits of extra
-      // This iterates for less than 16 iters, meaning this won't drop any bits
-      extra <<= 1;
-      extra |= word_val & 1;
-      word_val >>= 1;
-
-      // 10 bits per word creates a [8, 2], [6, 4], [4, 6], [2, 8] cycle
-      // 15 % 4 is 3, leaving 2 bits off, and 152 (19 * 8) - 2 is 150, the amount of bits in the
-      // secret
-      let mut word_bits = GF_BITS - 1;
-      while word_bits > 0 {
-        if entropy_bits == BITS_PER_BYTE {
-          entropy_idx += 1;
-          entropy_bits = 0;
-        }
-        let chunk_bits = word_bits.min(BITS_PER_BYTE - entropy_bits);
-        word_bits -= chunk_bits;
-        let chunk_mask = (1u16 << chunk_bits) - 1;
-        if chunk_bits < BITS_PER_BYTE {
-          entropy[entropy_idx] <<= chunk_bits;
-        }
-        entropy[entropy_idx] |=
-          u8::try_from((word_val >> word_bits) & chunk_mask).expect("chunk exceeded u8");
-        entropy_bits += chunk_bits;
-      }
-    }
-
-    let birthday = extra & DATE_MASK;
-    // extra is contained to u16, and DATE_BITS > 8
-    let features =
-      u8::try_from(extra >> DATE_BITS).expect("couldn't convert extra >> DATE_BITS to u8");
-
-    let res = Self::from_internal(lang, features, birthday, entropy);
-    if let Ok(res) = res.as_ref() {
-      debug_assert_eq!(res.checksum, checksum);
-    }
-    res
-  }
-
-  /// When this seed was created, defined in seconds since the epoch.
-  pub fn birthday(&self) -> u64 {
-    birthday_decode(self.birthday)
-  }
-
-  /// This seed's features.
-  pub fn features(&self) -> u8 {
-    self.features
-  }
-
-  /// This seed's entropy.
-  pub fn entropy(&self) -> &Zeroizing<[u8; 32]> {
-    &self.entropy
-  }
-
-  /// The key derived from this seed.
-  pub fn key(&self) -> Zeroizing<[u8; 32]> {
-    let mut key = Zeroizing::new([0; 32]);
-    pbkdf2_hmac::<Sha3_256>(
-      self.entropy.as_slice(),
-      POLYSEED_SALT,
-      POLYSEED_KEYGEN_ITERATIONS,
-      key.as_mut(),
-    );
-    key
-  }
-
-  /// The String representation of this seed.
-  pub fn to_string(&self) -> Zeroizing<String> {
-    // Encode the polynomial with the existing checksum
-    let mut poly = self.to_poly();
-    poly[0] = self.checksum;
-
-    // Embed the coin
-    poly[POLY_NUM_CHECK_DIGITS] ^= COIN;
-
-    // Output words
-    let mut seed = Zeroizing::new(String::new());
-    let words = &LANGUAGES[&self.language].words;
-    for i in 0 .. poly.len() {
-      seed.push_str(words[usize::from(poly[i])]);
-      if i < poly.len() - 1 {
-        seed.push(' ');
-      }
-    }
-
-    seed
-  }
-}

networks/monero/wallet/polyseed/src/tests.rs

@@ -1,218 +0,0 @@
-use zeroize::Zeroizing;
-use rand_core::OsRng;
-
-use crate::*;
-
-#[test]
-fn test_polyseed() {
-  struct Vector {
-    language: Language,
-    seed: String,
-    entropy: String,
-    birthday: u64,
-    has_prefix: bool,
-    has_accent: bool,
-  }
-
-  let vectors = [
-    Vector {
-      language: Language::English,
-      seed: "raven tail swear infant grief assist regular lamp \
-      duck valid someone little harsh puppy airport language"
-        .into(),
-      entropy: "dd76e7359a0ded37cd0ff0f3c829a5ae01673300000000000000000000000000".into(),
-      birthday: 1638446400,
-      has_prefix: true,
-      has_accent: false,
-    },
-    Vector {
-      language: Language::Spanish,
-      seed: "eje fin parte célebre tabú pestaña lienzo puma \
-      prisión hora regalo lengua existir lápiz lote sonoro"
-        .into(),
-      entropy: "5a2b02df7db21fcbe6ec6df137d54c7b20fd2b00000000000000000000000000".into(),
-      birthday: 3118651200,
-      has_prefix: true,
-      has_accent: true,
-    },
-    Vector {
-      language: Language::French,
-      seed: "valable arracher décaler jeudi amusant dresser mener épaissir risible \
-      prouesse réserve ampleur ajuster muter caméra enchère"
-        .into(),
-      entropy: "11cfd870324b26657342c37360c424a14a050b00000000000000000000000000".into(),
-      birthday: 1679314966,
-      has_prefix: true,
-      has_accent: true,
-    },
-    Vector {
-      language: Language::Italian,
-      seed: "caduco midollo copione meninge isotopo illogico riflesso tartaruga fermento \
-      olandese normale tristezza episodio voragine forbito achille"
-        .into(),
-      entropy: "7ecc57c9b4652d4e31428f62bec91cfd55500600000000000000000000000000".into(),
-      birthday: 1679316358,
-      has_prefix: true,
-      has_accent: false,
-    },
-    Vector {
-      language: Language::Portuguese,
-      seed: "caverna custear azedo adeus senador apertada sedoso omitir \
-      sujeito aurora videira molho cartaz gesso dentista tapar"
-        .into(),
-      entropy: "45473063711376cae38f1b3eba18c874124e1d00000000000000000000000000".into(),
-      birthday: 1679316657,
-      has_prefix: true,
-      has_accent: false,
-    },
-    Vector {
-      language: Language::Czech,
-      seed: "usmrtit nora dotaz komunita zavalit funkce mzda sotva akce \
-      vesta kabel herna stodola uvolnit ustrnout email"
-        .into(),
-      entropy: "7ac8a4efd62d9c3c4c02e350d32326df37821c00000000000000000000000000".into(),
-      birthday: 1679316898,
-      has_prefix: true,
-      has_accent: false,
-    },
-    Vector {
-      language: Language::Korean,
-      seed: "전망 선풍기 국제 무궁화 설사 기름 이론적 해안 절망 예선 \
-        지우개 보관 절망 말기 시각 귀신"
-        .into(),
-      entropy: "684663fda420298f42ed94b2c512ed38ddf12b00000000000000000000000000".into(),
-      birthday: 1679317073,
-      has_prefix: false,
-      has_accent: false,
-    },
-    Vector {
-      language: Language::Japanese,
-      seed: "うちあわせ　ちつじょ　つごう　しはい　けんこう　とおる　てみやげ　はんとし　たんとう \
-      といれ　おさない　おさえる　むかう　ぬぐう　なふだ　せまる"
-        .into(),
-      entropy: "94e6665518a6286c6e3ba508a2279eb62b771f00000000000000000000000000".into(),
-      birthday: 1679318722,
-      has_prefix: false,
-      has_accent: false,
-    },
-    Vector {
-      language: Language::ChineseTraditional,
-      seed: "亂 挖 斤 柄 代 圈 枝 轄 魯 論 函 開 勘 番 榮 壁".into(),
-      entropy: "b1594f585987ab0fd5a31da1f0d377dae5283f00000000000000000000000000".into(),
-      birthday: 1679426433,
-      has_prefix: false,
-      has_accent: false,
-    },
-    Vector {
-      language: Language::ChineseSimplified,
-      seed: "啊 百 族 府 票 划 伪 仓 叶 虾 借 溜 晨 左 等 鬼".into(),
-      entropy: "21cdd366f337b89b8d1bc1df9fe73047c22b0300000000000000000000000000".into(),
-      birthday: 1679426817,
-      has_prefix: false,
-      has_accent: false,
-    },
-    // The following seed requires the language specification in order to calculate
-    // a single valid checksum
-    Vector {
-      language: Language::Spanish,
-      seed: "impo sort usua cabi venu nobl oliv clim \
-        cont barr marc auto prod vaca torn fati"
-        .into(),
-      entropy: "dbfce25fe09b68a340e01c62417eeef43ad51800000000000000000000000000".into(),
-      birthday: 1701511650,
-      has_prefix: true,
-      has_accent: true,
-    },
-  ];
-
-  for vector in vectors {
-    let add_whitespace = |mut seed: String| {
-      seed.push(' ');
-      seed
-    };
-
-    let seed_without_accents = |seed: &str| {
-      seed
-        .split_whitespace()
-        .map(|w| w.chars().filter(char::is_ascii).collect::<String>())
-        .collect::<Vec<_>>()
-        .join(" ")
-    };
-
-    let trim_seed = |seed: &str| {
-      let seed_to_trim =
-        if vector.has_accent { seed_without_accents(seed) } else { seed.to_string() };
-      seed_to_trim
-        .split_whitespace()
-        .map(|w| {
-          let mut ascii = 0;
-          let mut to_take = w.len();
-          for (i, char) in w.chars().enumerate() {
-            if char.is_ascii() {
-              ascii += 1;
-            }
-            if ascii == PREFIX_LEN {
-              // +1 to include this character, which put us at the prefix length
-              to_take = i + 1;
-              break;
-            }
-          }
-          w.chars().take(to_take).collect::<String>()
-        })
-        .collect::<Vec<_>>()
-        .join(" ")
-    };
-
-    // String -> Seed
-    println!("{}. language: {:?}, seed: {}", line!(), vector.language, vector.seed.clone());
-    let seed = Polyseed::from_string(vector.language, Zeroizing::new(vector.seed.clone())).unwrap();
-    let trim = trim_seed(&vector.seed);
-    let add_whitespace = add_whitespace(vector.seed.clone());
-    let seed_without_accents = seed_without_accents(&vector.seed);
-
-    // Make sure a version with added whitespace still works
-    let whitespaced_seed =
-      Polyseed::from_string(vector.language, Zeroizing::new(add_whitespace)).unwrap();
-    assert_eq!(seed, whitespaced_seed);
-    // Check trimmed versions works
-    if vector.has_prefix {
-      let trimmed_seed = Polyseed::from_string(vector.language, Zeroizing::new(trim)).unwrap();
-      assert_eq!(seed, trimmed_seed);
-    }
-    // Check versions without accents work
-    if vector.has_accent {
-      let seed_without_accents =
-        Polyseed::from_string(vector.language, Zeroizing::new(seed_without_accents)).unwrap();
-      assert_eq!(seed, seed_without_accents);
-    }
-
-    let entropy = Zeroizing::new(hex::decode(vector.entropy).unwrap().try_into().unwrap());
-    assert_eq!(*seed.entropy(), entropy);
-    assert!(seed.birthday().abs_diff(vector.birthday) < TIME_STEP);
-
-    // Entropy -> Seed
-    let from_entropy = Polyseed::from(vector.language, 0, seed.birthday(), entropy).unwrap();
-    assert_eq!(seed.to_string(), from_entropy.to_string());
-
-    // Check against ourselves
-    {
-      let seed = Polyseed::new(&mut OsRng, vector.language);
-      println!("{}. seed: {}", line!(), *seed.to_string());
-      assert_eq!(seed, Polyseed::from_string(vector.language, seed.to_string()).unwrap());
-      assert_eq!(
-        seed,
-        Polyseed::from(vector.language, 0, seed.birthday(), seed.entropy().clone(),).unwrap()
-      );
-    }
-  }
-}
-
-#[test]
-fn test_invalid_polyseed() {
-  // This seed includes unsupported features bits and should error on decode
-  let seed = "include domain claim resemble urban hire lunch bird \
-    crucial fire best wife ring warm ignore model"
-    .into();
-  let res = Polyseed::from_string(Language::English, Zeroizing::new(seed));
-  assert_eq!(res, Err(PolyseedError::UnsupportedFeatures));
-}