Make transactions undroppable

coordinator/cosign/src/delay.rs literally demonstrates how we'd need to rewrite
our handling of transactions with this change. It can be cleaned up a bit but
already identifies ergonomic issues. It also doesn't model passing an &mut txn
to an async function, which would also require using the droppable wrapper
struct.

To locally see this build, run

RUSTFLAGS="-Zpanic_abort_tests -C panic=abort" cargo +nightly build -p serai-cosign --all-targets

To locally see this fail to build, run

cargo build -p serai-cosign --all-targets

While it doesn't say which line causes it fail to build, the only distinction
is panic=unwind.

For more context, please see #578.

.github/workflows/msrv.yml

@@ -173,9 +173,13 @@ jobs:
 
       - name: Run cargo msrv on coordinator
         run: |
-          cargo msrv verify --manifest-path coordinator/tributary/tendermint/Cargo.toml
-          cargo msrv verify --manifest-path coordinator/tributary/Cargo.toml
+          cargo msrv verify --manifest-path coordinator/tributary-sdk/tendermint/Cargo.toml
+          cargo msrv verify --manifest-path coordinator/tributary-sdk/Cargo.toml
           cargo msrv verify --manifest-path coordinator/cosign/Cargo.toml
+          cargo msrv verify --manifest-path coordinator/substrate/Cargo.toml
+          cargo msrv verify --manifest-path coordinator/tributary/Cargo.toml
+          cargo msrv verify --manifest-path coordinator/p2p/Cargo.toml
+          cargo msrv verify --manifest-path coordinator/p2p/libp2p/Cargo.toml
           cargo msrv verify --manifest-path coordinator/Cargo.toml
 
   msrv-substrate:

.github/workflows/tests.yml

@@ -60,8 +60,12 @@ jobs:
             -p serai-ethereum-processor \
             -p serai-monero-processor \
             -p tendermint-machine \
-            -p tributary-chain \
+            -p tributary-sdk \
             -p serai-cosign \
+            -p serai-coordinator-substrate \
+            -p serai-coordinator-tributary \
+            -p serai-coordinator-p2p \
+            -p serai-coordinator-libp2p-p2p \
             -p serai-coordinator \
             -p serai-orchestrator \
             -p serai-docker-tests

Cargo.toml

@@ -96,9 +96,13 @@ members = [
   "processor/ethereum",
   "processor/monero",
 
-  "coordinator/tributary/tendermint",
-  "coordinator/tributary",
+  "coordinator/tributary-sdk/tendermint",
+  "coordinator/tributary-sdk",
   "coordinator/cosign",
+  "coordinator/substrate",
+  "coordinator/tributary",
+  "coordinator/p2p",
+  "coordinator/p2p/libp2p",
   "coordinator",
 
   "substrate/primitives",

common/db/src/lib.rs

@@ -30,13 +30,53 @@ pub trait Get {
 /// is undefined. The transaction may block, deadlock, panic, overwrite one of the two values
 /// randomly, or any other action, at time of write or at time of commit.
 #[must_use]
-pub trait DbTxn: Send + Get {
+pub trait DbTxn: Sized + Send + Get {
   /// Write a value to this key.
   fn put(&mut self, key: impl AsRef<[u8]>, value: impl AsRef<[u8]>);
   /// Delete the value from this key.
   fn del(&mut self, key: impl AsRef<[u8]>);
   /// Commit this transaction.
   fn commit(self);
+  /// Close this transaction.
+  ///
+  /// This is equivalent to `Drop` on transactions which can be dropped. This is explicit and works
+  /// with transactions which can't be dropped.
+  fn close(self) {
+    drop(self);
+  }
+}
+
+// Credit for the idea goes to https://jack.wrenn.fyi/blog/undroppable
+pub struct Undroppable<T>(Option<T>);
+impl<T> Drop for Undroppable<T> {
+  fn drop(&mut self) {
+    // Use an assertion at compile time to prevent this code from compiling if generated
+    #[allow(clippy::assertions_on_constants)]
+    const {
+      assert!(false, "Undroppable DbTxn was dropped. Ensure all code paths call commit or close");
+    }
+  }
+}
+impl<T: DbTxn> Get for Undroppable<T> {
+  fn get(&self, key: impl AsRef<[u8]>) -> Option<Vec<u8>> {
+    self.0.as_ref().unwrap().get(key)
+  }
+}
+impl<T: DbTxn> DbTxn for Undroppable<T> {
+  fn put(&mut self, key: impl AsRef<[u8]>, value: impl AsRef<[u8]>) {
+    self.0.as_mut().unwrap().put(key, value);
+  }
+  fn del(&mut self, key: impl AsRef<[u8]>) {
+    self.0.as_mut().unwrap().del(key);
+  }
+  fn commit(mut self) {
+    self.0.take().unwrap().commit();
+    let _ = core::mem::ManuallyDrop::new(self);
+  }
+  fn close(mut self) {
+    drop(self.0.take().unwrap());
+    let _ = core::mem::ManuallyDrop::new(self);
+  }
 }
 
 /// A database supporting atomic transaction.
@@ -51,6 +91,10 @@ pub trait Db: 'static + Send + Sync + Clone + Get {
     let dst_len = u8::try_from(item_dst.len()).unwrap();
     [[db_len].as_ref(), db_dst, [dst_len].as_ref(), item_dst, key.as_ref()].concat()
   }
-  /// Open a new transaction.
-  fn txn(&mut self) -> Self::Transaction<'_>;
+  /// Open a new transaction which may be dropped.
+  fn unsafe_txn(&mut self) -> Self::Transaction<'_>;
+  /// Open a new transaction which must be committed or closed.
+  fn txn(&mut self) -> Undroppable<Self::Transaction<'_>> {
+    Undroppable(Some(self.unsafe_txn()))
+  }
 }

common/db/src/mem.rs

@@ -74,7 +74,7 @@ impl Get for MemDb {
 }
 impl Db for MemDb {
   type Transaction<'a> = MemDbTxn<'a>;
-  fn txn(&mut self) -> MemDbTxn<'_> {
+  fn unsafe_txn(&mut self) -> MemDbTxn<'_> {
     MemDbTxn(self, HashMap::new(), HashSet::new())
   }
 }

common/db/src/parity_db.rs

@@ -37,7 +37,7 @@ impl Get for Arc<ParityDb> {
 }
 impl Db for Arc<ParityDb> {
   type Transaction<'a> = Transaction<'a>;
-  fn txn(&mut self) -> Self::Transaction<'_> {
+  fn unsafe_txn(&mut self) -> Self::Transaction<'_> {
     Transaction(self, vec![])
   }
 }

common/db/src/rocks.rs

@@ -39,7 +39,7 @@ impl<T: ThreadMode> Get for Arc<OptimisticTransactionDB<T>> {
 }
 impl<T: Send + ThreadMode + 'static> Db for Arc<OptimisticTransactionDB<T>> {
   type Transaction<'a> = Transaction<'a, T>;
-  fn txn(&mut self) -> Self::Transaction<'_> {
+  fn unsafe_txn(&mut self) -> Self::Transaction<'_> {
     let mut opts = WriteOptions::default();
     opts.set_sync(true);
     Transaction(self.transaction_opt(&opts, &Default::default()), &**self)

common/task/src/lib.rs

@@ -2,28 +2,36 @@
 #![doc = include_str!("../README.md")]
 #![deny(missing_docs)]
 
-use core::{future::Future, time::Duration};
-use std::sync::Arc;
+use core::{
+  fmt::{self, Debug},
+  future::Future,
+  time::Duration,
+};
 
-use tokio::sync::{mpsc, oneshot, Mutex};
+use tokio::sync::mpsc;
 
-enum Closed {
-  NotClosed(Option<oneshot::Receiver<()>>),
-  Closed,
-}
+mod type_name;
 
 /// A handle for a task.
+///
+/// The task will only stop running once all handles for it are dropped.
+//
+// `run_now` isn't infallible if the task may have been closed. `run_now` on a closed task would
+// either need to panic (historic behavior), silently drop the fact the task can't be run, or
+// return an error. Instead of having a potential panic, and instead of modeling the error
+// behavior, this task can't be closed unless all handles are dropped, ensuring calls to `run_now`
+// are infallible.
 #[derive(Clone)]
 pub struct TaskHandle {
   run_now: mpsc::Sender<()>,
+  #[allow(dead_code)] // This is used to track if all handles have been dropped
   close: mpsc::Sender<()>,
-  closed: Arc<Mutex<Closed>>,
 }
+
 /// A task's internal structures.
 pub struct Task {
   run_now: mpsc::Receiver<()>,
   close: mpsc::Receiver<()>,
-  closed: oneshot::Sender<()>,
 }
 
 impl Task {
@@ -34,22 +42,15 @@ impl Task {
     let (run_now_send, run_now_recv) = mpsc::channel(1);
     // And any call to close satisfies all calls to close
     let (close_send, close_recv) = mpsc::channel(1);
-    let (closed_send, closed_recv) = oneshot::channel();
     (
-      Self { run_now: run_now_recv, close: close_recv, closed: closed_send },
-      TaskHandle {
-        run_now: run_now_send,
-        close: close_send,
-        closed: Arc::new(Mutex::new(Closed::NotClosed(Some(closed_recv)))),
-      },
+      Self { run_now: run_now_recv, close: close_recv },
+      TaskHandle { run_now: run_now_send, close: close_send },
     )
   }
 }
 
 impl TaskHandle {
   /// Tell the task to run now (and not whenever its next iteration on a timer is).
-  ///
-  /// Panics if the task has been dropped.
   pub fn run_now(&self) {
     #[allow(clippy::match_same_arms)]
     match self.run_now.try_send(()) {
@@ -57,27 +58,19 @@ impl TaskHandle {
       // NOP on full, as this task will already be ran as soon as possible
       Err(mpsc::error::TrySendError::Full(())) => {}
       Err(mpsc::error::TrySendError::Closed(())) => {
+        // The task should only be closed if all handles are dropped, and this one hasn't been
         panic!("task was unexpectedly closed when calling run_now")
       }
     }
   }
+}
 
-  /// Close the task.
-  ///
-  /// Returns once the task shuts down after it finishes its current iteration (which may be of
-  /// unbounded time).
-  pub async fn close(self) {
-    // If another instance of the handle called tfhis, don't error
-    let _ = self.close.send(()).await;
-    // Wait until we receive the closed message
-    let mut closed = self.closed.lock().await;
-    match &mut *closed {
-      Closed::NotClosed(ref mut recv) => {
-        assert_eq!(recv.take().unwrap().await, Ok(()), "continually ran task dropped itself?");
-        *closed = Closed::Closed;
-      }
-      Closed::Closed => {}
-    }
+/// An enum which can't be constructed, representing that the task does not error.
+pub enum DoesNotError {}
+impl Debug for DoesNotError {
+  fn fmt(&self, _: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
+    // This type can't be constructed so we'll never have a `&self` to call this fn with
+    unreachable!()
   }
 }
 
@@ -90,11 +83,14 @@ pub trait ContinuallyRan: Sized + Send {
   /// Upon error, the amount of time waited will be linearly increased until this limit.
   const MAX_DELAY_BETWEEN_ITERATIONS: u64 = 120;
 
+  /// The error potentially yielded upon running an iteration of this task.
+  type Error: Debug;
+
   /// Run an iteration of the task.
   ///
   /// If this returns `true`, all dependents of the task will immediately have a new iteration ran
   /// (without waiting for whatever timer they were already on).
-  fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, String>>;
+  fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, Self::Error>>;
 
   /// Continually run the task.
   fn continually_run(
@@ -136,12 +132,20 @@ pub trait ContinuallyRan: Sized + Send {
             }
           }
           Err(e) => {
-            log::warn!("{}", e);
+            // Get the type name
+            let type_name = type_name::strip_type_name(core::any::type_name::<Self>());
+            // Print the error as a warning, prefixed by the task's type
+            log::warn!("{type_name}: {e:?}");
             increase_sleep_before_next_task(&mut current_sleep_before_next_task);
           }
         }
 
         // Don't run the task again for another few seconds UNLESS told to run now
+        /*
+          We could replace tokio::mpsc with async_channel, tokio::time::sleep with
+          patchable_async_sleep::sleep, and tokio::select with futures_lite::future::or
+          It isn't worth the effort when patchable_async_sleep::sleep will still resolve to tokio
+        */
         tokio::select! {
           () = tokio::time::sleep(Duration::from_secs(current_sleep_before_next_task)) => {},
           msg = task.run_now.recv() => {
@@ -152,8 +156,6 @@ pub trait ContinuallyRan: Sized + Send {
           },
         }
       }
-
-      task.closed.send(()).unwrap();
     }
   }
 }

common/task/src/type_name.rs

@@ -0,0 +1,31 @@
+/// Strip the modules from a type name.
+// This may be of the form `a::b::C`, in which case we only want `C`
+pub(crate) fn strip_type_name(full_type_name: &'static str) -> String {
+  // It also may be `a::b::C<d::e::F>`, in which case, we only attempt to strip `a::b`
+  let mut by_generics = full_type_name.split('<');
+
+  // Strip to just `C`
+  let full_outer_object_name = by_generics.next().unwrap();
+  let mut outer_object_name_parts = full_outer_object_name.split("::");
+  let mut last_part_in_outer_object_name = outer_object_name_parts.next().unwrap();
+  for part in outer_object_name_parts {
+    last_part_in_outer_object_name = part;
+  }
+
+  // Push back on the generic terms
+  let mut type_name = last_part_in_outer_object_name.to_string();
+  for generic in by_generics {
+    type_name.push('<');
+    type_name.push_str(generic);
+  }
+  type_name
+}
+
+#[test]
+fn test_strip_type_name() {
+  assert_eq!(strip_type_name("core::option::Option"), "Option");
+  assert_eq!(
+    strip_type_name("core::option::Option<alloc::string::String>"),
+    "Option<alloc::string::String>"
+  );
+}

coordinator/Cargo.toml

@@ -18,31 +18,29 @@ rustdoc-args = ["--cfg", "docsrs"]
 workspace = true
 
 [dependencies]
-async-trait = { version = "0.1", default-features = false }
-
 zeroize = { version = "^1.5", default-features = false, features = ["std"] }
 bitvec = { version = "1", default-features = false, features = ["std"] }
 rand_core = { version = "0.6", default-features = false, features = ["std"] }
 
 blake2 = { version = "0.10", default-features = false, features = ["std"] }
+schnorrkel = { version = "0.11", default-features = false, features = ["std"] }
 
-transcript = { package = "flexible-transcript", path = "../crypto/transcript", default-features = false, features = ["std", "recommended"] }
 ciphersuite = { path = "../crypto/ciphersuite", default-features = false, features = ["std"] }
 schnorr = { package = "schnorr-signatures", path = "../crypto/schnorr", default-features = false, features = ["std"] }
 frost = { package = "modular-frost", path = "../crypto/frost" }
 frost-schnorrkel = { path = "../crypto/schnorrkel" }
 
-scale = { package = "parity-scale-codec", version = "3", default-features = false, features = ["std", "derive"] }
+scale = { package = "parity-scale-codec", version = "3", default-features = false, features = ["std", "derive", "bit-vec"] }
 
 zalloc = { path = "../common/zalloc" }
 serai-db = { path = "../common/db" }
 serai-env = { path = "../common/env" }
+serai-task = { path = "../common/task", version = "0.1" }
 
-processor-messages = { package = "serai-processor-messages", path = "../processor/messages" }
+messages = { package = "serai-processor-messages", path = "../processor/messages" }
 message-queue = { package = "serai-message-queue", path = "../message-queue" }
-tributary = { package = "tributary-chain", path = "./tributary" }
+tributary-sdk = { path = "./tributary-sdk" }
 
-sp-application-crypto = { git = "https://github.com/serai-dex/substrate", default-features = false, features = ["std"] }
 serai-client = { path = "../substrate/client", default-features = false, features = ["serai", "borsh"] }
 
 hex = { version = "0.4", default-features = false, features = ["std"] }
@@ -51,16 +49,15 @@ borsh = { version = "1", default-features = false, features = ["std", "derive",
 log = { version = "0.4", default-features = false, features = ["std"] }
 env_logger = { version = "0.10", default-features = false, features = ["humantime"] }
 
-futures-util = { version = "0.3", default-features = false, features = ["std"] }
-tokio = { version = "1", default-features = false, features = ["rt-multi-thread", "sync", "time", "macros"] }
-libp2p = { version = "0.52", default-features = false, features = ["tokio", "tcp", "noise", "yamux", "request-response", "gossipsub", "macros"] }
+tokio = { version = "1", default-features = false, features = ["time", "sync", "macros", "rt-multi-thread"] }
 
-[dev-dependencies]
-tributary = { package = "tributary-chain", path = "./tributary", features = ["tests"] }
-sp-application-crypto = { git = "https://github.com/serai-dex/substrate", default-features = false, features = ["std"] }
-sp-runtime = { git = "https://github.com/serai-dex/substrate", default-features = false, features = ["std"] }
+serai-cosign = { path = "./cosign" }
+serai-coordinator-substrate = { path = "./substrate" }
+serai-coordinator-tributary = { path = "./tributary" }
+serai-coordinator-p2p = { path = "./p2p" }
+serai-coordinator-libp2p-p2p = { path = "./p2p/libp2p" }
 
 [features]
-longer-reattempts = []
+longer-reattempts = ["serai-coordinator-tributary/longer-reattempts"]
 parity-db = ["serai-db/parity-db"]
 rocksdb = ["serai-db/rocksdb"]

coordinator/LICENSE

@@ -1,6 +1,6 @@
 AGPL-3.0-only license
 
-Copyright (c) 2023 Luke Parker
+Copyright (c) 2023-2025 Luke Parker
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License Version 3 as

coordinator/README.md

@@ -1,7 +1,29 @@
 # Coordinator
 
-The Serai coordinator communicates with other coordinators to prepare batches
-for Serai and sign transactions.
+- [`tendermint`](/tributary/tendermint) is an implementation of the Tendermint
+  BFT algorithm.
 
-In order to achieve consensus over gossip, and order certain events, a
-micro-blockchain is instantiated.
+- [`tributary-sdk`](./tributary-sdk) is a micro-blockchain framework. Instead
+  of a producing a blockchain daemon like the Polkadot SDK or Cosmos SDK intend
+  to, `tributary` is solely intended to be an embedded asynchronous task within
+  an application.
+
+  The Serai coordinator spawns a tributary for each validator set it's
+  coordinating. This allows the participating validators to communicate in a
+  byzantine-fault-tolerant manner (relying on Tendermint for consensus).
+
+- [`cosign`](./cosign) contains a library to decide which Substrate blocks
+  should be cosigned and to evaluate cosigns.
+
+- [`substrate`](./substrate) contains a library to index the Substrate
+  blockchain and handle its events.
+
+- [`tributary`](./tributary) is our instantiation of the Tributary SDK for the
+  Serai processor. It includes the `Transaction` definition and deferred
+  execution logic.
+
+- [`p2p`](./p2p) is our abstract P2P API to service the Coordinator.
+
+- [`libp2p`](./p2p/libp2p) is our libp2p-backed implementation of the P2P API.
+
+- [`src`](./src) contains the source code for the Coordinator binary itself.

coordinator/cosign/Cargo.toml

@@ -14,9 +14,6 @@ rust-version = "1.81"
 all-features = true
 rustdoc-args = ["--cfg", "docsrs"]
 
-[package.metadata.cargo-machete]
-ignored = ["scale"]
-
 [lints]
 workspace = true
 
@@ -30,7 +27,7 @@ serai-client = { path = "../../substrate/client", default-features = false, feat
 
 log = { version = "0.4", default-features = false, features = ["std"] }
 
-tokio = { version = "1", default-features = false, features = [] }
+tokio = { version = "1", default-features = false }
 
-serai-db = { path = "../../common/db" }
-serai-task = { path = "../../common/task" }
+serai-db = { path = "../../common/db", version = "0.1.1" }
+serai-task = { path = "../../common/task", version = "0.1" }

coordinator/cosign/src/delay.rs

@@ -2,7 +2,7 @@ use core::future::Future;
 use std::time::{Duration, SystemTime};
 
 use serai_db::*;
-use serai_task::ContinuallyRan;
+use serai_task::{DoesNotError, ContinuallyRan};
 
 use crate::evaluator::CosignedBlocks;
 
@@ -24,8 +24,19 @@ pub(crate) struct CosignDelayTask<D: Db> {
   pub(crate) db: D,
 }
 
+struct AwaitUndroppable<T: DbTxn>(Option<core::mem::ManuallyDrop<Undroppable<T>>>);
+impl<T: DbTxn> Drop for AwaitUndroppable<T> {
+  fn drop(&mut self) {
+    if let Some(mut txn) = self.0.take() {
+      (unsafe { core::mem::ManuallyDrop::take(&mut txn) }).close();
+    }
+  }
+}
+
 impl<D: Db> ContinuallyRan for CosignDelayTask<D> {
-  fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, String>> {
+  type Error = DoesNotError;
+
+  fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, Self::Error>> {
     async move {
       let mut made_progress = false;
       loop {
@@ -33,14 +44,18 @@ impl<D: Db> ContinuallyRan for CosignDelayTask<D> {
 
         // Receive the next block to mark as cosigned
         let Some((block_number, time_evaluated)) = CosignedBlocks::try_recv(&mut txn) else {
+          txn.close();
           break;
         };
+
         // Calculate when we should mark it as valid
         let time_valid =
           SystemTime::UNIX_EPOCH + Duration::from_secs(time_evaluated) + ACKNOWLEDGEMENT_DELAY;
         // Sleep until then
+        let mut txn = AwaitUndroppable(Some(core::mem::ManuallyDrop::new(txn)));
         tokio::time::sleep(SystemTime::now().duration_since(time_valid).unwrap_or(Duration::ZERO))
           .await;
+        let mut txn = core::mem::ManuallyDrop::into_inner(txn.0.take().unwrap());
 
         // Set the cosigned block
         LatestCosignedBlockNumber::set(&mut txn, &block_number);

coordinator/cosign/src/evaluator.rs

@@ -24,7 +24,7 @@ db_channel!(
 );
 
 // This is a strict function which won't panic, even with a malicious Serai node, so long as:
-// - It's called incrementally
+// - It's called incrementally (with an increment of 1)
 // - It's only called for block numbers we've completed indexing on within the intend task
 // - It's only called for block numbers after a global session has started
 // - The global sessions channel is populated as the block declaring the session is indexed
@@ -69,6 +69,10 @@ fn currently_evaluated_global_session_strict(
   res
 }
 
+pub(crate) fn currently_evaluated_global_session(getter: &impl Get) -> Option<[u8; 32]> {
+  CurrentlyEvaluatedGlobalSession::get(getter).map(|(id, _info)| id)
+}
+
 /// A task to determine if a block has been cosigned and we should handle it.
 pub(crate) struct CosignEvaluatorTask<D: Db, R: RequestNotableCosigns> {
   pub(crate) db: D,
@@ -76,24 +80,27 @@ pub(crate) struct CosignEvaluatorTask<D: Db, R: RequestNotableCosigns> {
 }
 
 impl<D: Db, R: RequestNotableCosigns> ContinuallyRan for CosignEvaluatorTask<D, R> {
-  fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, String>> {
+  type Error = String;
+
+  fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, Self::Error>> {
     async move {
       let mut known_cosign = None;
       let mut made_progress = false;
       loop {
-        let mut txn = self.db.txn();
+        let mut txn = self.db.unsafe_txn();
         let Some(BlockEventData { block_number, has_events }) = BlockEvents::try_recv(&mut txn)
         else {
           break;
         };
 
+        // Fetch the global session information
+        let (global_session, global_session_info) =
+          currently_evaluated_global_session_strict(&mut txn, block_number);
+
         match has_events {
           // Because this had notable events, we require an explicit cosign for this block by a
           // supermajority of the prior block's validator sets
           HasEvents::Notable => {
-            let (global_session, global_session_info) =
-              currently_evaluated_global_session_strict(&mut txn, block_number);
-
             let mut weight_cosigned = 0;
             for set in global_session_info.sets {
               // Check if we have the cosign from this set
@@ -122,6 +129,8 @@ impl<D: Db, R: RequestNotableCosigns> ContinuallyRan for CosignEvaluatorTask<D,
                 "notable block (#{block_number}) wasn't yet cosigned. this should resolve shortly",
               ));
             }
+
+            log::info!("marking notable block #{block_number} as cosigned");
           }
           // Since this block didn't have any notable events, we simply require a cosign for this
           // block or a greater block by the current validator sets
@@ -143,10 +152,6 @@ impl<D: Db, R: RequestNotableCosigns> ContinuallyRan for CosignEvaluatorTask<D,
                 is during the latest global session we've evaluated the start of.
               */
 
-              // Get the global session for this block
-              let (global_session, global_session_info) =
-                currently_evaluated_global_session_strict(&mut txn, block_number);
-
               let mut weight_cosigned = 0;
               let mut lowest_common_block: Option<u64> = None;
               for set in global_session_info.sets {
@@ -194,6 +199,8 @@ impl<D: Db, R: RequestNotableCosigns> ContinuallyRan for CosignEvaluatorTask<D,
               */
               known_cosign = lowest_common_block;
             }
+
+            log::debug!("marking non-notable block #{block_number} as cosigned");
           }
           // If this block has no events necessitating cosigning, we can immediately consider the
           // block cosigned (making this block a NOP)
@@ -213,6 +220,10 @@ impl<D: Db, R: RequestNotableCosigns> ContinuallyRan for CosignEvaluatorTask<D,
         );
         txn.commit();
 
+        if (block_number % 500) == 0 {
+          log::info!("marking block #{block_number} as cosigned");
+        }
+
         made_progress = true;
       }
 

coordinator/cosign/src/intend.rs

@@ -1,5 +1,5 @@
 use core::future::Future;
-use std::collections::HashMap;
+use std::{sync::Arc, collections::HashMap};
 
 use serai_client::{
   primitives::{SeraiAddress, Amount},
@@ -57,18 +57,20 @@ async fn block_has_events_justifying_a_cosign(
 /// A task to determine which blocks we should intend to cosign.
 pub(crate) struct CosignIntendTask<D: Db> {
   pub(crate) db: D,
-  pub(crate) serai: Serai,
+  pub(crate) serai: Arc<Serai>,
 }
 
 impl<D: Db> ContinuallyRan for CosignIntendTask<D> {
-  fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, String>> {
+  type Error = String;
+
+  fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, Self::Error>> {
     async move {
       let start_block_number = ScanCosignFrom::get(&self.db).unwrap_or(1);
       let latest_block_number =
         self.serai.latest_finalized_block().await.map_err(|e| format!("{e:?}"))?.number();
 
       for block_number in start_block_number ..= latest_block_number {
-        let mut txn = self.db.txn();
+        let mut txn = self.db.unsafe_txn();
 
         let (block, mut has_events) =
           block_has_events_justifying_a_cosign(&self.serai, block_number)
@@ -78,7 +80,7 @@ impl<D: Db> ContinuallyRan for CosignIntendTask<D> {
         // Check we are indexing a linear chain
         if (block_number > 1) &&
           (<[u8; 32]>::from(block.header.parent_hash) !=
-            SubstrateBlocks::get(&txn, block_number - 1)
+            SubstrateBlockHash::get(&txn, block_number - 1)
               .expect("indexing a block but haven't indexed its parent"))
         {
           Err(format!(
@@ -86,14 +88,15 @@ impl<D: Db> ContinuallyRan for CosignIntendTask<D> {
             block_number - 1
           ))?;
         }
-        SubstrateBlocks::set(&mut txn, block_number, &block.hash());
+        let block_hash = block.hash();
+        SubstrateBlockHash::set(&mut txn, block_number, &block_hash);
 
         let global_session_for_this_block = LatestGlobalSessionIntended::get(&txn);
 
         // If this is notable, it creates a new global session, which we index into the database
         // now
         if has_events == HasEvents::Notable {
-          let serai = self.serai.as_of(block.hash());
+          let serai = self.serai.as_of(block_hash);
           let sets_and_keys = cosigning_sets(&serai).await?;
           let global_session =
             GlobalSession::id(sets_and_keys.iter().map(|(set, _key)| *set).collect());
@@ -159,7 +162,7 @@ impl<D: Db> ContinuallyRan for CosignIntendTask<D> {
                 &CosignIntent {
                   global_session: global_session_for_this_block,
                   block_number,
-                  block_hash: block.hash(),
+                  block_hash,
                   notable: has_events == HasEvents::Notable,
                 },
               );

coordinator/cosign/src/lib.rs

@@ -3,10 +3,11 @@
 #![deny(missing_docs)]
 
 use core::{fmt::Debug, future::Future};
-use std::collections::HashMap;
+use std::{sync::Arc, collections::HashMap};
 
 use blake2::{Digest, Blake2s256};
 
+use scale::{Encode, Decode};
 use borsh::{BorshSerialize, BorshDeserialize};
 
 use serai_client::{
@@ -28,7 +29,7 @@ pub use delay::BROADCAST_FREQUENCY;
 use delay::LatestCosignedBlockNumber;
 
 /// The schnorrkel context to used when signing a cosign.
-pub const COSIGN_CONTEXT: &[u8] = b"serai-cosign";
+pub const COSIGN_CONTEXT: &[u8] = b"/serai/coordinator/cosign";
 
 /// A 'global session', defined as all validator sets used for cosigning at a given moment.
 ///
@@ -63,12 +64,70 @@ impl GlobalSession {
   }
 }
 
+/// If the block has events.
+#[derive(Clone, Copy, PartialEq, Eq, Debug, BorshSerialize, BorshDeserialize)]
+enum HasEvents {
+  /// The block had a notable event.
+  ///
+  /// This is a special case as blocks with key gen events change the keys used for cosigning, and
+  /// accordingly must be cosigned before we advance past them.
+  Notable,
+  /// The block had an non-notable event justifying a cosign.
+  NonNotable,
+  /// The block didn't have an event justifying a cosign.
+  No,
+}
+
+/// An intended cosign.
+#[derive(Clone, Copy, PartialEq, Eq, Debug, BorshSerialize, BorshDeserialize)]
+pub struct CosignIntent {
+  /// The global session this cosign is being performed under.
+  pub global_session: [u8; 32],
+  /// The number of the block to cosign.
+  pub block_number: u64,
+  /// The hash of the block to cosign.
+  pub block_hash: [u8; 32],
+  /// If this cosign must be handled before further cosigns are.
+  pub notable: bool,
+}
+
+/// A cosign.
+#[derive(Clone, PartialEq, Eq, Debug, Encode, Decode, BorshSerialize, BorshDeserialize)]
+pub struct Cosign {
+  /// The global session this cosign is being performed under.
+  pub global_session: [u8; 32],
+  /// The number of the block to cosign.
+  pub block_number: u64,
+  /// The hash of the block to cosign.
+  pub block_hash: [u8; 32],
+  /// The actual cosigner.
+  pub cosigner: NetworkId,
+}
+
+/// A signed cosign.
+#[derive(Clone, Debug, BorshSerialize, BorshDeserialize)]
+pub struct SignedCosign {
+  /// The cosign.
+  pub cosign: Cosign,
+  /// The signature for the cosign.
+  pub signature: [u8; 64],
+}
+
+impl SignedCosign {
+  fn verify_signature(&self, signer: serai_client::Public) -> bool {
+    let Ok(signer) = schnorrkel::PublicKey::from_bytes(&signer.0) else { return false };
+    let Ok(signature) = schnorrkel::Signature::from_bytes(&self.signature) else { return false };
+
+    signer.verify_simple(COSIGN_CONTEXT, &self.cosign.encode(), &signature).is_ok()
+  }
+}
+
 create_db! {
   Cosign {
     // The following are populated by the intend task and used throughout the library
 
     // An index of Substrate blocks
-    SubstrateBlocks: (block_number: u64) -> [u8; 32],
+    SubstrateBlockHash: (block_number: u64) -> [u8; 32],
     // A mapping from a global session's ID to its relevant information.
     GlobalSessions: (global_session: [u8; 32]) -> GlobalSession,
     // The last block to be cosigned by a global session.
@@ -97,69 +156,16 @@ create_db! {
   }
 }
 
-/// If the block has events.
-#[derive(Clone, Copy, PartialEq, Eq, Debug, BorshSerialize, BorshDeserialize)]
-enum HasEvents {
-  /// The block had a notable event.
-  ///
-  /// This is a special case as blocks with key gen events change the keys used for cosigning, and
-  /// accordingly must be cosigned before we advance past them.
-  Notable,
-  /// The block had an non-notable event justifying a cosign.
-  NonNotable,
-  /// The block didn't have an event justifying a cosign.
-  No,
-}
-
-/// An intended cosign.
-#[derive(Clone, Copy, PartialEq, Eq, Debug, BorshSerialize, BorshDeserialize)]
-struct CosignIntent {
-  /// The global session this cosign is being performed under.
-  global_session: [u8; 32],
-  /// The number of the block to cosign.
-  block_number: u64,
-  /// The hash of the block to cosign.
-  block_hash: [u8; 32],
-  /// If this cosign must be handled before further cosigns are.
-  notable: bool,
-}
-
-/// A cosign.
-#[derive(Clone, PartialEq, Eq, Debug, BorshSerialize, BorshDeserialize)]
-pub struct Cosign {
-  /// The global session this cosign is being performed under.
-  pub global_session: [u8; 32],
-  /// The number of the block to cosign.
-  pub block_number: u64,
-  /// The hash of the block to cosign.
-  pub block_hash: [u8; 32],
-  /// The actual cosigner.
-  pub cosigner: NetworkId,
-}
-
-/// A signed cosign.
-#[derive(Clone, Debug, BorshSerialize, BorshDeserialize)]
-pub struct SignedCosign {
-  /// The cosign.
-  pub cosign: Cosign,
-  /// The signature for the cosign.
-  pub signature: [u8; 64],
-}
-
-impl SignedCosign {
-  fn verify_signature(&self, signer: serai_client::Public) -> bool {
-    let Ok(signer) = schnorrkel::PublicKey::from_bytes(&signer.0) else { return false };
-    let Ok(signature) = schnorrkel::Signature::from_bytes(&self.signature) else { return false };
-
-    signer.verify_simple(COSIGN_CONTEXT, &borsh::to_vec(&self.cosign).unwrap(), &signature).is_ok()
-  }
-}
-
 /// Fetch the keys used for cosigning by a specific network.
 async fn keys_for_network(
   serai: &TemporalSerai<'_>,
   network: NetworkId,
 ) -> Result<Option<(Session, KeyPair)>, String> {
+  // The Serai network never cosigns so it has no keys for cosigning
+  if network == NetworkId::Serai {
+    return Ok(None);
+  }
+
   let Some(latest_session) =
     serai.validator_sets().session(network).await.map_err(|e| format!("{e:?}"))?
   else {
@@ -219,8 +225,46 @@ pub trait RequestNotableCosigns: 'static + Send {
 }
 
 /// An error used to indicate the cosigning protocol has faulted.
+#[derive(Debug)]
 pub struct Faulted;
 
+/// An error incurred while intaking a cosign.
+#[derive(Debug)]
+pub enum IntakeCosignError {
+  /// Cosign is for a not-yet-indexed block
+  NotYetIndexedBlock,
+  /// A later cosign for this cosigner has already been handled
+  StaleCosign,
+  /// The cosign's global session isn't recognized
+  UnrecognizedGlobalSession,
+  /// The cosign is for a block before its global session starts
+  BeforeGlobalSessionStart,
+  /// The cosign is for a block after its global session ends
+  AfterGlobalSessionEnd,
+  /// The cosign's signing network wasn't a participant in this global session
+  NonParticipatingNetwork,
+  /// The cosign had an invalid signature
+  InvalidSignature,
+  /// The cosign is for a global session which has yet to have its declaration block cosigned
+  FutureGlobalSession,
+}
+
+impl IntakeCosignError {
+  /// If this error is temporal to the local view
+  pub fn temporal(&self) -> bool {
+    match self {
+      IntakeCosignError::NotYetIndexedBlock |
+      IntakeCosignError::StaleCosign |
+      IntakeCosignError::UnrecognizedGlobalSession |
+      IntakeCosignError::FutureGlobalSession => true,
+      IntakeCosignError::BeforeGlobalSessionStart |
+      IntakeCosignError::AfterGlobalSessionEnd |
+      IntakeCosignError::NonParticipatingNetwork |
+      IntakeCosignError::InvalidSignature => false,
+    }
+  }
+}
+
 /// The interface to manage cosigning with.
 pub struct Cosigning<D: Db> {
   db: D,
@@ -232,7 +276,7 @@ impl<D: Db> Cosigning<D> {
   /// only used once at any given time.
   pub fn spawn<R: RequestNotableCosigns>(
     db: D,
-    serai: Serai,
+    serai: Arc<Serai>,
     request: R,
     tasks_to_run_upon_cosigning: Vec<TaskHandle>,
   ) -> Self {
@@ -255,22 +299,33 @@ impl<D: Db> Cosigning<D> {
   }
 
   /// The latest cosigned block number.
-  pub fn latest_cosigned_block_number(&self) -> Result<u64, Faulted> {
-    if FaultedSession::get(&self.db).is_some() {
+  pub fn latest_cosigned_block_number(getter: &impl Get) -> Result<u64, Faulted> {
+    if FaultedSession::get(getter).is_some() {
       Err(Faulted)?;
     }
 
-    Ok(LatestCosignedBlockNumber::get(&self.db).unwrap_or(0))
+    Ok(LatestCosignedBlockNumber::get(getter).unwrap_or(0))
+  }
+
+  /// Fetch a cosigned Substrate block's hash by its block number.
+  pub fn cosigned_block(getter: &impl Get, block_number: u64) -> Result<Option<[u8; 32]>, Faulted> {
+    if block_number > Self::latest_cosigned_block_number(getter)? {
+      return Ok(None);
+    }
+
+    Ok(Some(
+      SubstrateBlockHash::get(getter, block_number).expect("cosigned block but didn't index it"),
+    ))
   }
 
   /// Fetch the notable cosigns for a global session in order to respond to requests.
   ///
   /// If this global session hasn't produced any notable cosigns, this will return the latest
   /// cosigns for this session.
-  pub fn notable_cosigns(&self, global_session: [u8; 32]) -> Vec<SignedCosign> {
+  pub fn notable_cosigns(getter: &impl Get, global_session: [u8; 32]) -> Vec<SignedCosign> {
     let mut cosigns = Vec::with_capacity(serai_client::primitives::NETWORKS.len());
     for network in serai_client::primitives::NETWORKS {
-      if let Some(cosign) = NetworksLatestCosignedBlock::get(&self.db, global_session, network) {
+      if let Some(cosign) = NetworksLatestCosignedBlock::get(getter, global_session, network) {
         cosigns.push(cosign);
       }
     }
@@ -295,14 +350,12 @@ impl<D: Db> Cosigning<D> {
       }
       cosigns
     } else {
-      let Some(latest_global_session) = LatestGlobalSessionIntended::get(&self.db) else {
+      let Some(global_session) = evaluator::currently_evaluated_global_session(&self.db) else {
         return vec![];
       };
       let mut cosigns = Vec::with_capacity(serai_client::primitives::NETWORKS.len());
       for network in serai_client::primitives::NETWORKS {
-        if let Some(cosign) =
-          NetworksLatestCosignedBlock::get(&self.db, latest_global_session, network)
-        {
+        if let Some(cosign) = NetworksLatestCosignedBlock::get(&self.db, global_session, network) {
           cosigns.push(cosign);
         }
       }
@@ -310,27 +363,16 @@ impl<D: Db> Cosigning<D> {
     }
   }
 
-  /// Intake a cosign from the Serai network.
-  ///
-  /// - Returns Err(_) if there was an error trying to validate the cosign and it should be retired
-  ///   later.
-  /// - Returns Ok(true) if the cosign was successfully handled or could not be handled at this
-  ///   time.
-  /// - Returns Ok(false) if the cosign was invalid.
-  //
-  // We collapse a cosign which shouldn't be handled yet into a valid cosign (`Ok(true)`) as we
-  // assume we'll either explicitly request it if we need it or we'll naturally see it (or a later,
-  // more relevant, cosign) again.
+  /// Intake a cosign.
   //
   // Takes `&mut self` as this should only be called once at any given moment.
-  // TODO: Don't overload bool here
-  pub fn intake_cosign(&mut self, signed_cosign: &SignedCosign) -> Result<bool, String> {
+  pub fn intake_cosign(&mut self, signed_cosign: &SignedCosign) -> Result<(), IntakeCosignError> {
     let cosign = &signed_cosign.cosign;
     let network = cosign.cosigner;
 
     // Check our indexed blockchain includes a block with this block number
-    let Some(our_block_hash) = SubstrateBlocks::get(&self.db, cosign.block_number) else {
-      return Ok(true);
+    let Some(our_block_hash) = SubstrateBlockHash::get(&self.db, cosign.block_number) else {
+      Err(IntakeCosignError::NotYetIndexedBlock)?
     };
     let faulty = cosign.block_hash != our_block_hash;
 
@@ -340,20 +382,19 @@ impl<D: Db> Cosigning<D> {
         NetworksLatestCosignedBlock::get(&self.db, cosign.global_session, network)
       {
         if existing.cosign.block_number >= cosign.block_number {
-          return Ok(true);
+          Err(IntakeCosignError::StaleCosign)?;
         }
       }
     }
 
     let Some(global_session) = GlobalSessions::get(&self.db, cosign.global_session) else {
-      // Unrecognized global session
-      return Ok(true);
+      Err(IntakeCosignError::UnrecognizedGlobalSession)?
     };
 
     // Check the cosigned block number is in range to the global session
     if cosign.block_number < global_session.start_block_number {
       // Cosign is for a block predating the global session
-      return Ok(false);
+      Err(IntakeCosignError::BeforeGlobalSessionStart)?;
     }
     if !faulty {
       // This prevents a malicious validator set, on the same chain, from producing a cosign after
@@ -361,7 +402,7 @@ impl<D: Db> Cosigning<D> {
       if let Some(last_block) = GlobalSessionsLastBlock::get(&self.db, cosign.global_session) {
         if cosign.block_number > last_block {
           // Cosign is for a block after the last block this global session should have signed
-          return Ok(false);
+          Err(IntakeCosignError::AfterGlobalSessionEnd)?;
         }
       }
     }
@@ -370,20 +411,20 @@ impl<D: Db> Cosigning<D> {
     {
       let key = Public::from({
         let Some(key) = global_session.keys.get(&network) else {
-          return Ok(false);
+          Err(IntakeCosignError::NonParticipatingNetwork)?
         };
         *key
       });
 
       if !signed_cosign.verify_signature(key) {
-        return Ok(false);
+        Err(IntakeCosignError::InvalidSignature)?;
       }
     }
 
     // Since we verified this cosign's signature, and have a chain sufficiently long, handle the
     // cosign
 
-    let mut txn = self.db.txn();
+    let mut txn = self.db.unsafe_txn();
 
     if !faulty {
       // If this is for a future global session, we don't acknowledge this cosign at this time
@@ -392,7 +433,7 @@ impl<D: Db> Cosigning<D> {
       // block declaring it was cosigned
       if (global_session.start_block_number - 1) > latest_cosigned_block_number {
         drop(txn);
-        return Ok(true);
+        return Err(IntakeCosignError::FutureGlobalSession);
       }
 
       // This is safe as it's in-range and newer, as prior checked since it isn't faulty
@@ -406,9 +447,10 @@ impl<D: Db> Cosigning<D> {
 
         let mut weight_cosigned = 0;
         for fault in &faults {
-          let Some(stake) = global_session.stakes.get(&fault.cosign.cosigner) else {
-            Err("cosigner with recognized key didn't have a stake entry saved".to_string())?
-          };
+          let stake = global_session
+            .stakes
+            .get(&fault.cosign.cosigner)
+            .expect("cosigner with recognized key didn't have a stake entry saved");
           weight_cosigned += stake;
         }
 
@@ -420,6 +462,48 @@ impl<D: Db> Cosigning<D> {
     }
 
     txn.commit();
-    Ok(true)
+    Ok(())
+  }
+
+  /// Receive intended cosigns to produce for this ValidatorSet.
+  ///
+  /// All cosigns intended, up to and including the next notable cosign, are returned.
+  ///
+  /// This will drain the internal channel and not re-yield these intentions again.
+  pub fn intended_cosigns(txn: &mut impl DbTxn, set: ValidatorSet) -> Vec<CosignIntent> {
+    let mut res: Vec<CosignIntent> = vec![];
+    // While we have yet to find a notable cosign...
+    while !res.last().map(|cosign| cosign.notable).unwrap_or(false) {
+      let Some(intent) = intend::IntendedCosigns::try_recv(txn, set) else { break };
+      res.push(intent);
+    }
+    res
+  }
+}
+
+mod tests {
+  use super::*;
+
+  struct RNC;
+  impl RequestNotableCosigns for RNC {
+    /// The error type which may be encountered when requesting notable cosigns.
+    type Error = ();
+
+    /// Request the notable cosigns for this global session.
+    fn request_notable_cosigns(
+      &self,
+      global_session: [u8; 32],
+    ) -> impl Send + Future<Output = Result<(), Self::Error>> {
+      async move { Ok(()) }
+    }
+  }
+  #[tokio::test]
+  async fn test() {
+    let db: serai_db::MemDb = serai_db::MemDb::new();
+    let serai = unsafe { core::mem::transmute(0u64) };
+    let request = RNC;
+    let tasks = vec![];
+    let _ = Cosigning::spawn(db, serai, request, tasks);
+    core::future::pending().await
   }
 }

coordinator/p2p/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "serai-coordinator-p2p"
+version = "0.1.0"
+description = "Serai coordinator's P2P abstraction"
+license = "AGPL-3.0-only"
+repository = "https://github.com/serai-dex/serai/tree/develop/coordinator/p2p"
+authors = ["Luke Parker <lukeparker5132@gmail.com>"]
+keywords = []
+edition = "2021"
+publish = false
+rust-version = "1.81"
+
+[package.metadata.docs.rs]
+all-features = true
+rustdoc-args = ["--cfg", "docsrs"]
+
+[lints]
+workspace = true
+
+[dependencies]
+borsh = { version = "1", default-features = false, features = ["std", "derive", "de_strict_order"] }
+
+serai-db = { path = "../../common/db", version = "0.1" }
+
+serai-client = { path = "../../substrate/client", default-features = false, features = ["serai", "borsh"] }
+serai-cosign = { path = "../cosign" }
+tributary-sdk = { path = "../tributary-sdk" }
+
+futures-lite = { version = "2", default-features = false, features = ["std"] }
+tokio = { version = "1", default-features = false, features = ["sync", "macros"] }
+
+log = { version = "0.4", default-features = false, features = ["std"] }
+serai-task = { path = "../../common/task", version = "0.1" }

coordinator/p2p/LICENSE

@@ -0,0 +1,15 @@
+AGPL-3.0-only license
+
+Copyright (c) 2023-2025 Luke Parker
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License Version 3 as
+published by the Free Software Foundation.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.

coordinator/p2p/README.md

@@ -0,0 +1,3 @@
+# Serai Coordinator P2P
+
+The P2P abstraction used by Serai's coordinator, and tasks over it.

coordinator/p2p/libp2p/Cargo.toml

@@ -0,0 +1,42 @@
+[package]
+name = "serai-coordinator-libp2p-p2p"
+version = "0.1.0"
+description = "Serai coordinator's libp2p-based P2P backend"
+license = "AGPL-3.0-only"
+repository = "https://github.com/serai-dex/serai/tree/develop/coordinator/p2p/libp2p"
+authors = ["Luke Parker <lukeparker5132@gmail.com>"]
+keywords = []
+edition = "2021"
+publish = false
+rust-version = "1.81"
+
+[package.metadata.docs.rs]
+all-features = true
+rustdoc-args = ["--cfg", "docsrs"]
+
+[lints]
+workspace = true
+
+[dependencies]
+async-trait = { version = "0.1", default-features = false }
+
+rand_core = { version = "0.6", default-features = false, features = ["std"] }
+
+zeroize = { version = "^1.5", default-features = false, features = ["std"] }
+blake2 = { version = "0.10", default-features = false, features = ["std"] }
+schnorrkel = { version = "0.11", default-features = false, features = ["std"] }
+
+hex = { version = "0.4", default-features = false, features = ["std"] }
+borsh = { version = "1", default-features = false, features = ["std", "derive", "de_strict_order"] }
+
+serai-client = { path = "../../../substrate/client", default-features = false, features = ["serai", "borsh"] }
+serai-cosign = { path = "../../cosign" }
+tributary-sdk = { path = "../../tributary-sdk" }
+
+futures-util = { version = "0.3", default-features = false, features = ["std"] }
+tokio = { version = "1", default-features = false, features = ["sync"] }
+libp2p = { version = "0.52", default-features = false, features = ["tokio", "tcp", "noise", "yamux", "ping", "request-response", "gossipsub", "macros"] }
+
+log = { version = "0.4", default-features = false, features = ["std"] }
+serai-task = { path = "../../../common/task", version = "0.1" }
+serai-coordinator-p2p = { path = "../" }

coordinator/p2p/libp2p/LICENSE

@@ -0,0 +1,15 @@
+AGPL-3.0-only license
+
+Copyright (c) 2023-2025 Luke Parker
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License Version 3 as
+published by the Free Software Foundation.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.

coordinator/p2p/libp2p/README.md

@@ -0,0 +1,14 @@
+# Serai Coordinator libp2p P2P
+
+A libp2p-backed P2P instantiation for Serai's coordinator.
+
+The libp2p swarm is limited to validators from the Serai network. The swarm
+does not maintain any of its own peer finding/routing infrastructure, instead
+relying on the Serai network's connection information to dial peers. This does
+limit the listening peers to only the peers immediately reachable via the same
+IP address (despite the two distinct services), not hidden behind a NAT, yet is
+also quite simple and gives full control of who to connect to to us.
+
+Peers are decided via the internal `DialTask` which aims to maintain a target
+amount of peers for each external network. This ensures cosigns are able to
+propagate across the external networks which sign them.

coordinator/p2p/libp2p/src/authenticate.rs

@@ -0,0 +1,176 @@
+use core::{pin::Pin, future::Future};
+use std::io;
+
+use zeroize::Zeroizing;
+use rand_core::{RngCore, OsRng};
+
+use blake2::{Digest, Blake2s256};
+use schnorrkel::{Keypair, PublicKey, Signature};
+
+use serai_client::primitives::PublicKey as Public;
+
+use futures_util::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
+use libp2p::{
+  core::UpgradeInfo,
+  InboundUpgrade, OutboundUpgrade,
+  identity::{self, PeerId},
+  noise,
+};
+
+use crate::peer_id_from_public;
+
+const PROTOCOL: &str = "/serai/coordinator/validators";
+
+#[derive(Clone)]
+pub(crate) struct OnlyValidators {
+  pub(crate) serai_key: Zeroizing<Keypair>,
+  pub(crate) noise_keypair: identity::Keypair,
+}
+
+impl OnlyValidators {
+  /// The ephemeral challenge protocol for authentication.
+  ///
+  /// We use ephemeral challenges to prevent replaying signatures from historic sessions.
+  ///
+  /// We don't immediately send the challenge. We only send a commitment to it. This prevents our
+  /// remote peer from choosing their challenge in response to our challenge, in case there was any
+  /// benefit to doing so.
+  async fn challenges<S: 'static + Send + Unpin + AsyncRead + AsyncWrite>(
+    socket: &mut noise::Output<S>,
+  ) -> io::Result<([u8; 32], [u8; 32])> {
+    let mut our_challenge = [0; 32];
+    OsRng.fill_bytes(&mut our_challenge);
+
+    // Write the hash of our challenge
+    socket.write_all(&Blake2s256::digest(our_challenge)).await?;
+
+    // Read the hash of their challenge
+    let mut their_challenge_commitment = [0; 32];
+    socket.read_exact(&mut their_challenge_commitment).await?;
+
+    // Reveal our challenge
+    socket.write_all(&our_challenge).await?;
+
+    // Read their challenge
+    let mut their_challenge = [0; 32];
+    socket.read_exact(&mut their_challenge).await?;
+
+    // Verify their challenge
+    if <[u8; 32]>::from(Blake2s256::digest(their_challenge)) != their_challenge_commitment {
+      Err(io::Error::other("challenge didn't match challenge commitment"))?;
+    }
+
+    Ok((our_challenge, their_challenge))
+  }
+
+  // We sign the two noise peer IDs and the ephemeral challenges.
+  //
+  // Signing the noise peer IDs ensures we're authenticating this noise connection. The only
+  // expectations placed on noise are for it to prevent a MITM from impersonating the other end or
+  // modifying any messages sent.
+  //
+  // Signing the ephemeral challenges prevents any replays. While that should be unnecessary, as
+  // noise MAY prevent replays across sessions (even when the same key is used), and noise IDs
+  // shouldn't be reused (so it should be fine to reuse an existing signature for these noise IDs),
+  // it doesn't hurt.
+  async fn authenticate<S: 'static + Send + Unpin + AsyncRead + AsyncWrite>(
+    &self,
+    socket: &mut noise::Output<S>,
+    dialer_peer_id: PeerId,
+    dialer_challenge: [u8; 32],
+    listener_peer_id: PeerId,
+    listener_challenge: [u8; 32],
+  ) -> io::Result<PeerId> {
+    // Write our public key
+    socket.write_all(&self.serai_key.public.to_bytes()).await?;
+
+    let msg = borsh::to_vec(&(
+      dialer_peer_id.to_bytes(),
+      dialer_challenge,
+      listener_peer_id.to_bytes(),
+      listener_challenge,
+    ))
+    .unwrap();
+    let signature = self.serai_key.sign_simple(PROTOCOL.as_bytes(), &msg);
+    socket.write_all(&signature.to_bytes()).await?;
+
+    let mut public_key_and_sig = [0; 96];
+    socket.read_exact(&mut public_key_and_sig).await?;
+    let public_key = PublicKey::from_bytes(&public_key_and_sig[.. 32])
+      .map_err(|_| io::Error::other("invalid public key"))?;
+    let sig = Signature::from_bytes(&public_key_and_sig[32 ..])
+      .map_err(|_| io::Error::other("invalid signature serialization"))?;
+
+    public_key
+      .verify_simple(PROTOCOL.as_bytes(), &msg, &sig)
+      .map_err(|_| io::Error::other("invalid signature"))?;
+
+    Ok(peer_id_from_public(Public::from_raw(public_key.to_bytes())))
+  }
+}
+
+impl UpgradeInfo for OnlyValidators {
+  type Info = <noise::Config as UpgradeInfo>::Info;
+  type InfoIter = <noise::Config as UpgradeInfo>::InfoIter;
+  fn protocol_info(&self) -> Self::InfoIter {
+    // A keypair only causes an error if its sign operation fails, which is only possible with RSA,
+    // which isn't used within this codebase
+    noise::Config::new(&self.noise_keypair).unwrap().protocol_info()
+  }
+}
+
+impl<S: 'static + Send + Unpin + AsyncRead + AsyncWrite> InboundUpgrade<S> for OnlyValidators {
+  type Output = (PeerId, noise::Output<S>);
+  type Error = io::Error;
+  type Future = Pin<Box<dyn Send + Future<Output = Result<Self::Output, Self::Error>>>>;
+
+  fn upgrade_inbound(self, socket: S, info: Self::Info) -> Self::Future {
+    Box::pin(async move {
+      let (dialer_noise_peer_id, mut socket) = noise::Config::new(&self.noise_keypair)
+        .unwrap()
+        .upgrade_inbound(socket, info)
+        .await
+        .map_err(io::Error::other)?;
+
+      let (our_challenge, dialer_challenge) = OnlyValidators::challenges(&mut socket).await?;
+      let dialer_serai_validator = self
+        .authenticate(
+          &mut socket,
+          dialer_noise_peer_id,
+          dialer_challenge,
+          PeerId::from_public_key(&self.noise_keypair.public()),
+          our_challenge,
+        )
+        .await?;
+      Ok((dialer_serai_validator, socket))
+    })
+  }
+}
+
+impl<S: 'static + Send + Unpin + AsyncRead + AsyncWrite> OutboundUpgrade<S> for OnlyValidators {
+  type Output = (PeerId, noise::Output<S>);
+  type Error = io::Error;
+  type Future = Pin<Box<dyn Send + Future<Output = Result<Self::Output, Self::Error>>>>;
+
+  fn upgrade_outbound(self, socket: S, info: Self::Info) -> Self::Future {
+    Box::pin(async move {
+      let (listener_noise_peer_id, mut socket) = noise::Config::new(&self.noise_keypair)
+        .unwrap()
+        .upgrade_outbound(socket, info)
+        .await
+        .map_err(io::Error::other)?;
+
+      let (our_challenge, listener_challenge) = OnlyValidators::challenges(&mut socket).await?;
+      let listener_serai_validator = self
+        .authenticate(
+          &mut socket,
+          PeerId::from_public_key(&self.noise_keypair.public()),
+          our_challenge,
+          listener_noise_peer_id,
+          listener_challenge,
+        )
+        .await?;
+      Ok((listener_serai_validator, socket))
+    })
+  }
+}

coordinator/p2p/libp2p/src/dial.rs

@@ -0,0 +1,127 @@
+use core::future::Future;
+use std::{sync::Arc, collections::HashSet};
+
+use rand_core::{RngCore, OsRng};
+
+use tokio::sync::mpsc;
+
+use serai_client::{SeraiError, Serai};
+
+use libp2p::{
+  core::multiaddr::{Protocol, Multiaddr},
+  swarm::dial_opts::DialOpts,
+};
+
+use serai_task::ContinuallyRan;
+
+use crate::{PORT, Peers, validators::Validators};
+
+const TARGET_PEERS_PER_NETWORK: usize = 5;
+/*
+  If we only tracked the target amount of peers per network, we'd risk being eclipsed by an
+  adversary who immediately connects to us with their array of validators upon our boot. Their
+  array would satisfy our target amount of peers, so we'd never seek more, enabling the adversary
+  to be the only entity we peered with.
+
+  We solve this by additionally requiring an explicit amount of peers we dialed. That means we
+  randomly chose to connect to these peers.
+*/
+// TODO const TARGET_DIALED_PEERS_PER_NETWORK: usize = 3;
+
+pub(crate) struct DialTask {
+  serai: Arc<Serai>,
+  validators: Validators,
+  peers: Peers,
+  to_dial: mpsc::UnboundedSender<DialOpts>,
+}
+
+impl DialTask {
+  pub(crate) fn new(
+    serai: Arc<Serai>,
+    peers: Peers,
+    to_dial: mpsc::UnboundedSender<DialOpts>,
+  ) -> Self {
+    DialTask { serai: serai.clone(), validators: Validators::new(serai).0, peers, to_dial }
+  }
+}
+
+impl ContinuallyRan for DialTask {
+  // Only run every five minutes, not the default of every five seconds
+  const DELAY_BETWEEN_ITERATIONS: u64 = 5 * 60;
+  const MAX_DELAY_BETWEEN_ITERATIONS: u64 = 10 * 60;
+
+  type Error = SeraiError;
+
+  fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, Self::Error>> {
+    async move {
+      self.validators.update().await?;
+
+      // If any of our peers is lacking, try to connect to more
+      let mut dialed = false;
+      let peer_counts = self
+        .peers
+        .peers
+        .read()
+        .await
+        .iter()
+        .map(|(network, peers)| (*network, peers.len()))
+        .collect::<Vec<_>>();
+      for (network, peer_count) in peer_counts {
+        /*
+          If we don't have the target amount of peers, and we don't have all the validators in the
+          set but one, attempt to connect to more validators within this set.
+
+          The latter clause is so if there's a set with only 3 validators, we don't infinitely try
+          to connect to the target amount of peers for this network as we never will. Instead, we
+          only try to connect to most of the validators actually present.
+        */
+        if (peer_count < TARGET_PEERS_PER_NETWORK) &&
+          (peer_count <
+            self
+              .validators
+              .by_network()
+              .get(&network)
+              .map(HashSet::len)
+              .unwrap_or(0)
+              .saturating_sub(1))
+        {
+          let mut potential_peers = self.serai.p2p_validators(network).await?;
+          for _ in 0 .. (TARGET_PEERS_PER_NETWORK - peer_count) {
+            if potential_peers.is_empty() {
+              break;
+            }
+            let index_to_dial =
+              usize::try_from(OsRng.next_u64() % u64::try_from(potential_peers.len()).unwrap())
+                .unwrap();
+            let randomly_selected_peer = potential_peers.swap_remove(index_to_dial);
+
+            log::info!("found peer from substrate: {randomly_selected_peer}");
+
+            // Map the peer from a Substrate P2P network peer to a Coordinator P2P network peer
+            let mapped_peer = randomly_selected_peer
+              .into_iter()
+              .filter_map(|protocol| match protocol {
+                // Drop PeerIds from the Substrate P2p network
+                Protocol::P2p(_) => None,
+                // Use our own TCP port
+                Protocol::Tcp(_) => Some(Protocol::Tcp(PORT)),
+                // Pass-through any other specifications (IPv4, IPv6, etc)
+                other => Some(other),
+              })
+              .collect::<Multiaddr>();
+
+            log::debug!("mapped found peer: {mapped_peer}");
+
+            self
+              .to_dial
+              .send(DialOpts::unknown_peer_id().address(mapped_peer).build())
+              .expect("dial receiver closed?");
+            dialed = true;
+          }
+        }
+      }
+
+      Ok(dialed)
+    }
+  }
+}

coordinator/p2p/libp2p/src/gossip.rs

@@ -0,0 +1,75 @@
+use core::time::Duration;
+
+use blake2::{Digest, Blake2s256};
+
+use borsh::{BorshSerialize, BorshDeserialize};
+
+use libp2p::gossipsub::{
+  IdentTopic, MessageId, MessageAuthenticity, ValidationMode, ConfigBuilder, IdentityTransform,
+  AllowAllSubscriptionFilter, Behaviour,
+};
+pub use libp2p::gossipsub::Event;
+
+use serai_cosign::SignedCosign;
+
+// Block size limit + 16 KB of space for signatures/metadata
+pub(crate) const MAX_LIBP2P_GOSSIP_MESSAGE_SIZE: usize = tributary_sdk::BLOCK_SIZE_LIMIT + 16384;
+
+const LIBP2P_PROTOCOL: &str = "/serai/coordinator/gossip/1.0.0";
+const BASE_TOPIC: &str = "/";
+
+fn topic_for_tributary(tributary: [u8; 32]) -> IdentTopic {
+  IdentTopic::new(format!("/tributary/{}", hex::encode(tributary)))
+}
+
+#[derive(Clone, BorshSerialize, BorshDeserialize)]
+pub(crate) enum Message {
+  Tributary { tributary: [u8; 32], message: Vec<u8> },
+  Cosign(SignedCosign),
+}
+
+impl Message {
+  pub(crate) fn topic(&self) -> IdentTopic {
+    match self {
+      Message::Tributary { tributary, .. } => topic_for_tributary(*tributary),
+      Message::Cosign(_) => IdentTopic::new(BASE_TOPIC),
+    }
+  }
+}
+
+pub(crate) type Behavior = Behaviour<IdentityTransform, AllowAllSubscriptionFilter>;
+
+pub(crate) fn new_behavior() -> Behavior {
+  // The latency used by the Tendermint protocol, used here as the gossip epoch duration
+  // libp2p-rs defaults to 1 second, whereas ours will be ~2
+  let heartbeat_interval = tributary_sdk::tendermint::LATENCY_TIME;
+  // The amount of heartbeats which will occur within a single Tributary block
+  let heartbeats_per_block =
+    tributary_sdk::tendermint::TARGET_BLOCK_TIME.div_ceil(heartbeat_interval);
+  // libp2p-rs defaults to 5, whereas ours will be ~8
+  let heartbeats_to_keep = 2 * heartbeats_per_block;
+  // libp2p-rs defaults to 3 whereas ours will be ~4
+  let heartbeats_to_gossip = heartbeats_per_block;
+
+  let config = ConfigBuilder::default()
+    .protocol_id_prefix(LIBP2P_PROTOCOL)
+    .history_length(usize::try_from(heartbeats_to_keep).unwrap())
+    .history_gossip(usize::try_from(heartbeats_to_gossip).unwrap())
+    .heartbeat_interval(Duration::from_millis(heartbeat_interval.into()))
+    .max_transmit_size(MAX_LIBP2P_GOSSIP_MESSAGE_SIZE)
+    .duplicate_cache_time(Duration::from_millis((heartbeats_to_keep * heartbeat_interval).into()))
+    .validation_mode(ValidationMode::Anonymous)
+    // Uses a content based message ID to avoid duplicates as much as possible
+    .message_id_fn(|msg| {
+      MessageId::new(&Blake2s256::digest([msg.topic.as_str().as_bytes(), &msg.data].concat()))
+    })
+    .build();
+
+  let mut gossip = Behavior::new(MessageAuthenticity::Anonymous, config.unwrap()).unwrap();
+
+  // Subscribe to the base topic
+  let topic = IdentTopic::new(BASE_TOPIC);
+  let _ = gossip.subscribe(&topic);
+
+  gossip
+}

coordinator/p2p/libp2p/src/lib.rs

@@ -0,0 +1,433 @@
+#![cfg_attr(docsrs, feature(doc_auto_cfg))]
+#![doc = include_str!("../README.md")]
+#![deny(missing_docs)]
+
+use core::{future::Future, time::Duration};
+use std::{
+  sync::Arc,
+  collections::{HashSet, HashMap},
+};
+
+use rand_core::{RngCore, OsRng};
+
+use zeroize::Zeroizing;
+use schnorrkel::Keypair;
+
+use serai_client::{
+  primitives::{NetworkId, PublicKey},
+  validator_sets::primitives::ValidatorSet,
+  Serai,
+};
+
+use tokio::sync::{mpsc, oneshot, Mutex, RwLock};
+
+use serai_task::{Task, ContinuallyRan};
+
+use serai_cosign::SignedCosign;
+
+use libp2p::{
+  multihash::Multihash,
+  identity::{self, PeerId},
+  tcp::Config as TcpConfig,
+  yamux, allow_block_list,
+  connection_limits::{self, ConnectionLimits},
+  swarm::NetworkBehaviour,
+  SwarmBuilder,
+};
+
+use serai_coordinator_p2p::{Heartbeat, TributaryBlockWithCommit};
+
+/// A struct to sync the validators from the Serai node in order to keep track of them.
+mod validators;
+use validators::UpdateValidatorsTask;
+
+/// The authentication protocol upgrade to limit the P2P network to active validators.
+mod authenticate;
+use authenticate::OnlyValidators;
+
+/// The ping behavior, used to ensure connection latency is below the limit
+mod ping;
+
+/// The request-response messages and behavior
+mod reqres;
+use reqres::{RequestId, Request, Response};
+
+/// The gossip messages and behavior
+mod gossip;
+use gossip::Message;
+
+/// The swarm task, running it and dispatching to/from it
+mod swarm;
+use swarm::SwarmTask;
+
+/// The dial task, to find new peers to connect to
+mod dial;
+use dial::DialTask;
+
+const PORT: u16 = 30563; // 5132 ^ (('c' << 8) | 'o')
+
+// usize::max, manually implemented, as max isn't a const fn
+const MAX_LIBP2P_MESSAGE_SIZE: usize =
+  if gossip::MAX_LIBP2P_GOSSIP_MESSAGE_SIZE > reqres::MAX_LIBP2P_REQRES_MESSAGE_SIZE {
+    gossip::MAX_LIBP2P_GOSSIP_MESSAGE_SIZE
+  } else {
+    reqres::MAX_LIBP2P_REQRES_MESSAGE_SIZE
+  };
+
+fn peer_id_from_public(public: PublicKey) -> PeerId {
+  // 0 represents the identity Multihash, that no hash was performed
+  // It's an internal constant so we can't refer to the constant inside libp2p
+  PeerId::from_multihash(Multihash::wrap(0, &public.0).unwrap()).unwrap()
+}
+
+/// The representation of a peer.
+pub struct Peer<'a> {
+  outbound_requests: &'a mpsc::UnboundedSender<(PeerId, Request, oneshot::Sender<Response>)>,
+  id: PeerId,
+}
+impl serai_coordinator_p2p::Peer<'_> for Peer<'_> {
+  fn send_heartbeat(
+    &self,
+    heartbeat: Heartbeat,
+  ) -> impl Send + Future<Output = Option<Vec<TributaryBlockWithCommit>>> {
+    async move {
+      const HEARTBEAT_TIMEOUT: Duration = Duration::from_secs(5);
+
+      let request = Request::Heartbeat(heartbeat);
+      let (sender, receiver) = oneshot::channel();
+      self
+        .outbound_requests
+        .send((self.id, request, sender))
+        .expect("outbound requests recv channel was dropped?");
+      if let Ok(Ok(Response::Blocks(blocks))) =
+        tokio::time::timeout(HEARTBEAT_TIMEOUT, receiver).await
+      {
+        Some(blocks)
+      } else {
+        None
+      }
+    }
+  }
+}
+
+#[derive(Clone)]
+struct Peers {
+  peers: Arc<RwLock<HashMap<NetworkId, HashSet<PeerId>>>>,
+}
+
+// Consider adding identify/kad/autonat/rendevous/(relay + dcutr). While we currently use the Serai
+// network for peers, we could use it solely for bootstrapping/as a fallback.
+#[derive(NetworkBehaviour)]
+struct Behavior {
+  // Used to only allow Serai validators as peers
+  allow_list: allow_block_list::Behaviour<allow_block_list::AllowedPeers>,
+  // Used to limit each peer to a single connection
+  connection_limits: connection_limits::Behaviour,
+  // Used to ensure connection latency is within tolerances
+  ping: ping::Behavior,
+  // Used to request data from specific peers
+  reqres: reqres::Behavior,
+  // Used to broadcast messages to all other peers subscribed to a topic
+  gossip: gossip::Behavior,
+}
+
+#[allow(clippy::type_complexity)]
+struct Libp2pInner {
+  peers: Peers,
+
+  gossip: mpsc::UnboundedSender<Message>,
+  outbound_requests: mpsc::UnboundedSender<(PeerId, Request, oneshot::Sender<Response>)>,
+
+  tributary_gossip: Mutex<mpsc::UnboundedReceiver<([u8; 32], Vec<u8>)>>,
+
+  signed_cosigns: Mutex<mpsc::UnboundedReceiver<SignedCosign>>,
+  signed_cosigns_send: mpsc::UnboundedSender<SignedCosign>,
+
+  heartbeat_requests: Mutex<mpsc::UnboundedReceiver<(RequestId, ValidatorSet, [u8; 32])>>,
+  notable_cosign_requests: Mutex<mpsc::UnboundedReceiver<(RequestId, [u8; 32])>>,
+  inbound_request_responses: mpsc::UnboundedSender<(RequestId, Response)>,
+}
+
+/// The libp2p-backed P2P implementation.
+///
+/// The P2p trait implementation does not support backpressure and is expected to be fully
+/// utilized. Failure to poll the entire API will cause unbounded memory growth.
+#[derive(Clone)]
+pub struct Libp2p(Arc<Libp2pInner>);
+
+impl Libp2p {
+  /// Create a new libp2p-backed P2P instance.
+  ///
+  /// This will spawn all of the internal tasks necessary for functioning.
+  pub fn new(serai_key: &Zeroizing<Keypair>, serai: Arc<Serai>) -> Libp2p {
+    // Define the object we track peers with
+    let peers = Peers { peers: Arc::new(RwLock::new(HashMap::new())) };
+
+    // Define the dial task
+    let (dial_task_def, dial_task) = Task::new();
+    let (to_dial_send, to_dial_recv) = mpsc::unbounded_channel();
+    tokio::spawn(
+      DialTask::new(serai.clone(), peers.clone(), to_dial_send)
+        .continually_run(dial_task_def, vec![]),
+    );
+
+    let swarm = {
+      let new_only_validators = |noise_keypair: &identity::Keypair| -> Result<_, ()> {
+        Ok(OnlyValidators { serai_key: serai_key.clone(), noise_keypair: noise_keypair.clone() })
+      };
+
+      let new_yamux = || {
+        let mut config = yamux::Config::default();
+        // 1 MiB default + max message size
+        config.set_max_buffer_size((1024 * 1024) + MAX_LIBP2P_MESSAGE_SIZE);
+        // 256 KiB default + max message size
+        config
+          .set_receive_window_size(((256 * 1024) + MAX_LIBP2P_MESSAGE_SIZE).try_into().unwrap());
+        config
+      };
+
+      let mut swarm = SwarmBuilder::with_existing_identity(identity::Keypair::generate_ed25519())
+        .with_tokio()
+        .with_tcp(TcpConfig::default().nodelay(true), new_only_validators, new_yamux)
+        .unwrap()
+        .with_behaviour(|_| Behavior {
+          allow_list: allow_block_list::Behaviour::default(),
+          // Limit each per to a single connection
+          connection_limits: connection_limits::Behaviour::new(
+            ConnectionLimits::default().with_max_established_per_peer(Some(1)),
+          ),
+          ping: ping::new_behavior(),
+          reqres: reqres::new_behavior(),
+          gossip: gossip::new_behavior(),
+        })
+        .unwrap()
+        .with_swarm_config(|config| {
+          config
+            .with_idle_connection_timeout(ping::INTERVAL + ping::TIMEOUT + Duration::from_secs(5))
+        })
+        .build();
+      swarm.listen_on(format!("/ip4/0.0.0.0/tcp/{PORT}").parse().unwrap()).unwrap();
+      swarm.listen_on(format!("/ip6/::/tcp/{PORT}").parse().unwrap()).unwrap();
+      swarm
+    };
+
+    let (swarm_validators, validator_changes) = UpdateValidatorsTask::spawn(serai);
+
+    let (gossip_send, gossip_recv) = mpsc::unbounded_channel();
+    let (signed_cosigns_send, signed_cosigns_recv) = mpsc::unbounded_channel();
+    let (tributary_gossip_send, tributary_gossip_recv) = mpsc::unbounded_channel();
+
+    let (outbound_requests_send, outbound_requests_recv) = mpsc::unbounded_channel();
+
+    let (heartbeat_requests_send, heartbeat_requests_recv) = mpsc::unbounded_channel();
+    let (notable_cosign_requests_send, notable_cosign_requests_recv) = mpsc::unbounded_channel();
+    let (inbound_request_responses_send, inbound_request_responses_recv) =
+      mpsc::unbounded_channel();
+
+    // Create the swarm task
+    SwarmTask::spawn(
+      dial_task,
+      to_dial_recv,
+      swarm_validators,
+      validator_changes,
+      peers.clone(),
+      swarm,
+      gossip_recv,
+      signed_cosigns_send.clone(),
+      tributary_gossip_send,
+      outbound_requests_recv,
+      heartbeat_requests_send,
+      notable_cosign_requests_send,
+      inbound_request_responses_recv,
+    );
+
+    Libp2p(Arc::new(Libp2pInner {
+      peers,
+
+      gossip: gossip_send,
+      outbound_requests: outbound_requests_send,
+
+      tributary_gossip: Mutex::new(tributary_gossip_recv),
+
+      signed_cosigns: Mutex::new(signed_cosigns_recv),
+      signed_cosigns_send,
+
+      heartbeat_requests: Mutex::new(heartbeat_requests_recv),
+      notable_cosign_requests: Mutex::new(notable_cosign_requests_recv),
+      inbound_request_responses: inbound_request_responses_send,
+    }))
+  }
+}
+
+impl tributary_sdk::P2p for Libp2p {
+  fn broadcast(&self, tributary: [u8; 32], message: Vec<u8>) -> impl Send + Future<Output = ()> {
+    async move {
+      self
+        .0
+        .gossip
+        .send(Message::Tributary { tributary, message })
+        .expect("gossip recv channel was dropped?");
+    }
+  }
+}
+
+impl serai_cosign::RequestNotableCosigns for Libp2p {
+  type Error = ();
+
+  fn request_notable_cosigns(
+    &self,
+    global_session: [u8; 32],
+  ) -> impl Send + Future<Output = Result<(), Self::Error>> {
+    async move {
+      const AMOUNT_OF_PEERS_TO_REQUEST_FROM: usize = 3;
+      const NOTABLE_COSIGNS_TIMEOUT: Duration = Duration::from_secs(5);
+
+      let request = Request::NotableCosigns { global_session };
+
+      let peers = self.0.peers.peers.read().await.clone();
+      // HashSet of all peers
+      let peers = peers.into_values().flat_map(<_>::into_iter).collect::<HashSet<_>>();
+      // Vec of all peers
+      let mut peers = peers.into_iter().collect::<Vec<_>>();
+
+      let mut channels = Vec::with_capacity(AMOUNT_OF_PEERS_TO_REQUEST_FROM);
+      for _ in 0 .. AMOUNT_OF_PEERS_TO_REQUEST_FROM {
+        if peers.is_empty() {
+          break;
+        }
+        let i = usize::try_from(OsRng.next_u64() % u64::try_from(peers.len()).unwrap()).unwrap();
+        let peer = peers.swap_remove(i);
+
+        let (sender, receiver) = oneshot::channel();
+        self
+          .0
+          .outbound_requests
+          .send((peer, request, sender))
+          .expect("outbound requests recv channel was dropped?");
+        channels.push(receiver);
+      }
+
+      // We could reduce our latency by using FuturesUnordered here but the latency isn't a concern
+      for channel in channels {
+        if let Ok(Ok(Response::NotableCosigns(cosigns))) =
+          tokio::time::timeout(NOTABLE_COSIGNS_TIMEOUT, channel).await
+        {
+          for cosign in cosigns {
+            self
+              .0
+              .signed_cosigns_send
+              .send(cosign)
+              .expect("signed_cosigns recv in this object was dropped?");
+          }
+        }
+      }
+
+      Ok(())
+    }
+  }
+}
+
+impl serai_coordinator_p2p::P2p for Libp2p {
+  type Peer<'a> = Peer<'a>;
+
+  fn peers(&self, network: NetworkId) -> impl Send + Future<Output = Vec<Self::Peer<'_>>> {
+    async move {
+      let Some(peer_ids) = self.0.peers.peers.read().await.get(&network).cloned() else {
+        return vec![];
+      };
+      let mut res = vec![];
+      for id in peer_ids {
+        res.push(Peer { outbound_requests: &self.0.outbound_requests, id });
+      }
+      res
+    }
+  }
+
+  fn publish_cosign(&self, cosign: SignedCosign) -> impl Send + Future<Output = ()> {
+    async move {
+      self.0.gossip.send(Message::Cosign(cosign)).expect("gossip recv channel was dropped?");
+    }
+  }
+
+  fn heartbeat(
+    &self,
+  ) -> impl Send + Future<Output = (Heartbeat, oneshot::Sender<Vec<TributaryBlockWithCommit>>)> {
+    async move {
+      let (request_id, set, latest_block_hash) = self
+        .0
+        .heartbeat_requests
+        .lock()
+        .await
+        .recv()
+        .await
+        .expect("heartbeat_requests_send was dropped?");
+      let (sender, receiver) = oneshot::channel();
+      tokio::spawn({
+        let respond = self.0.inbound_request_responses.clone();
+        async move {
+          // The swarm task expects us to respond to every request. If the caller drops this
+          // channel, we'll receive `Err` and respond with `vec![]`, safely satisfying that bound
+          // without requiring the caller send a value down this channel
+          let response = if let Ok(blocks) = receiver.await {
+            Response::Blocks(blocks)
+          } else {
+            Response::Blocks(vec![])
+          };
+          respond
+            .send((request_id, response))
+            .expect("inbound_request_responses_recv was dropped?");
+        }
+      });
+      (Heartbeat { set, latest_block_hash }, sender)
+    }
+  }
+
+  fn notable_cosigns_request(
+    &self,
+  ) -> impl Send + Future<Output = ([u8; 32], oneshot::Sender<Vec<SignedCosign>>)> {
+    async move {
+      let (request_id, global_session) = self
+        .0
+        .notable_cosign_requests
+        .lock()
+        .await
+        .recv()
+        .await
+        .expect("notable_cosign_requests_send was dropped?");
+      let (sender, receiver) = oneshot::channel();
+      tokio::spawn({
+        let respond = self.0.inbound_request_responses.clone();
+        async move {
+          let response = if let Ok(notable_cosigns) = receiver.await {
+            Response::NotableCosigns(notable_cosigns)
+          } else {
+            Response::NotableCosigns(vec![])
+          };
+          respond
+            .send((request_id, response))
+            .expect("inbound_request_responses_recv was dropped?");
+        }
+      });
+      (global_session, sender)
+    }
+  }
+
+  fn tributary_message(&self) -> impl Send + Future<Output = ([u8; 32], Vec<u8>)> {
+    async move {
+      self.0.tributary_gossip.lock().await.recv().await.expect("tributary_gossip send was dropped?")
+    }
+  }
+
+  fn cosign(&self) -> impl Send + Future<Output = SignedCosign> {
+    async move {
+      self
+        .0
+        .signed_cosigns
+        .lock()
+        .await
+        .recv()
+        .await
+        .expect("signed_cosigns couldn't recv despite send in same object?")
+    }
+  }
+}

coordinator/p2p/libp2p/src/ping.rs

@@ -0,0 +1,17 @@
+use core::time::Duration;
+
+use tributary_sdk::tendermint::LATENCY_TIME;
+
+use libp2p::ping::{self, Config, Behaviour};
+pub use ping::Event;
+
+pub(crate) const INTERVAL: Duration = Duration::from_secs(30);
+// LATENCY_TIME represents the maximum latency for message delivery. Sending the ping, and
+// receiving the pong, each have to occur within this time bound to validate the connection. We
+// enforce that, as best we can, by requiring the round-trip be within twice the allowed latency.
+pub(crate) const TIMEOUT: Duration = Duration::from_millis((2 * LATENCY_TIME) as u64);
+
+pub(crate) type Behavior = Behaviour;
+pub(crate) fn new_behavior() -> Behavior {
+  Behavior::new(Config::default().with_interval(INTERVAL).with_timeout(TIMEOUT))
+}

coordinator/p2p/libp2p/src/reqres.rs

@@ -0,0 +1,135 @@
+use core::{fmt, time::Duration};
+use std::io;
+
+use async_trait::async_trait;
+
+use borsh::{BorshSerialize, BorshDeserialize};
+
+use futures_util::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
+
+use libp2p::request_response::{
+  self, Codec as CodecTrait, Event as GenericEvent, Config, Behaviour, ProtocolSupport,
+};
+pub use request_response::{RequestId, Message};
+
+use serai_cosign::SignedCosign;
+
+use serai_coordinator_p2p::{Heartbeat, TributaryBlockWithCommit};
+
+/// The maximum message size for the request-response protocol
+// This is derived from the heartbeat message size as it's our largest message
+pub(crate) const MAX_LIBP2P_REQRES_MESSAGE_SIZE: usize =
+  1024 + serai_coordinator_p2p::heartbeat::BATCH_SIZE_LIMIT;
+
+const PROTOCOL: &str = "/serai/coordinator/reqres/1.0.0";
+
+/// Requests which can be made via the request-response protocol.
+#[derive(Clone, Copy, Debug, BorshSerialize, BorshDeserialize)]
+pub(crate) enum Request {
+  /// A heartbeat informing our peers of our latest block, for the specified blockchain, on regular
+  /// intervals.
+  ///
+  /// If our peers have more blocks than us, they're expected to respond with those blocks.
+  Heartbeat(Heartbeat),
+  /// A request for the notable cosigns for a global session.
+  NotableCosigns { global_session: [u8; 32] },
+}
+
+/// Responses which can be received via the request-response protocol.
+#[derive(Clone, BorshSerialize, BorshDeserialize)]
+pub(crate) enum Response {
+  None,
+  Blocks(Vec<TributaryBlockWithCommit>),
+  NotableCosigns(Vec<SignedCosign>),
+}
+impl fmt::Debug for Response {
+  fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
+    match self {
+      Response::None => fmt.debug_struct("Response::None").finish(),
+      Response::Blocks(_) => fmt.debug_struct("Response::Block").finish_non_exhaustive(),
+      Response::NotableCosigns(_) => {
+        fmt.debug_struct("Response::NotableCosigns").finish_non_exhaustive()
+      }
+    }
+  }
+}
+
+/// The codec used for the request-response protocol.
+///
+/// We don't use CBOR or JSON, but use borsh to create `Vec<u8>`s we then length-prefix. While
+/// ideally, we'd use borsh directly with the `io` traits defined here, they're async and there
+/// isn't an amenable API within borsh for incremental deserialization.
+#[derive(Default, Clone, Copy, Debug)]
+pub(crate) struct Codec;
+impl Codec {
+  async fn read<M: BorshDeserialize>(io: &mut (impl Unpin + AsyncRead)) -> io::Result<M> {
+    let mut len = [0; 4];
+    io.read_exact(&mut len).await?;
+    let len = usize::try_from(u32::from_le_bytes(len)).expect("not at least a 32-bit platform?");
+    if len > MAX_LIBP2P_REQRES_MESSAGE_SIZE {
+      Err(io::Error::other("request length exceeded MAX_LIBP2P_REQRES_MESSAGE_SIZE"))?;
+    }
+    // This may be a non-trivial allocation easily causable
+    // While we could chunk the read, meaning we only perform the allocation as bandwidth is used,
+    // the max message size should be sufficiently sane
+    let mut buf = vec![0; len];
+    io.read_exact(&mut buf).await?;
+    let mut buf = buf.as_slice();
+    let res = M::deserialize(&mut buf)?;
+    if !buf.is_empty() {
+      Err(io::Error::other("p2p message had extra data appended to it"))?;
+    }
+    Ok(res)
+  }
+  async fn write(io: &mut (impl Unpin + AsyncWrite), msg: &impl BorshSerialize) -> io::Result<()> {
+    let msg = borsh::to_vec(msg).unwrap();
+    io.write_all(&u32::try_from(msg.len()).unwrap().to_le_bytes()).await?;
+    io.write_all(&msg).await
+  }
+}
+#[async_trait]
+impl CodecTrait for Codec {
+  type Protocol = &'static str;
+  type Request = Request;
+  type Response = Response;
+
+  async fn read_request<R: Send + Unpin + AsyncRead>(
+    &mut self,
+    _: &Self::Protocol,
+    io: &mut R,
+  ) -> io::Result<Request> {
+    Self::read(io).await
+  }
+  async fn read_response<R: Send + Unpin + AsyncRead>(
+    &mut self,
+    _: &Self::Protocol,
+    io: &mut R,
+  ) -> io::Result<Response> {
+    Self::read(io).await
+  }
+  async fn write_request<W: Send + Unpin + AsyncWrite>(
+    &mut self,
+    _: &Self::Protocol,
+    io: &mut W,
+    req: Request,
+  ) -> io::Result<()> {
+    Self::write(io, &req).await
+  }
+  async fn write_response<W: Send + Unpin + AsyncWrite>(
+    &mut self,
+    _: &Self::Protocol,
+    io: &mut W,
+    res: Response,
+  ) -> io::Result<()> {
+    Self::write(io, &res).await
+  }
+}
+
+pub(crate) type Event = GenericEvent<Request, Response>;
+
+pub(crate) type Behavior = Behaviour<Codec>;
+pub(crate) fn new_behavior() -> Behavior {
+  let mut config = Config::default();
+  config.set_request_timeout(Duration::from_secs(5));
+  Behavior::new([(PROTOCOL, ProtocolSupport::Full)], config)
+}

coordinator/p2p/libp2p/src/swarm.rs

@@ -0,0 +1,356 @@
+use std::{
+  sync::Arc,
+  collections::{HashSet, HashMap},
+  time::{Duration, Instant},
+};
+
+use borsh::BorshDeserialize;
+
+use serai_client::validator_sets::primitives::ValidatorSet;
+
+use tokio::sync::{mpsc, oneshot, RwLock};
+
+use serai_task::TaskHandle;
+
+use serai_cosign::SignedCosign;
+
+use futures_util::StreamExt;
+use libp2p::{
+  identity::PeerId,
+  request_response::{RequestId, ResponseChannel},
+  swarm::{dial_opts::DialOpts, SwarmEvent, Swarm},
+};
+
+use serai_coordinator_p2p::Heartbeat;
+
+use crate::{
+  Peers, BehaviorEvent, Behavior,
+  validators::{self, Validators},
+  ping,
+  reqres::{self, Request, Response},
+  gossip,
+};
+
+const TIME_BETWEEN_REBUILD_PEERS: Duration = Duration::from_secs(10 * 60);
+
+/*
+  `SwarmTask` handles everything we need the `Swarm` object for. The goal is to minimize the
+  contention on this task. Unfortunately, the `Swarm` object itself is needed for a variety of
+  purposes making this a rather large task.
+
+  Responsibilities include:
+  - Actually dialing new peers (the selection process occurs in another task)
+  - Maintaining the peers structure (as we need the Swarm object to see who our peers are)
+  - Gossiping messages
+  - Dispatching gossiped messages
+  - Sending requests
+  - Dispatching responses to requests
+  - Dispatching received requests
+  - Sending responses
+*/
+pub(crate) struct SwarmTask {
+  dial_task: TaskHandle,
+  to_dial: mpsc::UnboundedReceiver<DialOpts>,
+  last_dial_task_run: Instant,
+
+  validators: Arc<RwLock<Validators>>,
+  validator_changes: mpsc::UnboundedReceiver<validators::Changes>,
+  peers: Peers,
+  rebuild_peers_at: Instant,
+
+  swarm: Swarm<Behavior>,
+
+  gossip: mpsc::UnboundedReceiver<gossip::Message>,
+  signed_cosigns: mpsc::UnboundedSender<SignedCosign>,
+  tributary_gossip: mpsc::UnboundedSender<([u8; 32], Vec<u8>)>,
+
+  outbound_requests: mpsc::UnboundedReceiver<(PeerId, Request, oneshot::Sender<Response>)>,
+  outbound_request_responses: HashMap<RequestId, oneshot::Sender<Response>>,
+
+  inbound_request_response_channels: HashMap<RequestId, ResponseChannel<Response>>,
+  heartbeat_requests: mpsc::UnboundedSender<(RequestId, ValidatorSet, [u8; 32])>,
+  notable_cosign_requests: mpsc::UnboundedSender<(RequestId, [u8; 32])>,
+  inbound_request_responses: mpsc::UnboundedReceiver<(RequestId, Response)>,
+}
+
+impl SwarmTask {
+  fn handle_gossip(&mut self, event: gossip::Event) {
+    match event {
+      gossip::Event::Message { message, .. } => {
+        let Ok(message) = gossip::Message::deserialize(&mut message.data.as_slice()) else {
+          // TODO: Penalize the PeerId which created this message, which requires authenticating
+          // each message OR moving to explicit acknowledgement before re-gossiping
+          return;
+        };
+        match message {
+          gossip::Message::Tributary { tributary, message } => {
+            let _: Result<_, _> = self.tributary_gossip.send((tributary, message));
+          }
+          gossip::Message::Cosign(signed_cosign) => {
+            let _: Result<_, _> = self.signed_cosigns.send(signed_cosign);
+          }
+        }
+      }
+      gossip::Event::Subscribed { .. } | gossip::Event::Unsubscribed { .. } => {}
+      gossip::Event::GossipsubNotSupported { peer_id } => {
+        let _: Result<_, _> = self.swarm.disconnect_peer_id(peer_id);
+      }
+    }
+  }
+
+  fn handle_reqres(&mut self, event: reqres::Event) {
+    match event {
+      reqres::Event::Message { message, .. } => match message {
+        reqres::Message::Request { request_id, request, channel } => match request {
+          reqres::Request::Heartbeat(Heartbeat { set, latest_block_hash }) => {
+            self.inbound_request_response_channels.insert(request_id, channel);
+            let _: Result<_, _> =
+              self.heartbeat_requests.send((request_id, set, latest_block_hash));
+          }
+          reqres::Request::NotableCosigns { global_session } => {
+            self.inbound_request_response_channels.insert(request_id, channel);
+            let _: Result<_, _> = self.notable_cosign_requests.send((request_id, global_session));
+          }
+        },
+        reqres::Message::Response { request_id, response } => {
+          if let Some(channel) = self.outbound_request_responses.remove(&request_id) {
+            let _: Result<_, _> = channel.send(response);
+          }
+        }
+      },
+      reqres::Event::OutboundFailure { request_id, .. } => {
+        // Send None as the response for the request
+        if let Some(channel) = self.outbound_request_responses.remove(&request_id) {
+          let _: Result<_, _> = channel.send(Response::None);
+        }
+      }
+      reqres::Event::InboundFailure { .. } | reqres::Event::ResponseSent { .. } => {}
+    }
+  }
+
+  async fn run(mut self) {
+    loop {
+      let time_till_rebuild_peers = self.rebuild_peers_at.saturating_duration_since(Instant::now());
+
+      tokio::select! {
+        // If the validators have changed, update the allow list
+        validator_changes = self.validator_changes.recv() => {
+          let validator_changes = validator_changes.expect("validators update task shut down?");
+          let behavior = &mut self.swarm.behaviour_mut().allow_list;
+          for removed in validator_changes.removed {
+            behavior.disallow_peer(removed);
+          }
+          for added in validator_changes.added {
+            behavior.allow_peer(added);
+          }
+        }
+
+        // Dial peers we're instructed to
+        dial_opts = self.to_dial.recv() => {
+          let dial_opts = dial_opts.expect("DialTask was closed?");
+          let _: Result<_, _> = self.swarm.dial(dial_opts);
+        }
+
+        /*
+          Rebuild the peers every 10 minutes.
+
+          This protects against any race conditions/edge cases we have in our logic to track peers,
+          along with unrepresented behavior such as when a peer changes the networks they're active
+          in. This lets the peer tracking logic simply be 'good enough' to not become horribly
+          corrupt over the span of `TIME_BETWEEN_REBUILD_PEERS`.
+
+          We also use this to disconnect all peers who are no longer active in any network.
+        */
+        () = tokio::time::sleep(time_till_rebuild_peers) => {
+          let validators_by_network = self.validators.read().await.by_network().clone();
+          let connected_peers = self.swarm.connected_peers().copied().collect::<HashSet<_>>();
+
+          // Build the new peers object
+          let mut peers = HashMap::new();
+          for (network, validators) in validators_by_network {
+            peers.insert(network, validators.intersection(&connected_peers).copied().collect());
+          }
+
+          // Write the new peers object
+          *self.peers.peers.write().await = peers;
+          self.rebuild_peers_at = Instant::now() + TIME_BETWEEN_REBUILD_PEERS;
+        }
+
+        // Handle swarm events
+        event = self.swarm.next() => {
+          // `Swarm::next` will never return `Poll::Ready(None)`
+          // https://docs.rs/
+          //   libp2p/0.54.1/libp2p/struct.Swarm.html#impl-Stream-for-Swarm%3CTBehaviour%3E
+          let event = event.unwrap();
+          match event {
+            // New connection, so update peers
+            SwarmEvent::ConnectionEstablished { peer_id, .. } => {
+              let Some(networks) =
+                self.validators.read().await.networks(&peer_id).cloned() else { continue };
+              let mut peers = self.peers.peers.write().await;
+              for network in networks {
+                peers.entry(network).or_insert_with(HashSet::new).insert(peer_id);
+              }
+            }
+
+            // Connection closed, so update peers
+            SwarmEvent::ConnectionClosed { peer_id, .. } => {
+              let Some(networks) =
+                self.validators.read().await.networks(&peer_id).cloned() else { continue };
+              let mut peers = self.peers.peers.write().await;
+              for network in networks {
+                peers.entry(network).or_insert_with(HashSet::new).remove(&peer_id);
+              }
+
+              /*
+                We want to re-run the dial task, since we lost a peer, in case we should find new
+                peers. This opens a DoS where a validator repeatedly opens/closes connections to
+                force iterations of the dial task. We prevent this by setting a minimum distance
+                since the last explicit iteration.
+
+                This is suboptimal. If we have several disconnects in immediate proximity, we'll
+                trigger the dial task upon the first (where we may still have enough peers we
+                shouldn't dial more) but not the last (where we may have so few peers left we
+                should dial more). This is accepted as the dial task will eventually run on its
+                natural timer.
+              */
+              const MINIMUM_TIME_SINCE_LAST_EXPLICIT_DIAL: Duration = Duration::from_secs(60);
+              let now = Instant::now();
+              if (self.last_dial_task_run + MINIMUM_TIME_SINCE_LAST_EXPLICIT_DIAL) < now {
+                self.dial_task.run_now();
+                self.last_dial_task_run = now;
+              }
+            }
+
+            SwarmEvent::Behaviour(
+              BehaviorEvent::AllowList(event) | BehaviorEvent::ConnectionLimits(event)
+            ) => {
+              // This *is* an exhaustive match as these events are empty enums
+              match event {}
+            }
+            SwarmEvent::Behaviour(
+              BehaviorEvent::Ping(ping::Event { peer: _, connection, result, })
+            ) => {
+              if result.is_err() {
+                self.swarm.close_connection(connection);
+              }
+            }
+            SwarmEvent::Behaviour(BehaviorEvent::Reqres(event)) => {
+              self.handle_reqres(event)
+            }
+            SwarmEvent::Behaviour(BehaviorEvent::Gossip(event)) => {
+              self.handle_gossip(event)
+            }
+
+            // We don't handle any of these
+            SwarmEvent::IncomingConnection { .. } |
+            SwarmEvent::IncomingConnectionError { .. } |
+            SwarmEvent::OutgoingConnectionError { .. } |
+            SwarmEvent::NewListenAddr { .. } |
+            SwarmEvent::ExpiredListenAddr { .. } |
+            SwarmEvent::ListenerClosed { .. } |
+            SwarmEvent::ListenerError { .. } |
+            SwarmEvent::Dialing { .. } => {}
+          }
+        }
+
+        message = self.gossip.recv() => {
+          let message = message.expect("channel for messages to gossip was closed?");
+          let topic = message.topic();
+          let message = borsh::to_vec(&message).unwrap();
+
+          /*
+            If we're sending a message for this topic, it's because this topic is relevant to us.
+            Subscribe to it.
+
+            We create topics roughly weekly, one per validator set/session. Once present in a
+            topic, we're interested in all messages for it until the validator set/session retires.
+            Then there should no longer be any messages for the topic as we should drop the
+            Tributary which creates the messages.
+
+            We use this as an argument to not bother implement unsubscribing from topics. They're
+            incredibly infrequently created and old topics shouldn't still have messages published
+            to them. Having the coordinator reboot being our method of unsubscribing is fine.
+
+            Alternatively, we could route an API to determine when a topic is retired, or retire
+            any topics we haven't sent messages on in the past hour.
+          */
+          let behavior = self.swarm.behaviour_mut();
+          let _: Result<_, _> = behavior.gossip.subscribe(&topic);
+          /*
+            This may be an error of `InsufficientPeers`. If so, we could ask DialTask to dial more
+            peers for this network. We don't as we assume DialTask will detect the lack of peers
+            for this network, and will already successfully handle this.
+          */
+          let _: Result<_, _> = behavior.gossip.publish(topic.hash(), message);
+        }
+
+        request = self.outbound_requests.recv() => {
+          let (peer, request, response_channel) =
+            request.expect("channel for requests was closed?");
+          let request_id = self.swarm.behaviour_mut().reqres.send_request(&peer, request);
+          self.outbound_request_responses.insert(request_id, response_channel);
+        }
+
+        response = self.inbound_request_responses.recv() => {
+          let (request_id, response) =
+            response.expect("channel for inbound request responses was closed?");
+          if let Some(channel) = self.inbound_request_response_channels.remove(&request_id) {
+            let _: Result<_, _> =
+              self.swarm.behaviour_mut().reqres.send_response(channel, response);
+          }
+        }
+      }
+    }
+  }
+
+  #[allow(clippy::too_many_arguments)]
+  pub(crate) fn spawn(
+    dial_task: TaskHandle,
+    to_dial: mpsc::UnboundedReceiver<DialOpts>,
+
+    validators: Arc<RwLock<Validators>>,
+    validator_changes: mpsc::UnboundedReceiver<validators::Changes>,
+    peers: Peers,
+
+    swarm: Swarm<Behavior>,
+
+    gossip: mpsc::UnboundedReceiver<gossip::Message>,
+    signed_cosigns: mpsc::UnboundedSender<SignedCosign>,
+    tributary_gossip: mpsc::UnboundedSender<([u8; 32], Vec<u8>)>,
+
+    outbound_requests: mpsc::UnboundedReceiver<(PeerId, Request, oneshot::Sender<Response>)>,
+
+    heartbeat_requests: mpsc::UnboundedSender<(RequestId, ValidatorSet, [u8; 32])>,
+    notable_cosign_requests: mpsc::UnboundedSender<(RequestId, [u8; 32])>,
+    inbound_request_responses: mpsc::UnboundedReceiver<(RequestId, Response)>,
+  ) {
+    tokio::spawn(
+      SwarmTask {
+        dial_task,
+        to_dial,
+        last_dial_task_run: Instant::now(),
+
+        validators,
+        validator_changes,
+        peers,
+        rebuild_peers_at: Instant::now() + TIME_BETWEEN_REBUILD_PEERS,
+
+        swarm,
+
+        gossip,
+        signed_cosigns,
+        tributary_gossip,
+
+        outbound_requests,
+        outbound_request_responses: HashMap::new(),
+
+        inbound_request_response_channels: HashMap::new(),
+        heartbeat_requests,
+        notable_cosign_requests,
+        inbound_request_responses,
+      }
+      .run(),
+    );
+  }
+}

coordinator/p2p/libp2p/src/validators.rs

@@ -0,0 +1,214 @@
+use core::{borrow::Borrow, future::Future};
+use std::{
+  sync::Arc,
+  collections::{HashSet, HashMap},
+};
+
+use serai_client::{primitives::NetworkId, validator_sets::primitives::Session, SeraiError, Serai};
+
+use serai_task::{Task, ContinuallyRan};
+
+use libp2p::PeerId;
+
+use futures_util::stream::{StreamExt, FuturesUnordered};
+use tokio::sync::{mpsc, RwLock};
+
+use crate::peer_id_from_public;
+
+pub(crate) struct Changes {
+  pub(crate) removed: HashSet<PeerId>,
+  pub(crate) added: HashSet<PeerId>,
+}
+
+pub(crate) struct Validators {
+  serai: Arc<Serai>,
+
+  // A cache for which session we're populated with the validators of
+  sessions: HashMap<NetworkId, Session>,
+  // The validators by network
+  by_network: HashMap<NetworkId, HashSet<PeerId>>,
+  // The validators and their networks
+  validators: HashMap<PeerId, HashSet<NetworkId>>,
+
+  // The channel to send the changes down
+  changes: mpsc::UnboundedSender<Changes>,
+}
+
+impl Validators {
+  pub(crate) fn new(serai: Arc<Serai>) -> (Self, mpsc::UnboundedReceiver<Changes>) {
+    let (send, recv) = mpsc::unbounded_channel();
+    let validators = Validators {
+      serai,
+      sessions: HashMap::new(),
+      by_network: HashMap::new(),
+      validators: HashMap::new(),
+      changes: send,
+    };
+    (validators, recv)
+  }
+
+  async fn session_changes(
+    serai: impl Borrow<Serai>,
+    sessions: impl Borrow<HashMap<NetworkId, Session>>,
+  ) -> Result<Vec<(NetworkId, Session, HashSet<PeerId>)>, SeraiError> {
+    let temporal_serai = serai.borrow().as_of_latest_finalized_block().await?;
+    let temporal_serai = temporal_serai.validator_sets();
+
+    let mut session_changes = vec![];
+    {
+      // FuturesUnordered can be bad practice as it'll cause timeouts if infrequently polled, but
+      // we poll it till it yields all futures with the most minimal processing possible
+      let mut futures = FuturesUnordered::new();
+      for network in serai_client::primitives::NETWORKS {
+        if network == NetworkId::Serai {
+          continue;
+        }
+        let sessions = sessions.borrow();
+        futures.push(async move {
+          let session = match temporal_serai.session(network).await {
+            Ok(Some(session)) => session,
+            Ok(None) => return Ok(None),
+            Err(e) => return Err(e),
+          };
+
+          if sessions.get(&network) == Some(&session) {
+            Ok(None)
+          } else {
+            match temporal_serai.active_network_validators(network).await {
+              Ok(validators) => Ok(Some((
+                network,
+                session,
+                validators.into_iter().map(peer_id_from_public).collect(),
+              ))),
+              Err(e) => Err(e),
+            }
+          }
+        });
+      }
+      while let Some(session_change) = futures.next().await {
+        if let Some(session_change) = session_change? {
+          session_changes.push(session_change);
+        }
+      }
+    }
+
+    Ok(session_changes)
+  }
+
+  fn incorporate_session_changes(
+    &mut self,
+    session_changes: Vec<(NetworkId, Session, HashSet<PeerId>)>,
+  ) {
+    let mut removed = HashSet::new();
+    let mut added = HashSet::new();
+
+    for (network, session, validators) in session_changes {
+      // Remove the existing validators
+      for validator in self.by_network.remove(&network).unwrap_or_else(HashSet::new) {
+        // Get all networks this validator is in
+        let mut networks = self.validators.remove(&validator).unwrap();
+        // Remove this one
+        networks.remove(&network);
+        if !networks.is_empty() {
+          // Insert the networks back if the validator was present in other networks
+          self.validators.insert(validator, networks);
+        } else {
+          // Because this validator is no longer present in any network, mark them as removed
+          /*
+            This isn't accurate. The validator isn't present in the latest session for this
+            network. The validator was present in the prior session which has yet to retire. Our
+            lack of explicit inclusion for both the prior session and the current session causes
+            only the validators mutually present in both sessions to be responsible for all actions
+            still ongoing as the prior validator set retires.
+
+            TODO: Fix this
+          */
+          removed.insert(validator);
+        }
+      }
+
+      // Add the new validators
+      for validator in validators.iter().copied() {
+        self.validators.entry(validator).or_insert_with(HashSet::new).insert(network);
+        added.insert(validator);
+      }
+      self.by_network.insert(network, validators);
+
+      // Update the session we have populated
+      self.sessions.insert(network, session);
+    }
+
+    // Only flag validators for removal if they weren't simultaneously added by these changes
+    removed.retain(|validator| !added.contains(validator));
+    // Send the changes, dropping the error
+    // This lets the caller opt-out of change notifications by dropping the receiver
+    let _: Result<_, _> = self.changes.send(Changes { removed, added });
+  }
+
+  /// Update the view of the validators.
+  pub(crate) async fn update(&mut self) -> Result<(), SeraiError> {
+    let session_changes = Self::session_changes(&*self.serai, &self.sessions).await?;
+    self.incorporate_session_changes(session_changes);
+    Ok(())
+  }
+
+  pub(crate) fn by_network(&self) -> &HashMap<NetworkId, HashSet<PeerId>> {
+    &self.by_network
+  }
+
+  pub(crate) fn networks(&self, peer_id: &PeerId) -> Option<&HashSet<NetworkId>> {
+    self.validators.get(peer_id)
+  }
+}
+
+/// A task which updates a set of validators.
+///
+/// The validators managed by this tak will have their exclusive lock held for a minimal amount of
+/// time while the update occurs to minimize the disruption to the services relying on it.
+pub(crate) struct UpdateValidatorsTask {
+  validators: Arc<RwLock<Validators>>,
+}
+
+impl UpdateValidatorsTask {
+  /// Spawn a new instance of the UpdateValidatorsTask.
+  ///
+  /// This returns a reference to the Validators it updates after spawning itself.
+  pub(crate) fn spawn(
+    serai: Arc<Serai>,
+  ) -> (Arc<RwLock<Validators>>, mpsc::UnboundedReceiver<Changes>) {
+    // The validators which will be updated
+    let (validators, changes) = Validators::new(serai);
+    let validators = Arc::new(RwLock::new(validators));
+
+    // Define the task
+    let (update_validators_task, update_validators_task_handle) = Task::new();
+    // Forget the handle, as dropping the handle would stop the task
+    core::mem::forget(update_validators_task_handle);
+    // Spawn the task
+    tokio::spawn(
+      (Self { validators: validators.clone() }).continually_run(update_validators_task, vec![]),
+    );
+
+    // Return the validators
+    (validators, changes)
+  }
+}
+
+impl ContinuallyRan for UpdateValidatorsTask {
+  // Only run every minute, not the default of every five seconds
+  const DELAY_BETWEEN_ITERATIONS: u64 = 60;
+  const MAX_DELAY_BETWEEN_ITERATIONS: u64 = 5 * 60;
+
+  type Error = SeraiError;
+
+  fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, Self::Error>> {
+    async move {
+      let session_changes = {
+        let validators = self.validators.read().await;
+        Validators::session_changes(validators.serai.clone(), validators.sessions.clone()).await?
+      };
+      self.validators.write().await.incorporate_session_changes(session_changes);
+      Ok(true)
+    }
+  }
+}

coordinator/p2p/src/heartbeat.rs

@@ -0,0 +1,151 @@
+use core::future::Future;
+use std::time::{Duration, SystemTime};
+
+use serai_client::validator_sets::primitives::{MAX_KEY_SHARES_PER_SET, ValidatorSet};
+
+use futures_lite::FutureExt;
+
+use tributary_sdk::{ReadWrite, TransactionTrait, Block, Tributary, TributaryReader};
+
+use serai_db::*;
+use serai_task::ContinuallyRan;
+
+use crate::{Heartbeat, Peer, P2p};
+
+// Amount of blocks in a minute
+const BLOCKS_PER_MINUTE: usize =
+  (60 / (tributary_sdk::tendermint::TARGET_BLOCK_TIME / 1000)) as usize;
+
+/// The minimum amount of blocks to include/included within a batch, assuming there's blocks to
+/// include in the batch.
+///
+/// This decides the size limit of the Batch (the Block size limit multiplied by the minimum amount
+/// of blocks we'll send). The actual amount of blocks sent will be the amount which fits within
+/// the size limit.
+pub const MIN_BLOCKS_PER_BATCH: usize = BLOCKS_PER_MINUTE + 1;
+
+/// The size limit for a batch of blocks sent in response to a Heartbeat.
+///
+/// This estimates the size of a commit as `32 + (MAX_VALIDATORS * 128)`. At the time of writing, a
+/// commit is `8 + (validators * 32) + (32 + (validators * 32))` (for the time, list of validators,
+/// and aggregate signature). Accordingly, this should be a safe over-estimate.
+pub const BATCH_SIZE_LIMIT: usize = MIN_BLOCKS_PER_BATCH *
+  (tributary_sdk::BLOCK_SIZE_LIMIT + 32 + ((MAX_KEY_SHARES_PER_SET as usize) * 128));
+
+/// Sends a heartbeat to other validators on regular intervals informing them of our Tributary's
+/// tip.
+///
+/// If the other validator has more blocks then we do, they're expected to inform us. This forms
+/// the sync protocol for our Tributaries.
+pub(crate) struct HeartbeatTask<TD: Db, Tx: TransactionTrait, P: P2p> {
+  pub(crate) set: ValidatorSet,
+  pub(crate) tributary: Tributary<TD, Tx, P>,
+  pub(crate) reader: TributaryReader<TD, Tx>,
+  pub(crate) p2p: P,
+}
+
+impl<TD: Db, Tx: TransactionTrait, P: P2p> ContinuallyRan for HeartbeatTask<TD, Tx, P> {
+  type Error = String;
+
+  fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, Self::Error>> {
+    async move {
+      // If our blockchain hasn't had a block in the past minute, trigger the heartbeat protocol
+      const TIME_TO_TRIGGER_SYNCING: Duration = Duration::from_secs(60);
+
+      let mut tip = self.reader.tip();
+      let time_since = {
+        let block_time = if let Some(time_of_block) = self.reader.time_of_block(&tip) {
+          SystemTime::UNIX_EPOCH + Duration::from_secs(time_of_block)
+        } else {
+          // If we couldn't fetch this block's time, assume it's old
+          // We don't want to declare its unix time as 0 and claim it's 50+ years old though
+          log::warn!(
+            "heartbeat task couldn't fetch the time of a block, flagging it as a minute old"
+          );
+          SystemTime::now() - TIME_TO_TRIGGER_SYNCING
+        };
+        SystemTime::now().duration_since(block_time).unwrap_or(Duration::ZERO)
+      };
+      let mut tip_is_stale = false;
+
+      let mut synced_block = false;
+      if TIME_TO_TRIGGER_SYNCING <= time_since {
+        log::warn!(
+          "last known tributary block for {:?} was {} seconds ago",
+          self.set,
+          time_since.as_secs()
+        );
+
+        // This requests all peers for this network, without differentiating by session
+        // This should be fine as most validators should overlap across sessions
+        'peer: for peer in self.p2p.peers(self.set.network).await {
+          loop {
+            // Create the request for blocks
+            if tip_is_stale {
+              tip = self.reader.tip();
+              tip_is_stale = false;
+            }
+            // Necessary due to https://github.com/rust-lang/rust/issues/100013
+            let Some(blocks) = peer
+              .send_heartbeat(Heartbeat { set: self.set, latest_block_hash: tip })
+              .boxed()
+              .await
+            else {
+              continue 'peer;
+            };
+
+            // This is the final batch if it has less than the maximum amount of blocks
+            // (signifying there weren't more blocks after this to fill the batch with)
+            let final_batch = blocks.len() < MIN_BLOCKS_PER_BATCH;
+
+            // Sync each block
+            for block_with_commit in blocks {
+              let Ok(block) = Block::read(&mut block_with_commit.block.as_slice()) else {
+                // TODO: Disconnect/slash this peer
+                log::warn!("received invalid Block inside response to heartbeat");
+                continue 'peer;
+              };
+
+              // Attempt to sync the block
+              if !self.tributary.sync_block(block, block_with_commit.commit).await {
+                // The block may be invalid or stale if we added a block elsewhere
+                if (!tip_is_stale) && (tip != self.reader.tip()) {
+                  // Since the Tributary's tip advanced on its own, return
+                  return Ok(false);
+                }
+
+                // Since this block was invalid or stale in a way non-trivial to detect, try to
+                // sync with the next peer
+                continue 'peer;
+              }
+
+              // Because we synced a block, flag the tip as stale
+              tip_is_stale = true;
+              // And that we did sync a block
+              synced_block = true;
+            }
+
+            // If this was the final batch, move on from this peer
+            // We could assume they were honest and we are done syncing the chain, but this is a
+            // bit more robust
+            if final_batch {
+              continue 'peer;
+            }
+          }
+        }
+
+        // This will cause the tak to be run less and less often, ensuring we aren't spamming the
+        // net if we legitimately aren't making progress
+        if !synced_block {
+          Err(format!(
+            "tried to sync blocks for {:?} since we haven't seen one in {} seconds but didn't",
+            self.set,
+            time_since.as_secs(),
+          ))?;
+        }
+      }
+
+      Ok(synced_block)
+    }
+  }
+}

coordinator/p2p/src/lib.rs

@@ -0,0 +1,204 @@
+#![cfg_attr(docsrs, feature(doc_auto_cfg))]
+#![doc = include_str!("../README.md")]
+#![deny(missing_docs)]
+
+use core::future::Future;
+use std::collections::HashMap;
+
+use borsh::{BorshSerialize, BorshDeserialize};
+
+use serai_client::{primitives::NetworkId, validator_sets::primitives::ValidatorSet};
+
+use serai_db::Db;
+use tributary_sdk::{ReadWrite, TransactionTrait, Tributary, TributaryReader};
+use serai_cosign::{SignedCosign, Cosigning};
+
+use tokio::sync::{mpsc, oneshot};
+
+use serai_task::{Task, ContinuallyRan};
+
+/// The heartbeat task, effecting sync of Tributaries
+pub mod heartbeat;
+use crate::heartbeat::HeartbeatTask;
+
+/// A heartbeat for a Tributary.
+#[derive(Clone, Copy, BorshSerialize, BorshDeserialize, Debug)]
+pub struct Heartbeat {
+  /// The Tributary this is the heartbeat of.
+  pub set: ValidatorSet,
+  /// The hash of the latest block added to the Tributary.
+  pub latest_block_hash: [u8; 32],
+}
+
+/// A tributary block and its commit.
+#[derive(Clone, BorshSerialize, BorshDeserialize)]
+pub struct TributaryBlockWithCommit {
+  /// The serialized block.
+  pub block: Vec<u8>,
+  /// The serialized commit.
+  pub commit: Vec<u8>,
+}
+
+/// A representation of a peer.
+pub trait Peer<'a>: Send {
+  /// Send a heartbeat to this peer.
+  fn send_heartbeat(
+    &self,
+    heartbeat: Heartbeat,
+  ) -> impl Send + Future<Output = Option<Vec<TributaryBlockWithCommit>>>;
+}
+
+/// The representation of the P2P network.
+pub trait P2p:
+  Send + Sync + Clone + tributary_sdk::P2p + serai_cosign::RequestNotableCosigns
+{
+  /// The representation of a peer.
+  type Peer<'a>: Peer<'a>;
+
+  /// Fetch the peers for this network.
+  fn peers(&self, network: NetworkId) -> impl Send + Future<Output = Vec<Self::Peer<'_>>>;
+
+  /// Broadcast a cosign.
+  fn publish_cosign(&self, cosign: SignedCosign) -> impl Send + Future<Output = ()>;
+
+  /// A cancel-safe future for the next heartbeat received over the P2P network.
+  ///
+  /// Yields the validator set its for, the latest block hash observed, and a channel to return the
+  /// descending blocks. This channel MUST NOT and will not have its receiver dropped before a
+  /// message is sent.
+  fn heartbeat(
+    &self,
+  ) -> impl Send + Future<Output = (Heartbeat, oneshot::Sender<Vec<TributaryBlockWithCommit>>)>;
+
+  /// A cancel-safe future for the next request for the notable cosigns of a gloabl session.
+  ///
+  /// Yields the global session the request is for and a channel to return the notable cosigns.
+  /// This channel MUST NOT and will not have its receiver dropped before a message is sent.
+  fn notable_cosigns_request(
+    &self,
+  ) -> impl Send + Future<Output = ([u8; 32], oneshot::Sender<Vec<SignedCosign>>)>;
+
+  /// A cancel-safe future for the next message regarding a Tributary.
+  ///
+  /// Yields the message's Tributary's genesis block hash and the message.
+  fn tributary_message(&self) -> impl Send + Future<Output = ([u8; 32], Vec<u8>)>;
+
+  /// A cancel-safe future for the next cosign received.
+  fn cosign(&self) -> impl Send + Future<Output = SignedCosign>;
+}
+
+fn handle_notable_cosigns_request<D: Db>(
+  db: &D,
+  global_session: [u8; 32],
+  channel: oneshot::Sender<Vec<SignedCosign>>,
+) {
+  let cosigns = Cosigning::<D>::notable_cosigns(db, global_session);
+  channel.send(cosigns).expect("channel listening for cosign oneshot response was dropped?");
+}
+
+fn handle_heartbeat<D: Db, T: TransactionTrait>(
+  reader: &TributaryReader<D, T>,
+  mut latest_block_hash: [u8; 32],
+  channel: oneshot::Sender<Vec<TributaryBlockWithCommit>>,
+) {
+  let mut res_size = 8;
+  let mut res = vec![];
+  // This former case should be covered by this latter case
+  while (res.len() < heartbeat::MIN_BLOCKS_PER_BATCH) || (res_size < heartbeat::BATCH_SIZE_LIMIT) {
+    let Some(block_after) = reader.block_after(&latest_block_hash) else { break };
+
+    // These `break` conditions should only occur under edge cases, such as if we're actively
+    // deleting this Tributary due to being done with it
+    let Some(block) = reader.block(&block_after) else { break };
+    let block = block.serialize();
+    let Some(commit) = reader.commit(&block_after) else { break };
+    res_size += 8 + block.len() + 8 + commit.len();
+    res.push(TributaryBlockWithCommit { block, commit });
+
+    latest_block_hash = block_after;
+  }
+  channel
+    .send(res)
+    .map_err(|_| ())
+    .expect("channel listening for heartbeat oneshot response was dropped?");
+}
+
+/// Run the P2P instance.
+///
+/// `add_tributary`'s and `retire_tributary's senders, along with `send_cosigns`'s receiver, must
+/// never be dropped. `retire_tributary` is not required to only be instructed with added
+/// Tributaries.
+pub async fn run<TD: Db, Tx: TransactionTrait, P: P2p>(
+  db: impl Db,
+  p2p: P,
+  mut add_tributary: mpsc::UnboundedReceiver<(ValidatorSet, Tributary<TD, Tx, P>)>,
+  mut retire_tributary: mpsc::UnboundedReceiver<ValidatorSet>,
+  send_cosigns: mpsc::UnboundedSender<SignedCosign>,
+) {
+  let mut readers = HashMap::<ValidatorSet, TributaryReader<TD, Tx>>::new();
+  let mut tributaries = HashMap::<[u8; 32], mpsc::UnboundedSender<Vec<u8>>>::new();
+  let mut heartbeat_tasks = HashMap::<ValidatorSet, _>::new();
+
+  loop {
+    tokio::select! {
+      tributary = add_tributary.recv() => {
+        let (set, tributary) = tributary.expect("add_tributary send was dropped");
+        let reader = tributary.reader();
+        readers.insert(set, reader.clone());
+
+        let (heartbeat_task_def, heartbeat_task) = Task::new();
+        tokio::spawn(
+          (HeartbeatTask {
+            set,
+            tributary: tributary.clone(),
+            reader: reader.clone(),
+            p2p: p2p.clone(),
+          }).continually_run(heartbeat_task_def, vec![])
+        );
+        heartbeat_tasks.insert(set, heartbeat_task);
+
+        let (tributary_message_send, mut tributary_message_recv) = mpsc::unbounded_channel();
+        tributaries.insert(tributary.genesis(), tributary_message_send);
+        // For as long as this sender exists, handle the messages from it on a dedicated task
+        tokio::spawn(async move {
+          while let Some(message) = tributary_message_recv.recv().await {
+            tributary.handle_message(&message).await;
+          }
+        });
+      }
+      set = retire_tributary.recv() => {
+        let set = set.expect("retire_tributary send was dropped");
+        let Some(reader) = readers.remove(&set) else { continue };
+        tributaries.remove(&reader.genesis()).expect("tributary reader but no tributary");
+        heartbeat_tasks.remove(&set).expect("tributary but no heartbeat task");
+      }
+
+      (heartbeat, channel) = p2p.heartbeat() => {
+        if let Some(reader) = readers.get(&heartbeat.set) {
+          let reader = reader.clone(); // This is a cheap clone
+          // We spawn this on a task due to the DB reads needed
+          tokio::spawn(async move {
+            handle_heartbeat(&reader, heartbeat.latest_block_hash, channel)
+          });
+        }
+      }
+      (global_session, channel) = p2p.notable_cosigns_request() => {
+        tokio::spawn({
+          let db = db.clone();
+          async move { handle_notable_cosigns_request(&db, global_session, channel) }
+        });
+      }
+      (tributary, message) = p2p.tributary_message() => {
+        if let Some(tributary) = tributaries.get(&tributary) {
+          tributary.send(message).expect("tributary message recv was dropped?");
+        }
+      }
+      cosign = p2p.cosign() => {
+        // We don't call `Cosigning::intake_cosign` here as that can only be called from a single
+        // location. We also need to intake the cosigns we produce, which means we need to merge
+        // these streams (signing, network) somehow. That's done with this mpsc channel
+        send_cosigns.send(cosign).expect("channel receiving cosigns was dropped");
+      }
+    }
+  }
+}

coordinator/src/db.rs

@@ -1,134 +1,113 @@
-use blake2::{
-  digest::{consts::U32, Digest},
-  Blake2b,
-};
+use std::{path::Path, fs};
+
+pub(crate) use serai_db::{Get, DbTxn, Db as DbTrait};
+use serai_db::{create_db, db_channel};
 
-use scale::Encode;
-use borsh::{BorshSerialize, BorshDeserialize};
 use serai_client::{
   primitives::NetworkId,
   validator_sets::primitives::{Session, ValidatorSet},
-  in_instructions::primitives::{Batch, SignedBatch},
 };
 
-pub use serai_db::*;
+use serai_cosign::SignedCosign;
+use serai_coordinator_substrate::NewSetInformation;
+use serai_coordinator_tributary::Transaction;
 
-use ::tributary::ReadWrite;
-use crate::tributary::{TributarySpec, Transaction, scanner::RecognizedIdType};
+#[cfg(all(feature = "parity-db", not(feature = "rocksdb")))]
+pub(crate) type Db = serai_db::ParityDb;
+#[cfg(feature = "rocksdb")]
+pub(crate) type Db = serai_db::RocksDB;
 
-create_db!(
-  MainDb {
-    HandledMessageDb: (network: NetworkId) -> u64,
-    ActiveTributaryDb: () -> Vec<u8>,
-    RetiredTributaryDb: (set: ValidatorSet) -> (),
-    FirstPreprocessDb: (
-      network: NetworkId,
-      id_type: RecognizedIdType,
-      id: &[u8]
-    ) -> Vec<Vec<u8>>,
-    LastReceivedBatchDb: (network: NetworkId) -> u32,
-    ExpectedBatchDb: (network: NetworkId, id: u32) -> [u8; 32],
-    BatchDb: (network: NetworkId, id: u32)  -> SignedBatch,
-    LastVerifiedBatchDb: (network: NetworkId) -> u32,
-    HandoverBatchDb: (set: ValidatorSet) -> u32,
-    LookupHandoverBatchDb: (network: NetworkId, batch: u32) -> Session,
-    QueuedBatchesDb: (set: ValidatorSet) -> Vec<u8>
-  }
-);
-
-impl ActiveTributaryDb {
-  pub fn active_tributaries<G: Get>(getter: &G) -> (Vec<u8>, Vec<TributarySpec>) {
-    let bytes = Self::get(getter).unwrap_or_default();
-    let mut bytes_ref: &[u8] = bytes.as_ref();
-
-    let mut tributaries = vec![];
-    while !bytes_ref.is_empty() {
-      tributaries.push(TributarySpec::deserialize_reader(&mut bytes_ref).unwrap());
-    }
-
-    (bytes, tributaries)
+#[allow(unused_variables, unreachable_code)]
+fn db(path: &str) -> Db {
+  {
+    let path: &Path = path.as_ref();
+    // This may error if this path already exists, which we shouldn't propagate/panic on. If this
+    // is a problem (such as we don't have the necessary permissions to write to this path), we
+    // expect the following DB opening to error.
+    let _: Result<_, _> = fs::create_dir_all(path.parent().unwrap());
   }
 
-  pub fn add_participating_in_tributary(txn: &mut impl DbTxn, spec: &TributarySpec) {
-    let (mut existing_bytes, existing) = ActiveTributaryDb::active_tributaries(txn);
-    for tributary in &existing {
-      if tributary == spec {
-        return;
-      }
-    }
+  #[cfg(all(feature = "parity-db", feature = "rocksdb"))]
+  panic!("built with parity-db and rocksdb");
+  #[cfg(all(feature = "parity-db", not(feature = "rocksdb")))]
+  let db = serai_db::new_parity_db(path);
+  #[cfg(feature = "rocksdb")]
+  let db = serai_db::new_rocksdb(path);
+  db
+}
 
-    spec.serialize(&mut existing_bytes).unwrap();
-    ActiveTributaryDb::set(txn, &existing_bytes);
-  }
+pub(crate) fn coordinator_db() -> Db {
+  let root_path = serai_env::var("DB_PATH").expect("path to DB wasn't specified");
+  db(&format!("{root_path}/coordinator/db"))
+}
 
-  pub fn retire_tributary(txn: &mut impl DbTxn, set: ValidatorSet) {
-    let mut active = Self::active_tributaries(txn).1;
-    for i in 0 .. active.len() {
-      if active[i].set() == set {
-        active.remove(i);
-        break;
-      }
-    }
+fn tributary_db_folder(set: ValidatorSet) -> String {
+  let root_path = serai_env::var("DB_PATH").expect("path to DB wasn't specified");
+  let network = match set.network {
+    NetworkId::Serai => panic!("creating Tributary for the Serai network"),
+    NetworkId::Bitcoin => "Bitcoin",
+    NetworkId::Ethereum => "Ethereum",
+    NetworkId::Monero => "Monero",
+  };
+  format!("{root_path}/tributary-{network}-{}", set.session.0)
+}
 
-    let mut bytes = vec![];
-    for active in active {
-      active.serialize(&mut bytes).unwrap();
-    }
-    Self::set(txn, &bytes);
-    RetiredTributaryDb::set(txn, set, &());
+pub(crate) fn tributary_db(set: ValidatorSet) -> Db {
+  db(&format!("{}/db", tributary_db_folder(set)))
+}
+
+pub(crate) fn prune_tributary_db(set: ValidatorSet) {
+  log::info!("pruning data directory for tributary {set:?}");
+  let db = tributary_db_folder(set);
+  if fs::exists(&db).expect("couldn't check if tributary DB exists") {
+    fs::remove_dir_all(db).unwrap();
   }
 }
 
-impl FirstPreprocessDb {
-  pub fn save_first_preprocess(
-    txn: &mut impl DbTxn,
-    network: NetworkId,
-    id_type: RecognizedIdType,
-    id: &[u8],
-    preprocess: &Vec<Vec<u8>>,
-  ) {
-    if let Some(existing) = FirstPreprocessDb::get(txn, network, id_type, id) {
-      assert_eq!(&existing, preprocess, "saved a distinct first preprocess");
-      return;
+create_db! {
+  Coordinator {
+    // The currently active Tributaries
+    ActiveTributaries: () -> Vec<NewSetInformation>,
+    // The latest Tributary to have been retired for a network
+    // Since Tributaries are retired sequentially, this is informative to if any Tributary has been
+    // retired
+    RetiredTributary: (network: NetworkId) -> Session,
+    // The last handled message from a Processor
+    LastProcessorMessage: (network: NetworkId) -> u64,
+    // Cosigns we produced and tried to intake yet incurred an error while doing so
+    ErroneousCosigns: () -> Vec<SignedCosign>,
+  }
+}
+
+db_channel! {
+  Coordinator {
+    // Cosigns we produced
+    SignedCosigns: () -> SignedCosign,
+    // Tributaries to clean up upon reboot
+    TributaryCleanup: () -> ValidatorSet,
+  }
+}
+
+mod _internal_db {
+  use super::*;
+
+  db_channel! {
+    Coordinator {
+      // Tributary transactions to publish
+      TributaryTransactions: (set: ValidatorSet) -> Transaction,
     }
-    FirstPreprocessDb::set(txn, network, id_type, id, preprocess);
   }
 }
 
-impl ExpectedBatchDb {
-  pub fn save_expected_batch(txn: &mut impl DbTxn, batch: &Batch) {
-    LastReceivedBatchDb::set(txn, batch.network, &batch.id);
-    Self::set(
-      txn,
-      batch.network,
-      batch.id,
-      &Blake2b::<U32>::digest(batch.instructions.encode()).into(),
-    );
-  }
-}
-
-impl HandoverBatchDb {
-  pub fn set_handover_batch(txn: &mut impl DbTxn, set: ValidatorSet, batch: u32) {
-    Self::set(txn, set, &batch);
-    LookupHandoverBatchDb::set(txn, set.network, batch, &set.session);
-  }
-}
-impl QueuedBatchesDb {
-  pub fn queue(txn: &mut impl DbTxn, set: ValidatorSet, batch: &Transaction) {
-    let mut batches = Self::get(txn, set).unwrap_or_default();
-    batch.write(&mut batches).unwrap();
-    Self::set(txn, set, &batches);
-  }
-
-  pub fn take(txn: &mut impl DbTxn, set: ValidatorSet) -> Vec<Transaction> {
-    let batches_vec = Self::get(txn, set).unwrap_or_default();
-    txn.del(Self::key(set));
-
-    let mut batches: &[u8] = &batches_vec;
-    let mut res = vec![];
-    while !batches.is_empty() {
-      res.push(Transaction::read(&mut batches).unwrap());
+pub(crate) struct TributaryTransactions;
+impl TributaryTransactions {
+  pub(crate) fn send(txn: &mut impl DbTxn, set: ValidatorSet, tx: &Transaction) {
+    // If this set has yet to be retired, send this transaction
+    if RetiredTributary::get(txn, set.network).map(|session| session.0) < Some(set.session.0) {
+      _internal_db::TributaryTransactions::send(txn, set, tx);
     }
-    res
+  }
+  pub(crate) fn try_recv(txn: &mut impl DbTxn, set: ValidatorSet) -> Option<Transaction> {
+    _internal_db::TributaryTransactions::try_recv(txn, set)
   }
 }

coordinator/src/processors.rs

@@ -1,46 +0,0 @@
-use std::sync::Arc;
-
-use serai_client::primitives::NetworkId;
-use processor_messages::{ProcessorMessage, CoordinatorMessage};
-
-use message_queue::{Service, Metadata, client::MessageQueue};
-
-#[derive(Clone, PartialEq, Eq, Debug)]
-pub struct Message {
-  pub id: u64,
-  pub network: NetworkId,
-  pub msg: ProcessorMessage,
-}
-
-#[async_trait::async_trait]
-pub trait Processors: 'static + Send + Sync + Clone {
-  async fn send(&self, network: NetworkId, msg: impl Send + Into<CoordinatorMessage>);
-  async fn recv(&self, network: NetworkId) -> Message;
-  async fn ack(&self, msg: Message);
-}
-
-#[async_trait::async_trait]
-impl Processors for Arc<MessageQueue> {
-  async fn send(&self, network: NetworkId, msg: impl Send + Into<CoordinatorMessage>) {
-    let msg: CoordinatorMessage = msg.into();
-    let metadata =
-      Metadata { from: self.service, to: Service::Processor(network), intent: msg.intent() };
-    let msg = borsh::to_vec(&msg).unwrap();
-    self.queue(metadata, msg).await;
-  }
-  async fn recv(&self, network: NetworkId) -> Message {
-    let msg = self.next(Service::Processor(network)).await;
-    assert_eq!(msg.from, Service::Processor(network));
-
-    let id = msg.id;
-
-    // Deserialize it into a ProcessorMessage
-    let msg: ProcessorMessage =
-      borsh::from_slice(&msg.msg).expect("message wasn't a borsh-encoded ProcessorMessage");
-
-    return Message { id, network, msg };
-  }
-  async fn ack(&self, msg: Message) {
-    MessageQueue::ack(self, Service::Processor(msg.network), msg.id).await
-  }
-}

coordinator/src/substrate.rs

@@ -0,0 +1,159 @@
+use core::future::Future;
+use std::sync::Arc;
+
+use zeroize::Zeroizing;
+
+use ciphersuite::{Ciphersuite, Ristretto};
+
+use tokio::sync::mpsc;
+
+use serai_db::{DbTxn, Db as DbTrait};
+
+use serai_client::validator_sets::primitives::{Session, ValidatorSet};
+use message_queue::{Service, Metadata, client::MessageQueue};
+
+use tributary_sdk::Tributary;
+
+use serai_task::ContinuallyRan;
+
+use serai_coordinator_tributary::Transaction;
+use serai_coordinator_p2p::P2p;
+
+use crate::Db;
+
+pub(crate) struct SubstrateTask<P: P2p> {
+  pub(crate) serai_key: Zeroizing<<Ristretto as Ciphersuite>::F>,
+  pub(crate) db: Db,
+  pub(crate) message_queue: Arc<MessageQueue>,
+  pub(crate) p2p: P,
+  pub(crate) p2p_add_tributary:
+    mpsc::UnboundedSender<(ValidatorSet, Tributary<Db, Transaction, P>)>,
+  pub(crate) p2p_retire_tributary: mpsc::UnboundedSender<ValidatorSet>,
+}
+
+impl<P: P2p> ContinuallyRan for SubstrateTask<P> {
+  type Error = String; // TODO
+  fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, Self::Error>> {
+    async move {
+      let mut made_progress = false;
+
+      // Handle the Canonical events
+      for network in serai_client::primitives::NETWORKS {
+        loop {
+          let mut txn = self.db.txn();
+          let Some(msg) = serai_coordinator_substrate::Canonical::try_recv(&mut txn, network)
+          else {
+            break;
+          };
+
+          match msg {
+            // TODO: Stop trying to confirm the DKG
+            messages::substrate::CoordinatorMessage::SetKeys { .. } => todo!("TODO"),
+            messages::substrate::CoordinatorMessage::SlashesReported { session } => {
+              let prior_retired = crate::db::RetiredTributary::get(&txn, network);
+              let next_to_be_retired =
+                prior_retired.map(|session| Session(session.0 + 1)).unwrap_or(Session(0));
+              assert_eq!(session, next_to_be_retired);
+              crate::db::RetiredTributary::set(&mut txn, network, &session);
+              self
+                .p2p_retire_tributary
+                .send(ValidatorSet { network, session })
+                .expect("p2p retire_tributary channel dropped?");
+            }
+            messages::substrate::CoordinatorMessage::Block { .. } => {}
+          }
+
+          let msg = messages::CoordinatorMessage::from(msg);
+          let metadata = Metadata {
+            from: Service::Coordinator,
+            to: Service::Processor(network),
+            intent: msg.intent(),
+          };
+          let msg = borsh::to_vec(&msg).unwrap();
+          self.message_queue.queue(metadata, msg).await?;
+          txn.commit();
+          made_progress = true;
+        }
+      }
+
+      // Handle the NewSet events
+      loop {
+        let mut txn = self.db.txn();
+        let Some(new_set) = serai_coordinator_substrate::NewSet::try_recv(&mut txn) else { break };
+
+        if let Some(historic_session) = new_set.set.session.0.checked_sub(2) {
+          // We should have retired this session if we're here
+          if crate::db::RetiredTributary::get(&txn, new_set.set.network).map(|session| session.0) <
+            Some(historic_session)
+          {
+            /*
+              If we haven't, it's because we're processing the NewSet event before the retiry
+              event from the Canonical event stream. This happens if the Canonical event, and
+              then the NewSet event, is fired while we're already iterating over NewSet events.
+
+              We break, dropping the txn, restoring this NewSet to the database, so we'll only
+              handle it once a future iteration of this loop handles the retiry event.
+            */
+            break;
+          }
+
+          /*
+            Queue this historical Tributary for deletion.
+
+            We explicitly don't queue this upon Tributary retire, instead here, to give time to
+            investigate retired Tributaries if questions are raised post-retiry. This gives a
+            week (the duration of the following session) after the Tributary has been retired to
+            make a backup of the data directory for any investigations.
+          */
+          crate::db::TributaryCleanup::send(
+            &mut txn,
+            &ValidatorSet { network: new_set.set.network, session: Session(historic_session) },
+          );
+        }
+
+        // Save this Tributary as active to the database
+        {
+          let mut active_tributaries =
+            crate::db::ActiveTributaries::get(&txn).unwrap_or(Vec::with_capacity(1));
+          active_tributaries.push(new_set.clone());
+          crate::db::ActiveTributaries::set(&mut txn, &active_tributaries);
+        }
+
+        // Send GenerateKey to the processor
+        let msg = messages::key_gen::CoordinatorMessage::GenerateKey {
+          session: new_set.set.session,
+          threshold: new_set.threshold,
+          evrf_public_keys: new_set.evrf_public_keys.clone(),
+        };
+        let msg = messages::CoordinatorMessage::from(msg);
+        let metadata = Metadata {
+          from: Service::Coordinator,
+          to: Service::Processor(new_set.set.network),
+          intent: msg.intent(),
+        };
+        let msg = borsh::to_vec(&msg).unwrap();
+        self.message_queue.queue(metadata, msg).await?;
+
+        // Commit the transaction for all of this
+        txn.commit();
+
+        // Now spawn the Tributary
+        // If we reboot after committing the txn, but before this is called, this will be called
+        // on boot
+        crate::tributary::spawn_tributary(
+          self.db.clone(),
+          self.message_queue.clone(),
+          self.p2p.clone(),
+          &self.p2p_add_tributary,
+          new_set,
+          self.serai_key.clone(),
+        )
+        .await;
+
+        made_progress = true;
+      }
+
+      Ok(made_progress)
+    }
+  }
+}

coordinator/src/substrate/db.rs

@@ -1,32 +0,0 @@
-use serai_client::primitives::NetworkId;
-
-pub use serai_db::*;
-
-mod inner_db {
-  use super::*;
-
-  create_db!(
-    SubstrateDb {
-      NextBlock: () -> u64,
-      HandledEvent: (block: [u8; 32]) -> u32,
-      BatchInstructionsHashDb: (network: NetworkId, id: u32) -> [u8; 32]
-    }
-  );
-}
-pub(crate) use inner_db::{NextBlock, BatchInstructionsHashDb};
-
-pub struct HandledEvent;
-impl HandledEvent {
-  fn next_to_handle_event(getter: &impl Get, block: [u8; 32]) -> u32 {
-    inner_db::HandledEvent::get(getter, block).map_or(0, |last| last + 1)
-  }
-  pub fn is_unhandled(getter: &impl Get, block: [u8; 32], event_id: u32) -> bool {
-    let next = Self::next_to_handle_event(getter, block);
-    assert!(next >= event_id);
-    next == event_id
-  }
-  pub fn handle_event(txn: &mut impl DbTxn, block: [u8; 32], index: u32) {
-    assert!(Self::next_to_handle_event(txn, block) == index);
-    inner_db::HandledEvent::set(txn, block, &index);
-  }
-}

coordinator/src/substrate/mod.rs

@@ -1,583 +0,0 @@
-use core::{ops::Deref, time::Duration};
-use std::{
-  sync::Arc,
-  collections::{HashSet, HashMap},
-};
-
-use zeroize::Zeroizing;
-
-use ciphersuite::{group::GroupEncoding, Ciphersuite, Ristretto};
-
-use serai_client::{
-  SeraiError, Block, Serai, TemporalSerai,
-  primitives::{BlockHash, EmbeddedEllipticCurve, NetworkId},
-  validator_sets::{primitives::ValidatorSet, ValidatorSetsEvent},
-  in_instructions::InInstructionsEvent,
-  coins::CoinsEvent,
-};
-
-use serai_db::DbTxn;
-
-use processor_messages::SubstrateContext;
-
-use tokio::{sync::mpsc, time::sleep};
-
-use crate::{
-  Db,
-  processors::Processors,
-  tributary::{TributarySpec, SeraiDkgCompleted},
-};
-
-mod db;
-pub use db::*;
-
-mod cosign;
-pub use cosign::*;
-
-async fn in_set(
-  key: &Zeroizing<<Ristretto as Ciphersuite>::F>,
-  serai: &TemporalSerai<'_>,
-  set: ValidatorSet,
-) -> Result<Option<bool>, SeraiError> {
-  let Some(participants) = serai.validator_sets().participants(set.network).await? else {
-    return Ok(None);
-  };
-  let key = (Ristretto::generator() * key.deref()).to_bytes();
-  Ok(Some(participants.iter().any(|(participant, _)| participant.0 == key)))
-}
-
-async fn handle_new_set<D: Db>(
-  txn: &mut D::Transaction<'_>,
-  key: &Zeroizing<<Ristretto as Ciphersuite>::F>,
-  new_tributary_spec: &mpsc::UnboundedSender<TributarySpec>,
-  serai: &Serai,
-  block: &Block,
-  set: ValidatorSet,
-) -> Result<(), SeraiError> {
-  if in_set(key, &serai.as_of(block.hash()), set)
-    .await?
-    .expect("NewSet for set which doesn't exist")
-  {
-    log::info!("present in set {:?}", set);
-
-    let validators;
-    let mut evrf_public_keys = vec![];
-    {
-      let serai = serai.as_of(block.hash());
-      let serai = serai.validator_sets();
-      let set_participants =
-        serai.participants(set.network).await?.expect("NewSet for set which doesn't exist");
-
-      validators = set_participants
-        .iter()
-        .map(|(k, w)| {
-          (
-            <Ristretto as Ciphersuite>::read_G::<&[u8]>(&mut k.0.as_ref())
-              .expect("invalid key registered as participant"),
-            u16::try_from(*w).unwrap(),
-          )
-        })
-        .collect::<Vec<_>>();
-      for (validator, _) in set_participants {
-        // This is only run for external networks which always do a DKG for Serai
-        let substrate = serai
-          .embedded_elliptic_curve_key(validator, EmbeddedEllipticCurve::Embedwards25519)
-          .await?
-          .expect("Serai called NewSet on a validator without an Embedwards25519 key");
-        // `embedded_elliptic_curves` is documented to have the second entry be the
-        // network-specific curve (if it exists and is distinct from Embedwards25519)
-        let network =
-          if let Some(embedded_elliptic_curve) = set.network.embedded_elliptic_curves().get(1) {
-            serai.embedded_elliptic_curve_key(validator, *embedded_elliptic_curve).await?.expect(
-            "Serai called NewSet on a validator without the embedded key required for the network",
-          )
-          } else {
-            substrate.clone()
-          };
-        evrf_public_keys.push((
-          <[u8; 32]>::try_from(substrate)
-            .expect("validator-sets pallet accepted a key of an invalid length"),
-          network,
-        ));
-      }
-    };
-
-    let time = if let Ok(time) = block.time() {
-      time
-    } else {
-      assert_eq!(block.number(), 0);
-      // Use the next block's time
-      loop {
-        let Ok(Some(res)) = serai.finalized_block_by_number(1).await else {
-          sleep(Duration::from_secs(5)).await;
-          continue;
-        };
-        break res.time().unwrap();
-      }
-    };
-    // The block time is in milliseconds yet the Tributary is in seconds
-    let time = time / 1000;
-    // Since this block is in the past, and Tendermint doesn't play nice with starting chains after
-    // their start time (though it does eventually work), delay the start time by 120 seconds
-    // This is meant to handle ~20 blocks of lack of finalization for this first block
-    const SUBSTRATE_TO_TRIBUTARY_TIME_DELAY: u64 = 120;
-    let time = time + SUBSTRATE_TO_TRIBUTARY_TIME_DELAY;
-
-    let spec = TributarySpec::new(block.hash(), time, set, validators, evrf_public_keys);
-
-    log::info!("creating new tributary for {:?}", spec.set());
-
-    // Save it to the database now, not on the channel receiver's side, so this is safe against
-    // reboots
-    // If this txn finishes, and we reboot, then this'll be reloaded from active Tributaries
-    // If this txn doesn't finish, this will be re-fired
-    // If we waited to save to the DB, this txn may be finished, preventing re-firing, yet the
-    // prior fired event may have not been received yet
-    crate::ActiveTributaryDb::add_participating_in_tributary(txn, &spec);
-
-    new_tributary_spec.send(spec).unwrap();
-  } else {
-    log::info!("not present in new set {:?}", set);
-  }
-
-  Ok(())
-}
-
-async fn handle_batch_and_burns<Pro: Processors>(
-  txn: &mut impl DbTxn,
-  processors: &Pro,
-  serai: &Serai,
-  block: &Block,
-) -> Result<(), SeraiError> {
-  // Track which networks had events with a Vec in ordr to preserve the insertion order
-  // While that shouldn't be needed, ensuring order never hurts, and may enable design choices
-  // with regards to Processor <-> Coordinator message passing
-  let mut networks_with_event = vec![];
-  let mut network_had_event = |burns: &mut HashMap<_, _>, batches: &mut HashMap<_, _>, network| {
-    // Don't insert this network multiple times
-    // A Vec is still used in order to maintain the insertion order
-    if !networks_with_event.contains(&network) {
-      networks_with_event.push(network);
-      burns.insert(network, vec![]);
-      batches.insert(network, vec![]);
-    }
-  };
-
-  let mut batch_block = HashMap::new();
-  let mut batches = HashMap::<NetworkId, Vec<u32>>::new();
-  let mut burns = HashMap::new();
-
-  let serai = serai.as_of(block.hash());
-  for batch in serai.in_instructions().batch_events().await? {
-    if let InInstructionsEvent::Batch { network, id, block: network_block, instructions_hash } =
-      batch
-    {
-      network_had_event(&mut burns, &mut batches, network);
-
-      BatchInstructionsHashDb::set(txn, network, id, &instructions_hash);
-
-      // Make sure this is the only Batch event for this network in this Block
-      assert!(batch_block.insert(network, network_block).is_none());
-
-      // Add the batch included by this block
-      batches.get_mut(&network).unwrap().push(id);
-    } else {
-      panic!("Batch event wasn't Batch: {batch:?}");
-    }
-  }
-
-  for burn in serai.coins().burn_with_instruction_events().await? {
-    if let CoinsEvent::BurnWithInstruction { from: _, instruction } = burn {
-      let network = instruction.balance.coin.network();
-      network_had_event(&mut burns, &mut batches, network);
-
-      // network_had_event should register an entry in burns
-      burns.get_mut(&network).unwrap().push(instruction);
-    } else {
-      panic!("Burn event wasn't Burn: {burn:?}");
-    }
-  }
-
-  assert_eq!(HashSet::<&_>::from_iter(networks_with_event.iter()).len(), networks_with_event.len());
-
-  for network in networks_with_event {
-    let network_latest_finalized_block = if let Some(block) = batch_block.remove(&network) {
-      block
-    } else {
-      // If it's had a batch or a burn, it must have had a block acknowledged
-      serai
-        .in_instructions()
-        .latest_block_for_network(network)
-        .await?
-        .expect("network had a batch/burn yet never set a latest block")
-    };
-
-    processors
-      .send(
-        network,
-        processor_messages::substrate::CoordinatorMessage::SubstrateBlock {
-          context: SubstrateContext {
-            serai_time: block.time().unwrap() / 1000,
-            network_latest_finalized_block,
-          },
-          block: block.number(),
-          burns: burns.remove(&network).unwrap(),
-          batches: batches.remove(&network).unwrap(),
-        },
-      )
-      .await;
-  }
-
-  Ok(())
-}
-
-// Handle a specific Substrate block, returning an error when it fails to get data
-// (not blocking / holding)
-#[allow(clippy::too_many_arguments)]
-async fn handle_block<D: Db, Pro: Processors>(
-  db: &mut D,
-  key: &Zeroizing<<Ristretto as Ciphersuite>::F>,
-  new_tributary_spec: &mpsc::UnboundedSender<TributarySpec>,
-  perform_slash_report: &mpsc::UnboundedSender<ValidatorSet>,
-  tributary_retired: &mpsc::UnboundedSender<ValidatorSet>,
-  processors: &Pro,
-  serai: &Serai,
-  block: Block,
-) -> Result<(), SeraiError> {
-  let hash = block.hash();
-
-  // Define an indexed event ID.
-  let mut event_id = 0;
-
-  // If a new validator set was activated, create tributary/inform processor to do a DKG
-  for new_set in serai.as_of(hash).validator_sets().new_set_events().await? {
-    // Individually mark each event as handled so on reboot, we minimize duplicates
-    // Additionally, if the Serai connection also fails 1/100 times, this means a block with 1000
-    // events will successfully be incrementally handled
-    // (though the Serai connection should be stable, making this unnecessary)
-    let ValidatorSetsEvent::NewSet { set } = new_set else {
-      panic!("NewSet event wasn't NewSet: {new_set:?}");
-    };
-
-    // If this is Serai, do nothing
-    // We only coordinate/process external networks
-    if set.network == NetworkId::Serai {
-      continue;
-    }
-
-    if HandledEvent::is_unhandled(db, hash, event_id) {
-      log::info!("found fresh new set event {:?}", new_set);
-      let mut txn = db.txn();
-      handle_new_set::<D>(&mut txn, key, new_tributary_spec, serai, &block, set).await?;
-      HandledEvent::handle_event(&mut txn, hash, event_id);
-      txn.commit();
-    }
-    event_id += 1;
-  }
-
-  // If a key pair was confirmed, inform the processor
-  for key_gen in serai.as_of(hash).validator_sets().key_gen_events().await? {
-    if HandledEvent::is_unhandled(db, hash, event_id) {
-      log::info!("found fresh key gen event {:?}", key_gen);
-      let ValidatorSetsEvent::KeyGen { set, key_pair } = key_gen else {
-        panic!("KeyGen event wasn't KeyGen: {key_gen:?}");
-      };
-      let substrate_key = key_pair.0 .0;
-      processors
-        .send(
-          set.network,
-          processor_messages::substrate::CoordinatorMessage::ConfirmKeyPair {
-            context: SubstrateContext {
-              serai_time: block.time().unwrap() / 1000,
-              network_latest_finalized_block: serai
-                .as_of(block.hash())
-                .in_instructions()
-                .latest_block_for_network(set.network)
-                .await?
-                // The processor treats this as a magic value which will cause it to find a network
-                // block which has a time greater than or equal to the Serai time
-                .unwrap_or(BlockHash([0; 32])),
-            },
-            session: set.session,
-            key_pair,
-          },
-        )
-        .await;
-
-      // TODO: If we were in the set, yet were removed, drop the tributary
-
-      let mut txn = db.txn();
-      SeraiDkgCompleted::set(&mut txn, set, &substrate_key);
-      HandledEvent::handle_event(&mut txn, hash, event_id);
-      txn.commit();
-    }
-    event_id += 1;
-  }
-
-  for accepted_handover in serai.as_of(hash).validator_sets().accepted_handover_events().await? {
-    let ValidatorSetsEvent::AcceptedHandover { set } = accepted_handover else {
-      panic!("AcceptedHandover event wasn't AcceptedHandover: {accepted_handover:?}");
-    };
-
-    if set.network == NetworkId::Serai {
-      continue;
-    }
-
-    if HandledEvent::is_unhandled(db, hash, event_id) {
-      log::info!("found fresh accepted handover event {:?}", accepted_handover);
-      // TODO: This isn't atomic with the event handling
-      // Send a oneshot receiver so we can await the response?
-      perform_slash_report.send(set).unwrap();
-      let mut txn = db.txn();
-      HandledEvent::handle_event(&mut txn, hash, event_id);
-      txn.commit();
-    }
-    event_id += 1;
-  }
-
-  for retired_set in serai.as_of(hash).validator_sets().set_retired_events().await? {
-    let ValidatorSetsEvent::SetRetired { set } = retired_set else {
-      panic!("SetRetired event wasn't SetRetired: {retired_set:?}");
-    };
-
-    if set.network == NetworkId::Serai {
-      continue;
-    }
-
-    if HandledEvent::is_unhandled(db, hash, event_id) {
-      log::info!("found fresh set retired event {:?}", retired_set);
-      let mut txn = db.txn();
-      crate::ActiveTributaryDb::retire_tributary(&mut txn, set);
-      tributary_retired.send(set).unwrap();
-      HandledEvent::handle_event(&mut txn, hash, event_id);
-      txn.commit();
-    }
-    event_id += 1;
-  }
-
-  // Finally, tell the processor of acknowledged blocks/burns
-  // This uses a single event as unlike prior events which individually executed code, all
-  // following events share data collection
-  if HandledEvent::is_unhandled(db, hash, event_id) {
-    let mut txn = db.txn();
-    handle_batch_and_burns(&mut txn, processors, serai, &block).await?;
-    HandledEvent::handle_event(&mut txn, hash, event_id);
-    txn.commit();
-  }
-
-  Ok(())
-}
-
-#[allow(clippy::too_many_arguments)]
-async fn handle_new_blocks<D: Db, Pro: Processors>(
-  db: &mut D,
-  key: &Zeroizing<<Ristretto as Ciphersuite>::F>,
-  new_tributary_spec: &mpsc::UnboundedSender<TributarySpec>,
-  perform_slash_report: &mpsc::UnboundedSender<ValidatorSet>,
-  tributary_retired: &mpsc::UnboundedSender<ValidatorSet>,
-  processors: &Pro,
-  serai: &Serai,
-  next_block: &mut u64,
-) -> Result<(), SeraiError> {
-  // Check if there's been a new Substrate block
-  let latest_number = serai.latest_finalized_block().await?.number();
-
-  // Advance the cosigning protocol
-  advance_cosign_protocol(db, key, serai, latest_number).await?;
-
-  // Reduce to the latest cosigned block
-  let latest_number = latest_number.min(LatestCosignedBlock::latest_cosigned_block(db));
-
-  if latest_number < *next_block {
-    return Ok(());
-  }
-
-  for b in *next_block ..= latest_number {
-    let block = serai
-      .finalized_block_by_number(b)
-      .await?
-      .expect("couldn't get block before the latest finalized block");
-
-    log::info!("handling substrate block {b}");
-    handle_block(
-      db,
-      key,
-      new_tributary_spec,
-      perform_slash_report,
-      tributary_retired,
-      processors,
-      serai,
-      block,
-    )
-    .await?;
-    *next_block += 1;
-
-    let mut txn = db.txn();
-    NextBlock::set(&mut txn, next_block);
-    txn.commit();
-
-    log::info!("handled substrate block {b}");
-  }
-
-  Ok(())
-}
-
-pub async fn scan_task<D: Db, Pro: Processors>(
-  mut db: D,
-  key: Zeroizing<<Ristretto as Ciphersuite>::F>,
-  processors: Pro,
-  serai: Arc<Serai>,
-  new_tributary_spec: mpsc::UnboundedSender<TributarySpec>,
-  perform_slash_report: mpsc::UnboundedSender<ValidatorSet>,
-  tributary_retired: mpsc::UnboundedSender<ValidatorSet>,
-) {
-  log::info!("scanning substrate");
-  let mut next_substrate_block = NextBlock::get(&db).unwrap_or_default();
-
-  /*
-  let new_substrate_block_notifier = {
-    let serai = &serai;
-    move || async move {
-      loop {
-        match serai.newly_finalized_block().await {
-          Ok(sub) => return sub,
-          Err(e) => {
-            log::error!("couldn't communicate with serai node: {e}");
-            sleep(Duration::from_secs(5)).await;
-          }
-        }
-      }
-    }
-  };
-  */
-  // TODO: Restore the above subscription-based system
-  // That would require moving serai-client from HTTP to websockets
-  let new_substrate_block_notifier = {
-    let serai = &serai;
-    move |next_substrate_block| async move {
-      loop {
-        match serai.latest_finalized_block().await {
-          Ok(latest) => {
-            if latest.header.number >= next_substrate_block {
-              return latest;
-            }
-            sleep(Duration::from_secs(3)).await;
-          }
-          Err(e) => {
-            log::error!("couldn't communicate with serai node: {e}");
-            sleep(Duration::from_secs(5)).await;
-          }
-        }
-      }
-    }
-  };
-
-  loop {
-    // await the next block, yet if our notifier had an error, re-create it
-    {
-      let Ok(_) = tokio::time::timeout(
-        Duration::from_secs(60),
-        new_substrate_block_notifier(next_substrate_block),
-      )
-      .await
-      else {
-        // Timed out, which may be because Serai isn't finalizing or may be some issue with the
-        // notifier
-        if serai.latest_finalized_block().await.map(|block| block.number()).ok() ==
-          Some(next_substrate_block.saturating_sub(1))
-        {
-          log::info!("serai hasn't finalized a block in the last 60s...");
-        }
-        continue;
-      };
-
-      /*
-      // next_block is a Option<Result>
-      if next_block.and_then(Result::ok).is_none() {
-        substrate_block_notifier = new_substrate_block_notifier(next_substrate_block);
-        continue;
-      }
-      */
-    }
-
-    match handle_new_blocks(
-      &mut db,
-      &key,
-      &new_tributary_spec,
-      &perform_slash_report,
-      &tributary_retired,
-      &processors,
-      &serai,
-      &mut next_substrate_block,
-    )
-    .await
-    {
-      Ok(()) => {}
-      Err(e) => {
-        log::error!("couldn't communicate with serai node: {e}");
-        sleep(Duration::from_secs(5)).await;
-      }
-    }
-  }
-}
-
-/// Gets the expected ID for the next Batch.
-///
-/// Will log an error and apply a slight sleep on error, letting the caller simply immediately
-/// retry.
-pub(crate) async fn expected_next_batch(
-  serai: &Serai,
-  network: NetworkId,
-) -> Result<u32, SeraiError> {
-  async fn expected_next_batch_inner(serai: &Serai, network: NetworkId) -> Result<u32, SeraiError> {
-    let serai = serai.as_of_latest_finalized_block().await?;
-    let last = serai.in_instructions().last_batch_for_network(network).await?;
-    Ok(if let Some(last) = last { last + 1 } else { 0 })
-  }
-  match expected_next_batch_inner(serai, network).await {
-    Ok(next) => Ok(next),
-    Err(e) => {
-      log::error!("couldn't get the expected next batch from substrate: {e:?}");
-      sleep(Duration::from_millis(100)).await;
-      Err(e)
-    }
-  }
-}
-
-/// Verifies `Batch`s which have already been indexed from Substrate.
-///
-/// Spins if a distinct `Batch` is detected on-chain.
-///
-/// This has a slight malleability in that doesn't verify *who* published a `Batch` is as expected.
-/// This is deemed fine.
-pub(crate) async fn verify_published_batches<D: Db>(
-  txn: &mut D::Transaction<'_>,
-  network: NetworkId,
-  optimistic_up_to: u32,
-) -> Option<u32> {
-  // TODO: Localize from MainDb to SubstrateDb
-  let last = crate::LastVerifiedBatchDb::get(txn, network);
-  for id in last.map_or(0, |last| last + 1) ..= optimistic_up_to {
-    let Some(on_chain) = BatchInstructionsHashDb::get(txn, network, id) else {
-      break;
-    };
-    let off_chain = crate::ExpectedBatchDb::get(txn, network, id).unwrap();
-    if on_chain != off_chain {
-      // Halt operations on this network and spin, as this is a critical fault
-      loop {
-        log::error!(
-          "{}! network: {:?} id: {} off-chain: {} on-chain: {}",
-          "on-chain batch doesn't match off-chain",
-          network,
-          id,
-          hex::encode(off_chain),
-          hex::encode(on_chain),
-        );
-        sleep(Duration::from_secs(60)).await;
-      }
-    }
-    crate::LastVerifiedBatchDb::set(txn, network, &id);
-  }
-
-  crate::LastVerifiedBatchDb::get(txn, network)
-}

coordinator/src/tests/mod.rs

@@ -1,125 +0,0 @@
-use core::fmt::Debug;
-use std::{
-  sync::Arc,
-  collections::{VecDeque, HashSet, HashMap},
-};
-
-use serai_client::{primitives::NetworkId, validator_sets::primitives::ValidatorSet};
-
-use processor_messages::CoordinatorMessage;
-
-use async_trait::async_trait;
-
-use tokio::sync::RwLock;
-
-use crate::{
-  processors::{Message, Processors},
-  TributaryP2p, ReqResMessageKind, GossipMessageKind, P2pMessageKind, Message as P2pMessage, P2p,
-};
-
-pub mod tributary;
-
-#[derive(Clone)]
-pub struct MemProcessors(pub Arc<RwLock<HashMap<NetworkId, VecDeque<CoordinatorMessage>>>>);
-impl MemProcessors {
-  #[allow(clippy::new_without_default)]
-  pub fn new() -> MemProcessors {
-    MemProcessors(Arc::new(RwLock::new(HashMap::new())))
-  }
-}
-
-#[async_trait::async_trait]
-impl Processors for MemProcessors {
-  async fn send(&self, network: NetworkId, msg: impl Send + Into<CoordinatorMessage>) {
-    let mut processors = self.0.write().await;
-    let processor = processors.entry(network).or_insert_with(VecDeque::new);
-    processor.push_back(msg.into());
-  }
-  async fn recv(&self, _: NetworkId) -> Message {
-    todo!()
-  }
-  async fn ack(&self, _: Message) {
-    todo!()
-  }
-}
-
-#[allow(clippy::type_complexity)]
-#[derive(Clone, Debug)]
-pub struct LocalP2p(
-  usize,
-  pub Arc<RwLock<(HashSet<Vec<u8>>, Vec<VecDeque<(usize, P2pMessageKind, Vec<u8>)>>)>>,
-);
-
-impl LocalP2p {
-  pub fn new(validators: usize) -> Vec<LocalP2p> {
-    let shared = Arc::new(RwLock::new((HashSet::new(), vec![VecDeque::new(); validators])));
-    let mut res = vec![];
-    for i in 0 .. validators {
-      res.push(LocalP2p(i, shared.clone()));
-    }
-    res
-  }
-}
-
-#[async_trait]
-impl P2p for LocalP2p {
-  type Id = usize;
-
-  async fn subscribe(&self, _set: ValidatorSet, _genesis: [u8; 32]) {}
-  async fn unsubscribe(&self, _set: ValidatorSet, _genesis: [u8; 32]) {}
-
-  async fn send_raw(&self, to: Self::Id, msg: Vec<u8>) {
-    let mut msg_ref = msg.as_slice();
-    let kind = ReqResMessageKind::read(&mut msg_ref).unwrap();
-    self.1.write().await.1[to].push_back((self.0, P2pMessageKind::ReqRes(kind), msg_ref.to_vec()));
-  }
-
-  async fn broadcast_raw(&self, kind: P2pMessageKind, msg: Vec<u8>) {
-    // Content-based deduplication
-    let mut lock = self.1.write().await;
-    {
-      let already_sent = &mut lock.0;
-      if already_sent.contains(&msg) {
-        return;
-      }
-      already_sent.insert(msg.clone());
-    }
-    let queues = &mut lock.1;
-
-    let kind_len = (match kind {
-      P2pMessageKind::ReqRes(kind) => kind.serialize(),
-      P2pMessageKind::Gossip(kind) => kind.serialize(),
-    })
-    .len();
-    let msg = msg[kind_len ..].to_vec();
-
-    for (i, msg_queue) in queues.iter_mut().enumerate() {
-      if i == self.0 {
-        continue;
-      }
-      msg_queue.push_back((self.0, kind, msg.clone()));
-    }
-  }
-
-  async fn receive(&self) -> P2pMessage<Self> {
-    // This is a cursed way to implement an async read from a Vec
-    loop {
-      if let Some((sender, kind, msg)) = self.1.write().await.1[self.0].pop_front() {
-        return P2pMessage { sender, kind, msg };
-      }
-      tokio::time::sleep(std::time::Duration::from_millis(100)).await;
-    }
-  }
-}
-
-#[async_trait]
-impl TributaryP2p for LocalP2p {
-  async fn broadcast(&self, genesis: [u8; 32], msg: Vec<u8>) {
-    <Self as P2p>::broadcast(
-      self,
-      P2pMessageKind::Gossip(GossipMessageKind::Tributary(genesis)),
-      msg,
-    )
-    .await
-  }
-}

coordinator/src/tests/tributary/chain.rs

@@ -1,243 +0,0 @@
-use std::{
-  time::{Duration, SystemTime},
-  collections::HashSet,
-};
-
-use zeroize::Zeroizing;
-use rand_core::{RngCore, CryptoRng, OsRng};
-use futures_util::{task::Poll, poll};
-
-use ciphersuite::{group::ff::Field, Ciphersuite, Ristretto};
-
-use borsh::BorshDeserialize;
-use serai_client::{
-  primitives::NetworkId,
-  validator_sets::primitives::{Session, ValidatorSet},
-};
-
-use tokio::time::sleep;
-
-use serai_db::MemDb;
-
-use tributary::Tributary;
-
-use crate::{
-  GossipMessageKind, P2pMessageKind, P2p,
-  tributary::{Transaction, TributarySpec},
-  tests::LocalP2p,
-};
-
-pub fn new_keys<R: RngCore + CryptoRng>(
-  rng: &mut R,
-) -> Vec<Zeroizing<<Ristretto as Ciphersuite>::F>> {
-  let mut keys = vec![];
-  for _ in 0 .. 5 {
-    keys.push(Zeroizing::new(<Ristretto as Ciphersuite>::F::random(&mut *rng)));
-  }
-  keys
-}
-
-pub fn new_spec<R: RngCore + CryptoRng>(
-  rng: &mut R,
-  keys: &[Zeroizing<<Ristretto as Ciphersuite>::F>],
-) -> TributarySpec {
-  let mut serai_block = [0; 32];
-  rng.fill_bytes(&mut serai_block);
-
-  let start_time = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).unwrap().as_secs();
-
-  let set = ValidatorSet { session: Session(0), network: NetworkId::Bitcoin };
-
-  let validators = keys
-    .iter()
-    .map(|key| ((<Ristretto as Ciphersuite>::generator() * **key), 1))
-    .collect::<Vec<_>>();
-
-  // Generate random eVRF keys as none of these test rely on them to have any structure
-  let mut evrf_keys = vec![];
-  for _ in 0 .. keys.len() {
-    let mut substrate = [0; 32];
-    OsRng.fill_bytes(&mut substrate);
-    let mut network = vec![0; 64];
-    OsRng.fill_bytes(&mut network);
-    evrf_keys.push((substrate, network));
-  }
-
-  let res = TributarySpec::new(serai_block, start_time, set, validators, evrf_keys);
-  assert_eq!(
-    TributarySpec::deserialize_reader(&mut borsh::to_vec(&res).unwrap().as_slice()).unwrap(),
-    res,
-  );
-  res
-}
-
-pub async fn new_tributaries(
-  keys: &[Zeroizing<<Ristretto as Ciphersuite>::F>],
-  spec: &TributarySpec,
-) -> Vec<(MemDb, LocalP2p, Tributary<MemDb, Transaction, LocalP2p>)> {
-  let p2p = LocalP2p::new(keys.len());
-  let mut res = vec![];
-  for (i, key) in keys.iter().enumerate() {
-    let db = MemDb::new();
-    res.push((
-      db.clone(),
-      p2p[i].clone(),
-      Tributary::<_, Transaction, _>::new(
-        db,
-        spec.genesis(),
-        spec.start_time(),
-        key.clone(),
-        spec.validators(),
-        p2p[i].clone(),
-      )
-      .await
-      .unwrap(),
-    ));
-  }
-  res
-}
-
-pub async fn run_tributaries(
-  mut tributaries: Vec<(LocalP2p, Tributary<MemDb, Transaction, LocalP2p>)>,
-) {
-  loop {
-    for (p2p, tributary) in &mut tributaries {
-      while let Poll::Ready(msg) = poll!(p2p.receive()) {
-        match msg.kind {
-          P2pMessageKind::Gossip(GossipMessageKind::Tributary(genesis)) => {
-            assert_eq!(genesis, tributary.genesis());
-            if tributary.handle_message(&msg.msg).await {
-              p2p.broadcast(msg.kind, msg.msg).await;
-            }
-          }
-          _ => panic!("unexpected p2p message found"),
-        }
-      }
-    }
-
-    sleep(Duration::from_millis(100)).await;
-  }
-}
-
-pub async fn wait_for_tx_inclusion(
-  tributary: &Tributary<MemDb, Transaction, LocalP2p>,
-  mut last_checked: [u8; 32],
-  hash: [u8; 32],
-) -> [u8; 32] {
-  let reader = tributary.reader();
-  loop {
-    let tip = tributary.tip().await;
-    if tip == last_checked {
-      sleep(Duration::from_secs(1)).await;
-      continue;
-    }
-
-    let mut queue = vec![reader.block(&tip).unwrap()];
-    let mut block = None;
-    while {
-      let parent = queue.last().unwrap().parent();
-      if parent == tributary.genesis() {
-        false
-      } else {
-        block = Some(reader.block(&parent).unwrap());
-        block.as_ref().unwrap().hash() != last_checked
-      }
-    } {
-      queue.push(block.take().unwrap());
-    }
-
-    while let Some(block) = queue.pop() {
-      for tx in &block.transactions {
-        if tx.hash() == hash {
-          return block.hash();
-        }
-      }
-    }
-
-    last_checked = tip;
-  }
-}
-
-#[tokio::test]
-async fn tributary_test() {
-  let keys = new_keys(&mut OsRng);
-  let spec = new_spec(&mut OsRng, &keys);
-
-  let mut tributaries = new_tributaries(&keys, &spec)
-    .await
-    .into_iter()
-    .map(|(_, p2p, tributary)| (p2p, tributary))
-    .collect::<Vec<_>>();
-
-  let mut blocks = 0;
-  let mut last_block = spec.genesis();
-
-  // Doesn't use run_tributaries as we want to wind these down at a certain point
-  // run_tributaries will run them ad infinitum
-  let timeout = SystemTime::now() + Duration::from_secs(65);
-  while (blocks < 10) && (SystemTime::now().duration_since(timeout).is_err()) {
-    for (p2p, tributary) in &mut tributaries {
-      while let Poll::Ready(msg) = poll!(p2p.receive()) {
-        match msg.kind {
-          P2pMessageKind::Gossip(GossipMessageKind::Tributary(genesis)) => {
-            assert_eq!(genesis, tributary.genesis());
-            tributary.handle_message(&msg.msg).await;
-          }
-          _ => panic!("unexpected p2p message found"),
-        }
-      }
-    }
-
-    let tip = tributaries[0].1.tip().await;
-    if tip != last_block {
-      last_block = tip;
-      blocks += 1;
-    }
-
-    sleep(Duration::from_millis(100)).await;
-  }
-
-  if blocks != 10 {
-    panic!("tributary chain test hit timeout");
-  }
-
-  // Handle all existing messages
-  for (p2p, tributary) in &mut tributaries {
-    while let Poll::Ready(msg) = poll!(p2p.receive()) {
-      match msg.kind {
-        P2pMessageKind::Gossip(GossipMessageKind::Tributary(genesis)) => {
-          assert_eq!(genesis, tributary.genesis());
-          tributary.handle_message(&msg.msg).await;
-        }
-        _ => panic!("unexpected p2p message found"),
-      }
-    }
-  }
-
-  // handle_message informed the Tendermint machine, yet it still has to process it
-  // Sleep for a second accordingly
-  // TODO: Is there a better way to handle this?
-  sleep(Duration::from_secs(1)).await;
-
-  // All tributaries should agree on the tip, within a block
-  let mut tips = HashSet::new();
-  for (_, tributary) in &tributaries {
-    tips.insert(tributary.tip().await);
-  }
-  assert!(tips.len() <= 2);
-  if tips.len() == 2 {
-    for tip in &tips {
-      // Find a Tributary where this isn't the tip
-      for (_, tributary) in &tributaries {
-        let Some(after) = tributary.reader().block_after(tip) else { continue };
-        // Make sure the block after is the other tip
-        assert!(tips.contains(&after));
-        return;
-      }
-    }
-  } else {
-    assert_eq!(tips.len(), 1);
-    return;
-  }
-  panic!("tributary had different tip with a variance exceeding one block");
-}

coordinator/src/tests/tributary/dkg.rs

@@ -1,282 +0,0 @@
-use core::time::Duration;
-
-use zeroize::Zeroizing;
-use rand_core::{RngCore, OsRng};
-
-use ciphersuite::{group::GroupEncoding, Ciphersuite, Ristretto};
-use frost::Participant;
-
-use sp_runtime::traits::Verify;
-use serai_client::{
-  primitives::Signature,
-  validator_sets::primitives::{ValidatorSet, KeyPair},
-};
-
-use tokio::time::sleep;
-
-use serai_db::{Get, DbTxn, Db, MemDb};
-
-use processor_messages::{key_gen, CoordinatorMessage};
-
-use tributary::{TransactionTrait, Tributary};
-
-use crate::{
-  tributary::{
-    Transaction, TributarySpec,
-    scanner::{PublishSeraiTransaction, handle_new_blocks},
-  },
-  tests::{
-    MemProcessors, LocalP2p,
-    tributary::{new_keys, new_spec, new_tributaries, run_tributaries, wait_for_tx_inclusion},
-  },
-};
-
-#[tokio::test]
-async fn dkg_test() {
-  env_logger::init();
-
-  let keys = new_keys(&mut OsRng);
-  let spec = new_spec(&mut OsRng, &keys);
-
-  let full_tributaries = new_tributaries(&keys, &spec).await;
-  let mut dbs = vec![];
-  let mut tributaries = vec![];
-  for (db, p2p, tributary) in full_tributaries {
-    dbs.push(db);
-    tributaries.push((p2p, tributary));
-  }
-
-  // Run the tributaries in the background
-  tokio::spawn(run_tributaries(tributaries.clone()));
-
-  let mut txs = vec![];
-  // Create DKG participation for each key
-  for key in &keys {
-    let mut participation = vec![0; 4096];
-    OsRng.fill_bytes(&mut participation);
-
-    let mut tx =
-      Transaction::DkgParticipation { participation, signed: Transaction::empty_signed() };
-    tx.sign(&mut OsRng, spec.genesis(), key);
-    txs.push(tx);
-  }
-
-  let block_before_tx = tributaries[0].1.tip().await;
-
-  // Publish t-1 participations
-  let t = ((keys.len() * 2) / 3) + 1;
-  for (i, tx) in txs.iter().take(t - 1).enumerate() {
-    assert_eq!(tributaries[i].1.add_transaction(tx.clone()).await, Ok(true));
-    wait_for_tx_inclusion(&tributaries[0].1, block_before_tx, tx.hash()).await;
-  }
-
-  let expected_participations = txs
-    .iter()
-    .enumerate()
-    .map(|(i, tx)| {
-      if let Transaction::DkgParticipation { participation, .. } = tx {
-        CoordinatorMessage::KeyGen(key_gen::CoordinatorMessage::Participation {
-          session: spec.set().session,
-          participant: Participant::new((i + 1).try_into().unwrap()).unwrap(),
-          participation: participation.clone(),
-        })
-      } else {
-        panic!("txs wasn't a DkgParticipation");
-      }
-    })
-    .collect::<Vec<_>>();
-
-  async fn new_processors(
-    db: &mut MemDb,
-    key: &Zeroizing<<Ristretto as Ciphersuite>::F>,
-    spec: &TributarySpec,
-    tributary: &Tributary<MemDb, Transaction, LocalP2p>,
-  ) -> MemProcessors {
-    let processors = MemProcessors::new();
-    handle_new_blocks::<_, _, _, _, _, LocalP2p>(
-      db,
-      key,
-      &|_, _, _, _| async {
-        panic!("provided TX caused recognized_id to be called in new_processors")
-      },
-      &processors,
-      &(),
-      &|_| async {
-        panic!(
-          "test tried to publish a new Tributary TX from handle_application_tx in new_processors"
-        )
-      },
-      spec,
-      &tributary.reader(),
-    )
-    .await;
-    processors
-  }
-
-  // Instantiate a scanner and verify it has the first two participations to report (and isn't
-  // waiting for `t`)
-  let processors = new_processors(&mut dbs[0], &keys[0], &spec, &tributaries[0].1).await;
-  assert_eq!(processors.0.read().await.get(&spec.set().network).unwrap().len(), t - 1);
-
-  // Publish the rest of the participations
-  let block_before_tx = tributaries[0].1.tip().await;
-  for tx in txs.iter().skip(t - 1) {
-    assert_eq!(tributaries[0].1.add_transaction(tx.clone()).await, Ok(true));
-    wait_for_tx_inclusion(&tributaries[0].1, block_before_tx, tx.hash()).await;
-  }
-
-  // Verify the scanner emits all KeyGen::Participations messages
-  handle_new_blocks::<_, _, _, _, _, LocalP2p>(
-    &mut dbs[0],
-    &keys[0],
-    &|_, _, _, _| async {
-      panic!("provided TX caused recognized_id to be called after DkgParticipation")
-    },
-    &processors,
-    &(),
-    &|_| async {
-      panic!(
-        "test tried to publish a new Tributary TX from handle_application_tx after DkgParticipation"
-      )
-    },
-    &spec,
-    &tributaries[0].1.reader(),
-  )
-  .await;
-  {
-    let mut msgs = processors.0.write().await;
-    let msgs = msgs.get_mut(&spec.set().network).unwrap();
-    assert_eq!(msgs.len(), keys.len());
-    for expected in &expected_participations {
-      assert_eq!(&msgs.pop_front().unwrap(), expected);
-    }
-    assert!(msgs.is_empty());
-  }
-
-  // Verify all keys exhibit this scanner behavior
-  for (i, key) in keys.iter().enumerate().skip(1) {
-    let processors = new_processors(&mut dbs[i], key, &spec, &tributaries[i].1).await;
-    let mut msgs = processors.0.write().await;
-    let msgs = msgs.get_mut(&spec.set().network).unwrap();
-    assert_eq!(msgs.len(), keys.len());
-    for expected in &expected_participations {
-      assert_eq!(&msgs.pop_front().unwrap(), expected);
-    }
-    assert!(msgs.is_empty());
-  }
-
-  let mut substrate_key = [0; 32];
-  OsRng.fill_bytes(&mut substrate_key);
-  let mut network_key = vec![0; usize::try_from((OsRng.next_u64() % 32) + 32).unwrap()];
-  OsRng.fill_bytes(&mut network_key);
-  let key_pair = KeyPair(serai_client::Public(substrate_key), network_key.try_into().unwrap());
-
-  let mut txs = vec![];
-  for (i, key) in keys.iter().enumerate() {
-    let mut txn = dbs[i].txn();
-
-    // Claim we've generated the key pair
-    crate::tributary::generated_key_pair::<MemDb>(&mut txn, spec.genesis(), &key_pair);
-
-    // Publish the nonces
-    let attempt = 0;
-    let mut tx = Transaction::DkgConfirmationNonces {
-      attempt,
-      confirmation_nonces: crate::tributary::dkg_confirmation_nonces(key, &spec, &mut txn, 0),
-      signed: Transaction::empty_signed(),
-    };
-    txn.commit();
-    tx.sign(&mut OsRng, spec.genesis(), key);
-    txs.push(tx);
-  }
-  let block_before_tx = tributaries[0].1.tip().await;
-  for (i, tx) in txs.iter().enumerate() {
-    assert_eq!(tributaries[i].1.add_transaction(tx.clone()).await, Ok(true));
-  }
-  for tx in &txs {
-    wait_for_tx_inclusion(&tributaries[0].1, block_before_tx, tx.hash()).await;
-  }
-
-  // This should not cause any new processor event as the processor doesn't handle DKG confirming
-  for (i, key) in keys.iter().enumerate() {
-    handle_new_blocks::<_, _, _, _, _, LocalP2p>(
-      &mut dbs[i],
-      key,
-      &|_, _, _, _| async {
-        panic!("provided TX caused recognized_id to be called after DkgConfirmationNonces")
-      },
-      &processors,
-      &(),
-      // The Tributary handler should publish ConfirmationShare itself after ConfirmationNonces
-      &|tx| async { assert_eq!(tributaries[i].1.add_transaction(tx).await, Ok(true)) },
-      &spec,
-      &tributaries[i].1.reader(),
-    )
-    .await;
-    {
-      assert!(processors.0.read().await.get(&spec.set().network).unwrap().is_empty());
-    }
-  }
-
-  // Yet once these TXs are on-chain, the tributary should itself publish the confirmation shares
-  // This means in the block after the next block, the keys should be set onto Serai
-  // Sleep twice as long as two blocks, in case there's some stability issue
-  sleep(Duration::from_secs(
-    2 * 2 * u64::from(Tributary::<MemDb, Transaction, LocalP2p>::block_time()),
-  ))
-  .await;
-
-  struct CheckPublishSetKeys {
-    spec: TributarySpec,
-    key_pair: KeyPair,
-  }
-  #[async_trait::async_trait]
-  impl PublishSeraiTransaction for CheckPublishSetKeys {
-    async fn publish_set_keys(
-      &self,
-      _db: &(impl Sync + Get),
-      set: ValidatorSet,
-      key_pair: KeyPair,
-      signature_participants: bitvec::vec::BitVec<u8, bitvec::order::Lsb0>,
-      signature: Signature,
-    ) {
-      assert_eq!(set, self.spec.set());
-      assert_eq!(self.key_pair, key_pair);
-      assert!(signature.verify(
-        &*serai_client::validator_sets::primitives::set_keys_message(&set, &key_pair),
-        &serai_client::Public(
-          frost::dkg::musig::musig_key::<Ristretto>(
-            &serai_client::validator_sets::primitives::musig_context(set),
-            &self
-              .spec
-              .validators()
-              .into_iter()
-              .zip(signature_participants)
-              .filter_map(|((validator, _), included)| included.then_some(validator))
-              .collect::<Vec<_>>()
-          )
-          .unwrap()
-          .to_bytes()
-        ),
-      ));
-    }
-  }
-
-  // The scanner should successfully try to publish a transaction with a validly signed signature
-  handle_new_blocks::<_, _, _, _, _, LocalP2p>(
-    &mut dbs[0],
-    &keys[0],
-    &|_, _, _, _| async {
-      panic!("provided TX caused recognized_id to be called after DKG confirmation")
-    },
-    &processors,
-    &CheckPublishSetKeys { spec: spec.clone(), key_pair: key_pair.clone() },
-    &|_| async { panic!("test tried to publish a new Tributary TX from handle_application_tx") },
-    &spec,
-    &tributaries[0].1.reader(),
-  )
-  .await;
-  {
-    assert!(processors.0.read().await.get(&spec.set().network).unwrap().is_empty());
-  }
-}

coordinator/src/tests/tributary/handle_p2p.rs

@@ -1,74 +0,0 @@
-use core::time::Duration;
-use std::sync::Arc;
-
-use rand_core::OsRng;
-
-use tokio::{
-  sync::{mpsc, broadcast},
-  time::sleep,
-};
-
-use serai_db::MemDb;
-
-use tributary::Tributary;
-
-use crate::{
-  tributary::Transaction,
-  ActiveTributary, TributaryEvent,
-  p2p::handle_p2p_task,
-  tests::{
-    LocalP2p,
-    tributary::{new_keys, new_spec, new_tributaries},
-  },
-};
-
-#[tokio::test]
-async fn handle_p2p_test() {
-  let keys = new_keys(&mut OsRng);
-  let spec = new_spec(&mut OsRng, &keys);
-
-  let mut tributaries = new_tributaries(&keys, &spec)
-    .await
-    .into_iter()
-    .map(|(_, p2p, tributary)| (p2p, tributary))
-    .collect::<Vec<_>>();
-
-  let mut tributary_senders = vec![];
-  let mut tributary_arcs = vec![];
-  for (p2p, tributary) in tributaries.drain(..) {
-    let tributary = Arc::new(tributary);
-    tributary_arcs.push(tributary.clone());
-    let (new_tributary_send, new_tributary_recv) = broadcast::channel(5);
-    let (cosign_send, _) = mpsc::unbounded_channel();
-    tokio::spawn(handle_p2p_task(p2p, cosign_send, new_tributary_recv));
-    new_tributary_send
-      .send(TributaryEvent::NewTributary(ActiveTributary { spec: spec.clone(), tributary }))
-      .map_err(|_| "failed to send ActiveTributary")
-      .unwrap();
-    tributary_senders.push(new_tributary_send);
-  }
-  let tributaries = tributary_arcs;
-
-  // After two blocks of time, we should have a new block
-  // We don't wait one block of time as we may have missed the chance for this block
-  sleep(Duration::from_secs((2 * Tributary::<MemDb, Transaction, LocalP2p>::block_time()).into()))
-    .await;
-  let tip = tributaries[0].tip().await;
-  assert!(tip != spec.genesis());
-
-  // Sleep one second to make sure this block propagates
-  sleep(Duration::from_secs(1)).await;
-  // Make sure every tributary has it
-  for tributary in &tributaries {
-    assert!(tributary.reader().block(&tip).is_some());
-  }
-
-  // Then after another block of time, we should have yet another new block
-  sleep(Duration::from_secs(Tributary::<MemDb, Transaction, LocalP2p>::block_time().into())).await;
-  let new_tip = tributaries[0].tip().await;
-  assert!(new_tip != tip);
-  sleep(Duration::from_secs(1)).await;
-  for tributary in tributaries {
-    assert!(tributary.reader().block(&new_tip).is_some());
-  }
-}

coordinator/src/tests/tributary/mod.rs

@@ -1,245 +0,0 @@
-use core::fmt::Debug;
-
-use rand_core::{RngCore, OsRng};
-
-use ciphersuite::{group::Group, Ciphersuite, Ristretto};
-
-use scale::{Encode, Decode};
-use serai_client::{
-  primitives::Signature,
-  validator_sets::primitives::{MAX_KEY_SHARES_PER_SET, ValidatorSet, KeyPair},
-};
-use processor_messages::coordinator::SubstrateSignableId;
-
-use tributary::{ReadWrite, tests::random_signed_with_nonce};
-
-use crate::tributary::{Label, SignData, Transaction, scanner::PublishSeraiTransaction};
-
-mod chain;
-pub use chain::*;
-
-mod tx;
-
-mod dkg;
-// TODO: Test the other transactions
-
-mod handle_p2p;
-mod sync;
-
-#[async_trait::async_trait]
-impl PublishSeraiTransaction for () {
-  async fn publish_set_keys(
-    &self,
-    _db: &(impl Sync + serai_db::Get),
-    _set: ValidatorSet,
-    _key_pair: KeyPair,
-    _signature_participants: bitvec::vec::BitVec<u8, bitvec::order::Lsb0>,
-    _signature: Signature,
-  ) {
-    panic!("publish_set_keys was called in test")
-  }
-}
-
-fn random_u32<R: RngCore>(rng: &mut R) -> u32 {
-  u32::try_from(rng.next_u64() >> 32).unwrap()
-}
-
-fn random_vec<R: RngCore>(rng: &mut R, limit: usize) -> Vec<u8> {
-  let len = usize::try_from(rng.next_u64() % u64::try_from(limit).unwrap()).unwrap();
-  let mut res = vec![0; len];
-  rng.fill_bytes(&mut res);
-  res
-}
-
-fn random_sign_data<R: RngCore, Id: Clone + PartialEq + Eq + Debug + Encode + Decode>(
-  rng: &mut R,
-  plan: Id,
-  label: Label,
-) -> SignData<Id> {
-  SignData {
-    plan,
-    attempt: random_u32(&mut OsRng),
-    label,
-
-    data: {
-      let mut res = vec![];
-      for _ in 0 ..= (rng.next_u64() % 255) {
-        res.push(random_vec(&mut OsRng, 512));
-      }
-      res
-    },
-
-    signed: random_signed_with_nonce(&mut OsRng, label.nonce()),
-  }
-}
-
-fn test_read_write<RW: Eq + Debug + ReadWrite>(value: &RW) {
-  assert_eq!(value, &RW::read::<&[u8]>(&mut value.serialize().as_ref()).unwrap());
-}
-
-#[test]
-fn tx_size_limit() {
-  use serai_client::validator_sets::primitives::MAX_KEY_LEN;
-
-  use tributary::TRANSACTION_SIZE_LIMIT;
-
-  let max_dkg_coefficients = (MAX_KEY_SHARES_PER_SET * 2).div_ceil(3) + 1;
-  // n coefficients
-  // 2 ECDH values per recipient, and the encrypted share
-  let elements_outside_of_proof = max_dkg_coefficients + ((2 + 1) * MAX_KEY_SHARES_PER_SET);
-  // Then Pedersen Vector Commitments for each DH done, and the associated overhead in the proof
-  // It's handwaved as one commitment per DH, where we do 2 per coefficient and 1 for the explicit
-  //  ECDHs
-  let vector_commitments = (2 * max_dkg_coefficients) + (2 * MAX_KEY_SHARES_PER_SET);
-  // Then we have commitments to the `t` polynomial of length 2 + 2 nc, where nc is the amount of
-  // commitments
-  let t_commitments = 2 + (2 * vector_commitments);
-  // The remainder of the proof should be ~30 elements
-  let proof_elements = 30;
-
-  let handwaved_dkg_size =
-    ((elements_outside_of_proof + vector_commitments + t_commitments + proof_elements) *
-      MAX_KEY_LEN) +
-      1024;
-  // Further scale by two in case of any errors in the above
-  assert!(u32::try_from(TRANSACTION_SIZE_LIMIT).unwrap() >= (2 * handwaved_dkg_size));
-}
-
-#[test]
-fn serialize_sign_data() {
-  fn test_read_write<Id: Clone + PartialEq + Eq + Debug + Encode + Decode>(value: &SignData<Id>) {
-    let mut buf = vec![];
-    value.write(&mut buf).unwrap();
-    assert_eq!(value, &SignData::read(&mut buf.as_slice()).unwrap())
-  }
-
-  let mut plan = [0; 3];
-  OsRng.fill_bytes(&mut plan);
-  test_read_write(&random_sign_data::<_, _>(
-    &mut OsRng,
-    plan,
-    if (OsRng.next_u64() % 2) == 0 { Label::Preprocess } else { Label::Share },
-  ));
-  let mut plan = [0; 5];
-  OsRng.fill_bytes(&mut plan);
-  test_read_write(&random_sign_data::<_, _>(
-    &mut OsRng,
-    plan,
-    if (OsRng.next_u64() % 2) == 0 { Label::Preprocess } else { Label::Share },
-  ));
-  let mut plan = [0; 8];
-  OsRng.fill_bytes(&mut plan);
-  test_read_write(&random_sign_data::<_, _>(
-    &mut OsRng,
-    plan,
-    if (OsRng.next_u64() % 2) == 0 { Label::Preprocess } else { Label::Share },
-  ));
-  let mut plan = [0; 24];
-  OsRng.fill_bytes(&mut plan);
-  test_read_write(&random_sign_data::<_, _>(
-    &mut OsRng,
-    plan,
-    if (OsRng.next_u64() % 2) == 0 { Label::Preprocess } else { Label::Share },
-  ));
-}
-
-#[test]
-fn serialize_transaction() {
-  test_read_write(&Transaction::RemoveParticipant {
-    participant: <Ristretto as Ciphersuite>::G::random(&mut OsRng),
-    signed: random_signed_with_nonce(&mut OsRng, 0),
-  });
-
-  test_read_write(&Transaction::DkgParticipation {
-    participation: random_vec(&mut OsRng, 4096),
-    signed: random_signed_with_nonce(&mut OsRng, 0),
-  });
-
-  test_read_write(&Transaction::DkgConfirmationNonces {
-    attempt: random_u32(&mut OsRng),
-    confirmation_nonces: {
-      let mut nonces = [0; 64];
-      OsRng.fill_bytes(&mut nonces);
-      nonces
-    },
-    signed: random_signed_with_nonce(&mut OsRng, 0),
-  });
-
-  test_read_write(&Transaction::DkgConfirmationShare {
-    attempt: random_u32(&mut OsRng),
-    confirmation_share: {
-      let mut share = [0; 32];
-      OsRng.fill_bytes(&mut share);
-      share
-    },
-    signed: random_signed_with_nonce(&mut OsRng, 1),
-  });
-
-  {
-    let mut block = [0; 32];
-    OsRng.fill_bytes(&mut block);
-    test_read_write(&Transaction::CosignSubstrateBlock(block));
-  }
-
-  {
-    let mut block = [0; 32];
-    OsRng.fill_bytes(&mut block);
-    let batch = u32::try_from(OsRng.next_u64() >> 32).unwrap();
-    test_read_write(&Transaction::Batch { block, batch });
-  }
-  test_read_write(&Transaction::SubstrateBlock(OsRng.next_u64()));
-
-  {
-    let batch = u32::try_from(OsRng.next_u64() >> 32).unwrap();
-    test_read_write(&Transaction::SubstrateSign(random_sign_data(
-      &mut OsRng,
-      SubstrateSignableId::Batch(batch),
-      Label::Preprocess,
-    )));
-  }
-  {
-    let batch = u32::try_from(OsRng.next_u64() >> 32).unwrap();
-    test_read_write(&Transaction::SubstrateSign(random_sign_data(
-      &mut OsRng,
-      SubstrateSignableId::Batch(batch),
-      Label::Share,
-    )));
-  }
-
-  {
-    let mut plan = [0; 32];
-    OsRng.fill_bytes(&mut plan);
-    test_read_write(&Transaction::Sign(random_sign_data(&mut OsRng, plan, Label::Preprocess)));
-  }
-  {
-    let mut plan = [0; 32];
-    OsRng.fill_bytes(&mut plan);
-    test_read_write(&Transaction::Sign(random_sign_data(&mut OsRng, plan, Label::Share)));
-  }
-
-  {
-    let mut plan = [0; 32];
-    OsRng.fill_bytes(&mut plan);
-    let mut tx_hash = vec![0; (OsRng.next_u64() % 64).try_into().unwrap()];
-    OsRng.fill_bytes(&mut tx_hash);
-    test_read_write(&Transaction::SignCompleted {
-      plan,
-      tx_hash,
-      first_signer: random_signed_with_nonce(&mut OsRng, 2).signer,
-      signature: random_signed_with_nonce(&mut OsRng, 2).signature,
-    });
-  }
-
-  test_read_write(&Transaction::SlashReport(
-    {
-      let amount =
-        usize::try_from(OsRng.next_u64() % u64::from(MAX_KEY_SHARES_PER_SET - 1)).unwrap();
-      let mut points = vec![];
-      for _ in 0 .. amount {
-        points.push((OsRng.next_u64() >> 32).try_into().unwrap());
-      }
-      points
-    },
-    random_signed_with_nonce(&mut OsRng, 0),
-  ));
-}

coordinator/src/tests/tributary/sync.rs

@@ -1,165 +0,0 @@
-use core::time::Duration;
-use std::{sync::Arc, collections::HashSet};
-
-use rand_core::OsRng;
-
-use ciphersuite::{group::GroupEncoding, Ciphersuite, Ristretto};
-
-use tokio::{
-  sync::{mpsc, broadcast},
-  time::sleep,
-};
-
-use serai_db::MemDb;
-
-use tributary::Tributary;
-
-use crate::{
-  tributary::Transaction,
-  ActiveTributary, TributaryEvent,
-  p2p::{heartbeat_tributaries_task, handle_p2p_task},
-  tests::{
-    LocalP2p,
-    tributary::{new_keys, new_spec, new_tributaries},
-  },
-};
-
-#[tokio::test]
-async fn sync_test() {
-  let mut keys = new_keys(&mut OsRng);
-  let spec = new_spec(&mut OsRng, &keys);
-  // Ensure this can have a node fail
-  assert!(spec.n() > spec.t());
-
-  let mut tributaries = new_tributaries(&keys, &spec)
-    .await
-    .into_iter()
-    .map(|(_, p2p, tributary)| (p2p, tributary))
-    .collect::<Vec<_>>();
-
-  // Keep a Tributary back, effectively having it offline
-  let syncer_key = keys.pop().unwrap();
-  let (syncer_p2p, syncer_tributary) = tributaries.pop().unwrap();
-
-  // Have the rest form a P2P net
-  let mut tributary_senders = vec![];
-  let mut tributary_arcs = vec![];
-  let mut p2p_threads = vec![];
-  for (p2p, tributary) in tributaries.drain(..) {
-    let tributary = Arc::new(tributary);
-    tributary_arcs.push(tributary.clone());
-    let (new_tributary_send, new_tributary_recv) = broadcast::channel(5);
-    let (cosign_send, _) = mpsc::unbounded_channel();
-    let thread = tokio::spawn(handle_p2p_task(p2p, cosign_send, new_tributary_recv));
-    new_tributary_send
-      .send(TributaryEvent::NewTributary(ActiveTributary { spec: spec.clone(), tributary }))
-      .map_err(|_| "failed to send ActiveTributary")
-      .unwrap();
-    tributary_senders.push(new_tributary_send);
-    p2p_threads.push(thread);
-  }
-  let tributaries = tributary_arcs;
-
-  // After four blocks of time, we should have a new block
-  // We don't wait one block of time as we may have missed the chance for the first block
-  // We don't wait two blocks because we may have missed the chance, and then had a failure to
-  // propose by our 'offline' validator, which would cause the Tendermint round time to increase,
-  // requiring a longer delay
-  let block_time = u64::from(Tributary::<MemDb, Transaction, LocalP2p>::block_time());
-  sleep(Duration::from_secs(4 * block_time)).await;
-  let tip = tributaries[0].tip().await;
-  assert!(tip != spec.genesis());
-
-  // Sleep one second to make sure this block propagates
-  sleep(Duration::from_secs(1)).await;
-  // Make sure every tributary has it
-  for tributary in &tributaries {
-    assert!(tributary.reader().block(&tip).is_some());
-  }
-
-  // Now that we've confirmed the other tributaries formed a net without issue, drop the syncer's
-  // pending P2P messages
-  syncer_p2p.1.write().await.1.last_mut().unwrap().clear();
-
-  // Have it join the net
-  let syncer_key = Ristretto::generator() * *syncer_key;
-  let syncer_tributary = Arc::new(syncer_tributary);
-  let (syncer_tributary_send, syncer_tributary_recv) = broadcast::channel(5);
-  let (cosign_send, _) = mpsc::unbounded_channel();
-  tokio::spawn(handle_p2p_task(syncer_p2p.clone(), cosign_send, syncer_tributary_recv));
-  syncer_tributary_send
-    .send(TributaryEvent::NewTributary(ActiveTributary {
-      spec: spec.clone(),
-      tributary: syncer_tributary.clone(),
-    }))
-    .map_err(|_| "failed to send ActiveTributary to syncer")
-    .unwrap();
-
-  // It shouldn't automatically catch up. If it somehow was, our test would be broken
-  // Sanity check this
-  let tip = tributaries[0].tip().await;
-  // Wait until a new block occurs
-  sleep(Duration::from_secs(3 * block_time)).await;
-  // Make sure a new block actually occurred
-  assert!(tributaries[0].tip().await != tip);
-  // Make sure the new block alone didn't trigger catching up
-  assert_eq!(syncer_tributary.tip().await, spec.genesis());
-
-  // Start the heartbeat protocol
-  let (syncer_heartbeat_tributary_send, syncer_heartbeat_tributary_recv) = broadcast::channel(5);
-  tokio::spawn(heartbeat_tributaries_task(syncer_p2p, syncer_heartbeat_tributary_recv));
-  syncer_heartbeat_tributary_send
-    .send(TributaryEvent::NewTributary(ActiveTributary {
-      spec: spec.clone(),
-      tributary: syncer_tributary.clone(),
-    }))
-    .map_err(|_| "failed to send ActiveTributary to heartbeat")
-    .unwrap();
-
-  // The heartbeat is once every 10 blocks, with some limitations
-  sleep(Duration::from_secs(20 * block_time)).await;
-  assert!(syncer_tributary.tip().await != spec.genesis());
-
-  // Verify it synced to the tip
-  let syncer_tip = {
-    let tributary = &tributaries[0];
-
-    let tip = tributary.tip().await;
-    let syncer_tip = syncer_tributary.tip().await;
-    // Allow a one block tolerance in case of race conditions
-    assert!(
-      HashSet::from([tip, tributary.reader().block(&tip).unwrap().parent()]).contains(&syncer_tip)
-    );
-    syncer_tip
-  };
-
-  sleep(Duration::from_secs(block_time)).await;
-
-  // Verify it's now keeping up
-  assert!(syncer_tributary.tip().await != syncer_tip);
-
-  // Verify it's now participating in consensus
-  // Because only `t` validators are used in a commit, take n - t nodes offline
-  // leaving only `t` nodes. Which should force it to participate in the consensus
-  // of next blocks.
-  let spares = usize::from(spec.n() - spec.t());
-  for thread in p2p_threads.iter().take(spares) {
-    thread.abort();
-  }
-
-  // wait for a block
-  sleep(Duration::from_secs(block_time)).await;
-
-  if syncer_tributary
-    .reader()
-    .parsed_commit(&syncer_tributary.tip().await)
-    .unwrap()
-    .validators
-    .iter()
-    .any(|signer| signer == &syncer_key.to_bytes())
-  {
-    return;
-  }
-
-  panic!("synced tributary didn't start participating in consensus");
-}

coordinator/src/tests/tributary/tx.rs

@@ -1,62 +0,0 @@
-use core::time::Duration;
-
-use rand_core::{RngCore, OsRng};
-
-use tokio::time::sleep;
-
-use serai_db::MemDb;
-
-use tributary::{
-  transaction::Transaction as TransactionTrait, Transaction as TributaryTransaction, Tributary,
-};
-
-use crate::{
-  tributary::Transaction,
-  tests::{
-    LocalP2p,
-    tributary::{new_keys, new_spec, new_tributaries, run_tributaries, wait_for_tx_inclusion},
-  },
-};
-
-#[tokio::test]
-async fn tx_test() {
-  let keys = new_keys(&mut OsRng);
-  let spec = new_spec(&mut OsRng, &keys);
-
-  let tributaries = new_tributaries(&keys, &spec)
-    .await
-    .into_iter()
-    .map(|(_, p2p, tributary)| (p2p, tributary))
-    .collect::<Vec<_>>();
-
-  // Run the tributaries in the background
-  tokio::spawn(run_tributaries(tributaries.clone()));
-
-  // Send a TX from a random Tributary
-  let sender =
-    usize::try_from(OsRng.next_u64() % u64::try_from(tributaries.len()).unwrap()).unwrap();
-  let key = keys[sender].clone();
-
-  let block_before_tx = tributaries[sender].1.tip().await;
-  // Create the TX with a null signature so we can get its sig hash
-  let mut tx = Transaction::DkgParticipation {
-    participation: {
-      let mut participation = vec![0; 4096];
-      OsRng.fill_bytes(&mut participation);
-      participation
-    },
-    signed: Transaction::empty_signed(),
-  };
-  tx.sign(&mut OsRng, spec.genesis(), &key);
-
-  assert_eq!(tributaries[sender].1.add_transaction(tx.clone()).await, Ok(true));
-  let included_in = wait_for_tx_inclusion(&tributaries[sender].1, block_before_tx, tx.hash()).await;
-  // Also sleep for the block time to ensure the block is synced around before we run checks on it
-  sleep(Duration::from_secs(Tributary::<MemDb, Transaction, LocalP2p>::block_time().into())).await;
-
-  // All tributaries should have acknowledged this transaction in a block
-  for (_, tributary) in tributaries {
-    let block = tributary.reader().block(&included_in).unwrap();
-    assert_eq!(block.transactions, vec![TributaryTransaction::Application(tx.clone())]);
-  }
-}

coordinator/src/tributary.rs

@@ -0,0 +1,453 @@
+use core::{future::Future, time::Duration};
+use std::sync::Arc;
+
+use zeroize::Zeroizing;
+use rand_core::OsRng;
+use blake2::{digest::typenum::U32, Digest, Blake2s};
+use ciphersuite::{Ciphersuite, Ristretto};
+
+use tokio::sync::mpsc;
+
+use serai_db::{Get, DbTxn, Db as DbTrait, create_db, db_channel};
+
+use scale::Encode;
+use serai_client::validator_sets::primitives::ValidatorSet;
+
+use tributary_sdk::{TransactionKind, TransactionError, ProvidedError, TransactionTrait, Tributary};
+
+use serai_task::{Task, TaskHandle, DoesNotError, ContinuallyRan};
+
+use message_queue::{Service, Metadata, client::MessageQueue};
+
+use serai_cosign::{Faulted, CosignIntent, Cosigning};
+use serai_coordinator_substrate::{NewSetInformation, SignSlashReport};
+use serai_coordinator_tributary::{Transaction, ProcessorMessages, CosignIntents, ScanTributaryTask};
+use serai_coordinator_p2p::P2p;
+
+use crate::{Db, TributaryTransactions};
+
+db_channel! {
+  Coordinator {
+    PendingCosigns: (set: ValidatorSet) -> CosignIntent,
+  }
+}
+
+/// Provide a Provided Transaction to the Tributary.
+///
+/// This is not a well-designed function. This is specific to the context in which its called,
+/// within this file. It should only be considered an internal helper for this domain alone.
+async fn provide_transaction<TD: DbTrait, P: P2p>(
+  set: ValidatorSet,
+  tributary: &Tributary<TD, Transaction, P>,
+  tx: Transaction,
+) {
+  match tributary.provide_transaction(tx.clone()).await {
+    // The Tributary uses its own DB, so we may provide this multiple times if we reboot before
+    // committing the txn which provoked this
+    Ok(()) | Err(ProvidedError::AlreadyProvided) => {}
+    Err(ProvidedError::NotProvided) => {
+      panic!("providing a Transaction which wasn't a Provided transaction: {tx:?}");
+    }
+    Err(ProvidedError::InvalidProvided(e)) => {
+      panic!("providing an invalid Provided transaction, tx: {tx:?}, error: {e:?}")
+    }
+    // The Tributary's scan task won't advance if we don't have the Provided transactions
+    // present on-chain, and this enters an infinite loop to block the calling task from
+    // advancing
+    Err(ProvidedError::LocalMismatchesOnChain) => loop {
+      log::error!(
+        "Tributary {:?} was supposed to provide {:?} but peers disagree, halting Tributary",
+        set,
+        tx,
+      );
+      // Print this every five minutes as this does need to be handled
+      tokio::time::sleep(Duration::from_secs(5 * 60)).await;
+    },
+  }
+}
+
+/// Provides Cosign/Cosigned Transactions onto the Tributary.
+pub(crate) struct ProvideCosignCosignedTransactionsTask<CD: DbTrait, TD: DbTrait, P: P2p> {
+  db: CD,
+  tributary_db: TD,
+  set: NewSetInformation,
+  tributary: Tributary<TD, Transaction, P>,
+}
+impl<CD: DbTrait, TD: DbTrait, P: P2p> ContinuallyRan
+  for ProvideCosignCosignedTransactionsTask<CD, TD, P>
+{
+  type Error = String;
+
+  fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, Self::Error>> {
+    async move {
+      let mut made_progress = false;
+
+      // Check if we produced any cosigns we were supposed to
+      let mut pending_notable_cosign = false;
+      loop {
+        let mut txn = self.db.txn();
+
+        // Fetch the next cosign this tributary should handle
+        let Some(cosign) = PendingCosigns::try_recv(&mut txn, self.set.set) else { break };
+        pending_notable_cosign = cosign.notable;
+
+        // If we (Serai) haven't cosigned this block, break as this is still pending
+        let latest = match Cosigning::<CD>::latest_cosigned_block_number(&txn) {
+          Ok(latest) => latest,
+          Err(Faulted) => {
+            log::error!("cosigning faulted");
+            Err("cosigning faulted")?
+          }
+        };
+        if latest < cosign.block_number {
+          break;
+        }
+
+        // Because we've cosigned it, provide the TX for that
+        {
+          let mut txn = self.tributary_db.txn();
+          CosignIntents::provide(&mut txn, self.set.set, &cosign);
+          txn.commit();
+        }
+        provide_transaction(
+          self.set.set,
+          &self.tributary,
+          Transaction::Cosigned { substrate_block_hash: cosign.block_hash },
+        )
+        .await;
+        // Clear pending_notable_cosign since this cosign isn't pending
+        pending_notable_cosign = false;
+
+        // Commit the txn to clear this from PendingCosigns
+        txn.commit();
+        made_progress = true;
+      }
+
+      // If we don't have any notable cosigns pending, provide the next set of cosign intents
+      if !pending_notable_cosign {
+        let mut txn = self.db.txn();
+        // intended_cosigns will only yield up to and including the next notable cosign
+        for cosign in Cosigning::<CD>::intended_cosigns(&mut txn, self.set.set) {
+          // Flag this cosign as pending
+          PendingCosigns::send(&mut txn, self.set.set, &cosign);
+          // Provide the transaction to queue it for work
+          provide_transaction(
+            self.set.set,
+            &self.tributary,
+            Transaction::Cosign { substrate_block_hash: cosign.block_hash },
+          )
+          .await;
+        }
+        txn.commit();
+        made_progress = true;
+      }
+
+      Ok(made_progress)
+    }
+  }
+}
+
+/// Adds all of the transactions sent via `TributaryTransactions`.
+pub(crate) struct AddTributaryTransactionsTask<CD: DbTrait, TD: DbTrait, P: P2p> {
+  db: CD,
+  tributary_db: TD,
+  tributary: Tributary<TD, Transaction, P>,
+  set: ValidatorSet,
+  key: Zeroizing<<Ristretto as Ciphersuite>::F>,
+}
+impl<CD: DbTrait, TD: DbTrait, P: P2p> ContinuallyRan for AddTributaryTransactionsTask<CD, TD, P> {
+  type Error = DoesNotError;
+
+  fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, Self::Error>> {
+    async move {
+      let mut made_progress = false;
+      loop {
+        let mut txn = self.db.txn();
+        let Some(mut tx) = TributaryTransactions::try_recv(&mut txn, self.set) else { break };
+
+        let kind = tx.kind();
+        match kind {
+          TransactionKind::Provided(_) => provide_transaction(self.set, &self.tributary, tx).await,
+          TransactionKind::Unsigned | TransactionKind::Signed(_, _) => {
+            // If this is a signed transaction, sign it
+            if matches!(kind, TransactionKind::Signed(_, _)) {
+              tx.sign(&mut OsRng, self.tributary.genesis(), &self.key);
+            }
+
+            // Actually add the transaction
+            // TODO: If this is a preprocess, make sure the topic has been recognized
+            let res = self.tributary.add_transaction(tx.clone()).await;
+            match &res {
+              // Fresh publication, already published
+              Ok(true | false) => {}
+              Err(
+                TransactionError::TooLargeTransaction |
+                TransactionError::InvalidSigner |
+                TransactionError::InvalidNonce |
+                TransactionError::InvalidSignature |
+                TransactionError::InvalidContent,
+              ) => {
+                panic!("created an invalid transaction, tx: {tx:?}, err: {res:?}");
+              }
+              // We've published too many transactions recently
+              // Drop this txn to try to publish it again later on a future iteration
+              Err(TransactionError::TooManyInMempool) => {
+                drop(txn);
+                break;
+              }
+              // This isn't a Provided transaction so this should never be hit
+              Err(TransactionError::ProvidedAddedToMempool) => unreachable!(),
+            }
+          }
+        }
+
+        made_progress = true;
+        txn.commit();
+      }
+      Ok(made_progress)
+    }
+  }
+}
+
+/// Takes the messages from ScanTributaryTask and publishes them to the message-queue.
+pub(crate) struct TributaryProcessorMessagesTask<TD: DbTrait> {
+  tributary_db: TD,
+  set: ValidatorSet,
+  message_queue: Arc<MessageQueue>,
+}
+impl<TD: DbTrait> ContinuallyRan for TributaryProcessorMessagesTask<TD> {
+  type Error = String; // TODO
+
+  fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, Self::Error>> {
+    async move {
+      let mut made_progress = false;
+      loop {
+        let mut txn = self.tributary_db.txn();
+        let Some(msg) = ProcessorMessages::try_recv(&mut txn, self.set) else { break };
+        let metadata = Metadata {
+          from: Service::Coordinator,
+          to: Service::Processor(self.set.network),
+          intent: msg.intent(),
+        };
+        let msg = borsh::to_vec(&msg).unwrap();
+        self.message_queue.queue(metadata, msg).await?;
+        txn.commit();
+        made_progress = true;
+      }
+      Ok(made_progress)
+    }
+  }
+}
+
+/// Checks for the notification to sign a slash report and does so if present.
+pub(crate) struct SignSlashReportTask<CD: DbTrait, TD: DbTrait, P: P2p> {
+  db: CD,
+  tributary_db: TD,
+  tributary: Tributary<TD, Transaction, P>,
+  set: NewSetInformation,
+  key: Zeroizing<<Ristretto as Ciphersuite>::F>,
+}
+impl<CD: DbTrait, TD: DbTrait, P: P2p> ContinuallyRan for SignSlashReportTask<CD, TD, P> {
+  type Error = DoesNotError;
+
+  fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, Self::Error>> {
+    async move {
+      let mut txn = self.db.txn();
+      let Some(()) = SignSlashReport::try_recv(&mut txn, self.set.set) else { return Ok(false) };
+
+      // Fetch the slash report for this Tributary
+      let mut tx =
+        serai_coordinator_tributary::slash_report_transaction(&self.tributary_db, &self.set);
+      tx.sign(&mut OsRng, self.tributary.genesis(), &self.key);
+
+      let res = self.tributary.add_transaction(tx.clone()).await;
+      match &res {
+        // Fresh publication, already published
+        Ok(true | false) => {}
+        Err(
+          TransactionError::TooLargeTransaction |
+          TransactionError::InvalidSigner |
+          TransactionError::InvalidNonce |
+          TransactionError::InvalidSignature |
+          TransactionError::InvalidContent,
+        ) => {
+          panic!("created an invalid SlashReport transaction, tx: {tx:?}, err: {res:?}");
+        }
+        // We've published too many transactions recently
+        // Drop this txn to try to publish it again later on a future iteration
+        Err(TransactionError::TooManyInMempool) => {
+          drop(txn);
+          return Ok(false);
+        }
+        // This isn't a Provided transaction so this should never be hit
+        Err(TransactionError::ProvidedAddedToMempool) => unreachable!(),
+      }
+
+      txn.commit();
+      Ok(true)
+    }
+  }
+}
+
+/// Run the scan task whenever the Tributary adds a new block.
+async fn scan_on_new_block<CD: DbTrait, TD: DbTrait, P: P2p>(
+  db: CD,
+  set: ValidatorSet,
+  tributary: Tributary<TD, Transaction, P>,
+  scan_tributary_task: TaskHandle,
+  tasks_to_keep_alive: Vec<TaskHandle>,
+) {
+  loop {
+    // Break once this Tributary is retired
+    if crate::RetiredTributary::get(&db, set.network).map(|session| session.0) >=
+      Some(set.session.0)
+    {
+      drop(tasks_to_keep_alive);
+      break;
+    }
+
+    // Have the tributary scanner run as soon as there's a new block
+    match tributary.next_block_notification().await.await {
+      Ok(()) => scan_tributary_task.run_now(),
+      // unreachable since this owns the tributary object and doesn't drop it
+      Err(_) => panic!("tributary was dropped causing notification to error"),
+    }
+  }
+}
+
+/// Spawn a Tributary.
+///
+/// This will:
+/// - Spawn the Tributary
+/// - Inform the P2P network of the Tributary
+/// - Spawn the ScanTributaryTask
+/// - Spawn the ProvideCosignCosignedTransactionsTask
+/// - Spawn the TributaryProcessorMessagesTask
+/// - Spawn the SignSlashReportTask
+/// - Iterate the scan task whenever a new block occurs (not just on the standard interval)
+pub(crate) async fn spawn_tributary<P: P2p>(
+  db: Db,
+  message_queue: Arc<MessageQueue>,
+  p2p: P,
+  p2p_add_tributary: &mpsc::UnboundedSender<(ValidatorSet, Tributary<Db, Transaction, P>)>,
+  set: NewSetInformation,
+  serai_key: Zeroizing<<Ristretto as Ciphersuite>::F>,
+) {
+  // Don't spawn retired Tributaries
+  if crate::db::RetiredTributary::get(&db, set.set.network).map(|session| session.0) >=
+    Some(set.set.session.0)
+  {
+    return;
+  }
+
+  let genesis = <[u8; 32]>::from(Blake2s::<U32>::digest((set.serai_block, set.set).encode()));
+
+  // Since the Serai block will be finalized, then cosigned, before we handle this, this time will
+  // be a couple of minutes stale. While the Tributary will still function with a start time in the
+  // past, the Tributary will immediately incur round timeouts. We reduce these by adding a
+  // constant delay of a couple of minutes.
+  const TRIBUTARY_START_TIME_DELAY: u64 = 120;
+  let start_time = set.declaration_time + TRIBUTARY_START_TIME_DELAY;
+
+  let mut tributary_validators = Vec::with_capacity(set.validators.len());
+  for (validator, weight) in set.validators.iter().copied() {
+    let validator_key = <Ristretto as Ciphersuite>::read_G(&mut validator.0.as_slice())
+      .expect("Serai validator had an invalid public key");
+    let weight = u64::from(weight);
+    tributary_validators.push((validator_key, weight));
+  }
+
+  // Spawn the Tributary
+  let tributary_db = crate::db::tributary_db(set.set);
+  let tributary = Tributary::new(
+    tributary_db.clone(),
+    genesis,
+    start_time,
+    serai_key.clone(),
+    tributary_validators,
+    p2p,
+  )
+  .await
+  .unwrap();
+  let reader = tributary.reader();
+
+  // Inform the P2P network
+  p2p_add_tributary
+    .send((set.set, tributary.clone()))
+    .expect("p2p's add_tributary channel was closed?");
+
+  // Spawn the task to provide Cosign/Cosigned transactions onto the Tributary
+  let (provide_cosign_cosigned_transactions_task_def, provide_cosign_cosigned_transactions_task) =
+    Task::new();
+  tokio::spawn(
+    (ProvideCosignCosignedTransactionsTask {
+      db: db.clone(),
+      tributary_db: tributary_db.clone(),
+      set: set.clone(),
+      tributary: tributary.clone(),
+    })
+    .continually_run(provide_cosign_cosigned_transactions_task_def, vec![]),
+  );
+
+  // Spawn the task to send all messages from the Tributary scanner to the message-queue
+  let (scan_tributary_messages_task_def, scan_tributary_messages_task) = Task::new();
+  tokio::spawn(
+    (TributaryProcessorMessagesTask {
+      tributary_db: tributary_db.clone(),
+      set: set.set,
+      message_queue,
+    })
+    .continually_run(scan_tributary_messages_task_def, vec![]),
+  );
+
+  // Spawn the scan task
+  let (scan_tributary_task_def, scan_tributary_task) = Task::new();
+  tokio::spawn(
+    ScanTributaryTask::<_, P>::new(tributary_db.clone(), &set, reader)
+      // This is the only handle for this TributaryProcessorMessagesTask, so when this task is
+      // dropped, it will be too
+      .continually_run(scan_tributary_task_def, vec![scan_tributary_messages_task]),
+  );
+
+  // Spawn the sign slash report task
+  let (sign_slash_report_task_def, sign_slash_report_task) = Task::new();
+  tokio::spawn(
+    (SignSlashReportTask {
+      db: db.clone(),
+      tributary_db: tributary_db.clone(),
+      tributary: tributary.clone(),
+      set: set.clone(),
+      key: serai_key.clone(),
+    })
+    .continually_run(sign_slash_report_task_def, vec![]),
+  );
+
+  // Spawn the add transactions task
+  let (add_tributary_transactions_task_def, add_tributary_transactions_task) = Task::new();
+  tokio::spawn(
+    (AddTributaryTransactionsTask {
+      db: db.clone(),
+      tributary_db,
+      tributary: tributary.clone(),
+      set: set.set,
+      key: serai_key,
+    })
+    .continually_run(add_tributary_transactions_task_def, vec![]),
+  );
+
+  // Whenever a new block occurs, immediately run the scan task
+  // This function also preserves the ProvideCosignCosignedTransactionsTask handle until the
+  // Tributary is retired, ensuring it isn't dropped prematurely and that the task don't run ad
+  // infinitum
+  tokio::spawn(scan_on_new_block(
+    db,
+    set.set,
+    tributary,
+    scan_tributary_task,
+    vec![
+      provide_cosign_cosigned_transactions_task,
+      sign_slash_report_task,
+      add_tributary_transactions_task,
+    ],
+  ));
+}

coordinator/src/tributary/db.rs

@@ -1,188 +0,0 @@
-use std::collections::HashMap;
-
-use scale::Encode;
-use borsh::{BorshSerialize, BorshDeserialize};
-
-use ciphersuite::{group::GroupEncoding, Ciphersuite, Ristretto};
-use frost::Participant;
-
-use serai_client::validator_sets::primitives::{KeyPair, ValidatorSet};
-
-use processor_messages::coordinator::SubstrateSignableId;
-
-pub use serai_db::*;
-
-use tributary::ReadWrite;
-
-use crate::tributary::{Label, Transaction};
-
-#[derive(Clone, Copy, PartialEq, Eq, Debug, Encode, BorshSerialize, BorshDeserialize)]
-pub enum Topic {
-  DkgConfirmation,
-  SubstrateSign(SubstrateSignableId),
-  Sign([u8; 32]),
-}
-
-// A struct to refer to a piece of data all validators will presumably provide a value for.
-#[derive(Clone, Copy, PartialEq, Eq, Debug, Encode)]
-pub struct DataSpecification {
-  pub topic: Topic,
-  pub label: Label,
-  pub attempt: u32,
-}
-
-pub enum DataSet {
-  Participating(HashMap<Participant, Vec<u8>>),
-  NotParticipating,
-}
-
-pub enum Accumulation {
-  Ready(DataSet),
-  NotReady,
-}
-
-// TODO: Move from genesis to set for indexing
-create_db!(
-  Tributary {
-    SeraiBlockNumber: (hash: [u8; 32]) -> u64,
-    SeraiDkgCompleted: (set: ValidatorSet) -> [u8; 32],
-
-    TributaryBlockNumber: (block: [u8; 32]) -> u32,
-    LastHandledBlock: (genesis: [u8; 32]) -> [u8; 32],
-
-    // TODO: Revisit the point of this
-    FatalSlashes: (genesis: [u8; 32]) -> Vec<[u8; 32]>,
-    // TODO: Combine these two
-    FatallySlashed: (genesis: [u8; 32], account: [u8; 32]) -> (),
-    SlashPoints: (genesis: [u8; 32], account: [u8; 32]) -> u32,
-
-    VotedToRemove: (genesis: [u8; 32], voter: [u8; 32], to_remove: [u8; 32]) -> (),
-    VotesToRemove: (genesis: [u8; 32], to_remove: [u8; 32]) -> u16,
-
-    AttemptDb: (genesis: [u8; 32], topic: &Topic) -> u32,
-    ReattemptDb: (genesis: [u8; 32], block: u32) -> Vec<Topic>,
-    DataReceived: (genesis: [u8; 32], data_spec: &DataSpecification) -> u16,
-    DataDb: (genesis: [u8; 32], data_spec: &DataSpecification, signer_bytes: &[u8; 32]) -> Vec<u8>,
-
-    DkgParticipation: (genesis: [u8; 32], from: u16) -> Vec<u8>,
-    ConfirmationNonces: (genesis: [u8; 32], attempt: u32) -> HashMap<Participant, Vec<u8>>,
-    DkgKeyPair: (genesis: [u8; 32]) -> KeyPair,
-
-    PlanIds: (genesis: &[u8], block: u64) -> Vec<[u8; 32]>,
-
-    SignedTransactionDb: (order: &[u8], nonce: u32) -> Vec<u8>,
-
-    SlashReports: (genesis: [u8; 32], signer: [u8; 32]) -> Vec<u32>,
-    SlashReported: (genesis: [u8; 32]) -> u16,
-    SlashReportCutOff: (genesis: [u8; 32]) -> u64,
-    SlashReport: (set: ValidatorSet) -> Vec<([u8; 32], u32)>,
-  }
-);
-
-impl FatalSlashes {
-  pub fn get_as_keys(getter: &impl Get, genesis: [u8; 32]) -> Vec<<Ristretto as Ciphersuite>::G> {
-    FatalSlashes::get(getter, genesis)
-      .unwrap_or(vec![])
-      .iter()
-      .map(|key| <Ristretto as Ciphersuite>::G::from_bytes(key).unwrap())
-      .collect::<Vec<_>>()
-  }
-}
-
-impl FatallySlashed {
-  pub fn set_fatally_slashed(txn: &mut impl DbTxn, genesis: [u8; 32], account: [u8; 32]) {
-    Self::set(txn, genesis, account, &());
-    let mut existing = FatalSlashes::get(txn, genesis).unwrap_or_default();
-
-    // Don't append if we already have it, which can occur upon multiple faults
-    if existing.iter().any(|existing| existing == &account) {
-      return;
-    }
-
-    existing.push(account);
-    FatalSlashes::set(txn, genesis, &existing);
-  }
-}
-
-impl AttemptDb {
-  pub fn recognize_topic(txn: &mut impl DbTxn, genesis: [u8; 32], topic: Topic) {
-    Self::set(txn, genesis, &topic, &0u32);
-  }
-
-  pub fn start_next_attempt(txn: &mut impl DbTxn, genesis: [u8; 32], topic: Topic) -> u32 {
-    let next =
-      Self::attempt(txn, genesis, topic).expect("starting next attempt for unknown topic") + 1;
-    Self::set(txn, genesis, &topic, &next);
-    next
-  }
-
-  pub fn attempt(getter: &impl Get, genesis: [u8; 32], topic: Topic) -> Option<u32> {
-    let attempt = Self::get(getter, genesis, &topic);
-    // Don't require explicit recognition of the DkgConfirmation topic as it starts when the chain
-    // does
-    // Don't require explicit recognition of the SlashReport topic as it isn't a DoS risk and it
-    // should always happen (eventually)
-    if attempt.is_none() &&
-      ((topic == Topic::DkgConfirmation) ||
-        (topic == Topic::SubstrateSign(SubstrateSignableId::SlashReport)))
-    {
-      return Some(0);
-    }
-    attempt
-  }
-}
-
-impl ReattemptDb {
-  pub fn schedule_reattempt(
-    txn: &mut impl DbTxn,
-    genesis: [u8; 32],
-    current_block_number: u32,
-    topic: Topic,
-  ) {
-    // 5 minutes
-    #[cfg(not(feature = "longer-reattempts"))]
-    const BASE_REATTEMPT_DELAY: u32 = (5 * 60 * 1000) / tributary::tendermint::TARGET_BLOCK_TIME;
-
-    // 10 minutes, intended for latent environments like the GitHub CI
-    #[cfg(feature = "longer-reattempts")]
-    const BASE_REATTEMPT_DELAY: u32 = (10 * 60 * 1000) / tributary::tendermint::TARGET_BLOCK_TIME;
-
-    // 5 minutes for attempts 0 ..= 2, 10 minutes for attempts 3 ..= 5, 15 minutes for attempts > 5
-    // Assumes no event will take longer than 15 minutes, yet grows the time in case there are
-    // network bandwidth issues
-    let reattempt_delay = BASE_REATTEMPT_DELAY *
-      ((AttemptDb::attempt(txn, genesis, topic)
-        .expect("scheduling re-attempt for unknown topic") /
-        3) +
-        1)
-      .min(3);
-    let upon_block = current_block_number + reattempt_delay;
-
-    let mut reattempts = Self::get(txn, genesis, upon_block).unwrap_or(vec![]);
-    reattempts.push(topic);
-    Self::set(txn, genesis, upon_block, &reattempts);
-  }
-
-  pub fn take(txn: &mut impl DbTxn, genesis: [u8; 32], block_number: u32) -> Vec<Topic> {
-    let res = Self::get(txn, genesis, block_number).unwrap_or(vec![]);
-    if !res.is_empty() {
-      Self::del(txn, genesis, block_number);
-    }
-    res
-  }
-}
-
-impl SignedTransactionDb {
-  pub fn take_signed_transaction(
-    txn: &mut impl DbTxn,
-    order: &[u8],
-    nonce: u32,
-  ) -> Option<Transaction> {
-    let res = SignedTransactionDb::get(txn, order, nonce)
-      .map(|bytes| Transaction::read(&mut bytes.as_slice()).unwrap());
-    if res.is_some() {
-      Self::del(txn, order, nonce);
-    }
-    res
-  }
-}

coordinator/src/tributary/handle.rs

@@ -1,554 +0,0 @@
-use core::ops::Deref;
-use std::collections::HashMap;
-
-use zeroize::Zeroizing;
-use rand_core::OsRng;
-
-use ciphersuite::{group::GroupEncoding, Ciphersuite, Ristretto};
-use frost::dkg::Participant;
-
-use scale::{Encode, Decode};
-use serai_client::{Signature, validator_sets::primitives::KeyPair};
-
-use tributary::{Signed, TransactionKind, TransactionTrait};
-
-use processor_messages::{
-  key_gen::self,
-  coordinator::{self, SubstrateSignableId, SubstrateSignId},
-  sign::{self, SignId},
-};
-
-use serai_db::*;
-
-use crate::{
-  processors::Processors,
-  tributary::{
-    *,
-    signing_protocol::DkgConfirmer,
-    scanner::{
-      RecognizedIdType, RIDTrait, PublishSeraiTransaction, PTTTrait, TributaryBlockHandler,
-    },
-  },
-  P2p,
-};
-
-pub fn dkg_confirmation_nonces(
-  key: &Zeroizing<<Ristretto as Ciphersuite>::F>,
-  spec: &TributarySpec,
-  txn: &mut impl DbTxn,
-  attempt: u32,
-) -> [u8; 64] {
-  DkgConfirmer::new(key, spec, txn, attempt).preprocess()
-}
-
-pub fn generated_key_pair<D: Db>(
-  txn: &mut D::Transaction<'_>,
-  genesis: [u8; 32],
-  key_pair: &KeyPair,
-) {
-  DkgKeyPair::set(txn, genesis, key_pair);
-}
-
-fn unflatten(spec: &TributarySpec, data: &mut HashMap<Participant, Vec<u8>>) {
-  for (validator, _) in spec.validators() {
-    let Some(range) = spec.i(validator) else { continue };
-    let Some(all_segments) = data.remove(&range.start) else {
-      continue;
-    };
-    let mut data_vec = Vec::<_>::decode(&mut all_segments.as_slice()).unwrap();
-    for i in u16::from(range.start) .. u16::from(range.end) {
-      let i = Participant::new(i).unwrap();
-      data.insert(i, data_vec.remove(0));
-    }
-  }
-}
-
-impl<
-    D: Db,
-    T: DbTxn,
-    Pro: Processors,
-    PST: PublishSeraiTransaction,
-    PTT: PTTTrait,
-    RID: RIDTrait,
-    P: P2p,
-  > TributaryBlockHandler<'_, D, T, Pro, PST, PTT, RID, P>
-{
-  fn accumulate(
-    &mut self,
-    data_spec: &DataSpecification,
-    signer: <Ristretto as Ciphersuite>::G,
-    data: &Vec<u8>,
-  ) -> Accumulation {
-    log::debug!("accumulating entry for {:?} attempt #{}", &data_spec.topic, &data_spec.attempt);
-    let genesis = self.spec.genesis();
-    if DataDb::get(self.txn, genesis, data_spec, &signer.to_bytes()).is_some() {
-      panic!("accumulating data for a participant multiple times");
-    }
-    let signer_shares = {
-      let signer_i = self.spec.i(signer).expect("transaction signer wasn't a member of the set");
-      u16::from(signer_i.end) - u16::from(signer_i.start)
-    };
-
-    let prior_received = DataReceived::get(self.txn, genesis, data_spec).unwrap_or_default();
-    let now_received = prior_received + signer_shares;
-    DataReceived::set(self.txn, genesis, data_spec, &now_received);
-    DataDb::set(self.txn, genesis, data_spec, &signer.to_bytes(), data);
-
-    let received_range = (prior_received + 1) ..= now_received;
-
-    // If 2/3rds of the network participated in this preprocess, queue it for an automatic
-    // re-attempt
-    if (data_spec.label == Label::Preprocess) && received_range.contains(&self.spec.t()) {
-      // Double check the attempt on this entry, as we don't want to schedule a re-attempt if this
-      // is an old entry
-      // This is an assert, not part of the if check, as old data shouldn't be here in the first
-      // place
-      assert_eq!(AttemptDb::attempt(self.txn, genesis, data_spec.topic), Some(data_spec.attempt));
-      ReattemptDb::schedule_reattempt(self.txn, genesis, self.block_number, data_spec.topic);
-    }
-
-    // If we have all the needed commitments/preprocesses/shares, tell the processor
-    if received_range.contains(&self.spec.t()) {
-      log::debug!(
-        "accumulation for entry {:?} attempt #{} is ready",
-        &data_spec.topic,
-        &data_spec.attempt
-      );
-
-      let mut data = HashMap::new();
-      for validator in self.spec.validators().iter().map(|validator| validator.0) {
-        let Some(i) = self.spec.i(validator) else { continue };
-        data.insert(
-          i.start,
-          if let Some(data) = DataDb::get(self.txn, genesis, data_spec, &validator.to_bytes()) {
-            data
-          } else {
-            continue;
-          },
-        );
-      }
-
-      assert_eq!(data.len(), usize::from(self.spec.t()));
-
-      // Remove our own piece of data, if we were involved
-      if let Some(i) = self.spec.i(Ristretto::generator() * self.our_key.deref()) {
-        if data.remove(&i.start).is_some() {
-          return Accumulation::Ready(DataSet::Participating(data));
-        }
-      }
-      return Accumulation::Ready(DataSet::NotParticipating);
-    }
-    Accumulation::NotReady
-  }
-
-  fn handle_data(
-    &mut self,
-    data_spec: &DataSpecification,
-    bytes: &Vec<u8>,
-    signed: &Signed,
-  ) -> Accumulation {
-    let genesis = self.spec.genesis();
-
-    let Some(curr_attempt) = AttemptDb::attempt(self.txn, genesis, data_spec.topic) else {
-      // Premature publication of a valid ID/publication of an invalid ID
-      self.fatal_slash(signed.signer.to_bytes(), "published data for ID without an attempt");
-      return Accumulation::NotReady;
-    };
-
-    // If they've already published a TX for this attempt, slash
-    // This shouldn't be reachable since nonces were made inserted by the coordinator, yet it's a
-    // cheap check to leave in for safety
-    if DataDb::get(self.txn, genesis, data_spec, &signed.signer.to_bytes()).is_some() {
-      self.fatal_slash(signed.signer.to_bytes(), "published data multiple times");
-      return Accumulation::NotReady;
-    }
-
-    // If the attempt is lesser than the blockchain's, return
-    if data_spec.attempt < curr_attempt {
-      log::debug!(
-        "dated attempt published onto tributary for topic {:?} (used attempt {}, current {})",
-        data_spec.topic,
-        data_spec.attempt,
-        curr_attempt
-      );
-      return Accumulation::NotReady;
-    }
-    // If the attempt is greater, this is a premature publication, full slash
-    if data_spec.attempt > curr_attempt {
-      self.fatal_slash(
-        signed.signer.to_bytes(),
-        "published data with an attempt which hasn't started",
-      );
-      return Accumulation::NotReady;
-    }
-
-    // TODO: We can also full slash if shares before all commitments, or share before the
-    // necessary preprocesses
-
-    // TODO: If this is shares, we need to check they are part of the selected signing set
-
-    // Accumulate this data
-    self.accumulate(data_spec, signed.signer, bytes)
-  }
-
-  fn check_sign_data_len(
-    &mut self,
-    signer: <Ristretto as Ciphersuite>::G,
-    len: usize,
-  ) -> Result<(), ()> {
-    let signer_i = self.spec.i(signer).expect("signer wasn't a member of the set");
-    if len != usize::from(u16::from(signer_i.end) - u16::from(signer_i.start)) {
-      self.fatal_slash(
-        signer.to_bytes(),
-        "signer published a distinct amount of sign data than they had shares",
-      );
-      Err(())?;
-    }
-    Ok(())
-  }
-
-  // TODO: Don't call fatal_slash in here, return the party to fatal_slash to ensure no further
-  // execution occurs
-  pub(crate) async fn handle_application_tx(&mut self, tx: Transaction) {
-    let genesis = self.spec.genesis();
-
-    // Don't handle transactions from fatally slashed participants
-    // This prevents removed participants from sabotaging the removal signing sessions and so on
-    // TODO: Because fatally slashed participants can still publish onto the blockchain, they have
-    // a notable DoS ability
-    if let TransactionKind::Signed(_, signed) = tx.kind() {
-      if FatallySlashed::get(self.txn, genesis, signed.signer.to_bytes()).is_some() {
-        return;
-      }
-    }
-
-    match tx {
-      Transaction::RemoveParticipant { participant, signed } => {
-        if self.spec.i(participant).is_none() {
-          self.fatal_slash(participant.to_bytes(), "RemoveParticipant vote for non-validator");
-          return;
-        }
-
-        let participant = participant.to_bytes();
-        let signer = signed.signer.to_bytes();
-
-        assert!(
-          VotedToRemove::get(self.txn, genesis, signer, participant).is_none(),
-          "VotedToRemove multiple times despite a single nonce being allocated",
-        );
-        VotedToRemove::set(self.txn, genesis, signer, participant, &());
-
-        let prior_votes = VotesToRemove::get(self.txn, genesis, participant).unwrap_or(0);
-        let signer_votes =
-          self.spec.i(signed.signer).expect("signer wasn't a validator for this network?");
-        let new_votes = prior_votes + u16::from(signer_votes.end) - u16::from(signer_votes.start);
-        VotesToRemove::set(self.txn, genesis, participant, &new_votes);
-        if ((prior_votes + 1) ..= new_votes).contains(&self.spec.t()) {
-          self.fatal_slash(participant, "RemoveParticipant vote")
-        }
-      }
-
-      Transaction::DkgParticipation { participation, signed } => {
-        // Send the participation to the processor
-        self
-          .processors
-          .send(
-            self.spec.set().network,
-            key_gen::CoordinatorMessage::Participation {
-              session: self.spec.set().session,
-              participant: self
-                .spec
-                .i(signed.signer)
-                .expect("signer wasn't a validator for this network?")
-                .start,
-              participation,
-            },
-          )
-          .await;
-      }
-
-      Transaction::DkgConfirmationNonces { attempt, confirmation_nonces, signed } => {
-        let data_spec =
-          DataSpecification { topic: Topic::DkgConfirmation, label: Label::Preprocess, attempt };
-        match self.handle_data(&data_spec, &confirmation_nonces.to_vec(), &signed) {
-          Accumulation::Ready(DataSet::Participating(confirmation_nonces)) => {
-            log::info!(
-              "got all DkgConfirmationNonces for {}, attempt {attempt}",
-              hex::encode(genesis)
-            );
-
-            ConfirmationNonces::set(self.txn, genesis, attempt, &confirmation_nonces);
-
-            // Send the expected DkgConfirmationShare
-            // TODO: Slight race condition here due to set, publish tx, then commit txn
-            let key_pair = DkgKeyPair::get(self.txn, genesis)
-              .expect("participating in confirming key we don't have");
-            let mut tx = match DkgConfirmer::new(self.our_key, self.spec, self.txn, attempt)
-              .share(confirmation_nonces, &key_pair)
-            {
-              Ok(confirmation_share) => Transaction::DkgConfirmationShare {
-                attempt,
-                confirmation_share,
-                signed: Transaction::empty_signed(),
-              },
-              Err(participant) => Transaction::RemoveParticipant {
-                participant: self.spec.reverse_lookup_i(participant).unwrap(),
-                signed: Transaction::empty_signed(),
-              },
-            };
-            tx.sign(&mut OsRng, genesis, self.our_key);
-            self.publish_tributary_tx.publish_tributary_tx(tx).await;
-          }
-          Accumulation::Ready(DataSet::NotParticipating) | Accumulation::NotReady => {}
-        }
-      }
-
-      Transaction::DkgConfirmationShare { attempt, confirmation_share, signed } => {
-        let data_spec =
-          DataSpecification { topic: Topic::DkgConfirmation, label: Label::Share, attempt };
-        match self.handle_data(&data_spec, &confirmation_share.to_vec(), &signed) {
-          Accumulation::Ready(DataSet::Participating(shares)) => {
-            log::info!(
-              "got all DkgConfirmationShare for {}, attempt {attempt}",
-              hex::encode(genesis)
-            );
-
-            let preprocesses = ConfirmationNonces::get(self.txn, genesis, attempt).unwrap();
-
-            // TODO: This can technically happen under very very very specific timing as the txn
-            // put happens before DkgConfirmationShare, yet the txn isn't guaranteed to be
-            // committed
-            let key_pair = DkgKeyPair::get(self.txn, genesis).expect(
-              "in DkgConfirmationShare handling, which happens after everyone \
-              (including us) fires DkgConfirmationShare, yet no confirming key pair",
-            );
-
-            // Determine the bitstring representing who participated before we move `shares`
-            let validators = self.spec.validators();
-            let mut signature_participants = bitvec::vec::BitVec::with_capacity(validators.len());
-            for (participant, _) in validators {
-              signature_participants.push(
-                (participant == (<Ristretto as Ciphersuite>::generator() * self.our_key.deref())) ||
-                  shares.contains_key(&self.spec.i(participant).unwrap().start),
-              );
-            }
-
-            // Produce the final signature
-            let mut confirmer = DkgConfirmer::new(self.our_key, self.spec, self.txn, attempt);
-            let sig = match confirmer.complete(preprocesses, &key_pair, shares) {
-              Ok(sig) => sig,
-              Err(p) => {
-                let mut tx = Transaction::RemoveParticipant {
-                  participant: self.spec.reverse_lookup_i(p).unwrap(),
-                  signed: Transaction::empty_signed(),
-                };
-                tx.sign(&mut OsRng, genesis, self.our_key);
-                self.publish_tributary_tx.publish_tributary_tx(tx).await;
-                return;
-              }
-            };
-
-            self
-              .publish_serai_tx
-              .publish_set_keys(
-                self.db,
-                self.spec.set(),
-                key_pair,
-                signature_participants,
-                Signature(sig),
-              )
-              .await;
-          }
-          Accumulation::Ready(DataSet::NotParticipating) | Accumulation::NotReady => {}
-        }
-      }
-
-      Transaction::CosignSubstrateBlock(hash) => {
-        AttemptDb::recognize_topic(
-          self.txn,
-          genesis,
-          Topic::SubstrateSign(SubstrateSignableId::CosigningSubstrateBlock(hash)),
-        );
-
-        let block_number = SeraiBlockNumber::get(self.txn, hash)
-          .expect("CosignSubstrateBlock yet didn't save Serai block number");
-        let msg = coordinator::CoordinatorMessage::CosignSubstrateBlock {
-          id: SubstrateSignId {
-            session: self.spec.set().session,
-            id: SubstrateSignableId::CosigningSubstrateBlock(hash),
-            attempt: 0,
-          },
-          block_number,
-        };
-        self.processors.send(self.spec.set().network, msg).await;
-      }
-
-      Transaction::Batch { block: _, batch } => {
-        // Because this Batch has achieved synchrony, its batch ID should be authorized
-        AttemptDb::recognize_topic(
-          self.txn,
-          genesis,
-          Topic::SubstrateSign(SubstrateSignableId::Batch(batch)),
-        );
-        self
-          .recognized_id
-          .recognized_id(
-            self.spec.set(),
-            genesis,
-            RecognizedIdType::Batch,
-            batch.to_le_bytes().to_vec(),
-          )
-          .await;
-      }
-
-      Transaction::SubstrateBlock(block) => {
-        let plan_ids = PlanIds::get(self.txn, &genesis, block).expect(
-          "synced a tributary block finalizing a substrate block in a provided transaction \
-          despite us not providing that transaction",
-        );
-
-        for id in plan_ids {
-          AttemptDb::recognize_topic(self.txn, genesis, Topic::Sign(id));
-          self
-            .recognized_id
-            .recognized_id(self.spec.set(), genesis, RecognizedIdType::Plan, id.to_vec())
-            .await;
-        }
-      }
-
-      Transaction::SubstrateSign(data) => {
-        let signer = data.signed.signer;
-        let Ok(()) = self.check_sign_data_len(signer, data.data.len()) else {
-          return;
-        };
-        let expected_len = match data.label {
-          Label::Preprocess => 64,
-          Label::Share => 32,
-        };
-        for data in &data.data {
-          if data.len() != expected_len {
-            self.fatal_slash(
-              signer.to_bytes(),
-              "unexpected length data for substrate signing protocol",
-            );
-            return;
-          }
-        }
-
-        let data_spec = DataSpecification {
-          topic: Topic::SubstrateSign(data.plan),
-          label: data.label,
-          attempt: data.attempt,
-        };
-        let Accumulation::Ready(DataSet::Participating(mut results)) =
-          self.handle_data(&data_spec, &data.data.encode(), &data.signed)
-        else {
-          return;
-        };
-        unflatten(self.spec, &mut results);
-
-        let id = SubstrateSignId {
-          session: self.spec.set().session,
-          id: data.plan,
-          attempt: data.attempt,
-        };
-        let msg = match data.label {
-          Label::Preprocess => coordinator::CoordinatorMessage::SubstratePreprocesses {
-            id,
-            preprocesses: results.into_iter().map(|(v, p)| (v, p.try_into().unwrap())).collect(),
-          },
-          Label::Share => coordinator::CoordinatorMessage::SubstrateShares {
-            id,
-            shares: results.into_iter().map(|(v, p)| (v, p.try_into().unwrap())).collect(),
-          },
-        };
-        self.processors.send(self.spec.set().network, msg).await;
-      }
-
-      Transaction::Sign(data) => {
-        let Ok(()) = self.check_sign_data_len(data.signed.signer, data.data.len()) else {
-          return;
-        };
-
-        let data_spec = DataSpecification {
-          topic: Topic::Sign(data.plan),
-          label: data.label,
-          attempt: data.attempt,
-        };
-        if let Accumulation::Ready(DataSet::Participating(mut results)) =
-          self.handle_data(&data_spec, &data.data.encode(), &data.signed)
-        {
-          unflatten(self.spec, &mut results);
-          let id =
-            SignId { session: self.spec.set().session, id: data.plan, attempt: data.attempt };
-          self
-            .processors
-            .send(
-              self.spec.set().network,
-              match data.label {
-                Label::Preprocess => {
-                  sign::CoordinatorMessage::Preprocesses { id, preprocesses: results }
-                }
-                Label::Share => sign::CoordinatorMessage::Shares { id, shares: results },
-              },
-            )
-            .await;
-        }
-      }
-
-      Transaction::SignCompleted { plan, tx_hash, first_signer, signature: _ } => {
-        log::info!(
-          "on-chain SignCompleted claims {} completes {}",
-          hex::encode(&tx_hash),
-          hex::encode(plan)
-        );
-
-        if AttemptDb::attempt(self.txn, genesis, Topic::Sign(plan)).is_none() {
-          self.fatal_slash(first_signer.to_bytes(), "claimed an unrecognized plan was completed");
-          return;
-        };
-
-        // TODO: Confirm this signer hasn't prior published a completion
-
-        let msg = sign::CoordinatorMessage::Completed {
-          session: self.spec.set().session,
-          id: plan,
-          tx: tx_hash,
-        };
-        self.processors.send(self.spec.set().network, msg).await;
-      }
-
-      Transaction::SlashReport(points, signed) => {
-        let signer_range = self.spec.i(signed.signer).unwrap();
-        let signer_len = u16::from(signer_range.end) - u16::from(signer_range.start);
-        if points.len() != (self.spec.validators().len() - 1) {
-          self.fatal_slash(
-            signed.signer.to_bytes(),
-            "submitted a distinct amount of slash points to participants",
-          );
-          return;
-        }
-
-        if SlashReports::get(self.txn, genesis, signed.signer.to_bytes()).is_some() {
-          self.fatal_slash(signed.signer.to_bytes(), "submitted multiple slash points");
-          return;
-        }
-        SlashReports::set(self.txn, genesis, signed.signer.to_bytes(), &points);
-
-        let prior_reported = SlashReported::get(self.txn, genesis).unwrap_or(0);
-        let now_reported = prior_reported + signer_len;
-        SlashReported::set(self.txn, genesis, &now_reported);
-
-        if (prior_reported < self.spec.t()) && (now_reported >= self.spec.t()) {
-          SlashReportCutOff::set(
-            self.txn,
-            genesis,
-            // 30 minutes into the future
-            &(u64::from(self.block_number) +
-              ((30 * 60 * 1000) / u64::from(tributary::tendermint::TARGET_BLOCK_TIME))),
-          );
-        }
-      }
-    }
-  }
-}

coordinator/src/tributary/mod.rs

@@ -1,63 +0,0 @@
-use tributary::{
-  ReadWrite,
-  transaction::{TransactionError, TransactionKind, Transaction as TransactionTrait},
-  Tributary,
-};
-
-mod db;
-pub use db::*;
-
-mod spec;
-pub use spec::TributarySpec;
-
-mod transaction;
-pub use transaction::{Label, SignData, Transaction};
-
-mod signing_protocol;
-
-mod handle;
-pub use handle::*;
-
-pub mod scanner;
-
-pub async fn publish_signed_transaction<D: Db, P: crate::P2p>(
-  txn: &mut D::Transaction<'_>,
-  tributary: &Tributary<D, Transaction, P>,
-  tx: Transaction,
-) {
-  log::debug!("publishing transaction {}", hex::encode(tx.hash()));
-
-  let (order, signer) = if let TransactionKind::Signed(order, signed) = tx.kind() {
-    let signer = signed.signer;
-
-    // Safe as we should deterministically create transactions, meaning if this is already on-disk,
-    // it's what we're saving now
-    SignedTransactionDb::set(txn, &order, signed.nonce, &tx.serialize());
-
-    (order, signer)
-  } else {
-    panic!("non-signed transaction passed to publish_signed_transaction");
-  };
-
-  // If we're trying to publish 5, when the last transaction published was 3, this will delay
-  // publication until the point in time we publish 4
-  while let Some(tx) = SignedTransactionDb::take_signed_transaction(
-    txn,
-    &order,
-    tributary
-      .next_nonce(&signer, &order)
-      .await
-      .expect("we don't have a nonce, meaning we aren't a participant on this tributary"),
-  ) {
-    // We need to return a proper error here to enable that, due to a race condition around
-    // multiple publications
-    match tributary.add_transaction(tx.clone()).await {
-      Ok(_) => {}
-      // Some asynchonicity if InvalidNonce, assumed safe to deterministic nonces
-      Err(TransactionError::InvalidNonce) => {
-        log::warn!("publishing TX {tx:?} returned InvalidNonce. was it already added?")
-      }
-      Err(e) => panic!("created an invalid transaction: {e:?}"),
-    }
-  }
-}

coordinator/src/tributary/scanner.rs

@@ -1,685 +0,0 @@
-use core::{marker::PhantomData, future::Future, time::Duration};
-use std::sync::Arc;
-
-use zeroize::Zeroizing;
-
-use rand_core::OsRng;
-
-use ciphersuite::{group::GroupEncoding, Ciphersuite, Ristretto};
-
-use tokio::sync::broadcast;
-
-use scale::{Encode, Decode};
-use serai_client::{
-  primitives::Signature,
-  validator_sets::primitives::{KeyPair, ValidatorSet},
-  Serai,
-};
-
-use serai_db::DbTxn;
-
-use processor_messages::coordinator::{SubstrateSignId, SubstrateSignableId};
-
-use tributary::{
-  TransactionKind, Transaction as TributaryTransaction, TransactionError, Block, TributaryReader,
-  tendermint::{
-    tx::{TendermintTx, Evidence, decode_signed_message},
-    TendermintNetwork,
-  },
-};
-
-use crate::{Db, processors::Processors, substrate::BatchInstructionsHashDb, tributary::*, P2p};
-
-#[derive(Clone, Copy, PartialEq, Eq, Debug, Encode, Decode)]
-pub enum RecognizedIdType {
-  Batch,
-  Plan,
-}
-
-#[async_trait::async_trait]
-pub trait RIDTrait {
-  async fn recognized_id(
-    &self,
-    set: ValidatorSet,
-    genesis: [u8; 32],
-    kind: RecognizedIdType,
-    id: Vec<u8>,
-  );
-}
-#[async_trait::async_trait]
-impl<
-    FRid: Send + Future<Output = ()>,
-    F: Sync + Fn(ValidatorSet, [u8; 32], RecognizedIdType, Vec<u8>) -> FRid,
-  > RIDTrait for F
-{
-  async fn recognized_id(
-    &self,
-    set: ValidatorSet,
-    genesis: [u8; 32],
-    kind: RecognizedIdType,
-    id: Vec<u8>,
-  ) {
-    (self)(set, genesis, kind, id).await
-  }
-}
-
-#[async_trait::async_trait]
-pub trait PublishSeraiTransaction {
-  async fn publish_set_keys(
-    &self,
-    db: &(impl Sync + Get),
-    set: ValidatorSet,
-    key_pair: KeyPair,
-    signature_participants: bitvec::vec::BitVec<u8, bitvec::order::Lsb0>,
-    signature: Signature,
-  );
-}
-
-mod impl_pst_for_serai {
-  use super::*;
-
-  use serai_client::SeraiValidatorSets;
-
-  // Uses a macro because Rust can't resolve the lifetimes/generics around the check function
-  // check is expected to return true if the effect has already occurred
-  // The generated publish function will return true if *we* published the transaction
-  macro_rules! common_pst {
-    ($Meta: ty, $check: ident) => {
-      async fn publish(
-        serai: &Serai,
-        db: &impl Get,
-        set: ValidatorSet,
-        tx: serai_client::Transaction,
-        meta: $Meta,
-      ) -> bool {
-        loop {
-          match serai.publish(&tx).await {
-            Ok(_) => return true,
-            // This is assumed to be some ephemeral error due to the assumed fault-free
-            // creation
-            // TODO2: Differentiate connection errors from invariants
-            Err(e) => {
-              // The following block is irrelevant, and can/likely will fail, if we're publishing
-              // a TX for an old session
-              // If we're on a newer session, move on
-              if crate::RetiredTributaryDb::get(db, set).is_some() {
-                log::warn!("trying to publish a TX relevant to set {set:?} which isn't the latest");
-                return false;
-              }
-
-              if let Ok(serai) = serai.as_of_latest_finalized_block().await {
-                let serai = serai.validator_sets();
-
-                // Check if someone else published the TX in question
-                if $check(serai, set, meta).await {
-                  return false;
-                }
-              }
-
-              log::error!("couldn't connect to Serai node to publish TX: {e:?}");
-              tokio::time::sleep(core::time::Duration::from_secs(5)).await;
-            }
-          }
-        }
-      }
-    };
-  }
-
-  #[async_trait::async_trait]
-  impl PublishSeraiTransaction for Serai {
-    async fn publish_set_keys(
-      &self,
-      db: &(impl Sync + Get),
-      set: ValidatorSet,
-      key_pair: KeyPair,
-      signature_participants: bitvec::vec::BitVec<u8, bitvec::order::Lsb0>,
-      signature: Signature,
-    ) {
-      let tx =
-        SeraiValidatorSets::set_keys(set.network, key_pair, signature_participants, signature);
-      async fn check(serai: SeraiValidatorSets<'_>, set: ValidatorSet, (): ()) -> bool {
-        if matches!(serai.keys(set).await, Ok(Some(_))) {
-          log::info!("another coordinator set key pair for {:?}", set);
-          return true;
-        }
-        false
-      }
-      common_pst!((), check);
-      if publish(self, db, set, tx, ()).await {
-        log::info!("published set keys for {set:?}");
-      }
-    }
-  }
-}
-
-#[async_trait::async_trait]
-pub trait PTTTrait {
-  async fn publish_tributary_tx(&self, tx: Transaction);
-}
-#[async_trait::async_trait]
-impl<FPtt: Send + Future<Output = ()>, F: Sync + Fn(Transaction) -> FPtt> PTTTrait for F {
-  async fn publish_tributary_tx(&self, tx: Transaction) {
-    (self)(tx).await
-  }
-}
-
-pub struct TributaryBlockHandler<
-  'a,
-  D: Db,
-  T: DbTxn,
-  Pro: Processors,
-  PST: PublishSeraiTransaction,
-  PTT: PTTTrait,
-  RID: RIDTrait,
-  P: P2p,
-> {
-  pub db: &'a D,
-  pub txn: &'a mut T,
-  pub our_key: &'a Zeroizing<<Ristretto as Ciphersuite>::F>,
-  pub recognized_id: &'a RID,
-  pub processors: &'a Pro,
-  pub publish_serai_tx: &'a PST,
-  pub publish_tributary_tx: &'a PTT,
-  pub spec: &'a TributarySpec,
-  block: Block<Transaction>,
-  pub block_number: u32,
-  _p2p: PhantomData<P>,
-}
-
-impl<
-    D: Db,
-    T: DbTxn,
-    Pro: Processors,
-    PST: PublishSeraiTransaction,
-    PTT: PTTTrait,
-    RID: RIDTrait,
-    P: P2p,
-  > TributaryBlockHandler<'_, D, T, Pro, PST, PTT, RID, P>
-{
-  pub fn fatal_slash(&mut self, slashing: [u8; 32], reason: &str) {
-    let genesis = self.spec.genesis();
-
-    log::warn!("fatally slashing {}. reason: {}", hex::encode(slashing), reason);
-    FatallySlashed::set_fatally_slashed(self.txn, genesis, slashing);
-
-    // TODO: disconnect the node from network/ban from further participation in all Tributaries
-  }
-
-  // TODO: Once Substrate confirms a key, we need to rotate our validator set OR form a second
-  // Tributary post-DKG
-  // https://github.com/serai-dex/serai/issues/426
-
-  async fn handle(mut self) {
-    log::info!("found block for Tributary {:?}", self.spec.set());
-
-    let transactions = self.block.transactions.clone();
-    for tx in transactions {
-      match tx {
-        TributaryTransaction::Tendermint(TendermintTx::SlashEvidence(ev)) => {
-          // Since the evidence is on the chain, it should already have been validated
-          // We can just punish the signer
-          let data = match ev {
-            Evidence::ConflictingMessages(first, second) => (first, Some(second)),
-            Evidence::InvalidPrecommit(first) | Evidence::InvalidValidRound(first) => (first, None),
-          };
-          let msgs = (
-            decode_signed_message::<TendermintNetwork<D, Transaction, P>>(&data.0).unwrap(),
-            if data.1.is_some() {
-              Some(
-                decode_signed_message::<TendermintNetwork<D, Transaction, P>>(&data.1.unwrap())
-                  .unwrap(),
-              )
-            } else {
-              None
-            },
-          );
-
-          // Since anything with evidence is fundamentally faulty behavior, not just temporal
-          // errors, mark the node as fatally slashed
-          self.fatal_slash(msgs.0.msg.sender, &format!("invalid tendermint messages: {msgs:?}"));
-        }
-        TributaryTransaction::Application(tx) => {
-          self.handle_application_tx(tx).await;
-        }
-      }
-    }
-
-    let genesis = self.spec.genesis();
-
-    // Calculate the shares still present, spinning if not enough are
-    {
-      // Start with the original n value
-      let mut present_shares = self.spec.n();
-      // Remove everyone fatally slashed
-      let current_fatal_slashes = FatalSlashes::get_as_keys(self.txn, genesis);
-      for removed in &current_fatal_slashes {
-        let original_i_for_removed =
-          self.spec.i(*removed).expect("removed party was never present");
-        let removed_shares =
-          u16::from(original_i_for_removed.end) - u16::from(original_i_for_removed.start);
-        present_shares -= removed_shares;
-      }
-
-      // Spin if the present shares don't satisfy the required threshold
-      if present_shares < self.spec.t() {
-        loop {
-          log::error!(
-            "fatally slashed so many participants for {:?} we no longer meet the threshold",
-            self.spec.set()
-          );
-          tokio::time::sleep(core::time::Duration::from_secs(60)).await;
-        }
-      }
-    }
-
-    for topic in ReattemptDb::take(self.txn, genesis, self.block_number) {
-      let attempt = AttemptDb::start_next_attempt(self.txn, genesis, topic);
-      log::info!("potentially re-attempting {topic:?} with attempt {attempt}");
-
-      // Slash people who failed to participate as expected in the prior attempt
-      {
-        let prior_attempt = attempt - 1;
-        // TODO: If 67% sent preprocesses, this should be them. Else, this should be vec![]
-        let expected_participants: Vec<<Ristretto as Ciphersuite>::G> = vec![];
-
-        let mut did_not_participate = vec![];
-        for expected_participant in expected_participants {
-          if DataDb::get(
-            self.txn,
-            genesis,
-            &DataSpecification {
-              topic,
-              // Since we got the preprocesses, we were supposed to get the shares
-              label: Label::Share,
-              attempt: prior_attempt,
-            },
-            &expected_participant.to_bytes(),
-          )
-          .is_none()
-          {
-            did_not_participate.push(expected_participant);
-          }
-        }
-
-        // If a supermajority didn't participate as expected, the protocol was likely aborted due
-        // to detection of a completion or some larger networking error
-        // Accordingly, clear did_not_participate
-        // TODO
-
-        // TODO: Increment the slash points of people who didn't preprocess in some expected window
-        // of time
-
-        // Slash everyone who didn't participate as expected
-        // This may be overzealous as if a minority detects a completion, they'll abort yet the
-        // supermajority will cause the above allowance to not trigger, causing an honest minority
-        // to be slashed
-        // At the end of the protocol, the accumulated slashes are reduced by the amount obtained
-        // by the worst-performing member of the supermajority, and this is expected to
-        // sufficiently compensate for slashes which occur under normal operation
-        // TODO
-      }
-
-      /*
-        All of these have the same common flow:
-
-        1) Check if this re-attempt is actually needed
-        2) If so, dispatch whatever events as needed
-
-        This is because we *always* re-attempt any protocol which had participation. That doesn't
-        mean we *should* re-attempt this protocol.
-
-        The alternatives were:
-        1) Note on-chain we completed a protocol, halting re-attempts upon 34%.
-        2) Vote on-chain to re-attempt a protocol.
-
-        This schema doesn't have any additional messages upon the success case (whereas
-        alternative #1 does) and doesn't have overhead (as alternative #2 does, sending votes and
-        then preprocesses. This only sends preprocesses).
-      */
-      match topic {
-        Topic::DkgConfirmation => {
-          if SeraiDkgCompleted::get(self.txn, self.spec.set()).is_none() {
-            log::info!("re-attempting DKG confirmation with attempt {attempt}");
-
-            // Since it wasn't completed, publish our nonces for the next attempt
-            let confirmation_nonces =
-              crate::tributary::dkg_confirmation_nonces(self.our_key, self.spec, self.txn, attempt);
-            let mut tx = Transaction::DkgConfirmationNonces {
-              attempt,
-              confirmation_nonces,
-              signed: Transaction::empty_signed(),
-            };
-            tx.sign(&mut OsRng, genesis, self.our_key);
-            self.publish_tributary_tx.publish_tributary_tx(tx).await;
-          }
-        }
-        Topic::SubstrateSign(inner_id) => {
-          let id = processor_messages::coordinator::SubstrateSignId {
-            session: self.spec.set().session,
-            id: inner_id,
-            attempt,
-          };
-          match inner_id {
-            SubstrateSignableId::CosigningSubstrateBlock(block) => {
-              let block_number = SeraiBlockNumber::get(self.txn, block)
-                .expect("couldn't get the block number for prior attempted cosign");
-
-              // Check if the cosigner has a signature from our set for this block/a newer one
-              let latest_cosign =
-                crate::cosign_evaluator::LatestCosign::get(self.txn, self.spec.set().network)
-                  .map_or(0, |cosign| cosign.block_number);
-              if latest_cosign < block_number {
-                log::info!("re-attempting cosigning {block_number:?} with attempt {attempt}");
-
-                // Instruct the processor to start the next attempt
-                self
-                  .processors
-                  .send(
-                    self.spec.set().network,
-                    processor_messages::coordinator::CoordinatorMessage::CosignSubstrateBlock {
-                      id,
-                      block_number,
-                    },
-                  )
-                  .await;
-              }
-            }
-            SubstrateSignableId::Batch(batch) => {
-              // If the Batch hasn't appeared on-chain...
-              if BatchInstructionsHashDb::get(self.txn, self.spec.set().network, batch).is_none() {
-                log::info!("re-attempting signing batch {batch:?} with attempt {attempt}");
-
-                // Instruct the processor to start the next attempt
-                // The processor won't continue if it's already signed a Batch
-                // Prior checking if the Batch is on-chain just may reduce the non-participating
-                // 33% from publishing their re-attempt messages
-                self
-                  .processors
-                  .send(
-                    self.spec.set().network,
-                    processor_messages::coordinator::CoordinatorMessage::BatchReattempt { id },
-                  )
-                  .await;
-              }
-            }
-            SubstrateSignableId::SlashReport => {
-              // If this Tributary hasn't been retired...
-              // (published SlashReport/took too long to do so)
-              if crate::RetiredTributaryDb::get(self.txn, self.spec.set()).is_none() {
-                log::info!(
-                  "re-attempting signing slash report for {:?} with attempt {attempt}",
-                  self.spec.set()
-                );
-
-                let report = SlashReport::get(self.txn, self.spec.set())
-                  .expect("re-attempting signing a SlashReport we don't have?");
-                self
-                  .processors
-                  .send(
-                    self.spec.set().network,
-                    processor_messages::coordinator::CoordinatorMessage::SignSlashReport {
-                      id,
-                      report,
-                    },
-                  )
-                  .await;
-              }
-            }
-          }
-        }
-        Topic::Sign(id) => {
-          // Instruct the processor to start the next attempt
-          // If it has already noted a completion, it won't send a preprocess and will simply drop
-          // the re-attempt message
-          self
-            .processors
-            .send(
-              self.spec.set().network,
-              processor_messages::sign::CoordinatorMessage::Reattempt {
-                id: processor_messages::sign::SignId {
-                  session: self.spec.set().session,
-                  id,
-                  attempt,
-                },
-              },
-            )
-            .await;
-        }
-      }
-    }
-
-    if Some(u64::from(self.block_number)) == SlashReportCutOff::get(self.txn, genesis) {
-      // Grab every slash report
-      let mut all_reports = vec![];
-      for (i, (validator, _)) in self.spec.validators().into_iter().enumerate() {
-        let Some(mut report) = SlashReports::get(self.txn, genesis, validator.to_bytes()) else {
-          continue;
-        };
-        // Assign them 0 points for themselves
-        report.insert(i, 0);
-        let signer_i = self.spec.i(validator).unwrap();
-        let signer_len = u16::from(signer_i.end) - u16::from(signer_i.start);
-        // Push `n` copies, one for each of their shares
-        for _ in 0 .. signer_len {
-          all_reports.push(report.clone());
-        }
-      }
-
-      // For each participant, grab their median
-      let mut medians = vec![];
-      for p in 0 .. self.spec.validators().len() {
-        let mut median_calc = vec![];
-        for report in &all_reports {
-          median_calc.push(report[p]);
-        }
-        median_calc.sort_unstable();
-        medians.push(median_calc[median_calc.len() / 2]);
-      }
-
-      // Grab the points of the last party within the best-performing threshold
-      // This is done by first expanding the point values by the amount of shares
-      let mut sorted_medians = vec![];
-      for (i, (_, shares)) in self.spec.validators().into_iter().enumerate() {
-        for _ in 0 .. shares {
-          sorted_medians.push(medians[i]);
-        }
-      }
-      // Then performing the sort
-      sorted_medians.sort_unstable();
-      let worst_points_by_party_within_threshold = sorted_medians[usize::from(self.spec.t()) - 1];
-
-      // Reduce everyone's points by this value
-      for median in &mut medians {
-        *median = median.saturating_sub(worst_points_by_party_within_threshold);
-      }
-
-      // The threshold now has the proper incentive to report this as they no longer suffer
-      // negative effects
-      //
-      // Additionally, if all validators had degraded performance, they don't all get penalized for
-      // what's likely outside their control (as it occurred universally)
-
-      // Mark everyone fatally slashed with u32::MAX
-      for (i, (validator, _)) in self.spec.validators().into_iter().enumerate() {
-        if FatallySlashed::get(self.txn, genesis, validator.to_bytes()).is_some() {
-          medians[i] = u32::MAX;
-        }
-      }
-
-      let mut report = vec![];
-      for (i, (validator, _)) in self.spec.validators().into_iter().enumerate() {
-        if medians[i] != 0 {
-          report.push((validator.to_bytes(), medians[i]));
-        }
-      }
-
-      // This does lock in the report, meaning further slash point accumulations won't be reported
-      // They still have value to be locally tracked due to local decisions made based off
-      // accumulated slash reports
-      SlashReport::set(self.txn, self.spec.set(), &report);
-
-      // Start a signing protocol for this
-      self
-        .processors
-        .send(
-          self.spec.set().network,
-          processor_messages::coordinator::CoordinatorMessage::SignSlashReport {
-            id: SubstrateSignId {
-              session: self.spec.set().session,
-              id: SubstrateSignableId::SlashReport,
-              attempt: 0,
-            },
-            report,
-          },
-        )
-        .await;
-    }
-  }
-}
-
-#[allow(clippy::too_many_arguments)]
-pub(crate) async fn handle_new_blocks<
-  D: Db,
-  Pro: Processors,
-  PST: PublishSeraiTransaction,
-  PTT: PTTTrait,
-  RID: RIDTrait,
-  P: P2p,
->(
-  db: &mut D,
-  key: &Zeroizing<<Ristretto as Ciphersuite>::F>,
-  recognized_id: &RID,
-  processors: &Pro,
-  publish_serai_tx: &PST,
-  publish_tributary_tx: &PTT,
-  spec: &TributarySpec,
-  tributary: &TributaryReader<D, Transaction>,
-) {
-  let genesis = tributary.genesis();
-  let mut last_block = LastHandledBlock::get(db, genesis).unwrap_or(genesis);
-  let mut block_number = TributaryBlockNumber::get(db, last_block).unwrap_or(0);
-  while let Some(next) = tributary.block_after(&last_block) {
-    let block = tributary.block(&next).unwrap();
-    block_number += 1;
-
-    // Make sure we have all of the provided transactions for this block
-    for tx in &block.transactions {
-      // Provided TXs will appear first in the Block, so we can break after we hit a non-Provided
-      let TransactionKind::Provided(order) = tx.kind() else {
-        break;
-      };
-
-      // make sure we have all the provided txs in this block locally
-      if !tributary.locally_provided_txs_in_block(&block.hash(), order) {
-        return;
-      }
-    }
-
-    let mut db_clone = db.clone();
-    let mut txn = db_clone.txn();
-    TributaryBlockNumber::set(&mut txn, next, &block_number);
-    (TributaryBlockHandler {
-      db,
-      txn: &mut txn,
-      spec,
-      our_key: key,
-      recognized_id,
-      processors,
-      publish_serai_tx,
-      publish_tributary_tx,
-      block,
-      block_number,
-      _p2p: PhantomData::<P>,
-    })
-    .handle()
-    .await;
-    last_block = next;
-    LastHandledBlock::set(&mut txn, genesis, &next);
-    txn.commit();
-  }
-}
-
-pub(crate) async fn scan_tributaries_task<
-  D: Db,
-  Pro: Processors,
-  P: P2p,
-  RID: 'static + Send + Sync + Clone + RIDTrait,
->(
-  raw_db: D,
-  key: Zeroizing<<Ristretto as Ciphersuite>::F>,
-  recognized_id: RID,
-  processors: Pro,
-  serai: Arc<Serai>,
-  mut tributary_event: broadcast::Receiver<crate::TributaryEvent<D, P>>,
-) {
-  log::info!("scanning tributaries");
-
-  loop {
-    match tributary_event.recv().await {
-      Ok(crate::TributaryEvent::NewTributary(crate::ActiveTributary { spec, tributary })) => {
-        // For each Tributary, spawn a dedicated scanner task
-        tokio::spawn({
-          let raw_db = raw_db.clone();
-          let key = key.clone();
-          let recognized_id = recognized_id.clone();
-          let processors = processors.clone();
-          let serai = serai.clone();
-          async move {
-            let spec = &spec;
-            let reader = tributary.reader();
-            let mut tributary_db = raw_db.clone();
-            loop {
-              // Check if the set was retired, and if so, don't further operate
-              if crate::db::RetiredTributaryDb::get(&raw_db, spec.set()).is_some() {
-                break;
-              }
-
-              // Obtain the next block notification now to prevent obtaining it immediately after
-              // the next block occurs
-              let next_block_notification = tributary.next_block_notification().await;
-
-              handle_new_blocks::<_, _, _, _, _, P>(
-                &mut tributary_db,
-                &key,
-                &recognized_id,
-                &processors,
-                &*serai,
-                &|tx: Transaction| {
-                  let tributary = tributary.clone();
-                  async move {
-                    match tributary.add_transaction(tx.clone()).await {
-                      Ok(_) => {}
-                      // Can happen as this occurs on a distinct DB TXN
-                      Err(TransactionError::InvalidNonce) => {
-                        log::warn!(
-                          "publishing TX {tx:?} returned InvalidNonce. was it already added?"
-                        )
-                      }
-                      Err(e) => panic!("created an invalid transaction: {e:?}"),
-                    }
-                  }
-                },
-                spec,
-                &reader,
-              )
-              .await;
-
-              // Run either when the notification fires, or every interval of block_time
-              let _ = tokio::time::timeout(
-                Duration::from_secs(tributary::Tributary::<D, Transaction, P>::block_time().into()),
-                next_block_notification,
-              )
-              .await;
-            }
-          }
-        });
-      }
-      // The above loop simply checks the DB every few seconds, voiding the need for this event
-      Ok(crate::TributaryEvent::TributaryRetired(_)) => {}
-      Err(broadcast::error::RecvError::Lagged(_)) => {
-        panic!("scan_tributaries lagged to handle tributary_event")
-      }
-      Err(broadcast::error::RecvError::Closed) => panic!("tributary_event sender closed"),
-    }
-  }
-}

coordinator/src/tributary/signing_protocol.rs

@@ -1,361 +0,0 @@
-/*
-  A MuSig-based signing protocol executed with the validators' keys.
-
-  This is used for confirming the results of a DKG on-chain, an operation requiring all validators
-  which aren't specified as removed while still satisfying a supermajority.
-
-  Since we're using the validator's keys, as needed for their being the root of trust, the
-  coordinator must perform the signing. This is distinct from all other group-signing operations,
-  as they're all done by the processor.
-
-  The MuSig-aggregation achieves on-chain efficiency and enables a more secure design pattern.
-  While we could individually tack votes, that'd require logic to prevent voting multiple times and
-  tracking the accumulated votes. MuSig-aggregation simply requires checking the list is sorted and
-  the list's weight exceeds the threshold.
-
-  Instead of maintaining state in memory, a combination of the DB and re-execution are used. This
-  is deemed acceptable re: performance as:
-
-  1) This is only done prior to a DKG being confirmed on Substrate and is assumed infrequent.
-  2) This is an O(n) algorithm.
-  3) The size of the validator set is bounded by MAX_KEY_SHARES_PER_SET.
-
-  Accordingly, this should be tolerable.
-
-  As for safety, it is explicitly unsafe to reuse nonces across signing sessions. This raises
-  concerns regarding our re-execution which is dependent on fixed nonces. Safety is derived from
-  the nonces being context-bound under a BFT protocol. The flow is as follows:
-
-  1) Decide the nonce.
-  2) Publish the nonces' commitments, receiving everyone elses *and potentially the message to be
-     signed*.
-  3) Sign and publish the signature share.
-
-  In order for nonce re-use to occur, the received nonce commitments (or the message to be signed)
-  would have to be distinct and sign would have to be called again.
-
-  Before we act on any received messages, they're ordered and finalized by a BFT algorithm. The
-  only way to operate on distinct received messages would be if:
-
-  1) A logical flaw exists, letting new messages over write prior messages
-  2) A reorganization occurred from chain A to chain B, and with it, different messages
-
-  Reorganizations are not supported, as BFT is assumed by the presence of a BFT algorithm. While
-  a significant amount of processes may be byzantine, leading to BFT being broken, that still will
-  not trigger a reorganization. The only way to move to a distinct chain, with distinct messages,
-  would be by rebuilding the local process (this time following chain B). Upon any complete
-  rebuild, we'd re-decide nonces, achieving safety. This does set a bound preventing partial
-  rebuilds which is accepted.
-
-  Additionally, to ensure a rebuilt service isn't flagged as malicious, we have to check the
-  commitments generated from the decided nonces are in fact its commitments on-chain (TODO).
-
-  TODO: We also need to review how we're handling Processor preprocesses and likely implement the
-  same on-chain-preprocess-matches-presumed-preprocess check before publishing shares.
-*/
-
-use core::ops::Deref;
-use std::collections::{HashSet, HashMap};
-
-use zeroize::{Zeroize, Zeroizing};
-
-use rand_core::OsRng;
-
-use blake2::{Digest, Blake2s256};
-
-use ciphersuite::{group::ff::PrimeField, Ciphersuite, Ristretto};
-use frost::{
-  FrostError,
-  dkg::{Participant, musig::musig},
-  ThresholdKeys,
-  sign::*,
-};
-use frost_schnorrkel::Schnorrkel;
-
-use scale::Encode;
-
-#[rustfmt::skip]
-use serai_client::validator_sets::primitives::{ValidatorSet, KeyPair, musig_context, set_keys_message};
-
-use serai_db::*;
-
-use crate::tributary::TributarySpec;
-
-create_db!(
-  SigningProtocolDb {
-    CachedPreprocesses: (context: &impl Encode) -> [u8; 32]
-    DataSignedWith: (context: &impl Encode) -> (Vec<u8>, HashMap<Participant, Vec<u8>>),
-  }
-);
-
-struct SigningProtocol<'a, T: DbTxn, C: Encode> {
-  pub(crate) key: &'a Zeroizing<<Ristretto as Ciphersuite>::F>,
-  pub(crate) spec: &'a TributarySpec,
-  pub(crate) txn: &'a mut T,
-  pub(crate) context: C,
-}
-
-impl<T: DbTxn, C: Encode> SigningProtocol<'_, T, C> {
-  fn preprocess_internal(
-    &mut self,
-    participants: &[<Ristretto as Ciphersuite>::G],
-  ) -> (AlgorithmSignMachine<Ristretto, Schnorrkel>, [u8; 64]) {
-    // Encrypt the cached preprocess as recovery of it will enable recovering the private key
-    // While the DB isn't expected to be arbitrarily readable, it isn't a proper secret store and
-    // shouldn't be trusted as one
-    let mut encryption_key = {
-      let mut encryption_key_preimage =
-        Zeroizing::new(b"Cached Preprocess Encryption Key".to_vec());
-      encryption_key_preimage.extend(self.context.encode());
-      let repr = Zeroizing::new(self.key.to_repr());
-      encryption_key_preimage.extend(repr.deref());
-      Blake2s256::digest(&encryption_key_preimage)
-    };
-    let encryption_key_slice: &mut [u8] = encryption_key.as_mut();
-
-    // Create the MuSig keys
-    let keys: ThresholdKeys<Ristretto> =
-      musig(&musig_context(self.spec.set()), self.key, participants)
-        .expect("signing for a set we aren't in/validator present multiple times")
-        .into();
-
-    // Define the algorithm
-    let algorithm = Schnorrkel::new(b"substrate");
-
-    // Check if we've prior preprocessed
-    if CachedPreprocesses::get(self.txn, &self.context).is_none() {
-      // If we haven't, we create a machine solely to obtain the preprocess with
-      let (machine, _) =
-        AlgorithmMachine::new(algorithm.clone(), keys.clone()).preprocess(&mut OsRng);
-
-      // Cache and save the preprocess to disk
-      let mut cache = machine.cache();
-      assert_eq!(cache.0.len(), 32);
-      #[allow(clippy::needless_range_loop)]
-      for b in 0 .. 32 {
-        cache.0[b] ^= encryption_key_slice[b];
-      }
-
-      CachedPreprocesses::set(self.txn, &self.context, &cache.0);
-    }
-
-    // We're now guaranteed to have the preprocess, hence why this `unwrap` is safe
-    let cached = CachedPreprocesses::get(self.txn, &self.context).unwrap();
-    let mut cached = Zeroizing::new(cached);
-    #[allow(clippy::needless_range_loop)]
-    for b in 0 .. 32 {
-      cached[b] ^= encryption_key_slice[b];
-    }
-    encryption_key_slice.zeroize();
-    // Create the machine from the cached preprocess
-    let (machine, preprocess) =
-      AlgorithmSignMachine::from_cache(algorithm, keys, CachedPreprocess(cached));
-
-    (machine, preprocess.serialize().try_into().unwrap())
-  }
-
-  fn share_internal(
-    &mut self,
-    participants: &[<Ristretto as Ciphersuite>::G],
-    mut serialized_preprocesses: HashMap<Participant, Vec<u8>>,
-    msg: &[u8],
-  ) -> Result<(AlgorithmSignatureMachine<Ristretto, Schnorrkel>, [u8; 32]), Participant> {
-    // We can't clear the preprocess as we sitll need it to accumulate all of the shares
-    // We do save the message we signed so any future calls with distinct messages panic
-    // This assumes the txn deciding this data is committed before the share is broaadcast
-    if let Some((existing_msg, existing_preprocesses)) =
-      DataSignedWith::get(self.txn, &self.context)
-    {
-      assert_eq!(msg, &existing_msg, "obtaining a signature share for a distinct message");
-      assert_eq!(
-        &serialized_preprocesses, &existing_preprocesses,
-        "obtaining a signature share with a distinct set of preprocesses"
-      );
-    } else {
-      DataSignedWith::set(
-        self.txn,
-        &self.context,
-        &(msg.to_vec(), serialized_preprocesses.clone()),
-      );
-    }
-
-    // Get the preprocessed machine
-    let (machine, _) = self.preprocess_internal(participants);
-
-    // Deserialize all the preprocesses
-    let mut participants = serialized_preprocesses.keys().copied().collect::<Vec<_>>();
-    participants.sort();
-    let mut preprocesses = HashMap::new();
-    for participant in participants {
-      preprocesses.insert(
-        participant,
-        machine
-          .read_preprocess(&mut serialized_preprocesses.remove(&participant).unwrap().as_slice())
-          .map_err(|_| participant)?,
-      );
-    }
-
-    // Sign the share
-    let (machine, share) = machine.sign(preprocesses, msg).map_err(|e| match e {
-      FrostError::InternalError(e) => unreachable!("FrostError::InternalError {e}"),
-      FrostError::InvalidParticipant(_, _) |
-      FrostError::InvalidSigningSet(_) |
-      FrostError::InvalidParticipantQuantity(_, _) |
-      FrostError::DuplicatedParticipant(_) |
-      FrostError::MissingParticipant(_) => panic!("unexpected error during sign: {e:?}"),
-      FrostError::InvalidPreprocess(p) | FrostError::InvalidShare(p) => p,
-    })?;
-
-    Ok((machine, share.serialize().try_into().unwrap()))
-  }
-
-  fn complete_internal(
-    machine: AlgorithmSignatureMachine<Ristretto, Schnorrkel>,
-    shares: HashMap<Participant, Vec<u8>>,
-  ) -> Result<[u8; 64], Participant> {
-    let shares = shares
-      .into_iter()
-      .map(|(p, share)| {
-        machine.read_share(&mut share.as_slice()).map(|share| (p, share)).map_err(|_| p)
-      })
-      .collect::<Result<HashMap<_, _>, _>>()?;
-    let signature = machine.complete(shares).map_err(|e| match e {
-      FrostError::InternalError(e) => unreachable!("FrostError::InternalError {e}"),
-      FrostError::InvalidParticipant(_, _) |
-      FrostError::InvalidSigningSet(_) |
-      FrostError::InvalidParticipantQuantity(_, _) |
-      FrostError::DuplicatedParticipant(_) |
-      FrostError::MissingParticipant(_) => unreachable!("{e:?}"),
-      FrostError::InvalidPreprocess(p) | FrostError::InvalidShare(p) => p,
-    })?;
-    Ok(signature.to_bytes())
-  }
-}
-
-// Get the keys of the participants, noted by their threshold is, and return a new map indexed by
-// their MuSig is.
-fn threshold_i_map_to_keys_and_musig_i_map(
-  spec: &TributarySpec,
-  our_key: &Zeroizing<<Ristretto as Ciphersuite>::F>,
-  mut map: HashMap<Participant, Vec<u8>>,
-) -> (Vec<<Ristretto as Ciphersuite>::G>, HashMap<Participant, Vec<u8>>) {
-  // Insert our own index so calculations aren't offset
-  let our_threshold_i = spec
-    .i(<Ristretto as Ciphersuite>::generator() * our_key.deref())
-    .expect("not in a set we're signing for")
-    .start;
-  // Asserts we weren't unexpectedly already present
-  assert!(map.insert(our_threshold_i, vec![]).is_none());
-
-  let spec_validators = spec.validators();
-  let key_from_threshold_i = |threshold_i| {
-    for (key, _) in &spec_validators {
-      if threshold_i == spec.i(*key).expect("validator wasn't in a set they're in").start {
-        return *key;
-      }
-    }
-    panic!("requested info for threshold i which doesn't exist")
-  };
-
-  let mut sorted = vec![];
-  let mut threshold_is = map.keys().copied().collect::<Vec<_>>();
-  threshold_is.sort();
-  for threshold_i in threshold_is {
-    sorted.push((
-      threshold_i,
-      key_from_threshold_i(threshold_i),
-      map.remove(&threshold_i).unwrap(),
-    ));
-  }
-
-  // Now that signers are sorted, with their shares, create a map with the is needed for MuSig
-  let mut participants = vec![];
-  let mut map = HashMap::new();
-  let mut our_musig_i = None;
-  for (raw_i, (threshold_i, key, share)) in sorted.into_iter().enumerate() {
-    let musig_i = Participant::new(u16::try_from(raw_i).unwrap() + 1).unwrap();
-    if threshold_i == our_threshold_i {
-      our_musig_i = Some(musig_i);
-    }
-    participants.push(key);
-    map.insert(musig_i, share);
-  }
-
-  map.remove(&our_musig_i.unwrap()).unwrap();
-
-  (participants, map)
-}
-
-type DkgConfirmerSigningProtocol<'a, T> =
-  SigningProtocol<'a, T, (&'static [u8; 12], ValidatorSet, u32)>;
-
-pub(crate) struct DkgConfirmer<'a, T: DbTxn> {
-  key: &'a Zeroizing<<Ristretto as Ciphersuite>::F>,
-  spec: &'a TributarySpec,
-  txn: &'a mut T,
-  attempt: u32,
-}
-
-impl<T: DbTxn> DkgConfirmer<'_, T> {
-  pub(crate) fn new<'a>(
-    key: &'a Zeroizing<<Ristretto as Ciphersuite>::F>,
-    spec: &'a TributarySpec,
-    txn: &'a mut T,
-    attempt: u32,
-  ) -> DkgConfirmer<'a, T> {
-    DkgConfirmer { key, spec, txn, attempt }
-  }
-
-  fn signing_protocol(&mut self) -> DkgConfirmerSigningProtocol<'_, T> {
-    let context = (b"DkgConfirmer", self.spec.set(), self.attempt);
-    SigningProtocol { key: self.key, spec: self.spec, txn: self.txn, context }
-  }
-
-  fn preprocess_internal(&mut self) -> (AlgorithmSignMachine<Ristretto, Schnorrkel>, [u8; 64]) {
-    // This preprocesses with just us as we only decide the participants after obtaining
-    // preprocesses
-    let participants = vec![<Ristretto as Ciphersuite>::generator() * self.key.deref()];
-    self.signing_protocol().preprocess_internal(&participants)
-  }
-  // Get the preprocess for this confirmation.
-  pub(crate) fn preprocess(&mut self) -> [u8; 64] {
-    self.preprocess_internal().1
-  }
-
-  fn share_internal(
-    &mut self,
-    preprocesses: HashMap<Participant, Vec<u8>>,
-    key_pair: &KeyPair,
-  ) -> Result<(AlgorithmSignatureMachine<Ristretto, Schnorrkel>, [u8; 32]), Participant> {
-    let (participants, preprocesses) =
-      threshold_i_map_to_keys_and_musig_i_map(self.spec, self.key, preprocesses);
-    let msg = set_keys_message(&self.spec.set(), key_pair);
-    self.signing_protocol().share_internal(&participants, preprocesses, &msg)
-  }
-  // Get the share for this confirmation, if the preprocesses are valid.
-  pub(crate) fn share(
-    &mut self,
-    preprocesses: HashMap<Participant, Vec<u8>>,
-    key_pair: &KeyPair,
-  ) -> Result<[u8; 32], Participant> {
-    self.share_internal(preprocesses, key_pair).map(|(_, share)| share)
-  }
-
-  pub(crate) fn complete(
-    &mut self,
-    preprocesses: HashMap<Participant, Vec<u8>>,
-    key_pair: &KeyPair,
-    shares: HashMap<Participant, Vec<u8>>,
-  ) -> Result<[u8; 64], Participant> {
-    assert_eq!(preprocesses.keys().collect::<HashSet<_>>(), shares.keys().collect::<HashSet<_>>());
-
-    let shares = threshold_i_map_to_keys_and_musig_i_map(self.spec, self.key, shares).1;
-
-    let machine = self
-      .share_internal(preprocesses, key_pair)
-      .expect("trying to complete a machine which failed to preprocess")
-      .0;
-
-    DkgConfirmerSigningProtocol::<'_, T>::complete_internal(machine, shares)
-  }
-}

coordinator/src/tributary/spec.rs

@@ -1,124 +0,0 @@
-use core::{ops::Range, fmt::Debug};
-use std::{io, collections::HashMap};
-
-use transcript::{Transcript, RecommendedTranscript};
-
-use ciphersuite::{group::GroupEncoding, Ciphersuite, Ristretto};
-use frost::Participant;
-
-use scale::Encode;
-use borsh::{BorshSerialize, BorshDeserialize};
-
-use serai_client::validator_sets::primitives::ValidatorSet;
-
-fn borsh_serialize_validators<W: io::Write>(
-  validators: &Vec<(<Ristretto as Ciphersuite>::G, u16)>,
-  writer: &mut W,
-) -> Result<(), io::Error> {
-  let len = u16::try_from(validators.len()).unwrap();
-  BorshSerialize::serialize(&len, writer)?;
-  for validator in validators {
-    BorshSerialize::serialize(&validator.0.to_bytes(), writer)?;
-    BorshSerialize::serialize(&validator.1, writer)?;
-  }
-  Ok(())
-}
-
-fn borsh_deserialize_validators<R: io::Read>(
-  reader: &mut R,
-) -> Result<Vec<(<Ristretto as Ciphersuite>::G, u16)>, io::Error> {
-  let len: u16 = BorshDeserialize::deserialize_reader(reader)?;
-  let mut res = vec![];
-  for _ in 0 .. len {
-    let compressed: [u8; 32] = BorshDeserialize::deserialize_reader(reader)?;
-    let point = Option::from(<Ristretto as Ciphersuite>::G::from_bytes(&compressed))
-      .ok_or_else(|| io::Error::other("invalid point for validator"))?;
-    let weight: u16 = BorshDeserialize::deserialize_reader(reader)?;
-    res.push((point, weight));
-  }
-  Ok(res)
-}
-
-#[derive(Clone, PartialEq, Eq, Debug, BorshSerialize, BorshDeserialize)]
-pub struct TributarySpec {
-  serai_block: [u8; 32],
-  start_time: u64,
-  set: ValidatorSet,
-  #[borsh(
-    serialize_with = "borsh_serialize_validators",
-    deserialize_with = "borsh_deserialize_validators"
-  )]
-  validators: Vec<(<Ristretto as Ciphersuite>::G, u16)>,
-  evrf_public_keys: Vec<([u8; 32], Vec<u8>)>,
-}
-
-impl TributarySpec {
-  pub fn new(
-    serai_block: [u8; 32],
-    start_time: u64,
-    set: ValidatorSet,
-    validators: Vec<(<Ristretto as Ciphersuite>::G, u16)>,
-    evrf_public_keys: Vec<([u8; 32], Vec<u8>)>,
-  ) -> TributarySpec {
-    Self { serai_block, start_time, set, validators, evrf_public_keys }
-  }
-
-  pub fn set(&self) -> ValidatorSet {
-    self.set
-  }
-
-  pub fn genesis(&self) -> [u8; 32] {
-    // Calculate the genesis for this Tributary
-    let mut genesis = RecommendedTranscript::new(b"Serai Tributary Genesis");
-    // This locks it to a specific Serai chain
-    genesis.append_message(b"serai_block", self.serai_block);
-    genesis.append_message(b"session", self.set.session.0.to_le_bytes());
-    genesis.append_message(b"network", self.set.network.encode());
-    let genesis = genesis.challenge(b"genesis");
-    let genesis_ref: &[u8] = genesis.as_ref();
-    genesis_ref[.. 32].try_into().unwrap()
-  }
-
-  pub fn start_time(&self) -> u64 {
-    self.start_time
-  }
-
-  pub fn n(&self) -> u16 {
-    self.validators.iter().map(|(_, weight)| *weight).sum()
-  }
-
-  pub fn t(&self) -> u16 {
-    ((2 * self.n()) / 3) + 1
-  }
-
-  pub fn i(&self, key: <Ristretto as Ciphersuite>::G) -> Option<Range<Participant>> {
-    let mut all_is = HashMap::new();
-    let mut i = 1;
-    for (validator, weight) in &self.validators {
-      all_is.insert(
-        *validator,
-        Range { start: Participant::new(i).unwrap(), end: Participant::new(i + weight).unwrap() },
-      );
-      i += weight;
-    }
-
-    Some(all_is.get(&key)?.clone())
-  }
-
-  pub fn reverse_lookup_i(&self, i: Participant) -> Option<<Ristretto as Ciphersuite>::G> {
-    for (validator, _) in &self.validators {
-      if self.i(*validator).map_or(false, |range| range.contains(&i)) {
-        return Some(*validator);
-      }
-    }
-    None
-  }
-
-  pub fn validators(&self) -> Vec<(<Ristretto as Ciphersuite>::G, u64)> {
-    self.validators.iter().map(|(validator, weight)| (*validator, u64::from(*weight))).collect()
-  }
-
-  pub fn evrf_public_keys(&self) -> Vec<([u8; 32], Vec<u8>)> {
-    self.evrf_public_keys.clone()
-  }
-}

coordinator/src/tributary/transaction.rs

@@ -1,587 +0,0 @@
-use core::{ops::Deref, fmt::Debug};
-use std::io;
-
-use zeroize::Zeroizing;
-use rand_core::{RngCore, CryptoRng};
-
-use blake2::{Digest, Blake2s256};
-use transcript::{Transcript, RecommendedTranscript};
-
-use ciphersuite::{
-  group::{ff::Field, GroupEncoding},
-  Ciphersuite, Ristretto,
-};
-use schnorr::SchnorrSignature;
-
-use scale::{Encode, Decode};
-use processor_messages::coordinator::SubstrateSignableId;
-
-use tributary::{
-  TRANSACTION_SIZE_LIMIT, ReadWrite,
-  transaction::{Signed, TransactionError, TransactionKind, Transaction as TransactionTrait},
-};
-
-#[derive(Clone, Copy, PartialEq, Eq, Debug, Encode)]
-pub enum Label {
-  Preprocess,
-  Share,
-}
-
-impl Label {
-  // TODO: Should nonces be u8 thanks to our use of topics?
-  pub fn nonce(&self) -> u32 {
-    match self {
-      Label::Preprocess => 0,
-      Label::Share => 1,
-    }
-  }
-}
-
-#[derive(Clone, PartialEq, Eq)]
-pub struct SignData<Id: Clone + PartialEq + Eq + Debug + Encode + Decode> {
-  pub plan: Id,
-  pub attempt: u32,
-  pub label: Label,
-
-  pub data: Vec<Vec<u8>>,
-
-  pub signed: Signed,
-}
-
-impl<Id: Clone + PartialEq + Eq + Debug + Encode + Decode> Debug for SignData<Id> {
-  fn fmt(&self, fmt: &mut core::fmt::Formatter<'_>) -> Result<(), core::fmt::Error> {
-    fmt
-      .debug_struct("SignData")
-      .field("id", &hex::encode(self.plan.encode()))
-      .field("attempt", &self.attempt)
-      .field("label", &self.label)
-      .field("signer", &hex::encode(self.signed.signer.to_bytes()))
-      .finish_non_exhaustive()
-  }
-}
-
-impl<Id: Clone + PartialEq + Eq + Debug + Encode + Decode> SignData<Id> {
-  pub(crate) fn read<R: io::Read>(reader: &mut R) -> io::Result<Self> {
-    let plan = Id::decode(&mut scale::IoReader(&mut *reader))
-      .map_err(|_| io::Error::other("invalid plan in SignData"))?;
-
-    let mut attempt = [0; 4];
-    reader.read_exact(&mut attempt)?;
-    let attempt = u32::from_le_bytes(attempt);
-
-    let mut label = [0; 1];
-    reader.read_exact(&mut label)?;
-    let label = match label[0] {
-      0 => Label::Preprocess,
-      1 => Label::Share,
-      _ => Err(io::Error::other("invalid label in SignData"))?,
-    };
-
-    let data = {
-      let mut data_pieces = [0];
-      reader.read_exact(&mut data_pieces)?;
-      if data_pieces[0] == 0 {
-        Err(io::Error::other("zero pieces of data in SignData"))?;
-      }
-      let mut all_data = vec![];
-      for _ in 0 .. data_pieces[0] {
-        let mut data_len = [0; 2];
-        reader.read_exact(&mut data_len)?;
-        let mut data = vec![0; usize::from(u16::from_le_bytes(data_len))];
-        reader.read_exact(&mut data)?;
-        all_data.push(data);
-      }
-      all_data
-    };
-
-    let signed = Signed::read_without_nonce(reader, label.nonce())?;
-
-    Ok(SignData { plan, attempt, label, data, signed })
-  }
-
-  pub(crate) fn write<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
-    writer.write_all(&self.plan.encode())?;
-    writer.write_all(&self.attempt.to_le_bytes())?;
-    writer.write_all(&[match self.label {
-      Label::Preprocess => 0,
-      Label::Share => 1,
-    }])?;
-
-    writer.write_all(&[u8::try_from(self.data.len()).unwrap()])?;
-    for data in &self.data {
-      if data.len() > u16::MAX.into() {
-        // Currently, the largest individual preprocess is a Monero transaction
-        // It provides 4 commitments per input (128 bytes), a 64-byte proof for them, along with a
-        // key image and proof (96 bytes)
-        // Even with all of that, we could support 227 inputs in a single TX
-        // Monero is limited to ~120 inputs per TX
-        //
-        // Bitcoin has a much higher input count of 520, yet it only uses 64 bytes per preprocess
-        Err(io::Error::other("signing data exceeded 65535 bytes"))?;
-      }
-      writer.write_all(&u16::try_from(data.len()).unwrap().to_le_bytes())?;
-      writer.write_all(data)?;
-    }
-
-    self.signed.write_without_nonce(writer)
-  }
-}
-
-#[derive(Clone, PartialEq, Eq)]
-pub enum Transaction {
-  RemoveParticipant {
-    participant: <Ristretto as Ciphersuite>::G,
-    signed: Signed,
-  },
-
-  DkgParticipation {
-    participation: Vec<u8>,
-    signed: Signed,
-  },
-  DkgConfirmationNonces {
-    // The confirmation attempt
-    attempt: u32,
-    // The nonces for DKG confirmation attempt #attempt
-    confirmation_nonces: [u8; 64],
-    signed: Signed,
-  },
-  DkgConfirmationShare {
-    // The confirmation attempt
-    attempt: u32,
-    // The share for DKG confirmation attempt #attempt
-    confirmation_share: [u8; 32],
-    signed: Signed,
-  },
-
-  // Co-sign a Substrate block.
-  CosignSubstrateBlock([u8; 32]),
-
-  // When we have synchrony on a batch, we can allow signing it
-  // TODO (never?): This is less efficient compared to an ExternalBlock provided transaction,
-  // which would be binding over the block hash and automatically achieve synchrony on all
-  // relevant batches. ExternalBlock was removed for this due to complexity around the pipeline
-  // with the current processor, yet it would still be an improvement.
-  Batch {
-    block: [u8; 32],
-    batch: u32,
-  },
-  // When a Serai block is finalized, with the contained batches, we can allow the associated plan
-  // IDs
-  SubstrateBlock(u64),
-
-  SubstrateSign(SignData<SubstrateSignableId>),
-  Sign(SignData<[u8; 32]>),
-  // This is defined as an Unsigned transaction in order to de-duplicate SignCompleted amongst
-  // reporters (who should all report the same thing)
-  // We do still track the signer in order to prevent a single signer from publishing arbitrarily
-  // many TXs without penalty
-  // Here, they're denoted as the first_signer, as only the signer of the first TX to be included
-  // with this pairing will be remembered on-chain
-  SignCompleted {
-    plan: [u8; 32],
-    tx_hash: Vec<u8>,
-    first_signer: <Ristretto as Ciphersuite>::G,
-    signature: SchnorrSignature<Ristretto>,
-  },
-
-  SlashReport(Vec<u32>, Signed),
-}
-
-impl Debug for Transaction {
-  fn fmt(&self, fmt: &mut core::fmt::Formatter<'_>) -> Result<(), core::fmt::Error> {
-    match self {
-      Transaction::RemoveParticipant { participant, signed } => fmt
-        .debug_struct("Transaction::RemoveParticipant")
-        .field("participant", &hex::encode(participant.to_bytes()))
-        .field("signer", &hex::encode(signed.signer.to_bytes()))
-        .finish_non_exhaustive(),
-      Transaction::DkgParticipation { signed, .. } => fmt
-        .debug_struct("Transaction::DkgParticipation")
-        .field("signer", &hex::encode(signed.signer.to_bytes()))
-        .finish_non_exhaustive(),
-      Transaction::DkgConfirmationNonces { attempt, signed, .. } => fmt
-        .debug_struct("Transaction::DkgConfirmationNonces")
-        .field("attempt", attempt)
-        .field("signer", &hex::encode(signed.signer.to_bytes()))
-        .finish_non_exhaustive(),
-      Transaction::DkgConfirmationShare { attempt, signed, .. } => fmt
-        .debug_struct("Transaction::DkgConfirmationShare")
-        .field("attempt", attempt)
-        .field("signer", &hex::encode(signed.signer.to_bytes()))
-        .finish_non_exhaustive(),
-      Transaction::CosignSubstrateBlock(block) => fmt
-        .debug_struct("Transaction::CosignSubstrateBlock")
-        .field("block", &hex::encode(block))
-        .finish(),
-      Transaction::Batch { block, batch } => fmt
-        .debug_struct("Transaction::Batch")
-        .field("block", &hex::encode(block))
-        .field("batch", &batch)
-        .finish(),
-      Transaction::SubstrateBlock(block) => {
-        fmt.debug_struct("Transaction::SubstrateBlock").field("block", block).finish()
-      }
-      Transaction::SubstrateSign(sign_data) => {
-        fmt.debug_struct("Transaction::SubstrateSign").field("sign_data", sign_data).finish()
-      }
-      Transaction::Sign(sign_data) => {
-        fmt.debug_struct("Transaction::Sign").field("sign_data", sign_data).finish()
-      }
-      Transaction::SignCompleted { plan, tx_hash, .. } => fmt
-        .debug_struct("Transaction::SignCompleted")
-        .field("plan", &hex::encode(plan))
-        .field("tx_hash", &hex::encode(tx_hash))
-        .finish_non_exhaustive(),
-      Transaction::SlashReport(points, signed) => fmt
-        .debug_struct("Transaction::SignCompleted")
-        .field("points", points)
-        .field("signed", signed)
-        .finish(),
-    }
-  }
-}
-
-impl ReadWrite for Transaction {
-  fn read<R: io::Read>(reader: &mut R) -> io::Result<Self> {
-    let mut kind = [0];
-    reader.read_exact(&mut kind)?;
-
-    match kind[0] {
-      0 => Ok(Transaction::RemoveParticipant {
-        participant: Ristretto::read_G(reader)?,
-        signed: Signed::read_without_nonce(reader, 0)?,
-      }),
-
-      1 => {
-        let participation = {
-          let mut participation_len = [0; 4];
-          reader.read_exact(&mut participation_len)?;
-          let participation_len = u32::from_le_bytes(participation_len);
-
-          if participation_len > u32::try_from(TRANSACTION_SIZE_LIMIT).unwrap() {
-            Err(io::Error::other(
-              "participation present in transaction exceeded transaction size limit",
-            ))?;
-          }
-          let participation_len = usize::try_from(participation_len).unwrap();
-
-          let mut participation = vec![0; participation_len];
-          reader.read_exact(&mut participation)?;
-          participation
-        };
-
-        let signed = Signed::read_without_nonce(reader, 0)?;
-
-        Ok(Transaction::DkgParticipation { participation, signed })
-      }
-
-      2 => {
-        let mut attempt = [0; 4];
-        reader.read_exact(&mut attempt)?;
-        let attempt = u32::from_le_bytes(attempt);
-
-        let mut confirmation_nonces = [0; 64];
-        reader.read_exact(&mut confirmation_nonces)?;
-
-        let signed = Signed::read_without_nonce(reader, 0)?;
-
-        Ok(Transaction::DkgConfirmationNonces { attempt, confirmation_nonces, signed })
-      }
-
-      3 => {
-        let mut attempt = [0; 4];
-        reader.read_exact(&mut attempt)?;
-        let attempt = u32::from_le_bytes(attempt);
-
-        let mut confirmation_share = [0; 32];
-        reader.read_exact(&mut confirmation_share)?;
-
-        let signed = Signed::read_without_nonce(reader, 1)?;
-
-        Ok(Transaction::DkgConfirmationShare { attempt, confirmation_share, signed })
-      }
-
-      4 => {
-        let mut block = [0; 32];
-        reader.read_exact(&mut block)?;
-        Ok(Transaction::CosignSubstrateBlock(block))
-      }
-
-      5 => {
-        let mut block = [0; 32];
-        reader.read_exact(&mut block)?;
-        let mut batch = [0; 4];
-        reader.read_exact(&mut batch)?;
-        Ok(Transaction::Batch { block, batch: u32::from_le_bytes(batch) })
-      }
-
-      6 => {
-        let mut block = [0; 8];
-        reader.read_exact(&mut block)?;
-        Ok(Transaction::SubstrateBlock(u64::from_le_bytes(block)))
-      }
-
-      7 => SignData::read(reader).map(Transaction::SubstrateSign),
-      8 => SignData::read(reader).map(Transaction::Sign),
-
-      9 => {
-        let mut plan = [0; 32];
-        reader.read_exact(&mut plan)?;
-
-        let mut tx_hash_len = [0];
-        reader.read_exact(&mut tx_hash_len)?;
-        let mut tx_hash = vec![0; usize::from(tx_hash_len[0])];
-        reader.read_exact(&mut tx_hash)?;
-
-        let first_signer = Ristretto::read_G(reader)?;
-        let signature = SchnorrSignature::<Ristretto>::read(reader)?;
-
-        Ok(Transaction::SignCompleted { plan, tx_hash, first_signer, signature })
-      }
-
-      10 => {
-        let mut len = [0];
-        reader.read_exact(&mut len)?;
-        let len = len[0];
-        // If the set has as many validators as MAX_KEY_SHARES_PER_SET, then the amount of distinct
-        // validators (the amount of validators reported on) will be at most
-        // `MAX_KEY_SHARES_PER_SET - 1`
-        if u32::from(len) > (serai_client::validator_sets::primitives::MAX_KEY_SHARES_PER_SET - 1) {
-          Err(io::Error::other("more points reported than allowed validator"))?;
-        }
-        let mut points = vec![0u32; len.into()];
-        for points in &mut points {
-          let mut these_points = [0; 4];
-          reader.read_exact(&mut these_points)?;
-          *points = u32::from_le_bytes(these_points);
-        }
-        Ok(Transaction::SlashReport(points, Signed::read_without_nonce(reader, 0)?))
-      }
-
-      _ => Err(io::Error::other("invalid transaction type")),
-    }
-  }
-
-  fn write<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
-    match self {
-      Transaction::RemoveParticipant { participant, signed } => {
-        writer.write_all(&[0])?;
-        writer.write_all(&participant.to_bytes())?;
-        signed.write_without_nonce(writer)
-      }
-
-      Transaction::DkgParticipation { participation, signed } => {
-        writer.write_all(&[1])?;
-        writer.write_all(&u32::try_from(participation.len()).unwrap().to_le_bytes())?;
-        writer.write_all(participation)?;
-        signed.write_without_nonce(writer)
-      }
-
-      Transaction::DkgConfirmationNonces { attempt, confirmation_nonces, signed } => {
-        writer.write_all(&[2])?;
-        writer.write_all(&attempt.to_le_bytes())?;
-        writer.write_all(confirmation_nonces)?;
-        signed.write_without_nonce(writer)
-      }
-
-      Transaction::DkgConfirmationShare { attempt, confirmation_share, signed } => {
-        writer.write_all(&[3])?;
-        writer.write_all(&attempt.to_le_bytes())?;
-        writer.write_all(confirmation_share)?;
-        signed.write_without_nonce(writer)
-      }
-
-      Transaction::CosignSubstrateBlock(block) => {
-        writer.write_all(&[4])?;
-        writer.write_all(block)
-      }
-
-      Transaction::Batch { block, batch } => {
-        writer.write_all(&[5])?;
-        writer.write_all(block)?;
-        writer.write_all(&batch.to_le_bytes())
-      }
-
-      Transaction::SubstrateBlock(block) => {
-        writer.write_all(&[6])?;
-        writer.write_all(&block.to_le_bytes())
-      }
-
-      Transaction::SubstrateSign(data) => {
-        writer.write_all(&[7])?;
-        data.write(writer)
-      }
-      Transaction::Sign(data) => {
-        writer.write_all(&[8])?;
-        data.write(writer)
-      }
-      Transaction::SignCompleted { plan, tx_hash, first_signer, signature } => {
-        writer.write_all(&[9])?;
-        writer.write_all(plan)?;
-        writer
-          .write_all(&[u8::try_from(tx_hash.len()).expect("tx hash length exceed 255 bytes")])?;
-        writer.write_all(tx_hash)?;
-        writer.write_all(&first_signer.to_bytes())?;
-        signature.write(writer)
-      }
-      Transaction::SlashReport(points, signed) => {
-        writer.write_all(&[10])?;
-        writer.write_all(&[u8::try_from(points.len()).unwrap()])?;
-        for points in points {
-          writer.write_all(&points.to_le_bytes())?;
-        }
-        signed.write_without_nonce(writer)
-      }
-    }
-  }
-}
-
-impl TransactionTrait for Transaction {
-  fn kind(&self) -> TransactionKind<'_> {
-    match self {
-      Transaction::RemoveParticipant { participant, signed } => {
-        TransactionKind::Signed((b"remove", participant.to_bytes()).encode(), signed)
-      }
-
-      Transaction::DkgParticipation { signed, .. } => {
-        TransactionKind::Signed(b"dkg".to_vec(), signed)
-      }
-      Transaction::DkgConfirmationNonces { attempt, signed, .. } |
-      Transaction::DkgConfirmationShare { attempt, signed, .. } => {
-        TransactionKind::Signed((b"dkg_confirmation", attempt).encode(), signed)
-      }
-
-      Transaction::CosignSubstrateBlock(_) => TransactionKind::Provided("cosign"),
-
-      Transaction::Batch { .. } => TransactionKind::Provided("batch"),
-      Transaction::SubstrateBlock(_) => TransactionKind::Provided("serai"),
-
-      Transaction::SubstrateSign(data) => {
-        TransactionKind::Signed((b"substrate", data.plan, data.attempt).encode(), &data.signed)
-      }
-      Transaction::Sign(data) => {
-        TransactionKind::Signed((b"sign", data.plan, data.attempt).encode(), &data.signed)
-      }
-      Transaction::SignCompleted { .. } => TransactionKind::Unsigned,
-
-      Transaction::SlashReport(_, signed) => {
-        TransactionKind::Signed(b"slash_report".to_vec(), signed)
-      }
-    }
-  }
-
-  fn hash(&self) -> [u8; 32] {
-    let mut tx = self.serialize();
-    if let TransactionKind::Signed(_, signed) = self.kind() {
-      // Make sure the part we're cutting off is the signature
-      assert_eq!(tx.drain((tx.len() - 64) ..).collect::<Vec<_>>(), signed.signature.serialize());
-    }
-    Blake2s256::digest([b"Coordinator Tributary Transaction".as_slice(), &tx].concat()).into()
-  }
-
-  fn verify(&self) -> Result<(), TransactionError> {
-    // TODO: Check SubstrateSign's lengths here
-
-    if let Transaction::SignCompleted { first_signer, signature, .. } = self {
-      if !signature.verify(*first_signer, self.sign_completed_challenge()) {
-        Err(TransactionError::InvalidContent)?;
-      }
-    }
-
-    Ok(())
-  }
-}
-
-impl Transaction {
-  // Used to initially construct transactions so we can then get sig hashes and perform signing
-  pub fn empty_signed() -> Signed {
-    Signed {
-      signer: Ristretto::generator(),
-      nonce: 0,
-      signature: SchnorrSignature::<Ristretto> {
-        R: Ristretto::generator(),
-        s: <Ristretto as Ciphersuite>::F::ZERO,
-      },
-    }
-  }
-
-  // Sign a transaction
-  pub fn sign<R: RngCore + CryptoRng>(
-    &mut self,
-    rng: &mut R,
-    genesis: [u8; 32],
-    key: &Zeroizing<<Ristretto as Ciphersuite>::F>,
-  ) {
-    fn signed(tx: &mut Transaction) -> (u32, &mut Signed) {
-      #[allow(clippy::match_same_arms)] // Doesn't make semantic sense here
-      let nonce = match tx {
-        Transaction::RemoveParticipant { .. } => 0,
-
-        Transaction::DkgParticipation { .. } => 0,
-        // Uses a nonce of 0 as it has an internal attempt counter we distinguish by
-        Transaction::DkgConfirmationNonces { .. } => 0,
-        // Uses a nonce of 1 due to internal attempt counter and due to following
-        // DkgConfirmationNonces
-        Transaction::DkgConfirmationShare { .. } => 1,
-
-        Transaction::CosignSubstrateBlock(_) => panic!("signing CosignSubstrateBlock"),
-
-        Transaction::Batch { .. } => panic!("signing Batch"),
-        Transaction::SubstrateBlock(_) => panic!("signing SubstrateBlock"),
-
-        Transaction::SubstrateSign(data) => data.label.nonce(),
-        Transaction::Sign(data) => data.label.nonce(),
-
-        Transaction::SignCompleted { .. } => panic!("signing SignCompleted"),
-
-        Transaction::SlashReport(_, _) => 0,
-      };
-
-      (
-        nonce,
-        #[allow(clippy::match_same_arms)]
-        match tx {
-          Transaction::RemoveParticipant { ref mut signed, .. } |
-          Transaction::DkgParticipation { ref mut signed, .. } |
-          Transaction::DkgConfirmationNonces { ref mut signed, .. } => signed,
-          Transaction::DkgConfirmationShare { ref mut signed, .. } => signed,
-
-          Transaction::CosignSubstrateBlock(_) => panic!("signing CosignSubstrateBlock"),
-
-          Transaction::Batch { .. } => panic!("signing Batch"),
-          Transaction::SubstrateBlock(_) => panic!("signing SubstrateBlock"),
-
-          Transaction::SubstrateSign(ref mut data) => &mut data.signed,
-          Transaction::Sign(ref mut data) => &mut data.signed,
-
-          Transaction::SignCompleted { .. } => panic!("signing SignCompleted"),
-
-          Transaction::SlashReport(_, ref mut signed) => signed,
-        },
-      )
-    }
-
-    let (nonce, signed_ref) = signed(self);
-    signed_ref.signer = Ristretto::generator() * key.deref();
-    signed_ref.nonce = nonce;
-
-    let sig_nonce = Zeroizing::new(<Ristretto as Ciphersuite>::F::random(rng));
-    signed(self).1.signature.R = <Ristretto as Ciphersuite>::generator() * sig_nonce.deref();
-    let sig_hash = self.sig_hash(genesis);
-    signed(self).1.signature = SchnorrSignature::<Ristretto>::sign(key, sig_nonce, sig_hash);
-  }
-
-  pub fn sign_completed_challenge(&self) -> <Ristretto as Ciphersuite>::F {
-    if let Transaction::SignCompleted { plan, tx_hash, first_signer, signature } = self {
-      let mut transcript =
-        RecommendedTranscript::new(b"Coordinator Tributary Transaction SignCompleted");
-      transcript.append_message(b"plan", plan);
-      transcript.append_message(b"tx_hash", tx_hash);
-      transcript.append_message(b"signer", first_signer.to_bytes());
-      transcript.append_message(b"nonce", signature.R.to_bytes());
-      Ristretto::hash_to_F(b"SignCompleted signature", &transcript.challenge(b"challenge"))
-    } else {
-      panic!("sign_completed_challenge called on transaction which wasn't SignCompleted")
-    }
-  }
-}

coordinator/substrate/Cargo.toml

@@ -0,0 +1,37 @@
+[package]
+name = "serai-coordinator-substrate"
+version = "0.1.0"
+description = "Serai Coordinator's Substrate Scanner"
+license = "AGPL-3.0-only"
+repository = "https://github.com/serai-dex/serai/tree/develop/coordinator/substrate"
+authors = ["Luke Parker <lukeparker5132@gmail.com>"]
+keywords = []
+edition = "2021"
+publish = false
+rust-version = "1.81"
+
+[package.metadata.docs.rs]
+all-features = true
+rustdoc-args = ["--cfg", "docsrs"]
+
+[lints]
+workspace = true
+
+[dependencies]
+bitvec = { version = "1", default-features = false, features = ["std"] }
+
+scale = { package = "parity-scale-codec", version = "3", default-features = false, features = ["std", "derive", "bit-vec"] }
+borsh = { version = "1", default-features = false, features = ["std", "derive", "de_strict_order"] }
+serai-client = { path = "../../substrate/client", version = "0.1", default-features = false, features = ["serai", "borsh"] }
+
+log = { version = "0.4", default-features = false, features = ["std"] }
+
+futures = { version = "0.3", default-features = false, features = ["std"] }
+tokio = { version = "1", default-features = false }
+
+serai-db = { path = "../../common/db", version = "0.1.1" }
+serai-task = { path = "../../common/task", version = "0.1" }
+
+serai-cosign = { path = "../cosign", version = "0.1" }
+
+messages = { package = "serai-processor-messages", version = "0.1", path = "../../processor/messages" }

coordinator/substrate/LICENSE

@@ -0,0 +1,15 @@
+AGPL-3.0-only license
+
+Copyright (c) 2023-2024 Luke Parker
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License Version 3 as
+published by the Free Software Foundation.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.

coordinator/substrate/README.md

@@ -0,0 +1,20 @@
+# Serai Coordinator Substrate
+
+This crate manages the Serai coordinators's interactions with Serai's Substrate blockchain.
+
+Two event streams are defined:
+
+- Canonical events, which must be handled by every validator, regardless of the sets they're present
+  in. These are represented by `serai_processor_messages::substrate::CoordinatorMessage`.
+- Ephemeral events, which only need to be handled by the validators present within the sets they
+  relate to. These are represented by two channels, `NewSet` and `SignSlashReport`.
+
+The canonical event stream is available without provision of a validator's public key. The ephemeral
+event stream requires provision of a validator's public key. Both are ordered within themselves, yet
+there are no ordering guarantees across the two.
+
+Additionally, a collection of tasks are defined to publish data onto Serai:
+
+- `SetKeysTask`, which sets the keys generated via DKGs onto Serai.
+- `PublishBatchTask`, which publishes `Batch`s onto Serai.
+- `PublishSlashReportTask`, which publishes `SlashReport`s onto Serai.

coordinator/substrate/src/canonical.rs

@@ -0,0 +1,224 @@
+use core::future::Future;
+use std::sync::Arc;
+
+use futures::stream::{StreamExt, FuturesOrdered};
+
+use serai_client::Serai;
+
+use messages::substrate::{InInstructionResult, ExecutedBatch, CoordinatorMessage};
+
+use serai_db::*;
+use serai_task::ContinuallyRan;
+
+use serai_cosign::Cosigning;
+
+create_db!(
+  CoordinatorSubstrateCanonical {
+    NextBlock: () -> u64,
+  }
+);
+
+/// The event stream for canonical events.
+pub struct CanonicalEventStream<D: Db> {
+  db: D,
+  serai: Arc<Serai>,
+}
+
+impl<D: Db> CanonicalEventStream<D> {
+  /// Create a new canonical event stream.
+  ///
+  /// Only one of these may exist over the provided database.
+  pub fn new(db: D, serai: Arc<Serai>) -> Self {
+    Self { db, serai }
+  }
+}
+
+impl<D: Db> ContinuallyRan for CanonicalEventStream<D> {
+  type Error = String;
+
+  fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, Self::Error>> {
+    async move {
+      let next_block = NextBlock::get(&self.db).unwrap_or(0);
+      let latest_finalized_block =
+        Cosigning::<D>::latest_cosigned_block_number(&self.db).map_err(|e| format!("{e:?}"))?;
+
+      // These are all the events which generate canonical messages
+      struct CanonicalEvents {
+        time: u64,
+        key_gen_events: Vec<serai_client::validator_sets::ValidatorSetsEvent>,
+        set_retired_events: Vec<serai_client::validator_sets::ValidatorSetsEvent>,
+        batch_events: Vec<serai_client::in_instructions::InInstructionsEvent>,
+        burn_events: Vec<serai_client::coins::CoinsEvent>,
+      }
+
+      // For a cosigned block, fetch all relevant events
+      let scan = {
+        let db = self.db.clone();
+        let serai = &self.serai;
+        move |block_number| {
+          let block_hash = Cosigning::<D>::cosigned_block(&db, block_number);
+
+          async move {
+            let block_hash = match block_hash {
+              Ok(Some(block_hash)) => block_hash,
+              Ok(None) => {
+                panic!("iterating to latest cosigned block but couldn't get cosigned block")
+              }
+              Err(serai_cosign::Faulted) => return Err("cosigning process faulted".to_string()),
+            };
+            let temporal_serai = serai.as_of(block_hash);
+            let temporal_serai_validators = temporal_serai.validator_sets();
+            let temporal_serai_instructions = temporal_serai.in_instructions();
+            let temporal_serai_coins = temporal_serai.coins();
+
+            let (block, key_gen_events, set_retired_events, batch_events, burn_events) =
+              tokio::try_join!(
+                serai.block(block_hash),
+                temporal_serai_validators.key_gen_events(),
+                temporal_serai_validators.set_retired_events(),
+                temporal_serai_instructions.batch_events(),
+                temporal_serai_coins.burn_with_instruction_events(),
+              )
+              .map_err(|e| format!("{e:?}"))?;
+            let Some(block) = block else {
+              Err(format!("Serai node didn't have cosigned block #{block_number}"))?
+            };
+
+            let time = if block_number == 0 {
+              block.time().unwrap_or(0)
+            } else {
+              // Serai's block time is in milliseconds
+              block
+                .time()
+                .ok_or_else(|| "non-genesis Serai block didn't have a time".to_string())? /
+                1000
+            };
+
+            Ok((
+              block_number,
+              CanonicalEvents {
+                time,
+                key_gen_events,
+                set_retired_events,
+                batch_events,
+                burn_events,
+              },
+            ))
+          }
+        }
+      };
+
+      // Sync the next set of upcoming blocks all at once to minimize latency
+      const BLOCKS_TO_SYNC_AT_ONCE: u64 = 10;
+      // FuturesOrdered can be bad practice due to potentially causing tiemouts if it isn't
+      // sufficiently polled. Considering our processing loop is minimal and it does poll this,
+      // it's fine.
+      let mut set = FuturesOrdered::new();
+      for block_number in
+        next_block ..= latest_finalized_block.min(next_block + BLOCKS_TO_SYNC_AT_ONCE)
+      {
+        set.push_back(scan(block_number));
+      }
+
+      for block_number in next_block ..= latest_finalized_block {
+        // Get the next block in our queue
+        let (popped_block_number, block) = set.next().await.unwrap()?;
+        assert_eq!(block_number, popped_block_number);
+        // Re-populate the queue
+        if (block_number + BLOCKS_TO_SYNC_AT_ONCE) <= latest_finalized_block {
+          set.push_back(scan(block_number + BLOCKS_TO_SYNC_AT_ONCE));
+        }
+
+        let mut txn = self.db.txn();
+
+        for key_gen in block.key_gen_events {
+          let serai_client::validator_sets::ValidatorSetsEvent::KeyGen { set, key_pair } = &key_gen
+          else {
+            panic!("KeyGen event wasn't a KeyGen event: {key_gen:?}");
+          };
+          crate::Canonical::send(
+            &mut txn,
+            set.network,
+            &CoordinatorMessage::SetKeys {
+              serai_time: block.time,
+              session: set.session,
+              key_pair: key_pair.clone(),
+            },
+          );
+        }
+
+        for set_retired in block.set_retired_events {
+          let serai_client::validator_sets::ValidatorSetsEvent::SetRetired { set } = &set_retired
+          else {
+            panic!("SetRetired event wasn't a SetRetired event: {set_retired:?}");
+          };
+          crate::Canonical::send(
+            &mut txn,
+            set.network,
+            &CoordinatorMessage::SlashesReported { session: set.session },
+          );
+        }
+
+        for network in serai_client::primitives::NETWORKS {
+          let mut batch = None;
+          for this_batch in &block.batch_events {
+            let serai_client::in_instructions::InInstructionsEvent::Batch {
+              network: batch_network,
+              publishing_session,
+              id,
+              external_network_block_hash,
+              in_instructions_hash,
+              in_instruction_results,
+            } = this_batch
+            else {
+              panic!("Batch event wasn't a Batch event: {this_batch:?}");
+            };
+            if network == *batch_network {
+              if batch.is_some() {
+                Err("Serai block had multiple batches for the same network".to_string())?;
+              }
+              batch = Some(ExecutedBatch {
+                id: *id,
+                publisher: *publishing_session,
+                external_network_block_hash: *external_network_block_hash,
+                in_instructions_hash: *in_instructions_hash,
+                in_instruction_results: in_instruction_results
+                  .iter()
+                  .map(|bit| {
+                    if *bit {
+                      InInstructionResult::Succeeded
+                    } else {
+                      InInstructionResult::Failed
+                    }
+                  })
+                  .collect(),
+              });
+            }
+          }
+
+          let mut burns = vec![];
+          for burn in &block.burn_events {
+            let serai_client::coins::CoinsEvent::BurnWithInstruction { from: _, instruction } =
+              &burn
+            else {
+              panic!("Burn event wasn't a Burn.in event: {burn:?}");
+            };
+            if instruction.balance.coin.network() == network {
+              burns.push(instruction.clone());
+            }
+          }
+
+          crate::Canonical::send(
+            &mut txn,
+            network,
+            &CoordinatorMessage::Block { serai_block_number: block_number, batch, burns },
+          );
+        }
+
+        txn.commit();
+      }
+
+      Ok(next_block <= latest_finalized_block)
+    }
+  }
+}

coordinator/substrate/src/ephemeral.rs

@@ -0,0 +1,249 @@
+use core::future::Future;
+use std::sync::Arc;
+
+use futures::stream::{StreamExt, FuturesOrdered};
+
+use serai_client::{
+  primitives::{PublicKey, NetworkId, EmbeddedEllipticCurve},
+  validator_sets::primitives::MAX_KEY_SHARES_PER_SET,
+  Serai,
+};
+
+use serai_db::*;
+use serai_task::ContinuallyRan;
+
+use serai_cosign::Cosigning;
+
+use crate::NewSetInformation;
+
+create_db!(
+  CoordinatorSubstrateEphemeral {
+    NextBlock: () -> u64,
+  }
+);
+
+/// The event stream for ephemeral events.
+pub struct EphemeralEventStream<D: Db> {
+  db: D,
+  serai: Arc<Serai>,
+  validator: PublicKey,
+}
+
+impl<D: Db> EphemeralEventStream<D> {
+  /// Create a new ephemeral event stream.
+  ///
+  /// Only one of these may exist over the provided database.
+  pub fn new(db: D, serai: Arc<Serai>, validator: PublicKey) -> Self {
+    Self { db, serai, validator }
+  }
+}
+
+impl<D: Db> ContinuallyRan for EphemeralEventStream<D> {
+  type Error = String;
+
+  fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, Self::Error>> {
+    async move {
+      let next_block = NextBlock::get(&self.db).unwrap_or(0);
+      let latest_finalized_block =
+        Cosigning::<D>::latest_cosigned_block_number(&self.db).map_err(|e| format!("{e:?}"))?;
+
+      // These are all the events which generate canonical messages
+      struct EphemeralEvents {
+        block_hash: [u8; 32],
+        time: u64,
+        new_set_events: Vec<serai_client::validator_sets::ValidatorSetsEvent>,
+        accepted_handover_events: Vec<serai_client::validator_sets::ValidatorSetsEvent>,
+      }
+
+      // For a cosigned block, fetch all relevant events
+      let scan = {
+        let db = self.db.clone();
+        let serai = &self.serai;
+        move |block_number| {
+          let block_hash = Cosigning::<D>::cosigned_block(&db, block_number);
+
+          async move {
+            let block_hash = match block_hash {
+              Ok(Some(block_hash)) => block_hash,
+              Ok(None) => {
+                panic!("iterating to latest cosigned block but couldn't get cosigned block")
+              }
+              Err(serai_cosign::Faulted) => return Err("cosigning process faulted".to_string()),
+            };
+
+            let temporal_serai = serai.as_of(block_hash);
+            let temporal_serai_validators = temporal_serai.validator_sets();
+            let (block, new_set_events, accepted_handover_events) = tokio::try_join!(
+              serai.block(block_hash),
+              temporal_serai_validators.new_set_events(),
+              temporal_serai_validators.accepted_handover_events(),
+            )
+            .map_err(|e| format!("{e:?}"))?;
+            let Some(block) = block else {
+              Err(format!("Serai node didn't have cosigned block #{block_number}"))?
+            };
+
+            let time = if block_number == 0 {
+              block.time().unwrap_or(0)
+            } else {
+              // Serai's block time is in milliseconds
+              block
+                .time()
+                .ok_or_else(|| "non-genesis Serai block didn't have a time".to_string())? /
+                1000
+            };
+
+            Ok((
+              block_number,
+              EphemeralEvents { block_hash, time, new_set_events, accepted_handover_events },
+            ))
+          }
+        }
+      };
+
+      // Sync the next set of upcoming blocks all at once to minimize latency
+      const BLOCKS_TO_SYNC_AT_ONCE: u64 = 50;
+      // FuturesOrdered can be bad practice due to potentially causing tiemouts if it isn't
+      // sufficiently polled. Our processing loop isn't minimal, itself making multiple requests,
+      // but the loop body should only be executed a few times a week. It's better to get through
+      // most blocks with this optimization, and have timeouts a few times a week, than not have
+      // this at all.
+      let mut set = FuturesOrdered::new();
+      for block_number in
+        next_block ..= latest_finalized_block.min(next_block + BLOCKS_TO_SYNC_AT_ONCE)
+      {
+        set.push_back(scan(block_number));
+      }
+
+      for block_number in next_block ..= latest_finalized_block {
+        // Get the next block in our queue
+        let (popped_block_number, block) = set.next().await.unwrap()?;
+        assert_eq!(block_number, popped_block_number);
+        // Re-populate the queue
+        if (block_number + BLOCKS_TO_SYNC_AT_ONCE) <= latest_finalized_block {
+          set.push_back(scan(block_number + BLOCKS_TO_SYNC_AT_ONCE));
+        }
+
+        let mut txn = self.db.txn();
+
+        for new_set in block.new_set_events {
+          let serai_client::validator_sets::ValidatorSetsEvent::NewSet { set } = &new_set else {
+            panic!("NewSet event wasn't a NewSet event: {new_set:?}");
+          };
+
+          // We only coordinate over external networks
+          if set.network == NetworkId::Serai {
+            continue;
+          }
+
+          let serai = self.serai.as_of(block.block_hash);
+          let serai = serai.validator_sets();
+          let Some(validators) =
+            serai.participants(set.network).await.map_err(|e| format!("{e:?}"))?
+          else {
+            Err(format!(
+              "block #{block_number} declared a new set but didn't have the participants"
+            ))?
+          };
+          let in_set = validators.iter().any(|(validator, _)| *validator == self.validator);
+          if in_set {
+            if u16::try_from(validators.len()).is_err() {
+              Err("more than u16::MAX validators sent")?;
+            }
+
+            let Ok(validators) = validators
+              .into_iter()
+              .map(|(validator, weight)| u16::try_from(weight).map(|weight| (validator, weight)))
+              .collect::<Result<Vec<_>, _>>()
+            else {
+              Err("validator's weight exceeded u16::MAX".to_string())?
+            };
+
+            // Do the summation in u32 so we don't risk a u16 overflow
+            let total_weight = validators.iter().map(|(_, weight)| u32::from(*weight)).sum::<u32>();
+            if total_weight > u32::from(MAX_KEY_SHARES_PER_SET) {
+              Err(format!(
+                "{set:?} has {total_weight} key shares when the max is {MAX_KEY_SHARES_PER_SET}"
+              ))?;
+            }
+            let total_weight = u16::try_from(total_weight).unwrap();
+
+            // Fetch all of the validators' embedded elliptic curve keys
+            let mut embedded_elliptic_curve_keys = FuturesOrdered::new();
+            for (validator, _) in &validators {
+              let validator = *validator;
+              // try_join doesn't return a future so we need to wrap it in this additional async
+              // block
+              embedded_elliptic_curve_keys.push_back(async move {
+                tokio::try_join!(
+                  // One future to fetch the substrate embedded key
+                  serai
+                    .embedded_elliptic_curve_key(validator, EmbeddedEllipticCurve::Embedwards25519),
+                  // One future to fetch the external embedded key, if there is a distinct curve
+                  async {
+                    // `embedded_elliptic_curves` is documented to have the second entry be the
+                    // network-specific curve (if it exists and is distinct from Embedwards25519)
+                    if let Some(curve) = set.network.embedded_elliptic_curves().get(1) {
+                      serai.embedded_elliptic_curve_key(validator, *curve).await.map(Some)
+                    } else {
+                      Ok(None)
+                    }
+                  }
+                )
+                .map(|(substrate_embedded_key, external_embedded_key)| {
+                  (validator, substrate_embedded_key, external_embedded_key)
+                })
+              });
+            }
+
+            let mut evrf_public_keys = Vec::with_capacity(usize::from(total_weight));
+            for (validator, weight) in &validators {
+              let (future_validator, substrate_embedded_key, external_embedded_key) =
+                embedded_elliptic_curve_keys.next().await.unwrap().map_err(|e| format!("{e:?}"))?;
+              assert_eq!(*validator, future_validator);
+              let external_embedded_key =
+                external_embedded_key.unwrap_or(substrate_embedded_key.clone());
+              match (substrate_embedded_key, external_embedded_key) {
+                (Some(substrate_embedded_key), Some(external_embedded_key)) => {
+                  let substrate_embedded_key = <[u8; 32]>::try_from(substrate_embedded_key)
+                    .map_err(|_| "Embedwards25519 key wasn't 32 bytes".to_string())?;
+                  for _ in 0 .. *weight {
+                    evrf_public_keys.push((substrate_embedded_key, external_embedded_key.clone()));
+                  }
+                }
+                _ => Err("NewSet with validator missing an embedded key".to_string())?,
+              }
+            }
+
+            crate::NewSet::send(
+              &mut txn,
+              &NewSetInformation {
+                set: *set,
+                serai_block: block.block_hash,
+                declaration_time: block.time,
+                // TODO: Why do we have this as an explicit field here?
+                // Shouldn't thiis be inlined into the Processor's key gen code, where it's used?
+                threshold: ((total_weight * 2) / 3) + 1,
+                validators,
+                evrf_public_keys,
+              },
+            );
+          }
+        }
+
+        for accepted_handover in block.accepted_handover_events {
+          let serai_client::validator_sets::ValidatorSetsEvent::AcceptedHandover { set } =
+            &accepted_handover
+          else {
+            panic!("AcceptedHandover event wasn't a AcceptedHandover event: {accepted_handover:?}");
+          };
+          crate::SignSlashReport::send(&mut txn, *set);
+        }
+
+        txn.commit();
+      }
+
+      Ok(next_block <= latest_finalized_block)
+    }
+  }
+}

coordinator/substrate/src/lib.rs

@@ -0,0 +1,228 @@
+#![cfg_attr(docsrs, feature(doc_auto_cfg))]
+#![doc = include_str!("../README.md")]
+#![deny(missing_docs)]
+
+use scale::{Encode, Decode};
+use borsh::{io, BorshSerialize, BorshDeserialize};
+
+use serai_client::{
+  primitives::{NetworkId, PublicKey, Signature, SeraiAddress},
+  validator_sets::primitives::{Session, ValidatorSet, KeyPair},
+  in_instructions::primitives::SignedBatch,
+  Transaction,
+};
+
+use serai_db::*;
+
+mod canonical;
+pub use canonical::CanonicalEventStream;
+mod ephemeral;
+pub use ephemeral::EphemeralEventStream;
+
+mod set_keys;
+pub use set_keys::SetKeysTask;
+mod publish_batch;
+pub use publish_batch::PublishBatchTask;
+mod publish_slash_report;
+pub use publish_slash_report::PublishSlashReportTask;
+
+fn borsh_serialize_validators<W: io::Write>(
+  validators: &Vec<(PublicKey, u16)>,
+  writer: &mut W,
+) -> Result<(), io::Error> {
+  // This doesn't use `encode_to` as `encode_to` panics if the writer returns an error
+  writer.write_all(&validators.encode())
+}
+
+fn borsh_deserialize_validators<R: io::Read>(
+  reader: &mut R,
+) -> Result<Vec<(PublicKey, u16)>, io::Error> {
+  Decode::decode(&mut scale::IoReader(reader)).map_err(io::Error::other)
+}
+
+/// The information for a new set.
+#[derive(Clone, Debug, BorshSerialize, BorshDeserialize)]
+pub struct NewSetInformation {
+  /// The set.
+  pub set: ValidatorSet,
+  /// The Serai block which declared it.
+  pub serai_block: [u8; 32],
+  /// The time of the block which declared it, in seconds.
+  pub declaration_time: u64,
+  /// The threshold to use.
+  pub threshold: u16,
+  /// The validators, with the amount of key shares they have.
+  #[borsh(
+    serialize_with = "borsh_serialize_validators",
+    deserialize_with = "borsh_deserialize_validators"
+  )]
+  pub validators: Vec<(PublicKey, u16)>,
+  /// The eVRF public keys.
+  pub evrf_public_keys: Vec<([u8; 32], Vec<u8>)>,
+}
+
+mod _public_db {
+  use super::*;
+
+  db_channel!(
+    CoordinatorSubstrate {
+      // Canonical messages to send to the processor
+      Canonical: (network: NetworkId) -> messages::substrate::CoordinatorMessage,
+
+      // Relevant new set, from an ephemeral event stream
+      NewSet: () -> NewSetInformation,
+      // Potentially relevant sign slash report, from an ephemeral event stream
+      SignSlashReport: (set: ValidatorSet) -> (),
+
+      // Signed batches to publish onto the Serai network
+      SignedBatches: (network: NetworkId) -> SignedBatch,
+    }
+  );
+
+  create_db!(
+    CoordinatorSubstrate {
+      // Keys to set on the Serai network
+      Keys: (network: NetworkId) -> (Session, Vec<u8>),
+      // Slash reports to publish onto the Serai network
+      SlashReports: (network: NetworkId) -> (Session, Vec<u8>),
+    }
+  );
+}
+
+/// The canonical event stream.
+pub struct Canonical;
+impl Canonical {
+  pub(crate) fn send(
+    txn: &mut impl DbTxn,
+    network: NetworkId,
+    msg: &messages::substrate::CoordinatorMessage,
+  ) {
+    _public_db::Canonical::send(txn, network, msg);
+  }
+  /// Try to receive a canonical event, returning `None` if there is none to receive.
+  pub fn try_recv(
+    txn: &mut impl DbTxn,
+    network: NetworkId,
+  ) -> Option<messages::substrate::CoordinatorMessage> {
+    _public_db::Canonical::try_recv(txn, network)
+  }
+}
+
+/// The channel for new set events emitted by an ephemeral event stream.
+pub struct NewSet;
+impl NewSet {
+  pub(crate) fn send(txn: &mut impl DbTxn, msg: &NewSetInformation) {
+    _public_db::NewSet::send(txn, msg);
+  }
+  /// Try to receive a new set's information, returning `None` if there is none to receive.
+  pub fn try_recv(txn: &mut impl DbTxn) -> Option<NewSetInformation> {
+    _public_db::NewSet::try_recv(txn)
+  }
+}
+
+/// The channel for notifications to sign a slash report, as emitted by an ephemeral event stream.
+///
+/// These notifications MAY be for irrelevant validator sets. The only guarantee is the
+/// notifications for all relevant validator sets will be included.
+pub struct SignSlashReport;
+impl SignSlashReport {
+  pub(crate) fn send(txn: &mut impl DbTxn, set: ValidatorSet) {
+    _public_db::SignSlashReport::send(txn, set, &());
+  }
+  /// Try to receive a notification to sign a slash report, returning `None` if there is none to
+  /// receive.
+  pub fn try_recv(txn: &mut impl DbTxn, set: ValidatorSet) -> Option<()> {
+    _public_db::SignSlashReport::try_recv(txn, set)
+  }
+}
+
+/// The keys to set on Serai.
+pub struct Keys;
+impl Keys {
+  /// Set the keys to report for a validator set.
+  ///
+  /// This only saves the most recent keys as only a single session is eligible to have its keys
+  /// reported at once.
+  pub fn set(
+    txn: &mut impl DbTxn,
+    set: ValidatorSet,
+    key_pair: KeyPair,
+    signature_participants: bitvec::vec::BitVec<u8, bitvec::order::Lsb0>,
+    signature: Signature,
+  ) {
+    // If we have a more recent pair of keys, don't write this historic one
+    if let Some((existing_session, _)) = _public_db::Keys::get(txn, set.network) {
+      if existing_session.0 >= set.session.0 {
+        return;
+      }
+    }
+
+    let tx = serai_client::validator_sets::SeraiValidatorSets::set_keys(
+      set.network,
+      key_pair,
+      signature_participants,
+      signature,
+    );
+    _public_db::Keys::set(txn, set.network, &(set.session, tx.encode()));
+  }
+  pub(crate) fn take(txn: &mut impl DbTxn, network: NetworkId) -> Option<(Session, Transaction)> {
+    let (session, tx) = _public_db::Keys::take(txn, network)?;
+    Some((session, <_>::decode(&mut tx.as_slice()).unwrap()))
+  }
+}
+
+/// The signed batches to publish onto Serai.
+pub struct SignedBatches;
+impl SignedBatches {
+  /// Send a `SignedBatch` to publish onto Serai.
+  ///
+  /// These will be published sequentially. Out-of-order sending risks hanging the task.
+  pub fn send(txn: &mut impl DbTxn, batch: &SignedBatch) {
+    _public_db::SignedBatches::send(txn, batch.batch.network, batch);
+  }
+  pub(crate) fn try_recv(txn: &mut impl DbTxn, network: NetworkId) -> Option<SignedBatch> {
+    _public_db::SignedBatches::try_recv(txn, network)
+  }
+}
+
+/// The slash report was invalid.
+#[derive(Debug)]
+pub struct InvalidSlashReport;
+
+/// The slash reports to publish onto Serai.
+pub struct SlashReports;
+impl SlashReports {
+  /// Set the slashes to report for a validator set.
+  ///
+  /// This only saves the most recent slashes as only a single session is eligible to have its
+  /// slashes reported at once.
+  ///
+  /// Returns Err if the slashes are invalid. Returns Ok if the slashes weren't detected as
+  /// invalid. Slashes may be considered invalid by the Serai blockchain later even if not detected
+  /// as invalid here.
+  pub fn set(
+    txn: &mut impl DbTxn,
+    set: ValidatorSet,
+    slashes: Vec<(SeraiAddress, u32)>,
+    signature: Signature,
+  ) -> Result<(), InvalidSlashReport> {
+    // If we have a more recent slash report, don't write this historic one
+    if let Some((existing_session, _)) = _public_db::SlashReports::get(txn, set.network) {
+      if existing_session.0 >= set.session.0 {
+        return Ok(());
+      }
+    }
+
+    let tx = serai_client::validator_sets::SeraiValidatorSets::report_slashes(
+      set.network,
+      slashes.try_into().map_err(|_| InvalidSlashReport)?,
+      signature,
+    );
+    _public_db::SlashReports::set(txn, set.network, &(set.session, tx.encode()));
+    Ok(())
+  }
+  pub(crate) fn take(txn: &mut impl DbTxn, network: NetworkId) -> Option<(Session, Transaction)> {
+    let (session, tx) = _public_db::SlashReports::take(txn, network)?;
+    Some((session, <_>::decode(&mut tx.as_slice()).unwrap()))
+  }
+}

coordinator/substrate/src/publish_batch.rs

@@ -0,0 +1,66 @@
+use core::future::Future;
+use std::sync::Arc;
+
+use serai_db::{DbTxn, Db};
+
+use serai_client::{primitives::NetworkId, SeraiError, Serai};
+
+use serai_task::ContinuallyRan;
+
+use crate::SignedBatches;
+
+/// Publish `SignedBatch`s from `SignedBatches` onto Serai.
+pub struct PublishBatchTask<D: Db> {
+  db: D,
+  serai: Arc<Serai>,
+  network: NetworkId,
+}
+
+impl<D: Db> PublishBatchTask<D> {
+  /// Create a task to publish `SignedBatch`s onto Serai.
+  ///
+  /// Returns None if `network == NetworkId::Serai`.
+  // TODO: ExternalNetworkId
+  pub fn new(db: D, serai: Arc<Serai>, network: NetworkId) -> Option<Self> {
+    if network == NetworkId::Serai {
+      None?
+    };
+    Some(Self { db, serai, network })
+  }
+}
+
+impl<D: Db> ContinuallyRan for PublishBatchTask<D> {
+  type Error = SeraiError;
+
+  fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, Self::Error>> {
+    async move {
+      let mut made_progress = false;
+
+      loop {
+        let mut txn = self.db.txn();
+        let Some(batch) = SignedBatches::try_recv(&mut txn, self.network) else {
+          // No batch to publish at this time
+          break;
+        };
+
+        // Publish this Batch if it hasn't already been published
+        let serai = self.serai.as_of_latest_finalized_block().await?;
+        let last_batch = serai.in_instructions().last_batch_for_network(self.network).await?;
+        if last_batch < Some(batch.batch.id) {
+          // This stream of Batches *should* be sequential within the larger context of the Serai
+          // coordinator. In this library, we use a more relaxed definition and don't assert
+          // sequence. This does risk hanging the task, if Batch #n+1 is sent before Batch #n, but
+          // that is a documented fault of the `SignedBatches` API.
+          self
+            .serai
+            .publish(&serai_client::in_instructions::SeraiInInstructions::execute_batch(batch))
+            .await?;
+        }
+
+        txn.commit();
+        made_progress = true;
+      }
+      Ok(made_progress)
+    }
+  }
+}

coordinator/substrate/src/publish_slash_report.rs

@@ -0,0 +1,89 @@
+use core::future::Future;
+use std::sync::Arc;
+
+use serai_db::{DbTxn, Db};
+
+use serai_client::{primitives::NetworkId, validator_sets::primitives::Session, Serai};
+
+use serai_task::ContinuallyRan;
+
+use crate::SlashReports;
+
+/// Publish slash reports from `SlashReports` onto Serai.
+pub struct PublishSlashReportTask<D: Db> {
+  db: D,
+  serai: Arc<Serai>,
+}
+
+impl<D: Db> PublishSlashReportTask<D> {
+  /// Create a task to publish slash reports onto Serai.
+  pub fn new(db: D, serai: Arc<Serai>) -> Self {
+    Self { db, serai }
+  }
+}
+
+impl<D: Db> ContinuallyRan for PublishSlashReportTask<D> {
+  type Error = String;
+
+  fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, Self::Error>> {
+    async move {
+      let mut made_progress = false;
+      for network in serai_client::primitives::NETWORKS {
+        if network == NetworkId::Serai {
+          continue;
+        };
+
+        let mut txn = self.db.txn();
+        let Some((session, slash_report)) = SlashReports::take(&mut txn, network) else {
+          // No slash report to publish
+          continue;
+        };
+
+        let serai =
+          self.serai.as_of_latest_finalized_block().await.map_err(|e| format!("{e:?}"))?;
+        let serai = serai.validator_sets();
+        let session_after_slash_report = Session(session.0 + 1);
+        let current_session = serai.session(network).await.map_err(|e| format!("{e:?}"))?;
+        let current_session = current_session.map(|session| session.0);
+        // Only attempt to publish the slash report for session #n while session #n+1 is still
+        // active
+        let session_after_slash_report_retired =
+          current_session > Some(session_after_slash_report.0);
+        if session_after_slash_report_retired {
+          // Commit the txn to drain this slash report from the database and not try it again later
+          txn.commit();
+          continue;
+        }
+
+        if Some(session_after_slash_report.0) != current_session {
+          // We already checked the current session wasn't greater, and they're not equal
+          assert!(current_session < Some(session_after_slash_report.0));
+          // This would mean the Serai node is resyncing and is behind where it prior was
+          Err("have a slash report for a session Serai has yet to retire".to_string())?;
+        }
+
+        // If this session which should publish a slash report already has, move on
+        let key_pending_slash_report =
+          serai.key_pending_slash_report(network).await.map_err(|e| format!("{e:?}"))?;
+        if key_pending_slash_report.is_none() {
+          txn.commit();
+          continue;
+        };
+
+        match self.serai.publish(&slash_report).await {
+          Ok(()) => {
+            txn.commit();
+            made_progress = true;
+          }
+          // This could be specific to this TX (such as an already in mempool error) and it may be
+          // worthwhile to continue iteration with the other pending slash reports. We assume this
+          // error ephemeral and that the latency incurred for this ephemeral error to resolve is
+          // miniscule compared to the window available to publish the slash report. That makes
+          // this a non-issue.
+          Err(e) => Err(format!("couldn't publish slash report transaction: {e:?}"))?,
+        }
+      }
+      Ok(made_progress)
+    }
+  }
+}

coordinator/substrate/src/set_keys.rs

@@ -0,0 +1,88 @@
+use core::future::Future;
+use std::sync::Arc;
+
+use serai_db::{DbTxn, Db};
+
+use serai_client::{primitives::NetworkId, validator_sets::primitives::ValidatorSet, Serai};
+
+use serai_task::ContinuallyRan;
+
+use crate::Keys;
+
+/// Set keys from `Keys` on Serai.
+pub struct SetKeysTask<D: Db> {
+  db: D,
+  serai: Arc<Serai>,
+}
+
+impl<D: Db> SetKeysTask<D> {
+  /// Create a task to publish slash reports onto Serai.
+  pub fn new(db: D, serai: Arc<Serai>) -> Self {
+    Self { db, serai }
+  }
+}
+
+impl<D: Db> ContinuallyRan for SetKeysTask<D> {
+  type Error = String;
+
+  fn run_iteration(&mut self) -> impl Send + Future<Output = Result<bool, Self::Error>> {
+    async move {
+      let mut made_progress = false;
+      for network in serai_client::primitives::NETWORKS {
+        if network == NetworkId::Serai {
+          continue;
+        };
+
+        let mut txn = self.db.txn();
+        let Some((session, keys)) = Keys::take(&mut txn, network) else {
+          // No keys to set
+          continue;
+        };
+
+        let serai =
+          self.serai.as_of_latest_finalized_block().await.map_err(|e| format!("{e:?}"))?;
+        let serai = serai.validator_sets();
+        let current_session = serai.session(network).await.map_err(|e| format!("{e:?}"))?;
+        let current_session = current_session.map(|session| session.0);
+        // Only attempt to set these keys if this isn't a retired session
+        if Some(session.0) < current_session {
+          // Commit the txn to take these keys from the database and not try it again later
+          txn.commit();
+          continue;
+        }
+
+        if Some(session.0) != current_session {
+          // We already checked the current session wasn't greater, and they're not equal
+          assert!(current_session < Some(session.0));
+          // This would mean the Serai node is resyncing and is behind where it prior was
+          Err("have a keys for a session Serai has yet to start".to_string())?;
+        }
+
+        // If this session already has had its keys set, move on
+        if serai
+          .keys(ValidatorSet { network, session })
+          .await
+          .map_err(|e| format!("{e:?}"))?
+          .is_some()
+        {
+          txn.commit();
+          continue;
+        };
+
+        match self.serai.publish(&keys).await {
+          Ok(()) => {
+            txn.commit();
+            made_progress = true;
+          }
+          // This could be specific to this TX (such as an already in mempool error) and it may be
+          // worthwhile to continue iteration with the other pending slash reports. We assume this
+          // error ephemeral and that the latency incurred for this ephemeral error to resolve is
+          // miniscule compared to the window reasonable to set the keys. That makes this a
+          // non-issue.
+          Err(e) => Err(format!("couldn't publish set keys transaction: {e:?}"))?,
+        }
+      }
+      Ok(made_progress)
+    }
+  }
+}

coordinator/tributary-sdk/Cargo.toml

@@ -0,0 +1,49 @@
+[package]
+name = "tributary-sdk"
+version = "0.1.0"
+description = "A micro-blockchain to provide consensus and ordering to P2P communication"
+license = "AGPL-3.0-only"
+repository = "https://github.com/serai-dex/serai/tree/develop/coordinator/tributary-sdk"
+authors = ["Luke Parker <lukeparker5132@gmail.com>"]
+edition = "2021"
+rust-version = "1.81"
+
+[package.metadata.docs.rs]
+all-features = true
+rustdoc-args = ["--cfg", "docsrs"]
+
+[lints]
+workspace = true
+
+[dependencies]
+thiserror = { version = "2", default-features = false, features = ["std"] }
+
+subtle = { version = "^2", default-features = false, features = ["std"] }
+zeroize = { version = "^1.5", default-features = false, features = ["std"] }
+
+rand = { version = "0.8", default-features = false, features = ["std"] }
+rand_chacha = { version = "0.3", default-features = false, features = ["std"] }
+
+blake2 = { version = "0.10", default-features = false, features = ["std"] }
+transcript = { package = "flexible-transcript", path = "../../crypto/transcript", version = "0.3", default-features = false, features = ["std", "recommended"] }
+
+ciphersuite = { package = "ciphersuite", path = "../../crypto/ciphersuite", version = "0.4", default-features = false, features = ["std", "ristretto"] }
+schnorr = { package = "schnorr-signatures", path = "../../crypto/schnorr", version = "0.5", default-features = false, features = ["std"] }
+
+hex = { version = "0.4", default-features = false, features = ["std"] }
+log = { version = "0.4", default-features = false, features = ["std"] }
+
+serai-db = { path = "../../common/db", version = "0.1" }
+
+scale = { package = "parity-scale-codec", version = "3", default-features = false, features = ["std", "derive"] }
+futures-util = { version = "0.3", default-features = false, features = ["std", "sink", "channel"] }
+futures-channel = { version = "0.3", default-features = false, features = ["std", "sink"] }
+tendermint = { package = "tendermint-machine", path = "./tendermint", version = "0.2" }
+
+tokio = { version = "1", default-features = false, features = ["sync", "time", "rt"] }
+
+[dev-dependencies]
+tokio = { version = "1", features = ["macros"] }
+
+[features]
+tests = []

coordinator/tributary-sdk/LICENSE

@@ -0,0 +1,15 @@
+AGPL-3.0-only license
+
+Copyright (c) 2023 Luke Parker
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License Version 3 as
+published by the Free Software Foundation.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.

coordinator/tributary-sdk/README.md

@@ -0,0 +1,3 @@
+# Tributary
+
+A verifiable, ordered broadcast layer implemented as a BFT micro-blockchain.

coordinator/tributary-sdk/src/block.rs

@@ -135,7 +135,7 @@ impl<T: TransactionTrait> Block<T> {
     // Check TXs are sorted by nonce.
     let nonce = |tx: &Transaction<T>| {
       if let TransactionKind::Signed(_, Signed { nonce, .. }) = tx.kind() {
-        *nonce
+        nonce
       } else {
         0
       }

coordinator/tributary-sdk/src/blockchain.rs

@@ -323,7 +323,7 @@ impl<D: Db, T: TransactionTrait> Blockchain<D, T> {
         }
         TransactionKind::Signed(order, Signed { signer, nonce, .. }) => {
           let next_nonce = nonce + 1;
-          txn.put(Self::next_nonce_key(&self.genesis, signer, &order), next_nonce.to_le_bytes());
+          txn.put(Self::next_nonce_key(&self.genesis, &signer, &order), next_nonce.to_le_bytes());
           self.mempool.remove(&tx.hash());
         }
       }

coordinator/tributary-sdk/src/lib.rs

@@ -0,0 +1,388 @@
+use core::{marker::PhantomData, fmt::Debug, future::Future};
+use std::{sync::Arc, io};
+
+use zeroize::Zeroizing;
+
+use ciphersuite::{Ciphersuite, Ristretto};
+
+use scale::Decode;
+use futures_channel::mpsc::UnboundedReceiver;
+use futures_util::{StreamExt, SinkExt};
+use ::tendermint::{
+  ext::{BlockNumber, Commit, Block as BlockTrait, Network},
+  SignedMessageFor, SyncedBlock, SyncedBlockSender, SyncedBlockResultReceiver, MessageSender,
+  TendermintMachine, TendermintHandle,
+};
+
+pub use ::tendermint::Evidence;
+
+use serai_db::Db;
+
+use tokio::sync::RwLock;
+
+mod merkle;
+pub(crate) use merkle::*;
+
+pub mod transaction;
+pub use transaction::{TransactionError, Signed, TransactionKind, Transaction as TransactionTrait};
+
+use crate::tendermint::tx::TendermintTx;
+
+mod provided;
+pub(crate) use provided::*;
+pub use provided::ProvidedError;
+
+mod block;
+pub use block::*;
+
+mod blockchain;
+pub(crate) use blockchain::*;
+
+mod mempool;
+pub(crate) use mempool::*;
+
+pub mod tendermint;
+pub(crate) use crate::tendermint::*;
+
+#[cfg(any(test, feature = "tests"))]
+pub mod tests;
+
+/// Size limit for an individual transaction.
+// This needs to be big enough to participate in a 101-of-150 eVRF DKG with each element taking
+// `MAX_KEY_LEN`. This also needs to be big enough to pariticpate in signing 520 Bitcoin inputs
+// with 49 key shares, and signing 120 Monero inputs with 49 key shares.
+// TODO: Add a test for these properties
+pub const TRANSACTION_SIZE_LIMIT: usize = 2_000_000;
+/// Amount of transactions a single account may have in the mempool.
+pub const ACCOUNT_MEMPOOL_LIMIT: u32 = 50;
+/// Block size limit.
+// This targets a growth limit of roughly 30 GB a day, under load, in order to prevent a malicious
+// participant from flooding disks and causing out of space errors in order processes.
+pub const BLOCK_SIZE_LIMIT: usize = 2_001_000;
+
+pub(crate) const TENDERMINT_MESSAGE: u8 = 0;
+pub(crate) const TRANSACTION_MESSAGE: u8 = 1;
+
+#[allow(clippy::large_enum_variant)]
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub enum Transaction<T: TransactionTrait> {
+  Tendermint(TendermintTx),
+  Application(T),
+}
+
+impl<T: TransactionTrait> ReadWrite for Transaction<T> {
+  fn read<R: io::Read>(reader: &mut R) -> io::Result<Self> {
+    let mut kind = [0];
+    reader.read_exact(&mut kind)?;
+    match kind[0] {
+      0 => {
+        let tx = TendermintTx::read(reader)?;
+        Ok(Transaction::Tendermint(tx))
+      }
+      1 => {
+        let tx = T::read(reader)?;
+        Ok(Transaction::Application(tx))
+      }
+      _ => Err(io::Error::other("invalid transaction type")),
+    }
+  }
+  fn write<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
+    match self {
+      Transaction::Tendermint(tx) => {
+        writer.write_all(&[0])?;
+        tx.write(writer)
+      }
+      Transaction::Application(tx) => {
+        writer.write_all(&[1])?;
+        tx.write(writer)
+      }
+    }
+  }
+}
+
+impl<T: TransactionTrait> Transaction<T> {
+  pub fn hash(&self) -> [u8; 32] {
+    match self {
+      Transaction::Tendermint(tx) => tx.hash(),
+      Transaction::Application(tx) => tx.hash(),
+    }
+  }
+
+  pub fn kind(&self) -> TransactionKind {
+    match self {
+      Transaction::Tendermint(tx) => tx.kind(),
+      Transaction::Application(tx) => tx.kind(),
+    }
+  }
+}
+
+/// An item which can be read and written.
+pub trait ReadWrite: Sized {
+  fn read<R: io::Read>(reader: &mut R) -> io::Result<Self>;
+  fn write<W: io::Write>(&self, writer: &mut W) -> io::Result<()>;
+
+  fn serialize(&self) -> Vec<u8> {
+    // BlockHeader is 64 bytes and likely the smallest item in this system
+    let mut buf = Vec::with_capacity(64);
+    self.write(&mut buf).unwrap();
+    buf
+  }
+}
+
+pub trait P2p: 'static + Send + Sync + Clone {
+  /// Broadcast a message to all other members of the Tributary with the specified genesis.
+  ///
+  /// The Tributary will re-broadcast consensus messages on a fixed interval to ensure they aren't
+  /// prematurely dropped from the P2P layer. THe P2P layer SHOULD perform content-based
+  /// deduplication to ensure a sane amount of load.
+  fn broadcast(&self, genesis: [u8; 32], msg: Vec<u8>) -> impl Send + Future<Output = ()>;
+}
+
+impl<P: P2p> P2p for Arc<P> {
+  fn broadcast(&self, genesis: [u8; 32], msg: Vec<u8>) -> impl Send + Future<Output = ()> {
+    P::broadcast(self, genesis, msg)
+  }
+}
+
+#[derive(Clone)]
+pub struct Tributary<D: Db, T: TransactionTrait, P: P2p> {
+  db: D,
+
+  genesis: [u8; 32],
+  network: TendermintNetwork<D, T, P>,
+
+  synced_block: Arc<RwLock<SyncedBlockSender<TendermintNetwork<D, T, P>>>>,
+  synced_block_result: Arc<RwLock<SyncedBlockResultReceiver>>,
+  messages: Arc<RwLock<MessageSender<TendermintNetwork<D, T, P>>>>,
+}
+
+impl<D: Db, T: TransactionTrait, P: P2p> Tributary<D, T, P> {
+  pub async fn new(
+    db: D,
+    genesis: [u8; 32],
+    start_time: u64,
+    key: Zeroizing<<Ristretto as Ciphersuite>::F>,
+    validators: Vec<(<Ristretto as Ciphersuite>::G, u64)>,
+    p2p: P,
+  ) -> Option<Self> {
+    log::info!("new Tributary with genesis {}", hex::encode(genesis));
+
+    let validators_vec = validators.iter().map(|validator| validator.0).collect::<Vec<_>>();
+
+    let signer = Arc::new(Signer::new(genesis, key));
+    let validators = Arc::new(Validators::new(genesis, validators)?);
+
+    let mut blockchain = Blockchain::new(db.clone(), genesis, &validators_vec);
+    let block_number = BlockNumber(blockchain.block_number());
+
+    let start_time = if let Some(commit) = blockchain.commit(&blockchain.tip()) {
+      Commit::<Validators>::decode(&mut commit.as_ref()).unwrap().end_time
+    } else {
+      start_time
+    };
+    let proposal = TendermintBlock(
+      blockchain.build_block::<TendermintNetwork<D, T, P>>(&validators).serialize(),
+    );
+    let blockchain = Arc::new(RwLock::new(blockchain));
+
+    let network = TendermintNetwork { genesis, signer, validators, blockchain, p2p };
+
+    let TendermintHandle { synced_block, synced_block_result, messages, machine } =
+      TendermintMachine::new(
+        db.clone(),
+        network.clone(),
+        genesis,
+        block_number,
+        start_time,
+        proposal,
+      )
+      .await;
+    tokio::spawn(machine.run());
+
+    Some(Self {
+      db,
+      genesis,
+      network,
+      synced_block: Arc::new(RwLock::new(synced_block)),
+      synced_block_result: Arc::new(RwLock::new(synced_block_result)),
+      messages: Arc::new(RwLock::new(messages)),
+    })
+  }
+
+  pub fn block_time() -> u32 {
+    TendermintNetwork::<D, T, P>::block_time()
+  }
+
+  pub fn genesis(&self) -> [u8; 32] {
+    self.genesis
+  }
+
+  pub async fn block_number(&self) -> u64 {
+    self.network.blockchain.read().await.block_number()
+  }
+  pub async fn tip(&self) -> [u8; 32] {
+    self.network.blockchain.read().await.tip()
+  }
+
+  pub fn reader(&self) -> TributaryReader<D, T> {
+    TributaryReader(self.db.clone(), self.genesis, PhantomData)
+  }
+
+  pub async fn provide_transaction(&self, tx: T) -> Result<(), ProvidedError> {
+    self.network.blockchain.write().await.provide_transaction(tx)
+  }
+
+  pub async fn next_nonce(
+    &self,
+    signer: &<Ristretto as Ciphersuite>::G,
+    order: &[u8],
+  ) -> Option<u32> {
+    self.network.blockchain.read().await.next_nonce(signer, order)
+  }
+
+  // Returns Ok(true) if new, Ok(false) if an already present unsigned, or the error.
+  // Safe to be &self since the only meaningful usage of self is self.network.blockchain which
+  // successfully acquires its own write lock
+  pub async fn add_transaction(&self, tx: T) -> Result<bool, TransactionError> {
+    let tx = Transaction::Application(tx);
+    let mut to_broadcast = vec![TRANSACTION_MESSAGE];
+    tx.write(&mut to_broadcast).unwrap();
+    let res = self.network.blockchain.write().await.add_transaction::<TendermintNetwork<D, T, P>>(
+      true,
+      tx,
+      &self.network.signature_scheme(),
+    );
+    if res == Ok(true) {
+      self.network.p2p.broadcast(self.genesis, to_broadcast).await;
+    }
+    res
+  }
+
+  async fn sync_block_internal(
+    &self,
+    block: Block<T>,
+    commit: Vec<u8>,
+    result: &mut UnboundedReceiver<bool>,
+  ) -> bool {
+    let (tip, block_number) = {
+      let blockchain = self.network.blockchain.read().await;
+      (blockchain.tip(), blockchain.block_number())
+    };
+
+    if block.header.parent != tip {
+      log::debug!("told to sync a block whose parent wasn't our tip");
+      return false;
+    }
+
+    let block = TendermintBlock(block.serialize());
+    let mut commit_ref = commit.as_ref();
+    let Ok(commit) = Commit::<Arc<Validators>>::decode(&mut commit_ref) else {
+      log::error!("sent an invalidly serialized commit");
+      return false;
+    };
+    // Storage DoS vector. We *could* truncate to solely the relevant portion, trying to save this,
+    // yet then we'd have to test the truncation was performed correctly.
+    if !commit_ref.is_empty() {
+      log::error!("sent an commit with additional data after it");
+      return false;
+    }
+    if !self.network.verify_commit(block.id(), &commit) {
+      log::error!("sent an invalid commit");
+      return false;
+    }
+
+    let number = BlockNumber(block_number + 1);
+    self.synced_block.write().await.send(SyncedBlock { number, block, commit }).await.unwrap();
+    result.next().await.unwrap()
+  }
+
+  // Sync a block.
+  // TODO: Since we have a static validator set, we should only need the tail commit?
+  pub async fn sync_block(&self, block: Block<T>, commit: Vec<u8>) -> bool {
+    let mut result = self.synced_block_result.write().await;
+    self.sync_block_internal(block, commit, &mut result).await
+  }
+
+  // Return true if the message should be rebroadcasted.
+  pub async fn handle_message(&self, msg: &[u8]) -> bool {
+    match msg.first() {
+      Some(&TRANSACTION_MESSAGE) => {
+        let Ok(tx) = Transaction::read::<&[u8]>(&mut &msg[1 ..]) else {
+          log::error!("received invalid transaction message");
+          return false;
+        };
+
+        // TODO: Sync mempools with fellow peers
+        // Can we just rebroadcast transactions not included for at least two blocks?
+        let res =
+          self.network.blockchain.write().await.add_transaction::<TendermintNetwork<D, T, P>>(
+            false,
+            tx,
+            &self.network.signature_scheme(),
+          );
+        log::debug!("received transaction message. valid new transaction: {res:?}");
+        res == Ok(true)
+      }
+
+      Some(&TENDERMINT_MESSAGE) => {
+        let Ok(msg) =
+          SignedMessageFor::<TendermintNetwork<D, T, P>>::decode::<&[u8]>(&mut &msg[1 ..])
+        else {
+          log::error!("received invalid tendermint message");
+          return false;
+        };
+
+        self.messages.write().await.send(msg).await.unwrap();
+        false
+      }
+
+      _ => false,
+    }
+  }
+
+  /// Get a Future which will resolve once the next block has been added.
+  pub async fn next_block_notification(
+    &self,
+  ) -> impl Send + Sync + core::future::Future<Output = Result<(), impl Send + Sync>> {
+    let (tx, rx) = tokio::sync::oneshot::channel();
+    self.network.blockchain.write().await.next_block_notifications.push_back(tx);
+    rx
+  }
+}
+
+#[derive(Clone)]
+pub struct TributaryReader<D: Db, T: TransactionTrait>(D, [u8; 32], PhantomData<T>);
+impl<D: Db, T: TransactionTrait> TributaryReader<D, T> {
+  pub fn genesis(&self) -> [u8; 32] {
+    self.1
+  }
+
+  // Since these values are static once set, they can be safely read from the database without lock
+  // acquisition
+  pub fn block(&self, hash: &[u8; 32]) -> Option<Block<T>> {
+    Blockchain::<D, T>::block_from_db(&self.0, self.1, hash)
+  }
+  pub fn commit(&self, hash: &[u8; 32]) -> Option<Vec<u8>> {
+    Blockchain::<D, T>::commit_from_db(&self.0, self.1, hash)
+  }
+  pub fn parsed_commit(&self, hash: &[u8; 32]) -> Option<Commit<Validators>> {
+    self.commit(hash).map(|commit| Commit::<Validators>::decode(&mut commit.as_ref()).unwrap())
+  }
+  pub fn block_after(&self, hash: &[u8; 32]) -> Option<[u8; 32]> {
+    Blockchain::<D, T>::block_after(&self.0, self.1, hash)
+  }
+  pub fn time_of_block(&self, hash: &[u8; 32]) -> Option<u64> {
+    self
+      .commit(hash)
+      .map(|commit| Commit::<Validators>::decode(&mut commit.as_ref()).unwrap().end_time)
+  }
+
+  pub fn locally_provided_txs_in_block(&self, hash: &[u8; 32], order: &str) -> bool {
+    Blockchain::<D, T>::locally_provided_txs_in_block(&self.0, &self.1, hash, order)
+  }
+
+  // This isn't static, yet can be read with only minor discrepancy risks
+  pub fn tip(&self) -> [u8; 32] {
+    Blockchain::<D, T>::tip_from_db(&self.0, self.1)
+  }
+}

coordinator/tributary-sdk/src/mempool.rs

@@ -81,11 +81,11 @@ impl<D: Db, T: TransactionTrait> Mempool<D, T> {
         }
         Transaction::Application(tx) => match tx.kind() {
           TransactionKind::Signed(order, Signed { signer, nonce, .. }) => {
-            let amount = *res.txs_per_signer.get(signer).unwrap_or(&0) + 1;
-            res.txs_per_signer.insert(*signer, amount);
+            let amount = *res.txs_per_signer.get(&signer).unwrap_or(&0) + 1;
+            res.txs_per_signer.insert(signer, amount);
 
             if let Some(prior_nonce) =
-              res.last_nonce_in_mempool.insert((*signer, order.clone()), *nonce)
+              res.last_nonce_in_mempool.insert((signer, order.clone()), nonce)
             {
               assert_eq!(prior_nonce, nonce - 1);
             }
@@ -133,14 +133,14 @@ impl<D: Db, T: TransactionTrait> Mempool<D, T> {
         match app_tx.kind() {
           TransactionKind::Signed(order, Signed { signer, .. }) => {
             // Get the nonce from the blockchain
-            let Some(blockchain_next_nonce) = blockchain_next_nonce(*signer, order.clone()) else {
+            let Some(blockchain_next_nonce) = blockchain_next_nonce(signer, order.clone()) else {
               // Not a participant
               Err(TransactionError::InvalidSigner)?
             };
             let mut next_nonce = blockchain_next_nonce;
 
             if let Some(mempool_last_nonce) =
-              self.last_nonce_in_mempool.get(&(*signer, order.clone()))
+              self.last_nonce_in_mempool.get(&(signer, order.clone()))
             {
               assert!(*mempool_last_nonce >= blockchain_next_nonce);
               next_nonce = *mempool_last_nonce + 1;
@@ -148,14 +148,14 @@ impl<D: Db, T: TransactionTrait> Mempool<D, T> {
 
             // If we have too many transactions from this sender, don't add this yet UNLESS we are
             // this sender
-            let amount_in_pool = *self.txs_per_signer.get(signer).unwrap_or(&0) + 1;
+            let amount_in_pool = *self.txs_per_signer.get(&signer).unwrap_or(&0) + 1;
             if !internal && (amount_in_pool > ACCOUNT_MEMPOOL_LIMIT) {
               Err(TransactionError::TooManyInMempool)?;
             }
 
             verify_transaction(app_tx, self.genesis, &mut |_, _| Some(next_nonce))?;
-            self.last_nonce_in_mempool.insert((*signer, order.clone()), next_nonce);
-            self.txs_per_signer.insert(*signer, amount_in_pool);
+            self.last_nonce_in_mempool.insert((signer, order.clone()), next_nonce);
+            self.txs_per_signer.insert(signer, amount_in_pool);
           }
           TransactionKind::Unsigned => {
             // check we have the tx in the pool/chain
@@ -205,7 +205,7 @@ impl<D: Db, T: TransactionTrait> Mempool<D, T> {
     // Sort signed by nonce
     let nonce = |tx: &Transaction<T>| {
       if let TransactionKind::Signed(_, Signed { nonce, .. }) = tx.kind() {
-        *nonce
+        nonce
       } else {
         unreachable!()
       }
@@ -242,11 +242,11 @@ impl<D: Db, T: TransactionTrait> Mempool<D, T> {
 
     if let Some(tx) = self.txs.remove(tx) {
       if let TransactionKind::Signed(order, Signed { signer, nonce, .. }) = tx.kind() {
-        let amount = *self.txs_per_signer.get(signer).unwrap() - 1;
-        self.txs_per_signer.insert(*signer, amount);
+        let amount = *self.txs_per_signer.get(&signer).unwrap() - 1;
+        self.txs_per_signer.insert(signer, amount);
 
-        if self.last_nonce_in_mempool.get(&(*signer, order.clone())) == Some(nonce) {
-          self.last_nonce_in_mempool.remove(&(*signer, order));
+        if self.last_nonce_in_mempool.get(&(signer, order.clone())) == Some(&nonce) {
+          self.last_nonce_in_mempool.remove(&(signer, order));
         }
       }
     }

coordinator/tributary-sdk/src/tendermint/mod.rs

@@ -1,8 +1,6 @@
-use core::ops::Deref;
+use core::{ops::Deref, future::Future};
 use std::{sync::Arc, collections::HashMap};
 
-use async_trait::async_trait;
-
 use subtle::ConstantTimeEq;
 use zeroize::{Zeroize, Zeroizing};
 
@@ -74,50 +72,52 @@ impl Signer {
   }
 }
 
-#[async_trait]
 impl SignerTrait for Signer {
   type ValidatorId = [u8; 32];
   type Signature = [u8; 64];
 
   /// Returns the validator's current ID. Returns None if they aren't a current validator.
-  async fn validator_id(&self) -> Option<Self::ValidatorId> {
-    Some((Ristretto::generator() * self.key.deref()).to_bytes())
+  fn validator_id(&self) -> impl Send + Future<Output = Option<Self::ValidatorId>> {
+    async move { Some((Ristretto::generator() * self.key.deref()).to_bytes()) }
   }
 
   /// Sign a signature with the current validator's private key.
-  async fn sign(&self, msg: &[u8]) -> Self::Signature {
-    let mut nonce = Zeroizing::new(RecommendedTranscript::new(b"Tributary Chain Tendermint Nonce"));
-    nonce.append_message(b"genesis", self.genesis);
-    nonce.append_message(b"key", Zeroizing::new(self.key.deref().to_repr()).as_ref());
-    nonce.append_message(b"message", msg);
-    let mut nonce = nonce.challenge(b"nonce");
+  fn sign(&self, msg: &[u8]) -> impl Send + Future<Output = Self::Signature> {
+    async move {
+      let mut nonce =
+        Zeroizing::new(RecommendedTranscript::new(b"Tributary Chain Tendermint Nonce"));
+      nonce.append_message(b"genesis", self.genesis);
+      nonce.append_message(b"key", Zeroizing::new(self.key.deref().to_repr()).as_ref());
+      nonce.append_message(b"message", msg);
+      let mut nonce = nonce.challenge(b"nonce");
 
-    let mut nonce_arr = [0; 64];
-    nonce_arr.copy_from_slice(nonce.as_ref());
+      let mut nonce_arr = [0; 64];
+      nonce_arr.copy_from_slice(nonce.as_ref());
 
-    let nonce_ref: &mut [u8] = nonce.as_mut();
-    nonce_ref.zeroize();
-    let nonce_ref: &[u8] = nonce.as_ref();
-    assert_eq!(nonce_ref, [0; 64].as_ref());
+      let nonce_ref: &mut [u8] = nonce.as_mut();
+      nonce_ref.zeroize();
+      let nonce_ref: &[u8] = nonce.as_ref();
+      assert_eq!(nonce_ref, [0; 64].as_ref());
 
-    let nonce =
-      Zeroizing::new(<Ristretto as Ciphersuite>::F::from_bytes_mod_order_wide(&nonce_arr));
-    nonce_arr.zeroize();
+      let nonce =
+        Zeroizing::new(<Ristretto as Ciphersuite>::F::from_bytes_mod_order_wide(&nonce_arr));
+      nonce_arr.zeroize();
 
-    assert!(!bool::from(nonce.ct_eq(&<Ristretto as Ciphersuite>::F::ZERO)));
+      assert!(!bool::from(nonce.ct_eq(&<Ristretto as Ciphersuite>::F::ZERO)));
 
-    let challenge = challenge(
-      self.genesis,
-      (Ristretto::generator() * self.key.deref()).to_bytes(),
-      (Ristretto::generator() * nonce.deref()).to_bytes().as_ref(),
-      msg,
-    );
+      let challenge = challenge(
+        self.genesis,
+        (Ristretto::generator() * self.key.deref()).to_bytes(),
+        (Ristretto::generator() * nonce.deref()).to_bytes().as_ref(),
+        msg,
+      );
 
-    let sig = SchnorrSignature::<Ristretto>::sign(&self.key, nonce, challenge).serialize();
+      let sig = SchnorrSignature::<Ristretto>::sign(&self.key, nonce, challenge).serialize();
 
-    let mut res = [0; 64];
-    res.copy_from_slice(&sig);
-    res
+      let mut res = [0; 64];
+      res.copy_from_slice(&sig);
+      res
+    }
   }
 }
 
@@ -274,7 +274,6 @@ pub const BLOCK_PROCESSING_TIME: u32 = 999;
 pub const LATENCY_TIME: u32 = 1667;
 pub const TARGET_BLOCK_TIME: u32 = BLOCK_PROCESSING_TIME + (3 * LATENCY_TIME);
 
-#[async_trait]
 impl<D: Db, T: TransactionTrait, P: P2p> Network for TendermintNetwork<D, T, P> {
   type Db = D;
 
@@ -300,111 +299,126 @@ impl<D: Db, T: TransactionTrait, P: P2p> Network for TendermintNetwork<D, T, P>
     self.validators.clone()
   }
 
-  async fn broadcast(&mut self, msg: SignedMessageFor<Self>) {
-    let mut to_broadcast = vec![TENDERMINT_MESSAGE];
-    to_broadcast.extend(msg.encode());
-    self.p2p.broadcast(self.genesis, to_broadcast).await
-  }
-
-  async fn slash(&mut self, validator: Self::ValidatorId, slash_event: SlashEvent) {
-    log::error!(
-      "validator {} triggered a slash event on tributary {} (with evidence: {})",
-      hex::encode(validator),
-      hex::encode(self.genesis),
-      matches!(slash_event, SlashEvent::WithEvidence(_)),
-    );
-
-    let signer = self.signer();
-    let Some(tx) = (match slash_event {
-      SlashEvent::WithEvidence(evidence) => {
-        // create an unsigned evidence tx
-        Some(TendermintTx::SlashEvidence(evidence))
-      }
-      SlashEvent::Id(_reason, _block, _round) => {
-        // TODO: Increase locally observed slash points
-        None
-      }
-    }) else {
-      return;
-    };
-
-    // add tx to blockchain and broadcast to peers
-    let mut to_broadcast = vec![TRANSACTION_MESSAGE];
-    tx.write(&mut to_broadcast).unwrap();
-    if self.blockchain.write().await.add_transaction::<Self>(
-      true,
-      Transaction::Tendermint(tx),
-      &self.signature_scheme(),
-    ) == Ok(true)
-    {
-      self.p2p.broadcast(signer.genesis, to_broadcast).await;
+  fn broadcast(&mut self, msg: SignedMessageFor<Self>) -> impl Send + Future<Output = ()> {
+    async move {
+      let mut to_broadcast = vec![TENDERMINT_MESSAGE];
+      to_broadcast.extend(msg.encode());
+      self.p2p.broadcast(self.genesis, to_broadcast).await
     }
   }
 
-  async fn validate(&self, block: &Self::Block) -> Result<(), TendermintBlockError> {
-    let block =
-      Block::read::<&[u8]>(&mut block.0.as_ref()).map_err(|_| TendermintBlockError::Fatal)?;
-    self
-      .blockchain
-      .read()
-      .await
-      .verify_block::<Self>(&block, &self.signature_scheme(), false)
-      .map_err(|e| match e {
-        BlockError::NonLocalProvided(_) => TendermintBlockError::Temporal,
-        _ => {
-          log::warn!("Tributary Tendermint validate returning BlockError::Fatal due to {e:?}");
-          TendermintBlockError::Fatal
+  fn slash(
+    &mut self,
+    validator: Self::ValidatorId,
+    slash_event: SlashEvent,
+  ) -> impl Send + Future<Output = ()> {
+    async move {
+      log::error!(
+        "validator {} triggered a slash event on tributary {} (with evidence: {})",
+        hex::encode(validator),
+        hex::encode(self.genesis),
+        matches!(slash_event, SlashEvent::WithEvidence(_)),
+      );
+
+      let signer = self.signer();
+      let Some(tx) = (match slash_event {
+        SlashEvent::WithEvidence(evidence) => {
+          // create an unsigned evidence tx
+          Some(TendermintTx::SlashEvidence(evidence))
         }
-      })
+        SlashEvent::Id(_reason, _block, _round) => {
+          // TODO: Increase locally observed slash points
+          None
+        }
+      }) else {
+        return;
+      };
+
+      // add tx to blockchain and broadcast to peers
+      let mut to_broadcast = vec![TRANSACTION_MESSAGE];
+      tx.write(&mut to_broadcast).unwrap();
+      if self.blockchain.write().await.add_transaction::<Self>(
+        true,
+        Transaction::Tendermint(tx),
+        &self.signature_scheme(),
+      ) == Ok(true)
+      {
+        self.p2p.broadcast(signer.genesis, to_broadcast).await;
+      }
+    }
   }
 
-  async fn add_block(
+  fn validate(
+    &self,
+    block: &Self::Block,
+  ) -> impl Send + Future<Output = Result<(), TendermintBlockError>> {
+    async move {
+      let block =
+        Block::read::<&[u8]>(&mut block.0.as_ref()).map_err(|_| TendermintBlockError::Fatal)?;
+      self
+        .blockchain
+        .read()
+        .await
+        .verify_block::<Self>(&block, &self.signature_scheme(), false)
+        .map_err(|e| match e {
+          BlockError::NonLocalProvided(_) => TendermintBlockError::Temporal,
+          _ => {
+            log::warn!("Tributary Tendermint validate returning BlockError::Fatal due to {e:?}");
+            TendermintBlockError::Fatal
+          }
+        })
+    }
+  }
+
+  fn add_block(
     &mut self,
     serialized_block: Self::Block,
     commit: Commit<Self::SignatureScheme>,
-  ) -> Option<Self::Block> {
-    let invalid_block = || {
-      // There's a fatal flaw in the code, it's behind a hard fork, or the validators turned
-      // malicious
-      // All justify a halt to then achieve social consensus from
-      // TODO: Under multiple validator sets, a small validator set turning malicious knocks
-      // off the entire network. That's an unacceptable DoS.
-      panic!("validators added invalid block to tributary {}", hex::encode(self.genesis));
-    };
+  ) -> impl Send + Future<Output = Option<Self::Block>> {
+    async move {
+      let invalid_block = || {
+        // There's a fatal flaw in the code, it's behind a hard fork, or the validators turned
+        // malicious
+        // All justify a halt to then achieve social consensus from
+        // TODO: Under multiple validator sets, a small validator set turning malicious knocks
+        // off the entire network. That's an unacceptable DoS.
+        panic!("validators added invalid block to tributary {}", hex::encode(self.genesis));
+      };
 
-    // Tendermint should only produce valid commits
-    assert!(self.verify_commit(serialized_block.id(), &commit));
+      // Tendermint should only produce valid commits
+      assert!(self.verify_commit(serialized_block.id(), &commit));
 
-    let Ok(block) = Block::read::<&[u8]>(&mut serialized_block.0.as_ref()) else {
-      return invalid_block();
-    };
+      let Ok(block) = Block::read::<&[u8]>(&mut serialized_block.0.as_ref()) else {
+        return invalid_block();
+      };
 
-    let encoded_commit = commit.encode();
-    loop {
-      let block_res = self.blockchain.write().await.add_block::<Self>(
-        &block,
-        encoded_commit.clone(),
-        &self.signature_scheme(),
-      );
-      match block_res {
-        Ok(()) => {
-          // If we successfully added this block, break
-          break;
+      let encoded_commit = commit.encode();
+      loop {
+        let block_res = self.blockchain.write().await.add_block::<Self>(
+          &block,
+          encoded_commit.clone(),
+          &self.signature_scheme(),
+        );
+        match block_res {
+          Ok(()) => {
+            // If we successfully added this block, break
+            break;
+          }
+          Err(BlockError::NonLocalProvided(hash)) => {
+            log::error!(
+              "missing provided transaction {} which other validators on tributary {} had",
+              hex::encode(hash),
+              hex::encode(self.genesis)
+            );
+            tokio::time::sleep(core::time::Duration::from_secs(5)).await;
+          }
+          _ => return invalid_block(),
         }
-        Err(BlockError::NonLocalProvided(hash)) => {
-          log::error!(
-            "missing provided transaction {} which other validators on tributary {} had",
-            hex::encode(hash),
-            hex::encode(self.genesis)
-          );
-          tokio::time::sleep(core::time::Duration::from_secs(5)).await;
-        }
-        _ => return invalid_block(),
       }
-    }
 
-    Some(TendermintBlock(
-      self.blockchain.write().await.build_block::<Self>(&self.signature_scheme()).serialize(),
-    ))
+      Some(TendermintBlock(
+        self.blockchain.write().await.build_block::<Self>(&self.signature_scheme()).serialize(),
+      ))
+    }
   }
 }

coordinator/tributary-sdk/src/tendermint/tx.rs

@@ -39,7 +39,7 @@ impl ReadWrite for TendermintTx {
 }
 
 impl Transaction for TendermintTx {
-  fn kind(&self) -> TransactionKind<'_> {
+  fn kind(&self) -> TransactionKind {
     // There's an assert elsewhere in the codebase expecting this behavior
     // If we do want to add Provided/Signed TendermintTxs, review the implications carefully
     TransactionKind::Unsigned

coordinator/tributary-sdk/src/tests/block.rs

@@ -60,8 +60,8 @@ impl ReadWrite for NonceTransaction {
 }
 
 impl TransactionTrait for NonceTransaction {
-  fn kind(&self) -> TransactionKind<'_> {
-    TransactionKind::Signed(vec![], &self.2)
+  fn kind(&self) -> TransactionKind {
+    TransactionKind::Signed(vec![], self.2.clone())
   }
 
   fn hash(&self) -> [u8; 32] {

coordinator/tributary-sdk/src/tests/blockchain.rs

@@ -425,7 +425,7 @@ async fn block_tx_ordering() {
   }
 
   impl TransactionTrait for SignedTx {
-    fn kind(&self) -> TransactionKind<'_> {
+    fn kind(&self) -> TransactionKind {
       match self {
         SignedTx::Signed(signed) => signed.kind(),
         SignedTx::Provided(pro) => pro.kind(),

coordinator/tributary-sdk/src/tests/p2p.rs

@@ -0,0 +1,12 @@
+use core::future::Future;
+
+pub use crate::P2p;
+
+#[derive(Clone, Debug)]
+pub struct DummyP2p;
+
+impl P2p for DummyP2p {
+  fn broadcast(&self, _: [u8; 32], _: Vec<u8>) -> impl Send + Future<Output = ()> {
+    async move { unimplemented!() }
+  }
+}

coordinator/tributary-sdk/src/tests/tendermint.rs

@@ -1,4 +1,7 @@
+use core::future::Future;
+
 use tendermint::ext::Network;
+
 use crate::{
   P2p, TendermintTx,
   tendermint::{TARGET_BLOCK_TIME, TendermintNetwork},
@@ -11,10 +14,9 @@ fn assert_target_block_time() {
   #[derive(Clone, Debug)]
   pub struct DummyP2p;
 
-  #[async_trait::async_trait]
   impl P2p for DummyP2p {
-    async fn broadcast(&self, _: [u8; 32], _: Vec<u8>) {
-      unimplemented!()
+    fn broadcast(&self, _: [u8; 32], _: Vec<u8>) -> impl Send + Future<Output = ()> {
+      async move { unimplemented!() }
     }
   }
 

coordinator/tributary-sdk/src/tests/transaction/mod.rs

@@ -67,7 +67,7 @@ impl ReadWrite for ProvidedTransaction {
 }
 
 impl Transaction for ProvidedTransaction {
-  fn kind(&self) -> TransactionKind<'_> {
+  fn kind(&self) -> TransactionKind {
     match self.0[0] {
       1 => TransactionKind::Provided("order1"),
       2 => TransactionKind::Provided("order2"),
@@ -119,8 +119,8 @@ impl ReadWrite for SignedTransaction {
 }
 
 impl Transaction for SignedTransaction {
-  fn kind(&self) -> TransactionKind<'_> {
-    TransactionKind::Signed(vec![], &self.1)
+  fn kind(&self) -> TransactionKind {
+    TransactionKind::Signed(vec![], self.1.clone())
   }
 
   fn hash(&self) -> [u8; 32] {

coordinator/tributary-sdk/src/transaction.rs

@@ -0,0 +1,218 @@
+use core::fmt::Debug;
+use std::io;
+
+use zeroize::Zeroize;
+use thiserror::Error;
+
+use blake2::{Digest, Blake2b512};
+
+use ciphersuite::{
+  group::{Group, GroupEncoding},
+  Ciphersuite, Ristretto,
+};
+use schnorr::SchnorrSignature;
+
+use crate::{TRANSACTION_SIZE_LIMIT, ReadWrite};
+
+#[derive(Clone, PartialEq, Eq, Debug, Error)]
+pub enum TransactionError {
+  /// Transaction exceeded the size limit.
+  #[error("transaction is too large")]
+  TooLargeTransaction,
+  /// Transaction's signer isn't a participant.
+  #[error("invalid signer")]
+  InvalidSigner,
+  /// Transaction's nonce isn't the prior nonce plus one.
+  #[error("invalid nonce")]
+  InvalidNonce,
+  /// Transaction's signature is invalid.
+  #[error("invalid signature")]
+  InvalidSignature,
+  /// Transaction's content is invalid.
+  #[error("transaction content is invalid")]
+  InvalidContent,
+  /// Transaction's signer has too many transactions in the mempool.
+  #[error("signer has too many transactions in the mempool")]
+  TooManyInMempool,
+  /// Provided Transaction added to mempool.
+  #[error("provided transaction added to mempool")]
+  ProvidedAddedToMempool,
+}
+
+/// Data for a signed transaction.
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub struct Signed {
+  pub signer: <Ristretto as Ciphersuite>::G,
+  pub nonce: u32,
+  pub signature: SchnorrSignature<Ristretto>,
+}
+
+impl ReadWrite for Signed {
+  fn read<R: io::Read>(reader: &mut R) -> io::Result<Self> {
+    let signer = Ristretto::read_G(reader)?;
+
+    let mut nonce = [0; 4];
+    reader.read_exact(&mut nonce)?;
+    let nonce = u32::from_le_bytes(nonce);
+    if nonce >= (u32::MAX - 1) {
+      Err(io::Error::other("nonce exceeded limit"))?;
+    }
+
+    let mut signature = SchnorrSignature::<Ristretto>::read(reader)?;
+    if signature.R.is_identity().into() {
+      // Anyone malicious could remove this and try to find zero signatures
+      // We should never produce zero signatures though meaning this should never come up
+      // If it does somehow come up, this is a decent courtesy
+      signature.zeroize();
+      Err(io::Error::other("signature nonce was identity"))?;
+    }
+
+    Ok(Signed { signer, nonce, signature })
+  }
+
+  fn write<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
+    // This is either an invalid signature or a private key leak
+    if self.signature.R.is_identity().into() {
+      Err(io::Error::other("signature nonce was identity"))?;
+    }
+    writer.write_all(&self.signer.to_bytes())?;
+    writer.write_all(&self.nonce.to_le_bytes())?;
+    self.signature.write(writer)
+  }
+}
+
+impl Signed {
+  pub fn read_without_nonce<R: io::Read>(reader: &mut R, nonce: u32) -> io::Result<Self> {
+    let signer = Ristretto::read_G(reader)?;
+
+    let mut signature = SchnorrSignature::<Ristretto>::read(reader)?;
+    if signature.R.is_identity().into() {
+      // Anyone malicious could remove this and try to find zero signatures
+      // We should never produce zero signatures though meaning this should never come up
+      // If it does somehow come up, this is a decent courtesy
+      signature.zeroize();
+      Err(io::Error::other("signature nonce was identity"))?;
+    }
+
+    Ok(Signed { signer, nonce, signature })
+  }
+
+  pub fn write_without_nonce<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
+    // This is either an invalid signature or a private key leak
+    if self.signature.R.is_identity().into() {
+      Err(io::Error::other("signature nonce was identity"))?;
+    }
+    writer.write_all(&self.signer.to_bytes())?;
+    self.signature.write(writer)
+  }
+}
+
+#[allow(clippy::large_enum_variant)]
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub enum TransactionKind {
+  /// This transaction should be provided by every validator, in an exact order.
+  ///
+  /// The contained static string names the orderer to use. This allows two distinct provided
+  /// transaction kinds, without a synchronized order, to be ordered within their own kind without
+  /// requiring ordering with each other.
+  ///
+  /// The only malleability is in when this transaction appears on chain. The block producer will
+  /// include it when they have it. Block verification will fail for validators without it.
+  ///
+  /// If a supermajority of validators produce a commit for a block with a provided transaction
+  /// which isn't locally held, the block will be added to the local chain. When the transaction is
+  /// locally provided, it will be compared for correctness to the on-chain version
+  ///
+  /// In order to ensure TXs aren't accidentally provided multiple times, all provided transactions
+  /// must have a unique hash which is also unique to all Unsigned transactions.
+  Provided(&'static str),
+
+  /// An unsigned transaction, only able to be included by the block producer.
+  ///
+  /// Once an Unsigned transaction is included on-chain, it may not be included again. In order to
+  /// have multiple Unsigned transactions with the same values included on-chain, some distinct
+  /// nonce must be included in order to cause a distinct hash.
+  ///
+  /// The hash must also be unique with all Provided transactions.
+  Unsigned,
+
+  /// A signed transaction.
+  Signed(Vec<u8>, Signed),
+}
+
+// TODO: Should this be renamed TransactionTrait now that a literal Transaction exists?
+// Or should the literal Transaction be renamed to Event?
+pub trait Transaction: 'static + Send + Sync + Clone + Eq + Debug + ReadWrite {
+  /// Return what type of transaction this is.
+  fn kind(&self) -> TransactionKind;
+
+  /// Return the hash of this transaction.
+  ///
+  /// The hash must NOT commit to the signature.
+  fn hash(&self) -> [u8; 32];
+
+  /// Perform transaction-specific verification.
+  fn verify(&self) -> Result<(), TransactionError>;
+
+  /// Obtain the challenge for this transaction's signature.
+  ///
+  /// Do not override this unless you know what you're doing.
+  ///
+  /// Panics if called on non-signed transactions.
+  fn sig_hash(&self, genesis: [u8; 32]) -> <Ristretto as Ciphersuite>::F {
+    match self.kind() {
+      TransactionKind::Signed(order, Signed { signature, .. }) => {
+        <Ristretto as Ciphersuite>::F::from_bytes_mod_order_wide(
+          &Blake2b512::digest(
+            [
+              b"Tributary Signed Transaction",
+              genesis.as_ref(),
+              &self.hash(),
+              order.as_ref(),
+              signature.R.to_bytes().as_ref(),
+            ]
+            .concat(),
+          )
+          .into(),
+        )
+      }
+      _ => panic!("sig_hash called on non-signed transaction"),
+    }
+  }
+}
+
+pub trait GAIN: FnMut(&<Ristretto as Ciphersuite>::G, &[u8]) -> Option<u32> {}
+impl<F: FnMut(&<Ristretto as Ciphersuite>::G, &[u8]) -> Option<u32>> GAIN for F {}
+
+pub(crate) fn verify_transaction<F: GAIN, T: Transaction>(
+  tx: &T,
+  genesis: [u8; 32],
+  get_and_increment_nonce: &mut F,
+) -> Result<(), TransactionError> {
+  if tx.serialize().len() > TRANSACTION_SIZE_LIMIT {
+    Err(TransactionError::TooLargeTransaction)?;
+  }
+
+  tx.verify()?;
+
+  match tx.kind() {
+    TransactionKind::Provided(_) | TransactionKind::Unsigned => {}
+    TransactionKind::Signed(order, Signed { signer, nonce, signature }) => {
+      if let Some(next_nonce) = get_and_increment_nonce(&signer, &order) {
+        if nonce != next_nonce {
+          Err(TransactionError::InvalidNonce)?;
+        }
+      } else {
+        // Not a participant
+        Err(TransactionError::InvalidSigner)?;
+      }
+
+      // TODO: Use a batch verification here
+      if !signature.verify(signer, tx.sig_hash(genesis)) {
+        Err(TransactionError::InvalidSignature)?;
+      }
+    }
+  }
+
+  Ok(())
+}

coordinator/tributary-sdk/tendermint/Cargo.toml

@@ -16,7 +16,6 @@ rustdoc-args = ["--cfg", "docsrs"]
 workspace = true
 
 [dependencies]
-async-trait = { version = "0.1", default-features = false }
 thiserror = { version = "2", default-features = false, features = ["std"] }
 
 hex = { version = "0.4", default-features = false, features = ["std"] }

coordinator/tributary-sdk/tendermint/src/ext.rs

@@ -1,7 +1,6 @@
-use core::{hash::Hash, fmt::Debug};
+use core::{hash::Hash, fmt::Debug, future::Future};
 use std::{sync::Arc, collections::HashSet};
 
-use async_trait::async_trait;
 use thiserror::Error;
 
 use parity_scale_codec::{Encode, Decode};
@@ -34,7 +33,6 @@ pub struct BlockNumber(pub u64);
 pub struct RoundNumber(pub u32);
 
 /// A signer for a validator.
-#[async_trait]
 pub trait Signer: Send + Sync {
   // Type used to identify validators.
   type ValidatorId: ValidatorId;
@@ -42,22 +40,21 @@ pub trait Signer: Send + Sync {
   type Signature: Signature;
 
   /// Returns the validator's current ID. Returns None if they aren't a current validator.
-  async fn validator_id(&self) -> Option<Self::ValidatorId>;
+  fn validator_id(&self) -> impl Send + Future<Output = Option<Self::ValidatorId>>;
   /// Sign a signature with the current validator's private key.
-  async fn sign(&self, msg: &[u8]) -> Self::Signature;
+  fn sign(&self, msg: &[u8]) -> impl Send + Future<Output = Self::Signature>;
 }
 
-#[async_trait]
 impl<S: Signer> Signer for Arc<S> {
   type ValidatorId = S::ValidatorId;
   type Signature = S::Signature;
 
-  async fn validator_id(&self) -> Option<Self::ValidatorId> {
-    self.as_ref().validator_id().await
+  fn validator_id(&self) -> impl Send + Future<Output = Option<Self::ValidatorId>> {
+    self.as_ref().validator_id()
   }
 
-  async fn sign(&self, msg: &[u8]) -> Self::Signature {
-    self.as_ref().sign(msg).await
+  fn sign(&self, msg: &[u8]) -> impl Send + Future<Output = Self::Signature> {
+    self.as_ref().sign(msg)
   }
 }
 
@@ -210,7 +207,6 @@ pub trait Block: Send + Sync + Clone + PartialEq + Eq + Debug + Encode + Decode
 }
 
 /// Trait representing the distributed system Tendermint is providing consensus over.
-#[async_trait]
 pub trait Network: Sized + Send + Sync {
   /// The database used to back this.
   type Db: serai_db::Db;
@@ -229,6 +225,7 @@ pub trait Network: Sized + Send + Sync {
   /// This should include both the time to download the block and the actual processing time.
   ///
   /// BLOCK_PROCESSING_TIME + (3 * LATENCY_TIME) must be divisible by 1000.
+  // TODO: Redefine as Duration
   const BLOCK_PROCESSING_TIME: u32;
   /// Network latency time in milliseconds.
   ///
@@ -280,15 +277,19 @@ pub trait Network: Sized + Send + Sync {
   /// Switching to unauthenticated channels in a system already providing authenticated channels is
   /// not recommended as this is a minor, temporal inefficiency, while downgrading channels may
   /// have wider implications.
-  async fn broadcast(&mut self, msg: SignedMessageFor<Self>);
+  fn broadcast(&mut self, msg: SignedMessageFor<Self>) -> impl Send + Future<Output = ()>;
 
   /// Trigger a slash for the validator in question who was definitively malicious.
   ///
   /// The exact process of triggering a slash is undefined and left to the network as a whole.
-  async fn slash(&mut self, validator: Self::ValidatorId, slash_event: SlashEvent);
+  fn slash(
+    &mut self,
+    validator: Self::ValidatorId,
+    slash_event: SlashEvent,
+  ) -> impl Send + Future<Output = ()>;
 
   /// Validate a block.
-  async fn validate(&self, block: &Self::Block) -> Result<(), BlockError>;
+  fn validate(&self, block: &Self::Block) -> impl Send + Future<Output = Result<(), BlockError>>;
 
   /// Add a block, returning the proposal for the next one.
   ///
@@ -298,9 +299,9 @@ pub trait Network: Sized + Send + Sync {
   /// This deviates from the paper which will have a local node refuse to decide on a block it
   /// considers invalid. This library acknowledges the network did decide on it, leaving handling
   /// of it to the network, and outside of this scope.
-  async fn add_block(
+  fn add_block(
     &mut self,
     block: Self::Block,
     commit: Commit<Self::SignatureScheme>,
-  ) -> Option<Self::Block>;
+  ) -> impl Send + Future<Output = Option<Self::Block>>;
 }

coordinator/tributary-sdk/tendermint/tests/ext.rs

@@ -1,10 +1,9 @@
+use core::future::Future;
 use std::{
   sync::Arc,
   time::{UNIX_EPOCH, SystemTime, Duration},
 };
 
-use async_trait::async_trait;
-
 use parity_scale_codec::{Encode, Decode};
 
 use futures_util::sink::SinkExt;
@@ -21,20 +20,21 @@ type TestValidatorId = u16;
 type TestBlockId = [u8; 4];
 
 struct TestSigner(u16);
-#[async_trait]
 impl Signer for TestSigner {
   type ValidatorId = TestValidatorId;
   type Signature = [u8; 32];
 
-  async fn validator_id(&self) -> Option<TestValidatorId> {
-    Some(self.0)
+  fn validator_id(&self) -> impl Send + Future<Output = Option<TestValidatorId>> {
+    async move { Some(self.0) }
   }
 
-  async fn sign(&self, msg: &[u8]) -> [u8; 32] {
-    let mut sig = [0; 32];
-    sig[.. 2].copy_from_slice(&self.0.to_le_bytes());
-    sig[2 .. (2 + 30.min(msg.len()))].copy_from_slice(&msg[.. 30.min(msg.len())]);
-    sig
+  fn sign(&self, msg: &[u8]) -> impl Send + Future<Output = [u8; 32]> {
+    async move {
+      let mut sig = [0; 32];
+      sig[.. 2].copy_from_slice(&self.0.to_le_bytes());
+      sig[2 .. (2 + 30.min(msg.len()))].copy_from_slice(&msg[.. 30.min(msg.len())]);
+      sig
+    }
   }
 }
 
@@ -111,7 +111,6 @@ struct TestNetwork(
   Arc<RwLock<Vec<(MessageSender<Self>, SyncedBlockSender<Self>, SyncedBlockResultReceiver)>>>,
 );
 
-#[async_trait]
 impl Network for TestNetwork {
   type Db = MemDb;
 

coordinator/tributary/Cargo.toml

@@ -1,11 +1,13 @@
 [package]
-name = "tributary-chain"
+name = "serai-coordinator-tributary"
 version = "0.1.0"
-description = "A micro-blockchain to provide consensus and ordering to P2P communication"
+description = "The Tributary used by the Serai Coordinator"
 license = "AGPL-3.0-only"
 repository = "https://github.com/serai-dex/serai/tree/develop/coordinator/tributary"
 authors = ["Luke Parker <lukeparker5132@gmail.com>"]
+keywords = []
 edition = "2021"
+publish = false
 rust-version = "1.81"
 
 [package.metadata.docs.rs]
@@ -16,35 +18,29 @@ rustdoc-args = ["--cfg", "docsrs"]
 workspace = true
 
 [dependencies]
-async-trait = { version = "0.1", default-features = false }
-thiserror = { version = "2", default-features = false, features = ["std"] }
-
-subtle = { version = "^2", default-features = false, features = ["std"] }
 zeroize = { version = "^1.5", default-features = false, features = ["std"] }
-
-rand = { version = "0.8", default-features = false, features = ["std"] }
-rand_chacha = { version = "0.3", default-features = false, features = ["std"] }
+rand_core = { version = "0.6", default-features = false, features = ["std"] }
 
 blake2 = { version = "0.10", default-features = false, features = ["std"] }
-transcript = { package = "flexible-transcript", path = "../../crypto/transcript", default-features = false, features = ["std", "recommended"] }
-
-ciphersuite = { package = "ciphersuite", path = "../../crypto/ciphersuite", default-features = false, features = ["std", "ristretto"] }
+ciphersuite = { path = "../../crypto/ciphersuite", default-features = false, features = ["std"] }
 schnorr = { package = "schnorr-signatures", path = "../../crypto/schnorr", default-features = false, features = ["std"] }
 
-hex = { version = "0.4", default-features = false, features = ["std"] }
-log = { version = "0.4", default-features = false, features = ["std"] }
+scale = { package = "parity-scale-codec", version = "3", default-features = false, features = ["std", "derive"] }
+borsh = { version = "1", default-features = false, features = ["std", "derive", "de_strict_order"] }
+
+serai-client = { path = "../../substrate/client", default-features = false, features = ["serai", "borsh"] }
 
 serai-db = { path = "../../common/db" }
+serai-task = { path = "../../common/task", version = "0.1" }
 
-scale = { package = "parity-scale-codec", version = "3", default-features = false, features = ["std", "derive"] }
-futures-util = { version = "0.3", default-features = false, features = ["std", "sink", "channel"] }
-futures-channel = { version = "0.3", default-features = false, features = ["std", "sink"] }
-tendermint = { package = "tendermint-machine", path = "./tendermint" }
+tributary-sdk = { path = "../tributary-sdk" }
 
-tokio = { version = "1", default-features = false, features = ["sync", "time", "rt"] }
+serai-cosign = { path = "../cosign" }
+serai-coordinator-substrate = { path = "../substrate" }
 
-[dev-dependencies]
-tokio = { version = "1", features = ["macros"] }
+messages = { package = "serai-processor-messages", path = "../../processor/messages" }
+
+log = { version = "0.4", default-features = false, features = ["std"] }
 
 [features]
-tests = []
+longer-reattempts = []

coordinator/tributary/LICENSE

@@ -1,6 +1,6 @@
 AGPL-3.0-only license
 
-Copyright (c) 2023 Luke Parker
+Copyright (c) 2023-2025 Luke Parker
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License Version 3 as

coordinator/tributary/README.md

@@ -1,3 +1,4 @@
-# Tributary
+# Serai Coordinator Tributary
 
-A verifiable, ordered broadcast layer implemented as a BFT micro-blockchain.
+The Tributary used by the Serai Coordinator. This includes the `Transaction`
+definition and the code to handle blocks added on-chain.