Add a test to the coordinator for running a Tributary

Impls a LocalP2p for testing.

Moves rebroadcasting into Tendermint, since it's what knows if a message is
fully valid + original.

Removes TributarySpec::validators() HashMap, as its non-determinism caused
different instances to have different round robin schedules. It was already
prior moved to a Vec for this issue, so I'm unsure why this remnant existed.

Also renames the GH no-std workflow from the prior commit.

coordinator/src/main.rs

@@ -94,7 +94,7 @@ async fn main() {
   let db = MemDb::new(); // TODO
 
   let key = Zeroizing::new(<Ristretto as Ciphersuite>::F::ZERO); // TODO
-  let p2p = LocalP2p {}; // TODO
+  let p2p = LocalP2p::new(1).swap_remove(0); // TODO
 
   let processor = processor::MemProcessor::new(); // TODO
 

coordinator/src/p2p.rs

@@ -1,25 +1,79 @@
 use core::fmt::Debug;
+use std::{
+  sync::{Arc, RwLock},
+  collections::VecDeque,
+};
 
 use async_trait::async_trait;
 
-use tributary::P2p as TributaryP2p;
+pub use tributary::P2p as TributaryP2p;
 
-// TODO
-#[async_trait]
-pub trait P2p: Send + Sync + Clone + Debug + TributaryP2p {}
+#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
+pub enum P2pMessageKind {
+  Tributary,
+}
 
-// TODO
-#[derive(Clone, Debug)]
-pub struct LocalP2p {}
+impl P2pMessageKind {
+  fn to_byte(self) -> u8 {
+    match self {
+      P2pMessageKind::Tributary => 0,
+    }
+  }
 
-#[async_trait]
-impl TributaryP2p for LocalP2p {
-  async fn broadcast(&self, msg: Vec<u8>) {
-    // TODO
-    todo!()
+  fn from_byte(byte: u8) -> Option<P2pMessageKind> {
+    match byte {
+      0 => Some(P2pMessageKind::Tributary),
+      _ => None,
+    }
   }
 }
 
 // TODO
 #[async_trait]
-impl P2p for LocalP2p {}
+pub trait P2p: Send + Sync + Clone + Debug + TributaryP2p {
+  async fn broadcast(&self, msg: Vec<u8>);
+  async fn receive(&self) -> Option<(P2pMessageKind, Vec<u8>)>;
+}
+
+#[derive(Clone, Debug)]
+pub struct LocalP2p(usize, Arc<RwLock<Vec<VecDeque<Vec<u8>>>>>);
+
+impl LocalP2p {
+  pub fn new(validators: usize) -> Vec<LocalP2p> {
+    let shared = Arc::new(RwLock::new(vec![VecDeque::new(); validators]));
+    let mut res = vec![];
+    for i in 0 .. validators {
+      res.push(LocalP2p(i, shared.clone()));
+    }
+    res
+  }
+}
+
+#[async_trait]
+impl P2p for LocalP2p {
+  async fn broadcast(&self, msg: Vec<u8>) {
+    for (i, msg_queue) in self.1.write().unwrap().iter_mut().enumerate() {
+      if i == self.0 {
+        continue;
+      }
+      msg_queue.push_back(msg.clone());
+    }
+  }
+
+  async fn receive(&self) -> Option<(P2pMessageKind, Vec<u8>)> {
+    let mut msg = self.1.write().unwrap()[self.0].pop_front()?;
+    if msg.is_empty() {
+      log::error!("empty p2p message");
+      return None;
+    }
+    Some((P2pMessageKind::from_byte(msg.remove(0))?, msg))
+  }
+}
+
+#[async_trait]
+impl TributaryP2p for LocalP2p {
+  async fn broadcast(&self, mut msg: Vec<u8>) {
+    msg.insert(0, P2pMessageKind::Tributary.to_byte());
+    <Self as P2p>::broadcast(self, msg).await
+  }
+}

coordinator/src/tests/tributary/chain.rs

@@ -0,0 +1,128 @@
+use std::time::{Duration, SystemTime};
+
+use zeroize::Zeroizing;
+
+use rand_core::{RngCore, OsRng};
+
+use ciphersuite::{
+  group::{ff::Field, GroupEncoding},
+  Ciphersuite, Ristretto,
+};
+
+use sp_application_crypto::sr25519;
+
+use serai_client::{
+  primitives::{NETWORKS, NetworkId, Amount},
+  validator_sets::primitives::{Session, ValidatorSet, ValidatorSetData},
+};
+
+use tokio::time::sleep;
+
+use serai_db::MemDb;
+
+use tributary::Tributary;
+
+use crate::{P2pMessageKind, P2p, LocalP2p, processor::MemProcessor, tributary::TributarySpec};
+
+fn new_spec(keys: &[Zeroizing<<Ristretto as Ciphersuite>::F>]) -> TributarySpec {
+  let mut serai_block = [0; 32];
+  OsRng.fill_bytes(&mut serai_block);
+
+  let start_time = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).unwrap().as_secs();
+
+  let set = ValidatorSet { session: Session(0), network: NetworkId::Bitcoin };
+
+  let set_data = ValidatorSetData {
+    bond: Amount(100),
+    network: NETWORKS[&NetworkId::Bitcoin].clone(),
+    participants: keys
+      .iter()
+      .map(|key| {
+        (sr25519::Public((<Ristretto as Ciphersuite>::generator() * **key).to_bytes()), Amount(100))
+      })
+      .collect::<Vec<_>>()
+      .try_into()
+      .unwrap(),
+  };
+
+  TributarySpec::new(serai_block, start_time, set, set_data)
+}
+
+#[tokio::test]
+async fn tributary_test() {
+  let mut keys = vec![];
+  for _ in 0 .. 5 {
+    keys.push(Zeroizing::new(<Ristretto as Ciphersuite>::F::random(&mut OsRng)));
+  }
+
+  let processor = MemProcessor::new();
+
+  let spec = new_spec(&keys);
+
+  let p2p = LocalP2p::new(keys.len());
+
+  let mut tributaries = vec![];
+
+  for (i, key) in keys.iter().enumerate() {
+    tributaries.push(
+      Tributary::<_, crate::tributary::Transaction, _>::new(
+        MemDb::new(),
+        spec.genesis(),
+        spec.start_time(),
+        key.clone(),
+        spec.validators(),
+        p2p[i].clone(),
+      )
+      .await
+      .unwrap(),
+    );
+  }
+
+  let mut blocks = 0;
+  let mut last_block = spec.genesis();
+
+  let timeout = SystemTime::now() + Duration::from_secs(70);
+  while (blocks < 10) && (SystemTime::now().duration_since(timeout).is_err()) {
+    for (i, p2p) in p2p.iter().enumerate() {
+      while let Some(msg) = p2p.receive().await {
+        match msg.0 {
+          P2pMessageKind::Tributary => {
+            tributaries[i].handle_message(&msg.1).await;
+          }
+        }
+      }
+    }
+
+    let tip = tributaries[0].tip();
+    if tip != last_block {
+      last_block = tip;
+      blocks += 1;
+    }
+
+    sleep(Duration::from_millis(100)).await;
+  }
+
+  if blocks != 10 {
+    panic!("tributary chain test hit timeout");
+  }
+
+  // Handle all existing messages
+  for (i, p2p) in p2p.iter().enumerate() {
+    while let Some(msg) = p2p.receive().await {
+      match msg.0 {
+        P2pMessageKind::Tributary => {
+          tributaries[i].handle_message(&msg.1).await;
+        }
+      }
+    }
+  }
+
+  // All tributaries should agree on the tip
+  let mut final_block = None;
+  for tributary in tributaries {
+    final_block = final_block.or_else(|| Some(tributary.tip()));
+    if tributary.tip() != final_block.unwrap() {
+      panic!("tributary had different tip");
+    }
+  }
+}

coordinator/src/tests/tributary/mod.rs

@@ -9,6 +9,8 @@ use tributary::{ReadWrite, tests::random_signed};
 
 use crate::tributary::{SignData, Transaction};
 
+mod chain;
+
 fn random_u32<R: RngCore>(rng: &mut R) -> u32 {
   u32::try_from(rng.next_u64() >> 32).unwrap()
 }

coordinator/src/tributary/db.rs

@@ -97,7 +97,7 @@ impl<D: Db> TributaryDb<D> {
     genesis: [u8; 32],
     id: [u8; 32],
     attempt: u32,
-    signer: &<Ristretto as Ciphersuite>::G,
+    signer: <Ristretto as Ciphersuite>::G,
   ) -> Vec<u8> {
     Self::tributary_key(
       b"data",
@@ -117,7 +117,7 @@ impl<D: Db> TributaryDb<D> {
     genesis: [u8; 32],
     id: [u8; 32],
     attempt: u32,
-    signer: &<Ristretto as Ciphersuite>::G,
+    signer: <Ristretto as Ciphersuite>::G,
   ) -> Option<Vec<u8>> {
     getter.get(Self::data_key(label, genesis, id, attempt, signer))
   }
@@ -127,7 +127,7 @@ impl<D: Db> TributaryDb<D> {
     genesis: [u8; 32],
     id: [u8; 32],
     attempt: u32,
-    signer: &<Ristretto as Ciphersuite>::G,
+    signer: <Ristretto as Ciphersuite>::G,
     data: &[u8],
   ) -> u16 {
     let received_key = Self::data_received_key(label, genesis, id, attempt);

coordinator/src/tributary/mod.rs

@@ -91,12 +91,8 @@ impl TributarySpec {
     None
   }
 
-  pub fn validators(&self) -> HashMap<<Ristretto as Ciphersuite>::G, u64> {
-    let mut res = HashMap::new();
-    for (key, amount) in self.validators.clone() {
-      res.insert(key, amount);
-    }
-    res
+  pub fn validators(&self) -> Vec<(<Ristretto as Ciphersuite>::G, u64)> {
+    self.validators.clone()
   }
 }
 

coordinator/src/tributary/scanner.rs

@@ -70,7 +70,7 @@ async fn handle_block<D: Db, Pro: Processor, P: P2p>(
 
           // If they've already published a TX for this attempt, slash
           if let Some(data) =
-            TributaryDb::<D>::data(label, &txn, tributary.genesis(), id, attempt, &signed.signer)
+            TributaryDb::<D>::data(label, &txn, tributary.genesis(), id, attempt, signed.signer)
           {
             if data != bytes {
               // TODO: Full slash
@@ -99,17 +99,18 @@ async fn handle_block<D: Db, Pro: Processor, P: P2p>(
             tributary.genesis(),
             id,
             attempt,
-            &signed.signer,
+            signed.signer,
             &bytes,
           );
 
           // If we have all the needed commitments/preprocesses/shares, tell the processor
+          // TODO: This needs to be coded by weight, not by validator count
           if received == needed {
             let mut data = HashMap::new();
-            for validator in spec.validators().keys() {
+            for validator in spec.validators().iter().map(|validator| validator.0) {
               data.insert(
-                spec.i(*validator).unwrap(),
-                if validator == &signed.signer {
+                spec.i(validator).unwrap(),
+                if validator == signed.signer {
                   bytes.split_off(0)
                 } else if let Some(data) =
                   TributaryDb::<D>::data(label, &txn, tributary.genesis(), id, attempt, validator)