Rewrite tendermint's message handling loop to much more clearly match the paper (#560)

* Rewrite tendermint's message handling loop to much more clearly match the paper

No longer checks relevant branches upon messages, yet all branches upon any
state change. This is slower, yet easier to review and likely without one or
two rare edge cases.

When reviewing, please see page 5 of https://arxiv.org/pdf/1807.04938.pdf.
Lines from the specified algorithm can be found in the code by searching for
"// L".

* Sane rebroadcasting of consensus messages

Instead of broadcasting the last n messages on the Tributary side of things, we
now have the machine rebroadcast the message tape for the current block.

* Only rebroadcast messages which didn't error in some way

* Only rebroadcast our own messages for tendermint

coordinator/tributary/src/lib.rs

@@ -1,5 +1,5 @@
 use core::{marker::PhantomData, fmt::Debug};
-use std::{sync::Arc, io, collections::VecDeque};
+use std::{sync::Arc, io};
 
 use async_trait::async_trait;
 
@@ -154,14 +154,6 @@ pub struct Tributary<D: Db, T: TransactionTrait, P: P2p> {
   synced_block: Arc<RwLock<SyncedBlockSender<TendermintNetwork<D, T, P>>>>,
   synced_block_result: Arc<RwLock<SyncedBlockResultReceiver>>,
   messages: Arc<RwLock<MessageSender<TendermintNetwork<D, T, P>>>>,
-
-  p2p_meta_task_handle: Arc<tokio::task::AbortHandle>,
-}
-
-impl<D: Db, T: TransactionTrait, P: P2p> Drop for Tributary<D, T, P> {
-  fn drop(&mut self) {
-    self.p2p_meta_task_handle.abort();
-  }
 }
 
 impl<D: Db, T: TransactionTrait, P: P2p> Tributary<D, T, P> {
@@ -193,28 +185,7 @@ impl<D: Db, T: TransactionTrait, P: P2p> Tributary<D, T, P> {
     );
     let blockchain = Arc::new(RwLock::new(blockchain));
 
-    let to_rebroadcast = Arc::new(RwLock::new(VecDeque::new()));
-    // Actively rebroadcast consensus messages to ensure they aren't prematurely dropped from the
-    // P2P layer
-    let p2p_meta_task_handle = Arc::new(
-      tokio::spawn({
-        let to_rebroadcast = to_rebroadcast.clone();
-        let p2p = p2p.clone();
-        async move {
-          loop {
-            let to_rebroadcast = to_rebroadcast.read().await.clone();
-            for msg in to_rebroadcast {
-              p2p.broadcast(genesis, msg).await;
-            }
-            tokio::time::sleep(core::time::Duration::from_secs(60)).await;
-          }
-        }
-      })
-      .abort_handle(),
-    );
-
-    let network =
-      TendermintNetwork { genesis, signer, validators, blockchain, to_rebroadcast, p2p };
+    let network = TendermintNetwork { genesis, signer, validators, blockchain, p2p };
 
     let TendermintHandle { synced_block, synced_block_result, messages, machine } =
       TendermintMachine::new(
@@ -235,7 +206,6 @@ impl<D: Db, T: TransactionTrait, P: P2p> Tributary<D, T, P> {
       synced_block: Arc::new(RwLock::new(synced_block)),
       synced_block_result: Arc::new(RwLock::new(synced_block_result)),
       messages: Arc::new(RwLock::new(messages)),
-      p2p_meta_task_handle,
     })
   }
 

coordinator/tributary/src/tendermint/mod.rs

@@ -1,8 +1,5 @@
 use core::ops::Deref;
-use std::{
-  sync::Arc,
-  collections::{VecDeque, HashMap},
-};
+use std::{sync::Arc, collections::HashMap};
 
 use async_trait::async_trait;
 
@@ -270,8 +267,6 @@ pub struct TendermintNetwork<D: Db, T: TransactionTrait, P: P2p> {
   pub(crate) validators: Arc<Validators>,
   pub(crate) blockchain: Arc<RwLock<Blockchain<D, T>>>,
 
-  pub(crate) to_rebroadcast: Arc<RwLock<VecDeque<Vec<u8>>>>,
-
   pub(crate) p2p: P,
 }
 
@@ -308,26 +303,6 @@ impl<D: Db, T: TransactionTrait, P: P2p> Network for TendermintNetwork<D, T, P>
   async fn broadcast(&mut self, msg: SignedMessageFor<Self>) {
     let mut to_broadcast = vec![TENDERMINT_MESSAGE];
     to_broadcast.extend(msg.encode());
-
-    // Since we're broadcasting a Tendermint message, set it to be re-broadcasted every second
-    // until the block it's trying to build is complete
-    // If the P2P layer drops a message before all nodes obtained access, or a node had an
-    // intermittent failure, this will ensure reconcilliation
-    // This is atrocious if there's no content-based deduplication protocol for messages actively
-    // being gossiped
-    // LibP2p, as used by Serai, is configured to content-based deduplicate
-    {
-      let mut to_rebroadcast_lock = self.to_rebroadcast.write().await;
-      to_rebroadcast_lock.push_back(to_broadcast.clone());
-      // We should have, ideally, 3 * validators messages within a round
-      // Therefore, this should keep the most recent 2-rounds
-      // TODO: This isn't perfect. Each participant should just rebroadcast their latest round of
-      // messages
-      while to_rebroadcast_lock.len() > (6 * self.validators.weights.len()) {
-        to_rebroadcast_lock.pop_front();
-      }
-    }
-
     self.p2p.broadcast(self.genesis, to_broadcast).await
   }
 
@@ -366,7 +341,7 @@ impl<D: Db, T: TransactionTrait, P: P2p> Network for TendermintNetwork<D, T, P>
     }
   }
 
-  async fn validate(&mut self, block: &Self::Block) -> Result<(), TendermintBlockError> {
+  async fn validate(&self, block: &Self::Block) -> Result<(), TendermintBlockError> {
     let block =
       Block::read::<&[u8]>(&mut block.0.as_ref()).map_err(|_| TendermintBlockError::Fatal)?;
     self
@@ -428,9 +403,6 @@ impl<D: Db, T: TransactionTrait, P: P2p> Network for TendermintNetwork<D, T, P>
       }
     }
 
-    // Since we've added a valid block, clear to_rebroadcast
-    *self.to_rebroadcast.write().await = VecDeque::new();
-
     Some(TendermintBlock(
       self.blockchain.write().await.build_block::<Self>(&self.signature_scheme()).serialize(),
     ))