Improve batch handling (#316)

* restrict batch size to ~25kb

* add batch size check to node

* rate limit batches to 1 per serai block

* add support for multiple batches for block

* fix review comments

* Misc fixes

Doesn't yet update tests/processor until data flow is inspected.

* Move the block from SignId to ProcessorMessage::BatchPreprocesses

* Misc clean up

---------

Co-authored-by: Luke Parker <lukeparker5132@gmail.com>

processor/src/tests/addresses.rs

@@ -20,7 +20,6 @@ async fn spend<N: Network, D: Db>(
   network: &N,
   keys: &HashMap<Participant, ThresholdKeys<N::Curve>>,
   scanner: &mut ScannerHandle<N, D>,
-  batch: u32,
   outputs: Vec<N::Output>,
 ) -> Vec<N::Output> {
   let key = keys[&Participant::new(1).unwrap()].group_key();
@@ -52,9 +51,8 @@ async fn spend<N: Network, D: Db>(
     network.mine_block().await;
   }
   match timeout(Duration::from_secs(30), scanner.events.recv()).await.unwrap().unwrap() {
-    ScannerEvent::Block { key: this_key, block: _, batch: this_batch, outputs } => {
+    ScannerEvent::Block { key: this_key, block: _, outputs } => {
       assert_eq!(this_key, key);
-      assert_eq!(this_batch, batch);
       assert_eq!(outputs.len(), 1);
       // Make sure this is actually a change output
       assert_eq!(outputs[0].kind(), OutputType::Change);
@@ -91,10 +89,9 @@ pub async fn test_addresses<N: Network>(network: N) {
   // Verify the Scanner picked them up
   let outputs =
     match timeout(Duration::from_secs(30), scanner.events.recv()).await.unwrap().unwrap() {
-      ScannerEvent::Block { key: this_key, block, batch, outputs } => {
+      ScannerEvent::Block { key: this_key, block, outputs } => {
         assert_eq!(this_key, key);
         assert_eq!(block, block_id);
-        assert_eq!(batch, 0);
         assert_eq!(outputs.len(), 1);
         assert_eq!(outputs[0].kind(), OutputType::Branch);
         outputs
@@ -105,7 +102,7 @@ pub async fn test_addresses<N: Network>(network: N) {
     };
 
   // Spend the branch output, creating a change output and ensuring we actually get change
-  let outputs = spend(&network, &keys, &mut scanner, 1, outputs).await;
+  let outputs = spend(&network, &keys, &mut scanner, outputs).await;
   // Also test spending the change output
-  spend(&network, &keys, &mut scanner, 2, outputs).await;
+  spend(&network, &keys, &mut scanner, outputs).await;
 }

processor/src/tests/scanner.rs

@@ -55,10 +55,9 @@ pub async fn test_scanner<N: Network>(network: N) {
   let verify_event = |mut scanner: ScannerHandle<N, MemDb>| async {
     let outputs =
       match timeout(Duration::from_secs(30), scanner.events.recv()).await.unwrap().unwrap() {
-        ScannerEvent::Block { key, block, batch, outputs } => {
+        ScannerEvent::Block { key, block, outputs } => {
           assert_eq!(key, keys.group_key());
           assert_eq!(block, block_id);
-          assert_eq!(batch, 0);
           assert_eq!(outputs.len(), 1);
           assert_eq!(outputs[0].kind(), OutputType::External);
           outputs
@@ -90,10 +89,7 @@ pub async fn test_scanner<N: Network>(network: N) {
 
   let mut cloned_db = db.clone();
   let mut txn = cloned_db.txn();
-  assert_eq!(
-    scanner.ack_up_to_block(&mut txn, keys.group_key(), block_id).await,
-    (blocks, outputs)
-  );
+  assert_eq!(scanner.ack_up_to_block(&mut txn, keys.group_key(), block_id).await, outputs);
   txn.commit();
 
   // There should be no more events

processor/src/tests/substrate_signer.rs

@@ -24,13 +24,16 @@ async fn test_substrate_signer() {
 
   let participant_one = Participant::new(1).unwrap();
 
+  let id: u32 = 5;
+  let mut id_arr = [0u8; 32];
+  id_arr[.. 4].copy_from_slice(&id.to_le_bytes());
   let block = BlockHash([0xaa; 32]);
   let actual_id =
-    SignId { key: keys[&participant_one].group_key().to_bytes().to_vec(), id: block.0, attempt: 0 };
+    SignId { key: keys[&participant_one].group_key().to_bytes().to_vec(), id: id_arr, attempt: 0 };
 
   let batch = Batch {
     network: NetworkId::Monero,
-    id: 5,
+    id,
     block,
     instructions: vec![
       InInstructionWithBalance {
@@ -81,10 +84,12 @@ async fn test_substrate_signer() {
     let i = Participant::new(u16::try_from(i).unwrap()).unwrap();
     if let SubstrateSignerEvent::ProcessorMessage(ProcessorMessage::BatchPreprocess {
       id,
+      block: batch_block,
       preprocess,
     }) = signers.get_mut(&i).unwrap().events.pop_front().unwrap()
     {
       assert_eq!(id, actual_id);
+      assert_eq!(batch_block, block);
       if signing_set.contains(&i) {
         preprocesses.insert(i, preprocess);
       }

processor/src/tests/wallet.rs

@@ -36,10 +36,9 @@ pub async fn test_wallet<N: Network>(network: N) {
     let block_id = block.id();
 
     match timeout(Duration::from_secs(30), scanner.events.recv()).await.unwrap().unwrap() {
-      ScannerEvent::Block { key: this_key, block, batch, outputs } => {
+      ScannerEvent::Block { key: this_key, block, outputs } => {
         assert_eq!(this_key, key);
         assert_eq!(block, block_id);
-        assert_eq!(batch, 0);
         assert_eq!(outputs.len(), 1);
         (block_id, outputs)
       }
@@ -110,10 +109,9 @@ pub async fn test_wallet<N: Network>(network: N) {
   }
 
   match timeout(Duration::from_secs(30), scanner.events.recv()).await.unwrap().unwrap() {
-    ScannerEvent::Block { key: this_key, block: block_id, batch, outputs: these_outputs } => {
+    ScannerEvent::Block { key: this_key, block: block_id, outputs: these_outputs } => {
       assert_eq!(this_key, key);
       assert_eq!(block_id, block.id());
-      assert_eq!(batch, 1);
       assert_eq!(these_outputs, outputs);
     }
     ScannerEvent::Completed(_, _) => {
@@ -124,7 +122,7 @@ pub async fn test_wallet<N: Network>(network: N) {
   // Check the Scanner DB can reload the outputs
   let mut txn = db.txn();
   assert_eq!(
-    scanner.ack_up_to_block(&mut txn, key, block.id()).await.1,
+    scanner.ack_up_to_block(&mut txn, key, block.id()).await,
     [first_outputs, outputs].concat().to_vec()
   );
   txn.commit();