Replace bincode with borsh (#452)

* Add SignalsConfig to chain_spec

* Correct multiexp feature flagging for rand_core std

* Remove bincode for borsh

Replaces a non-canonical encoding with a canonical encoding which additionally
should be faster.

Also fixes an issue where we used bincode in transcripts where it cannot be
trusted.

This ended up fixing a myriad of other bugs observed, unfortunately.
Accordingly, it either has to be merged or the bug fixes from it must be ported
to a new PR.

* Make serde optional, minimize usage

* Make borsh an optional dependency of substrate/ crates

* Remove unused dependencies

* Use [u8; 64] where possible in the processor messages

* Correct borsh feature flagging

processor/Cargo.toml

@@ -28,7 +28,7 @@ rand_chacha = { version = "0.3", default-features = false, features = ["std"] }
 # Encoders
 hex = { version = "0.4", default-features = false, features = ["std"] }
 scale = { package = "parity-scale-codec", version = "3", default-features = false, features = ["std"] }
-bincode = { version = "1", default-features = false }
+borsh = { version = "1", default-features = false, features = ["std", "derive", "de_strict_order"] }
 serde_json = { version = "1", default-features = false, features = ["std"] }
 
 # Cryptography

processor/messages/Cargo.toml

@@ -17,11 +17,11 @@ rustdoc-args = ["--cfg", "docsrs"]
 zeroize = { version = "1", default-features = false, features = ["std", "derive"] }
 
 scale = { package = "parity-scale-codec", version = "3", default-features = false, features = ["std"] }
-serde = { version = "1", default-features = false, features = ["std", "derive"] }
+borsh = { version = "1", default-features = false, features = ["std", "derive", "de_strict_order"] }
 
-dkg = { path = "../../crypto/dkg", default-features = false, features = ["std", "serde"] }
+dkg = { path = "../../crypto/dkg", default-features = false, features = ["std", "borsh", "serde"] }
 
-serai-primitives = { path = "../../substrate/primitives", default-features = false, features = ["std"] }
-in-instructions-primitives = { package = "serai-in-instructions-primitives", path = "../../substrate/in-instructions/primitives", default-features = false, features = ["std"] }
-coins-primitives = { package = "serai-coins-primitives", path = "../../substrate/coins/primitives", default-features = false, features = ["std"] }
-validator-sets-primitives = { package = "serai-validator-sets-primitives", path = "../../substrate/validator-sets/primitives", default-features = false, features = ["std"] }
+serai-primitives = { path = "../../substrate/primitives", default-features = false, features = ["std", "borsh"] }
+in-instructions-primitives = { package = "serai-in-instructions-primitives", path = "../../substrate/in-instructions/primitives", default-features = false, features = ["std", "borsh"] }
+coins-primitives = { package = "serai-coins-primitives", path = "../../substrate/coins/primitives", default-features = false, features = ["std", "borsh"] }
+validator-sets-primitives = { package = "serai-validator-sets-primitives", path = "../../substrate/validator-sets/primitives", default-features = false, features = ["std", "borsh"] }

processor/messages/src/lib.rs

@@ -3,7 +3,7 @@ use std::collections::HashMap;
 use zeroize::Zeroize;
 
 use scale::{Encode, Decode};
-use serde::{Serialize, Deserialize};
+use borsh::{BorshSerialize, BorshDeserialize};
 
 use dkg::{Participant, ThresholdParams};
 
@@ -12,7 +12,9 @@ use in_instructions_primitives::{Batch, SignedBatch};
 use coins_primitives::OutInstructionWithBalance;
 use validator_sets_primitives::{ValidatorSet, KeyPair};
 
-#[derive(Clone, Copy, PartialEq, Eq, Debug, Zeroize, Encode, Decode, Serialize, Deserialize)]
+#[derive(
+  Clone, Copy, PartialEq, Eq, Debug, Zeroize, Encode, Decode, BorshSerialize, BorshDeserialize,
+)]
 pub struct SubstrateContext {
   pub serai_time: u64,
   pub network_latest_finalized_block: BlockHash,
@@ -22,14 +24,24 @@ pub mod key_gen {
   use super::*;
 
   #[derive(
-    Clone, Copy, PartialEq, Eq, Hash, Debug, Zeroize, Encode, Decode, Serialize, Deserialize,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    Hash,
+    Debug,
+    Zeroize,
+    Encode,
+    Decode,
+    BorshSerialize,
+    BorshDeserialize,
   )]
   pub struct KeyGenId {
     pub set: ValidatorSet,
     pub attempt: u32,
   }
 
-  #[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
+  #[derive(Clone, PartialEq, Eq, Debug, BorshSerialize, BorshDeserialize)]
   pub enum CoordinatorMessage {
     // Instructs the Processor to begin the key generation process.
     // TODO: Should this be moved under Substrate?
@@ -64,7 +76,7 @@ pub mod key_gen {
     }
   }
 
-  #[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
+  #[derive(Clone, PartialEq, Eq, Debug, BorshSerialize, BorshDeserialize)]
   pub enum ProcessorMessage {
     // Created commitments for the specified key generation protocol.
     Commitments {
@@ -106,14 +118,16 @@ pub mod key_gen {
 pub mod sign {
   use super::*;
 
-  #[derive(Clone, PartialEq, Eq, Hash, Debug, Zeroize, Encode, Decode, Serialize, Deserialize)]
+  #[derive(
+    Clone, PartialEq, Eq, Hash, Debug, Zeroize, Encode, Decode, BorshSerialize, BorshDeserialize,
+  )]
   pub struct SignId {
     pub key: Vec<u8>,
     pub id: [u8; 32],
     pub attempt: u32,
   }
 
-  #[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
+  #[derive(Clone, PartialEq, Eq, Debug, BorshSerialize, BorshDeserialize)]
   pub enum CoordinatorMessage {
     // Received preprocesses for the specified signing protocol.
     Preprocesses { id: SignId, preprocesses: HashMap<Participant, Vec<u8>> },
@@ -140,7 +154,7 @@ pub mod sign {
     }
   }
 
-  #[derive(Clone, PartialEq, Eq, Debug, Zeroize, Serialize, Deserialize)]
+  #[derive(Clone, PartialEq, Eq, Debug, Zeroize, BorshSerialize, BorshDeserialize)]
   pub enum ProcessorMessage {
     // Participant sent an invalid message during the sign protocol.
     InvalidParticipant { id: SignId, participant: Participant },
@@ -166,25 +180,36 @@ pub mod coordinator {
   }
 
   #[derive(
-    Clone, Copy, PartialEq, Eq, Hash, Debug, Zeroize, Encode, Decode, Serialize, Deserialize,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    Hash,
+    Debug,
+    Zeroize,
+    Encode,
+    Decode,
+    BorshSerialize,
+    BorshDeserialize,
   )]
   pub enum SubstrateSignableId {
     CosigningSubstrateBlock([u8; 32]),
     Batch([u8; 5]),
   }
 
-  #[derive(Clone, PartialEq, Eq, Hash, Debug, Zeroize, Encode, Decode, Serialize, Deserialize)]
+  #[derive(
+    Clone, PartialEq, Eq, Hash, Debug, Zeroize, Encode, Decode, BorshSerialize, BorshDeserialize,
+  )]
   pub struct SubstrateSignId {
     pub key: [u8; 32],
     pub id: SubstrateSignableId,
     pub attempt: u32,
   }
 
-  #[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
+  #[derive(Clone, PartialEq, Eq, Debug, BorshSerialize, BorshDeserialize)]
   pub enum CoordinatorMessage {
     CosignSubstrateBlock { id: SubstrateSignId, block_number: u64 },
-    // Uses Vec<u8> instead of [u8; 64] since serde Deserialize isn't implemented for [u8; 64]
-    SubstratePreprocesses { id: SubstrateSignId, preprocesses: HashMap<Participant, Vec<u8>> },
+    SubstratePreprocesses { id: SubstrateSignId, preprocesses: HashMap<Participant, [u8; 64]> },
     SubstrateShares { id: SubstrateSignId, shares: HashMap<Participant, [u8; 32]> },
     // Re-attempt a batch signing protocol.
     BatchReattempt { id: SubstrateSignId },
@@ -214,18 +239,20 @@ pub mod coordinator {
     }
   }
 
-  #[derive(Clone, PartialEq, Eq, Debug, Zeroize, Encode, Decode, Serialize, Deserialize)]
+  #[derive(
+    Clone, PartialEq, Eq, Debug, Zeroize, Encode, Decode, BorshSerialize, BorshDeserialize,
+  )]
   pub struct PlanMeta {
     pub key: Vec<u8>,
     pub id: [u8; 32],
   }
 
-  #[derive(Clone, PartialEq, Eq, Debug, Zeroize, Serialize, Deserialize)]
+  #[derive(Clone, PartialEq, Eq, Debug, Zeroize, BorshSerialize, BorshDeserialize)]
   pub enum ProcessorMessage {
     SubstrateBlockAck { network: NetworkId, block: u64, plans: Vec<PlanMeta> },
     InvalidParticipant { id: SubstrateSignId, participant: Participant },
-    CosignPreprocess { id: SubstrateSignId, preprocesses: Vec<Vec<u8>> },
-    BatchPreprocess { id: SubstrateSignId, block: BlockHash, preprocesses: Vec<Vec<u8>> },
+    CosignPreprocess { id: SubstrateSignId, preprocesses: Vec<[u8; 64]> },
+    BatchPreprocess { id: SubstrateSignId, block: BlockHash, preprocesses: Vec<[u8; 64]> },
     SubstrateShare { id: SubstrateSignId, shares: Vec<[u8; 32]> },
     CosignedBlock { block_number: u64, block: [u8; 32], signature: Vec<u8> },
   }
@@ -234,7 +261,7 @@ pub mod coordinator {
 pub mod substrate {
   use super::*;
 
-  #[derive(Clone, PartialEq, Eq, Debug, Encode, Decode, Serialize, Deserialize)]
+  #[derive(Clone, PartialEq, Eq, Debug, Encode, Decode, BorshSerialize, BorshDeserialize)]
   pub enum CoordinatorMessage {
     ConfirmKeyPair {
       context: SubstrateContext,
@@ -260,7 +287,9 @@ pub mod substrate {
     }
   }
 
-  #[derive(Clone, PartialEq, Eq, Debug, Zeroize, Encode, Decode, Serialize, Deserialize)]
+  #[derive(
+    Clone, PartialEq, Eq, Debug, Zeroize, Encode, Decode, BorshSerialize, BorshDeserialize,
+  )]
   pub enum ProcessorMessage {
     Batch { batch: Batch },
     SignedBatch { batch: SignedBatch },
@@ -277,7 +306,7 @@ macro_rules! impl_from {
   };
 }
 
-#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
+#[derive(Clone, PartialEq, Eq, Debug, BorshSerialize, BorshDeserialize)]
 pub enum CoordinatorMessage {
   KeyGen(key_gen::CoordinatorMessage),
   Sign(sign::CoordinatorMessage),
@@ -308,7 +337,7 @@ impl CoordinatorMessage {
   }
 }
 
-#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
+#[derive(Clone, PartialEq, Eq, Debug, BorshSerialize, BorshDeserialize)]
 pub enum ProcessorMessage {
   KeyGen(key_gen::ProcessorMessage),
   Sign(sign::ProcessorMessage),

processor/src/batch_signer.rs

@@ -191,7 +191,7 @@ impl<D: Db> BatchSigner<D> {
 
       let (machine, preprocess) = machine.preprocess(&mut OsRng);
       machines.push(machine);
-      serialized_preprocesses.push(preprocess.serialize());
+      serialized_preprocesses.push(preprocess.serialize().try_into().unwrap());
       preprocesses.push(preprocess);
     }
     self.preprocessing.insert(id, (machines, preprocesses));

processor/src/coordinator.rs

@@ -20,9 +20,9 @@ impl Coordinator for MessageQueue {
   async fn send(&mut self, msg: impl Send + Into<ProcessorMessage>) {
     let msg: ProcessorMessage = msg.into();
     let metadata = Metadata { from: self.service, to: Service::Coordinator, intent: msg.intent() };
-    let msg = serde_json::to_string(&msg).unwrap();
+    let msg = borsh::to_vec(&msg).unwrap();
 
-    self.queue(metadata, msg.into_bytes()).await;
+    self.queue(metadata, msg).await;
   }
 
   async fn recv(&mut self) -> Message {
@@ -32,7 +32,7 @@ impl Coordinator for MessageQueue {
 
     // Deserialize it into a CoordinatorMessage
     let msg: CoordinatorMessage =
-      serde_json::from_slice(&msg.msg).expect("message wasn't a JSON-encoded CoordinatorMessage");
+      borsh::from_slice(&msg.msg).expect("message wasn't a borsh-encoded CoordinatorMessage");
 
     return Message { id, msg };
   }

processor/src/cosigner.rs

@@ -95,7 +95,7 @@ impl Cosigner {
 
       let (machine, preprocess) = machine.preprocess(&mut OsRng);
       machines.push(machine);
-      serialized_preprocesses.push(preprocess.serialize());
+      serialized_preprocesses.push(preprocess.serialize().try_into().unwrap());
       preprocesses.push(preprocess);
     }
     let preprocessing = Some((machines, preprocesses));

processor/src/multisigs/db.rs

@@ -44,7 +44,7 @@ impl PlanDb {
       }
 
       signing.extend(&id);
-      SigningDb::set(txn, key, &id);
+      SigningDb::set(txn, key, &signing);
     }
 
     {

processor/src/signer.rs

@@ -51,7 +51,6 @@ impl CompletedOnChainDb {
         .unwrap_or_default()
         .into_iter()
         .filter(|active| active != id)
-        .flatten()
         .collect::<Vec<_>>(),
     );
   }

processor/src/tests/key_gen.rs

@@ -12,7 +12,7 @@ use serai_db::{DbTxn, Db, MemDb};
 use sp_application_crypto::sr25519;
 use serai_client::{
   primitives::NetworkId,
-  validator_sets::primitives::{Session, ValidatorSet},
+  validator_sets::primitives::{Session, ValidatorSet, KeyPair},
 };
 
 use messages::key_gen::*;
@@ -139,7 +139,7 @@ pub async fn test_key_gen<N: Network>() {
     let key_gen = key_gens.get_mut(&i).unwrap();
     let mut txn = dbs.get_mut(&i).unwrap().txn();
     let KeyConfirmed { mut substrate_keys, mut network_keys } = key_gen
-      .confirm(&mut txn, ID.set, (sr25519::Public(res.0), res.1.clone().try_into().unwrap()))
+      .confirm(&mut txn, ID.set, KeyPair(sr25519::Public(res.0), res.1.clone().try_into().unwrap()))
       .await;
     txn.commit();