Add a non-canonical SCALE derivations feature

Enables representing IUMT within `StorageValues`. Applied to a variety of
values.

Fixes a bug where `Some([0; 32])` would be considered a valid block anchor.

substrate/abi/Cargo.toml

@@ -47,6 +47,6 @@ std = [
 
   "serai-primitives/std",
 ]
-substrate = ["serde", "scale", "scale-info", "sp-runtime", "frame-support"]
+substrate = ["serde", "scale", "scale-info", "sp-runtime", "frame-support", "serai-primitives/non_canonical_scale_derivations"]
 try-runtime = ["sp-runtime/try-runtime"]
 default = ["std"]

substrate/primitives/Cargo.toml

@@ -32,5 +32,6 @@ bech32 = { version = "0.11", default-features = false }
 rand_core = { version = "0.6", default-features = false, features = ["std"] }
 
 [features]
+non_canonical_scale_derivations = []
 std = ["zeroize/std", "borsh/std", "ciphersuite/std", "dkg/std", "sp-core/std", "bech32/std"]
 default = ["std"]

substrate/primitives/src/balance.rs

@@ -10,18 +10,10 @@ use crate::coin::{ExternalCoin, Coin};
 pub type AmountRepr = u64;
 
 /// A wrapper used to represent amounts.
-#[derive(
+#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Zeroize, BorshSerialize, BorshDeserialize)]
-  Clone,
+#[cfg_attr(
-  Copy,
+  feature = "non_canonical_scale_derivations",
-  PartialEq,
+  derive(scale::Encode, scale::Decode, scale::MaxEncodedLen)
-  Eq,
-  PartialOrd,
-  Ord,
-  Hash,
-  Debug,
-  Zeroize,
-  BorshSerialize,
-  BorshDeserialize,
 )]
 pub struct Amount(pub AmountRepr);
 
@@ -48,6 +40,10 @@ impl Mul for Amount {
 
 /// An ExternalCoin and an Amount, forming a balance for an external coin.
 #[derive(Clone, Copy, PartialEq, Eq, Debug, Zeroize, BorshSerialize, BorshDeserialize)]
+#[cfg_attr(
+  feature = "non_canonical_scale_derivations",
+  derive(scale::Encode, scale::Decode, scale::MaxEncodedLen)
+)]
 pub struct ExternalBalance {
   /// The coin this is a balance for.
   pub coin: ExternalCoin,
@@ -78,6 +74,10 @@ impl Mul<Amount> for ExternalBalance {
 
 /// A Coin and an Amount, forming a balance for a coin.
 #[derive(Clone, Copy, PartialEq, Eq, Debug, Zeroize, BorshSerialize, BorshDeserialize)]
+#[cfg_attr(
+  feature = "non_canonical_scale_derivations",
+  derive(scale::Encode, scale::Decode, scale::MaxEncodedLen)
+)]
 pub struct Balance {
   /// The coin this is a balance for.
   pub coin: Coin,

substrate/primitives/src/coin.rs

@@ -9,6 +9,10 @@ use crate::network_id::{ExternalNetworkId, NetworkId};
 /// This type serializes to a subset of `Coin`.
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Zeroize, BorshSerialize, BorshDeserialize)]
 #[borsh(use_discriminant = true)]
+#[cfg_attr(
+  feature = "non_canonical_scale_derivations",
+  derive(scale::Encode, scale::Decode, scale::MaxEncodedLen)
+)]
 #[non_exhaustive]
 pub enum ExternalCoin {
   /// Bitcoin, from the Bitcoin network.
@@ -31,6 +35,10 @@ impl ExternalCoin {
 
 /// The type used to identify coins.
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Zeroize)]
+#[cfg_attr(
+  feature = "non_canonical_scale_derivations",
+  derive(scale::Encode, scale::Decode, scale::MaxEncodedLen)
+)]
 pub enum Coin {
   /// The Serai coin.
   Serai,

substrate/primitives/src/merkle.rs

@@ -67,7 +67,8 @@ impl UnbalancedMerkleTree {
 }
 
 /// An unbalanced Merkle tree which is incrementally created.
-#[derive(Clone, PartialEq, Eq, Debug, BorshSerialize, BorshDeserialize)]
+#[derive(Clone, PartialEq, Eq, Debug)]
+#[cfg_attr(feature = "non_canonical_scale_derivations", derive(scale::Encode, scale::Decode))]
 pub struct IncrementalUnbalancedMerkleTree {
   /// (number of children under branch, branch hash)
   branches: Vec<(u64, [u8; 32])>,

substrate/runtime/src/core_pallet.rs

@@ -1,7 +1,6 @@
 use core::marker::PhantomData;
-use alloc::{vec, vec::Vec};
 
-use borsh::{BorshSerialize, BorshDeserialize};
+use borsh::BorshSerialize;
 
 use frame_support::pallet_prelude::*;
 
@@ -10,13 +9,14 @@ use serai_abi::{
   *,
 };
 
+/// A wrapper around a `StorageValue` which offers a high-level API as an IUMT.
 struct IncrementalUnbalancedMerkleTree<
-  T: frame_support::StorageValue<Vec<u8>, Query = Option<Vec<u8>>>,
+  T: frame_support::StorageValue<Iumt, Query = Option<Iumt>>,
   const BRANCH_TAG: u8 = 1,
   const LEAF_TAG: u8 = 0,
 >(PhantomData<T>);
 impl<
-    T: frame_support::StorageValue<Vec<u8>, Query = Option<Vec<u8>>>,
+    T: frame_support::StorageValue<Iumt, Query = Option<Iumt>>,
     const BRANCH_TAG: u8,
     const LEAF_TAG: u8,
   > IncrementalUnbalancedMerkleTree<T, BRANCH_TAG, LEAF_TAG>
@@ -27,7 +27,7 @@ impl<
   fn new_expecting_none() {
     T::mutate(|value| {
       assert!(value.is_none());
-      *value = Some(borsh::to_vec(&Iumt::new()).unwrap());
+      *value = Some(Iumt::new());
     });
   }
   /// Append a leaf to the Merkle tree.
@@ -37,26 +37,21 @@ impl<
     let leaf = sp_core::blake2_256(&borsh::to_vec(&(LEAF_TAG, leaf)).unwrap());
 
     T::mutate(|value| {
-      let mut tree = Iumt::deserialize_reader(&mut value.as_ref().unwrap().as_slice()).unwrap();
+      let tree = value.as_mut().unwrap();
       tree.append(BRANCH_TAG, leaf);
-      *value = Some(borsh::to_vec(&tree).unwrap());
     })
   }
   /// Get the unbalanced merkle tree.
   ///
   /// Panics if no Merkle tree was present.
   fn get() -> UnbalancedMerkleTree {
-    Iumt::deserialize_reader(&mut T::get().unwrap().as_slice()).unwrap().calculate(BRANCH_TAG)
+    T::get().unwrap().calculate(BRANCH_TAG)
   }
   /// Take the Merkle tree.
   ///
   /// Panics if no Merkle tree was present.
   fn take() -> UnbalancedMerkleTree {
-    T::mutate(|value| {
+    T::mutate(|value| value.take().unwrap().calculate(BRANCH_TAG))
-      let tree = Iumt::deserialize_reader(&mut value.as_ref().unwrap().as_slice()).unwrap();
-      *value = None;
-      tree.calculate(BRANCH_TAG)
-    })
   }
 }
 
@@ -70,20 +65,20 @@ mod pallet {
   /// The Merkle tree of all blocks added to the blockchain.
   #[pallet::storage]
   #[pallet::unbounded]
-  pub(super) type BlocksCommitment<T: Config> = StorageValue<_, Vec<u8>, OptionQuery>;
+  pub(super) type BlocksCommitment<T: Config> = StorageValue<_, Iumt, OptionQuery>;
   pub(super) type BlocksCommitmentMerkle<T> = IncrementalUnbalancedMerkleTree<BlocksCommitment<T>>;
 
   /// The Merkle tree of all transactions within the current block.
   #[pallet::storage]
   #[pallet::unbounded]
-  pub(super) type BlockTransactionsCommitment<T: Config> = StorageValue<_, Vec<u8>, OptionQuery>;
+  pub(super) type BlockTransactionsCommitment<T: Config> = StorageValue<_, Iumt, OptionQuery>;
   pub(super) type BlockTransactionsCommitmentMerkle<T> =
     IncrementalUnbalancedMerkleTree<BlockTransactionsCommitment<T>>;
 
   /// The hashes of events caused by the current transaction.
   #[pallet::storage]
   #[pallet::unbounded]
-  pub(super) type TransactionEvents<T: Config> = StorageValue<_, Vec<u8>, OptionQuery>;
+  pub(super) type TransactionEvents<T: Config> = StorageValue<_, Iumt, OptionQuery>;
   pub(super) type TransactionEventsMerkle<T> = IncrementalUnbalancedMerkleTree<
     TransactionEvents<T>,
     TRANSACTION_EVENTS_COMMITMENT_BRANCH_TAG,
@@ -92,7 +87,7 @@ mod pallet {
   /// The roots of the Merkle trees of each transaction's events.
   #[pallet::storage]
   #[pallet::unbounded]
-  pub(super) type BlockEventsCommitment<T: Config> = StorageValue<_, Vec<u8>, OptionQuery>;
+  pub(super) type BlockEventsCommitment<T: Config> = StorageValue<_, Iumt, OptionQuery>;
   pub(super) type BlockEventsCommitmentMerkle<T> = IncrementalUnbalancedMerkleTree<
     BlockEventsCommitment<T>,
     EVENTS_COMMITMENT_BRANCH_TAG,
@@ -105,12 +100,7 @@ mod pallet {
     StorageMap<_, Blake2_128Concat, T::AccountId, T::Nonce, ValueQuery>;
 
   #[pallet::config]
-  pub trait Config:
+  pub trait Config: frame_system::Config<Hash: Into<[u8; 32]>> {}
-    frame_system::Config<
-    Block: sp_runtime::traits::Block<Header: sp_runtime::traits::Header<Hash: Into<[u8; 32]>>>,
-  >
-  {
-  }
 
   #[pallet::pallet]
   pub struct Pallet<T>(_);
@@ -141,12 +131,11 @@ pub(super) use pallet::*;
 pub struct StartOfBlock<T: Config>(PhantomData<T>);
 impl<T: Config> frame_support::traits::PreInherents for StartOfBlock<T> {
   fn pre_inherents() {
-    let parent_hash = frame_system::Pallet::<T>::parent_hash();
+    if frame_system::Pallet::<T>::block_number().is_zero() {
-    Blocks::<T>::set(parent_hash, Some(()));
-    // TODO: Better detection of genesis
-    if parent_hash == Default::default() {
       BlocksCommitmentMerkle::<T>::new_expecting_none();
     } else {
+      let parent_hash = frame_system::Pallet::<T>::parent_hash();
+      Blocks::<T>::set(parent_hash, Some(()));
       let parent_hash: [u8; 32] = parent_hash.into();
       BlocksCommitmentMerkle::<T>::append(&parent_hash);
     }