Move develop to patch-polkadot-sdk (#678)

* Update `build-dependencies` CI action

* Update `develop` to `patch-polkadot-sdk`

Allows us to finally remove the old `serai-dex/substrate` repository _and_
should have CI pass without issue on `develop` again.

The changes made here should be trivial and maintain all prior
behavior/functionality. The most notable are to `chain_spec.rs`, in order to
still use a SCALE-encoded `GenesisConfig` (avoiding `serde_json`).

* CI fixes

* Add `/usr/local/opt/llvm/lib` to paths on macOS hosts

* Attempt to use `LD_LIBRARY_PATH` in macOS GitHub CI

* Use `libp2p 0.56` in `serai-node`

* Correct Windows build dependencies

* Correct `llvm/lib` path on macOS

* Correct how macOS 13 and 14 have different homebrew paths

* Use `sw_vers` instead of `uname` on macOS

Yields the macOS version instead of the kernel's version.

* Replace hard-coded path with the intended env variable to fix macOS 13

* Add `libclang-dev` as dependency to the Debian Dockerfile

* Set the `CODE` storage slot

* Update to a version of substrate without `wasmtimer`

Turns out `wasmtimer` is WASM only. This should restore the node's functioning
on non-WASM environments.

* Restore `clang` as a dependency due to the Debian Dockerfile as we require a C++ compiler

* Move from Debian bookworm to trixie

* Restore `chain_getBlockBin` to the RPC

* Always generate a new key for the P2P network

* Mention every account on-chain before they publish a transaction

`CheckNonce` required accounts have a provider in order to even have their
nonce considered. This shims that by claiming every account has a provider at
the start of a block, if it signs a transaction.

The actual execution could presumably diverge between block building (which
sets the provider before each transaction) and execution (which sets the
providers at the start of the block). It doesn't diverge in our current
configuration and it won't be propagated to `next` (which doesn't use
`CheckNonce`).

Also uses explicit indexes for the `serai_abi::{Call, Event}` `enum`s.

* Adopt `patch-polkadot-sdk` with fixed peering

* Manually insert the authority discovery key into the keystore

I did try pulling in `pallet-authority-discovery` for this, updating
`SessionKeys`, but that was insufficient for whatever reason.

* Update to latest `substrate-wasm-builder`

* Fix timeline for incrementing providers

e1671dd71b incremented the providers for every
single transaction's sender before execution, noting the solution was fragile
but it worked for us at this time. It did not work for us at this time.

The new solution replaces `inc_providers` with direct access to the `Account`
`StorageMap` to increment the providers, achieving the desired goal, _without_
emitting an event (which is ordered, and the disparate order between building
and execution was causing mismatches of the state root).

This solution is also fragile and may also be insufficient. None of this code
exists anymore on `next` however. It just has to work sufficiently for now.

* clippy

substrate/validator-sets/pallet/Cargo.toml

@@ -13,7 +13,7 @@ all-features = true
 rustdoc-args = ["--cfg", "docsrs"]
 
 [package.metadata.cargo-machete]
-ignored = ["scale", "scale-info"]
+ignored = ["scale"]
 
 [lints]
 workspace = true
@@ -22,21 +22,21 @@ workspace = true
 hashbrown = { version = "0.14", default-features = false, features = ["ahash", "inline-more"] }
 
 scale = { package = "parity-scale-codec", version = "3", default-features = false, features = ["derive"] }
-scale-info = { version = "2", default-features = false, features = ["derive"] }
 
-sp-core = { git = "https://github.com/serai-dex/substrate", default-features = false }
-sp-io = { git = "https://github.com/serai-dex/substrate", default-features = false }
-sp-std = { git = "https://github.com/serai-dex/substrate", default-features = false }
-sp-application-crypto = { git = "https://github.com/serai-dex/substrate", default-features = false }
-sp-runtime = { git = "https://github.com/serai-dex/substrate", default-features = false }
-sp-session = { git = "https://github.com/serai-dex/substrate", default-features = false }
-sp-staking = { git = "https://github.com/serai-dex/substrate", default-features = false }
+sp-core = { git = "https://github.com/serai-dex/patch-polkadot-sdk", rev = "da19e1f8ca7a9e2cbf39fbfa493918eeeb45e10b", default-features = false }
+sp-io = { git = "https://github.com/serai-dex/patch-polkadot-sdk", rev = "da19e1f8ca7a9e2cbf39fbfa493918eeeb45e10b", default-features = false }
+sp-std = { git = "https://github.com/serai-dex/patch-polkadot-sdk", rev = "da19e1f8ca7a9e2cbf39fbfa493918eeeb45e10b", default-features = false }
+sp-application-crypto = { git = "https://github.com/serai-dex/patch-polkadot-sdk", rev = "da19e1f8ca7a9e2cbf39fbfa493918eeeb45e10b", default-features = false }
+sp-runtime = { git = "https://github.com/serai-dex/patch-polkadot-sdk", rev = "da19e1f8ca7a9e2cbf39fbfa493918eeeb45e10b", default-features = false }
+sp-session = { git = "https://github.com/serai-dex/patch-polkadot-sdk", rev = "da19e1f8ca7a9e2cbf39fbfa493918eeeb45e10b", default-features = false }
+sp-staking = { git = "https://github.com/serai-dex/patch-polkadot-sdk", rev = "da19e1f8ca7a9e2cbf39fbfa493918eeeb45e10b", default-features = false }
 
-frame-system = { git = "https://github.com/serai-dex/substrate", default-features = false }
-frame-support = { git = "https://github.com/serai-dex/substrate", default-features = false }
+frame-system = { git = "https://github.com/serai-dex/patch-polkadot-sdk", rev = "da19e1f8ca7a9e2cbf39fbfa493918eeeb45e10b", default-features = false }
+frame-support = { git = "https://github.com/serai-dex/patch-polkadot-sdk", rev = "da19e1f8ca7a9e2cbf39fbfa493918eeeb45e10b", default-features = false }
 
-pallet-babe = { git = "https://github.com/serai-dex/substrate", default-features = false }
-pallet-grandpa = { git = "https://github.com/serai-dex/substrate", default-features = false }
+pallet-session = { git = "https://github.com/serai-dex/patch-polkadot-sdk", rev = "da19e1f8ca7a9e2cbf39fbfa493918eeeb45e10b", default-features = false }
+pallet-babe = { git = "https://github.com/serai-dex/patch-polkadot-sdk", rev = "da19e1f8ca7a9e2cbf39fbfa493918eeeb45e10b", default-features = false }
+pallet-grandpa = { git = "https://github.com/serai-dex/patch-polkadot-sdk", rev = "da19e1f8ca7a9e2cbf39fbfa493918eeeb45e10b", default-features = false }
 
 serai-primitives = { path = "../../primitives", default-features = false }
 validator-sets-primitives = { package = "serai-validator-sets-primitives", path = "../primitives", default-features = false }
@@ -47,19 +47,19 @@ dex-pallet = { package = "serai-dex-pallet", path = "../../dex/pallet", default-
 [features]
 std = [
   "scale/std",
-  "scale-info/std",
 
   "sp-core/std",
   "sp-io/std",
   "sp-std/std",
   "sp-application-crypto/std",
-  "sp-runtime/std",
   "sp-session/std",
+  "sp-runtime/std",
   "sp-staking/std",
 
   "frame-system/std",
   "frame-support/std",
 
+  "pallet-session/std",
   "pallet-babe/std",
   "pallet-grandpa/std",
 

substrate/validator-sets/pallet/src/lib.rs

@@ -2,13 +2,13 @@
 
 use core::marker::PhantomData;
 
-use scale::{Encode, Decode};
-use scale_info::TypeInfo;
+use scale::{Encode, Decode, DecodeWithMemTracking};
 
 use sp_std::{vec, vec::Vec};
 use sp_core::sr25519::{Public, Signature};
 use sp_application_crypto::RuntimePublic;
-use sp_session::{ShouldEndSession, GetSessionNumber, GetValidatorCount};
+use sp_session::{GetSessionNumber, GetValidatorCount};
+use pallet_session::ShouldEndSession;
 use sp_runtime::{KeyTypeId, ConsensusEngineId, traits::IsMember};
 use sp_staking::offence::{ReportOffence, Offence, OffenceError};
 
@@ -16,7 +16,7 @@ use frame_system::{pallet_prelude::*, RawOrigin};
 use frame_support::{
   pallet_prelude::*,
   sp_runtime::SaturatedConversion,
-  traits::{DisabledValidators, KeyOwnerProofSystem, FindAuthor},
+  traits::{DisabledValidators, KeyOwnerProofSystem, FindAuthor, OneSessionHandler},
   BoundedVec, WeakBoundedVec, StoragePrefixedMap,
 };
 
@@ -35,7 +35,7 @@ use pallet_grandpa::{
   EquivocationOffence as GrandpaEquivocationOffence,
 };
 
-#[derive(Debug, Encode, Decode, TypeInfo, PartialEq, Eq, Clone)]
+#[derive(Debug, Encode, Decode, DecodeWithMemTracking, PartialEq, Eq, Clone)]
 pub struct MembershipProof<T: pallet::Config>(pub Public, pub PhantomData<T>);
 impl<T: pallet::Config> GetSessionNumber for MembershipProof<T> {
   fn session(&self) -> u32 {
@@ -74,17 +74,15 @@ pub mod pallet {
     frame_system::Config<AccountId = Public>
     + coins_pallet::Config
     + dex_pallet::Config
+    + pallet_session::Config
     + pallet_babe::Config
     + pallet_grandpa::Config
-    + TypeInfo
   {
-    type RuntimeEvent: IsType<<Self as frame_system::Config>::RuntimeEvent> + From<Event<Self>>;
-
     type ShouldEndSession: ShouldEndSession<BlockNumberFor<Self>>;
   }
 
   #[pallet::genesis_config]
-  #[derive(Clone, PartialEq, Eq, Debug, Encode, Decode)]
+  #[derive(Clone, Debug)]
   pub struct GenesisConfig<T: Config> {
     /// Networks to spawn Serai with, and the stake requirement per key share.
     ///
@@ -242,7 +240,8 @@ pub mod pallet {
       Amount(u64::from_be_bytes(raw))
     }
     fn recover_key_from_sorted_allocation_key(key: &[u8]) -> Public {
-      Public(key[(key.len() - 32) ..].try_into().unwrap())
+      let key: [u8; 32] = key[(key.len() - 32) ..].try_into().unwrap();
+      key.into()
     }
     // Returns if this validator already had an allocation set.
     fn set_allocation(network: NetworkId, key: Public, amount: Amount) -> bool {
@@ -810,10 +809,13 @@ pub mod pallet {
         ),
         Some(session),
       );
-      Grandpa::<T>::new_session(
+      fn grandpa_map(i: &(Public, u64)) -> (&Public, GrandpaAuthorityId) {
+        (&i.0, i.0.into())
+      }
+      Grandpa::<T>::on_new_session(
         true,
-        session,
-        now_validators.into_iter().map(|(id, w)| (GrandpaAuthorityId::from(id), w)).collect(),
+        now_validators.iter().map(grandpa_map),
+        next_validators.iter().map(grandpa_map),
       );
 
       // Clear SeraiDisabledIndices, only preserving keys still present in the new session
@@ -1292,6 +1294,17 @@ pub mod pallet {
     fn is_disabled(index: u32) -> bool {
       SeraiDisabledIndices::<T>::get(index).is_some()
     }
+    fn disabled_validators() -> Vec<u32> {
+      // TODO: Use a storage iterator here
+      let mut res = vec![];
+      for i in 0 .. MAX_KEY_SHARES_PER_SET {
+        let i = i.into();
+        if Self::is_disabled(i) {
+          res.push(i);
+        }
+      }
+      res
+    }
   }
 }
 

substrate/validator-sets/primitives/Cargo.toml

@@ -26,15 +26,14 @@ borsh = { version = "1", default-features = false, features = ["derive", "de_str
 serde = { version = "1", default-features = false, features = ["derive", "alloc"], optional = true }
 
 scale = { package = "parity-scale-codec", version = "3", default-features = false, features = ["derive", "max-encoded-len"] }
-scale-info = { version = "2", default-features = false, features = ["derive"] }
 
-sp-core = { git = "https://github.com/serai-dex/substrate", default-features = false }
-sp-std = { git = "https://github.com/serai-dex/substrate", default-features = false }
+sp-core = { git = "https://github.com/serai-dex/patch-polkadot-sdk", rev = "da19e1f8ca7a9e2cbf39fbfa493918eeeb45e10b", default-features = false }
+sp-std = { git = "https://github.com/serai-dex/patch-polkadot-sdk", rev = "da19e1f8ca7a9e2cbf39fbfa493918eeeb45e10b", default-features = false }
 
 serai-primitives = { path = "../../primitives", default-features = false }
 
 [features]
-std = ["zeroize", "ciphersuite/std", "dkg-musig/std", "borsh?/std", "serde?/std", "scale/std", "scale-info/std", "sp-core/std", "sp-std/std", "serai-primitives/std"]
+std = ["zeroize", "ciphersuite/std", "dkg-musig/std", "borsh?/std", "serde?/std", "scale/std", "sp-core/std", "sp-std/std", "serai-primitives/std"]
 borsh = ["dep:borsh", "serai-primitives/borsh"]
 serde = ["dep:serde", "serai-primitives/serde"]
 default = ["std"]

substrate/validator-sets/primitives/src/lib.rs

@@ -6,8 +6,7 @@ use zeroize::Zeroize;
 use dalek_ff_group::Ristretto;
 use ciphersuite::{group::GroupEncoding, Ciphersuite};
 
-use scale::{Encode, Decode, MaxEncodedLen};
-use scale_info::TypeInfo;
+use scale::{Encode, Decode, DecodeWithMemTracking, MaxEncodedLen};
 
 #[cfg(feature = "borsh")]
 use borsh::{BorshSerialize, BorshDeserialize};
@@ -27,7 +26,17 @@ pub const MAX_KEY_LEN: u32 = 96;
 
 /// The type used to identify a specific session of validators.
 #[derive(
-  Clone, Copy, PartialEq, Eq, Hash, Default, Debug, Encode, Decode, TypeInfo, MaxEncodedLen,
+  Clone,
+  Copy,
+  PartialEq,
+  Eq,
+  Hash,
+  Default,
+  Debug,
+  Encode,
+  Decode,
+  DecodeWithMemTracking,
+  MaxEncodedLen,
 )]
 #[cfg_attr(feature = "std", derive(Zeroize))]
 #[cfg_attr(feature = "borsh", derive(BorshSerialize, BorshDeserialize))]
@@ -35,7 +44,9 @@ pub const MAX_KEY_LEN: u32 = 96;
 pub struct Session(pub u32);
 
 /// The type used to identify a specific validator set during a specific session.
-#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Encode, Decode, TypeInfo, MaxEncodedLen)]
+#[derive(
+  Clone, Copy, PartialEq, Eq, Hash, Debug, Encode, Decode, DecodeWithMemTracking, MaxEncodedLen,
+)]
 #[cfg_attr(feature = "std", derive(Zeroize))]
 #[cfg_attr(feature = "borsh", derive(BorshSerialize, BorshDeserialize))]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
@@ -45,7 +56,9 @@ pub struct ValidatorSet {
 }
 
 /// The type used to identify a specific validator set during a specific session.
-#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Encode, Decode, TypeInfo, MaxEncodedLen)]
+#[derive(
+  Clone, Copy, PartialEq, Eq, Hash, Debug, Encode, Decode, DecodeWithMemTracking, MaxEncodedLen,
+)]
 #[cfg_attr(feature = "std", derive(Zeroize))]
 #[cfg_attr(feature = "borsh", derive(BorshSerialize, BorshDeserialize))]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
@@ -78,7 +91,7 @@ pub type ExternalKey = BoundedVec<u8, MaxKeyLen>;
 /// The key pair for a validator set.
 ///
 /// This is their Ristretto key, used for signing Batches, and their key on the external network.
-#[derive(Clone, PartialEq, Eq, Debug, Encode, Decode, TypeInfo, MaxEncodedLen)]
+#[derive(Clone, PartialEq, Eq, Debug, Encode, Decode, DecodeWithMemTracking, MaxEncodedLen)]
 #[cfg_attr(feature = "borsh", derive(BorshSerialize, BorshDeserialize))]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 pub struct KeyPair(
@@ -128,7 +141,7 @@ pub fn musig_key(set: ValidatorSet, set_keys: &[Public]) -> Public {
         .expect("invalid participant"),
     );
   }
-  Public(dkg_musig::musig_key_vartime::<Ristretto>(musig_context(set), &keys).unwrap().to_bytes())
+  dkg_musig::musig_key_vartime::<Ristretto>(musig_context(set), &keys).unwrap().to_bytes().into()
 }
 
 /// The message for the set_keys signature.