Move in instructions from inherent transactions to unsigned transactions

The original intent was to use inherent transactions to prevent needing to vote
on-chain, which would spam the chain with worthless votes. Inherent
transactions, and our Tendermint library, would use the BFT's processs voting
to also vote on all included transactions. This perfectly collapses integrity
voting creating *no additional on-chain costs*.

Unfortunately, this led to issues such as #6, along with questions of validator
scalability when all validators are expencted to participate in consensus (in
order to vote on if the included instructions are valid). This has been
summarized in #241.

With this change, we can remove Tendermint from Substrate. This greatly
decreases our complexity. While I'm unhappy with the amount of time spent on
it, just to reach this conclusion, thankfully tendermint-machine itself is
still usable for #163. This also has reached a tipping point recently as the
polkadot-v0.9.40 branch of substrate changed how syncing works, requiring
further changes to sc-tendermint. These have no value if we're just going to
get rid of it later, due to fundamental design issues, yet I would like to
keep Substrate updated.

This should be followed by moving back to GRANDPA, enabling closing most open
Tendermint issues.

Please note the current in-instructions-pallet does not actually verify the
included signature yet. It's marked TODO, despite this bing critical.

substrate/serai/client/Cargo.toml

@@ -45,5 +45,3 @@ lazy_static = "1"
 rand_core = "0.6"
 
 tokio = "1"
-
-jsonrpsee-server = "0.16"

substrate/serai/client/src/serai/in_instructions.rs

@@ -1,10 +1,10 @@
 use serai_runtime::{in_instructions, InInstructions, Runtime};
 pub use in_instructions::primitives;
+use primitives::SignedBatch;
 
-use crate::{
-  primitives::{Coin, BlockNumber},
-  Serai, SeraiError, scale_value,
-};
+use subxt::{tx, utils::Encoded};
+
+use crate::{Serai, SeraiError, scale_composite};
 
 const PALLET: &str = "InInstructions";
 
@@ -22,16 +22,11 @@ impl Serai {
       .await
   }
 
-  pub async fn get_coin_block_number(
-    &self,
-    coin: Coin,
-    block: [u8; 32],
-  ) -> Result<BlockNumber, SeraiError> {
-    Ok(
-      self
-        .storage(PALLET, "BlockNumbers", Some(vec![scale_value(coin)]), block)
-        .await?
-        .unwrap_or(BlockNumber(0)),
-    )
+  pub fn execute_batch(&self, batch: SignedBatch) -> Result<Encoded, SeraiError> {
+    self.unsigned(&tx::dynamic(
+      PALLET,
+      "execute_batch",
+      scale_composite(in_instructions::Call::<Runtime>::execute_batch { batch }),
+    ))
   }
 }

substrate/serai/client/src/serai/mod.rs

@@ -6,6 +6,8 @@ pub(crate) use scale_value::{scale_value, scale_composite};
 use subxt::ext::scale_value::Value;
 
 use sp_core::{Pair as PairTrait, sr25519::Pair};
+
+pub use subxt;
 use subxt::{
   error::Error as SubxtError,
   utils::Encoded,
@@ -14,6 +16,7 @@ use subxt::{
     extrinsic_params::{BaseExtrinsicParams, BaseExtrinsicParamsBuilder},
   },
   tx::{Signer, DynamicTxPayload, TxClient},
+  rpc::types::ChainBlock,
   Config as SubxtConfig, OnlineClient,
 };
 
@@ -36,6 +39,8 @@ pub struct Tip {
   pub tip: u64,
 }
 
+pub type Header = SubstrateHeader<<Runtime as Config>::BlockNumber, BlakeTwo256>;
+
 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
 pub struct SeraiConfig;
 impl SubxtConfig for SeraiConfig {
@@ -47,12 +52,14 @@ impl SubxtConfig for SeraiConfig {
   // TODO: Bech32m
   type Address = SeraiAddress;
 
-  type Header = SubstrateHeader<<Runtime as Config>::BlockNumber, BlakeTwo256>;
+  type Header = Header;
   type Signature = Signature;
 
   type ExtrinsicParams = BaseExtrinsicParams<SeraiConfig, Tip>;
 }
 
+pub type Block = ChainBlock<SeraiConfig>;
+
 #[derive(Error, Debug)]
 pub enum SeraiError {
   #[error("failed to communicate with serai: {0}")]
@@ -116,6 +123,41 @@ impl Serai {
     Ok(self.0.rpc().finalized_head().await.map_err(SeraiError::RpcError)?.into())
   }
 
+  pub async fn get_block(&self, hash: [u8; 32]) -> Result<Option<Block>, SeraiError> {
+    let Some(res) =
+      self.0.rpc().block(Some(hash.into())).await.map_err(SeraiError::RpcError)? else {
+        return Ok(None);
+      };
+
+    // Only return finalized blocks
+    let Some(justifications) = res.justifications.as_ref() else { return Ok(None); };
+    if justifications.is_empty() {
+      return Ok(None);
+    }
+
+    Ok(Some(res.block))
+  }
+
+  // Ideally, this would be get_block_hash, not get_block_by_number
+  // Unfortunately, in order to only operate over only finalized data, we have to check the
+  // returned hash is for a finalized block. We can only do that by calling subxt's `block`, which
+  // will return the block and any justifications
+  // If we're already putting in all the work to get the block, we may as well just return it here
+  pub async fn get_block_by_number(&self, number: u64) -> Result<Option<Block>, SeraiError> {
+    let Some(hash) =
+      self.0.rpc().block_hash(Some(number.into())).await.map_err(SeraiError::RpcError)? else {
+        return Ok(None);
+      };
+    self.get_block(hash.into()).await
+  }
+
+  pub fn unsigned(&self, payload: &DynamicTxPayload<'static>) -> Result<Encoded, SeraiError> {
+    TxClient::new(self.0.offline())
+      .create_unsigned(payload)
+      .map(|tx| Encoded(tx.into_encoded()))
+      .map_err(|_| SeraiError::InvalidRuntime)
+  }
+
   pub fn sign<S: Send + Sync + Signer<SeraiConfig>>(
     &self,
     signer: &S,

substrate/serai/client/tests/batch.rs

@@ -0,0 +1,56 @@
+use rand_core::{RngCore, OsRng};
+
+use sp_core::sr25519::Signature;
+
+use serai_client::{
+  primitives::{BITCOIN_NET_ID, BITCOIN, BlockHash, SeraiAddress, Amount, Balance},
+  tokens::TokensEvent,
+  in_instructions::{
+    primitives::{InInstruction, InInstructionWithBalance, Batch, SignedBatch},
+    InInstructionsEvent,
+  },
+  Serai,
+};
+
+mod runner;
+use runner::{URL, provide_batch};
+
+serai_test!(
+  async fn publish_batch() {
+    let network = BITCOIN_NET_ID;
+    let id = 0;
+
+    let mut block_hash = BlockHash([0; 32]);
+    OsRng.fill_bytes(&mut block_hash.0);
+
+    let mut address = SeraiAddress::new([0; 32]);
+    OsRng.fill_bytes(&mut address.0);
+
+    let coin = BITCOIN;
+    let amount = Amount(OsRng.next_u64().saturating_add(1));
+    let balance = Balance { coin, amount };
+
+    let batch = Batch {
+      network,
+      id,
+      block: block_hash,
+      instructions: vec![InInstructionWithBalance {
+        instruction: InInstruction::Transfer(address),
+        balance,
+      }],
+    };
+    let signed = SignedBatch { batch, signature: Signature::from_raw([0; 64]) };
+    let block = provide_batch(signed).await;
+
+    let serai = Serai::new(URL).await.unwrap();
+    let batches = serai.get_batch_events(block).await.unwrap();
+    assert_eq!(batches, vec![InInstructionsEvent::Batch { network, id, block: block_hash }]);
+
+    assert_eq!(
+      serai.get_mint_events(block).await.unwrap(),
+      vec![TokensEvent::Mint { address, balance }],
+    );
+    assert_eq!(serai.get_token_supply(block, coin).await.unwrap(), amount);
+    assert_eq!(serai.get_token_balance(block, coin, address).await.unwrap(), amount);
+  }
+);

substrate/serai/client/tests/burn.rs

@@ -4,36 +4,65 @@ use rand_core::{RngCore, OsRng};
 
 use tokio::time::sleep;
 
-use sp_core::Pair;
+use sp_core::{sr25519::Signature, Pair};
 use subxt::{config::extrinsic_params::BaseExtrinsicParamsBuilder};
 
 use serai_client::{
   primitives::{
-    BITCOIN, BlockNumber, BlockHash, SeraiAddress, Amount, WithAmount, Balance, Data,
-    ExternalAddress, insecure_pair_from_name,
+    BITCOIN_NET_ID, BITCOIN, BlockHash, SeraiAddress, Amount, Balance, Data, ExternalAddress,
+    insecure_pair_from_name,
+  },
+  in_instructions::{
+    InInstructionsEvent,
+    primitives::{InInstruction, InInstructionWithBalance, Batch, SignedBatch},
   },
-  in_instructions::primitives::{InInstruction, Batch, Update},
   tokens::{primitives::OutInstruction, TokensEvent},
   PairSigner, Serai,
 };
 
 mod runner;
-use runner::{URL, provide_updates};
+use runner::{URL, provide_batch};
 
 serai_test!(
   async fn burn() {
-    let coin = BITCOIN;
-    let mut id = BlockHash([0; 32]);
-    OsRng.fill_bytes(&mut id.0);
-    let block_number = BlockNumber(OsRng.next_u64());
+    let network = BITCOIN_NET_ID;
+    let id = 0;
+
+    let mut block_hash = BlockHash([0; 32]);
+    OsRng.fill_bytes(&mut block_hash.0);
 
     let pair = insecure_pair_from_name("Alice");
     let public = pair.public();
     let address = SeraiAddress::from(public);
 
-    let amount = Amount(OsRng.next_u64());
+    let coin = BITCOIN;
+    let amount = Amount(OsRng.next_u64().saturating_add(1));
     let balance = Balance { coin, amount };
 
+    let batch = Batch {
+      network,
+      id,
+      block: block_hash,
+      instructions: vec![InInstructionWithBalance {
+        instruction: InInstruction::Transfer(address),
+        balance,
+      }],
+    };
+    let signed = SignedBatch { batch, signature: Signature::from_raw([0; 64]) };
+    let block = provide_batch(signed).await;
+
+    let serai = Serai::new(URL).await.unwrap();
+    let batches = serai.get_batch_events(block).await.unwrap();
+    assert_eq!(batches, vec![InInstructionsEvent::Batch { network, id, block: block_hash }]);
+
+    assert_eq!(
+      serai.get_mint_events(block).await.unwrap(),
+      vec![TokensEvent::Mint { address, balance }]
+    );
+    assert_eq!(serai.get_token_supply(block, coin).await.unwrap(), amount);
+    assert_eq!(serai.get_token_balance(block, coin, address).await.unwrap(), amount);
+
+    // Now burn it
     let mut rand_bytes = vec![0; 32];
     OsRng.fill_bytes(&mut rand_bytes);
     let external_address = ExternalAddress::new(rand_bytes).unwrap();
@@ -42,16 +71,6 @@ serai_test!(
     OsRng.fill_bytes(&mut rand_bytes);
     let data = Data::new(rand_bytes).unwrap();
 
-    let batch = Batch {
-      id,
-      instructions: vec![WithAmount { data: InInstruction::Transfer(address), amount }],
-    };
-    let update = Update { block_number, batches: vec![batch] };
-    let block = provide_updates(vec![Some(update)]).await;
-
-    let serai = Serai::new(URL).await.unwrap();
-    assert_eq!(serai.get_token_balance(block, coin, address).await.unwrap(), amount);
-
     let out = OutInstruction { address: external_address, data: Some(data) };
     let burn = Serai::burn(balance, out.clone());
 

substrate/serai/client/tests/runner.rs

@@ -1,18 +1,15 @@
 use core::time::Duration;
-use std::sync::Arc;
 
 use lazy_static::lazy_static;
 
 use tokio::{sync::Mutex, time::sleep};
 
 use serai_client::{
-  primitives::Coin,
-  in_instructions::{primitives::Updates, InInstructionsEvent},
+  subxt::config::Header,
+  in_instructions::{primitives::SignedBatch, InInstructionsEvent},
   Serai,
 };
 
-use jsonrpsee_server::RpcModule;
-
 pub const URL: &str = "ws://127.0.0.1:9944";
 
 lazy_static! {
@@ -20,73 +17,64 @@ lazy_static! {
 }
 
 #[allow(dead_code)]
-pub async fn provide_updates(updates: Updates) -> [u8; 32] {
-  let done = Arc::new(Mutex::new(false));
-  let done_clone = done.clone();
-  let updates_clone = updates.clone();
+pub async fn provide_batch(batch: SignedBatch) -> [u8; 32] {
+  let serai = Serai::new(URL).await.unwrap();
 
-  let mut rpc = RpcModule::new(());
-  rpc
-    .register_async_method("processor_coinUpdates", move |_, _| {
-      let done_clone = done_clone.clone();
-      let updates_clone = updates_clone.clone();
-      async move {
-        // Sleep to prevent a race condition where we submit the inherents for this block and the
-        // next one, then remove them, making them unverifiable, causing the node to panic for
-        // being self-malicious
-        sleep(Duration::from_millis(500)).await;
-        if !*done_clone.lock().await {
-          Ok(updates_clone)
-        } else {
-          Ok(vec![])
-        }
-      }
-    })
-    .unwrap();
-  let handle = jsonrpsee_server::ServerBuilder::default()
-    .build("127.0.0.1:5134")
+  let mut latest = serai
+    .get_block(serai.get_latest_block_hash().await.unwrap())
     .await
     .unwrap()
-    .start(rpc)
-    .unwrap();
+    .unwrap()
+    .header()
+    .number();
 
-  let serai = Serai::new(URL).await.unwrap();
-  loop {
-    let latest = serai.get_latest_block_hash().await.unwrap();
-    let mut batches = serai.get_batch_events(latest).await.unwrap();
-    if batches.is_empty() {
-      sleep(Duration::from_millis(50)).await;
-      continue;
-    }
-    *done.lock().await = true;
+  let execution = serai.execute_batch(batch.clone()).unwrap();
+  serai.publish(&execution).await.unwrap();
 
-    for (index, update) in updates.iter().enumerate() {
-      if let Some(update) = update {
-        let coin_by_index = Coin(u32::try_from(index).unwrap() + 1);
+  // Get the block it was included in
+  let mut block;
+  let mut ticks = 0;
+  'get_block: loop {
+    latest += 1;
 
-        for expected in &update.batches {
-          match batches.swap_remove(0) {
-            InInstructionsEvent::Batch { coin, id } => {
-              assert_eq!(coin, coin_by_index);
-              assert_eq!(expected.id, id);
-            }
-            _ => panic!("get_batches returned non-batch"),
-          }
+    block = {
+      let mut block;
+      while {
+        block = serai.get_block_by_number(latest).await.unwrap();
+        block.is_none()
+      } {
+        sleep(Duration::from_secs(1)).await;
+        ticks += 1;
+
+        if ticks > 60 {
+          panic!("60 seconds without inclusion in a finalized block");
         }
-        assert_eq!(
-          serai.get_coin_block_number(coin_by_index, latest).await.unwrap(),
-          update.block_number
-        );
+      }
+      block.unwrap()
+    };
+
+    for extrinsic in block.extrinsics {
+      if extrinsic.0 == execution.0[2 ..] {
+        break 'get_block;
       }
     }
-    // This will fail if there were more batch events than expected
-    assert!(batches.is_empty());
-
-    handle.stop().unwrap();
-    handle.stopped().await;
-
-    return latest;
   }
+  let block = block.header.hash().into();
+
+  let batches = serai.get_batch_events(block).await.unwrap();
+  // TODO: impl From<Batch> for BatchEvent?
+  assert_eq!(
+    batches,
+    vec![InInstructionsEvent::Batch {
+      network: batch.batch.network,
+      id: batch.batch.id,
+      block: batch.batch.block,
+    }],
+  );
+
+  // TODO: Check the tokens events
+
+  block
 }
 
 #[macro_export]

substrate/serai/client/tests/updates.rs

@@ -1,46 +0,0 @@
-use rand_core::{RngCore, OsRng};
-
-use serai_client::{
-  primitives::{BITCOIN, BlockNumber, BlockHash, SeraiAddress, Amount, WithAmount, Balance},
-  tokens::TokensEvent,
-  in_instructions::{
-    primitives::{InInstruction, Batch, Update},
-    InInstructionsEvent,
-  },
-  Serai,
-};
-
-mod runner;
-use runner::{URL, provide_updates};
-
-serai_test!(
-  async fn publish_updates() {
-    let coin = BITCOIN;
-    let mut id = BlockHash([0; 32]);
-    OsRng.fill_bytes(&mut id.0);
-    let block_number = BlockNumber(OsRng.next_u64());
-
-    let mut address = SeraiAddress::new([0; 32]);
-    OsRng.fill_bytes(&mut address.0);
-    let amount = Amount(OsRng.next_u64());
-
-    let batch = Batch {
-      id,
-      instructions: vec![WithAmount { data: InInstruction::Transfer(address), amount }],
-    };
-    let update = Update { block_number, batches: vec![batch] };
-    let block = provide_updates(vec![Some(update)]).await;
-
-    let serai = Serai::new(URL).await.unwrap();
-    let batches = serai.get_batch_events(block).await.unwrap();
-    assert_eq!(batches, vec![InInstructionsEvent::Batch { coin, id }]);
-    assert_eq!(serai.get_coin_block_number(coin, block).await.unwrap(), block_number);
-
-    assert_eq!(
-      serai.get_mint_events(block).await.unwrap(),
-      vec![TokensEvent::Mint { address, balance: Balance { coin, amount } }]
-    );
-    assert_eq!(serai.get_token_supply(block, coin).await.unwrap(), amount);
-    assert_eq!(serai.get_token_balance(block, coin, address).await.unwrap(), amount);
-  }
-);

substrate/serai/primitives/src/amount.rs

@@ -45,12 +45,3 @@ impl Mul for Amount {
     Amount(self.0.checked_mul(other.0).unwrap())
   }
 }
-
-#[derive(Clone, PartialEq, Eq, Debug, Encode, Decode, MaxEncodedLen, TypeInfo)]
-#[cfg_attr(feature = "std", derive(Zeroize, Serialize, Deserialize))]
-pub struct WithAmount<
-  T: Clone + PartialEq + Eq + Debug + Encode + Decode + MaxEncodedLen + TypeInfo,
-> {
-  pub data: T,
-  pub amount: Amount,
-}

substrate/serai/primitives/src/coins.rs

@@ -10,7 +10,7 @@ use sp_core::{ConstU32, bounded::BoundedVec};
 use serde::{Serialize, Deserialize};
 
 /// The type used to identify networks.
-#[derive(Clone, Copy, PartialEq, Eq, Debug, Encode, Decode, MaxEncodedLen, TypeInfo)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Encode, Decode, MaxEncodedLen, TypeInfo)]
 #[cfg_attr(feature = "std", derive(Zeroize, Serialize, Deserialize))]
 pub struct NetworkId(pub u16);
 impl From<u16> for NetworkId {
@@ -24,7 +24,7 @@ pub const ETHEREUM_NET_ID: NetworkId = NetworkId(1);
 pub const MONERO_NET_ID: NetworkId = NetworkId(2);
 
 /// The type used to identify coins.
-#[derive(Clone, Copy, PartialEq, Eq, Debug, Encode, Decode, MaxEncodedLen, TypeInfo)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Encode, Decode, MaxEncodedLen, TypeInfo)]
 #[cfg_attr(feature = "std", derive(Zeroize, Serialize, Deserialize))]
 pub struct Coin(pub u32);
 impl From<u32> for Coin {