Add test for the integrity of headers

substrate/client/serai/Cargo.toml

@@ -29,6 +29,8 @@ serai-abi = { path = "../../abi", version = "0.1" }
 async-lock = "3"
 
 [dev-dependencies]
+blake2 = { version = "0.11.0-rc.3", default-features = false }
+
 tokio = { version = "1", default-features = false, features = ["rt", "macros"] }
 dockertest = "0.5"
 serai-docker-tests = { path = "../../../tests/docker" }

substrate/client/serai/src/lib.rs

@@ -59,7 +59,7 @@ pub struct Serai {
 pub struct TemporalSerai<'a> {
   serai: &'a Serai,
   block: BlockHash,
-  events: Arc<RwLock<Option<Vec<Event>>>>,
+  events: Arc<RwLock<Option<Vec<Vec<Event>>>>>,
 }
 
 impl Serai {
@@ -195,7 +195,9 @@ impl<'a> TemporalSerai<'a> {
   /// Fetch the events for this block.
   ///
   /// The returned `Option` will always be `Some(_)`.
-  async fn events(&self) -> Result<async_lock::RwLockReadGuard<'_, Option<Vec<Event>>>, RpcError> {
+  async fn events_borrowed(
+    &self,
+  ) -> Result<async_lock::RwLockReadGuard<'_, Option<Vec<Vec<Event>>>>, RpcError> {
     let mut events = self.events.read().await;
     if events.is_none() {
       drop(events);
@@ -204,8 +206,11 @@ impl<'a> TemporalSerai<'a> {
         if events_mut.is_none() {
           *events_mut = Some(
             self
-              .call::<Vec<String>>("blockchain/events", "")
+              .call::<Vec<Vec<String>>>("blockchain/events", "")
               .await?
+              .into_iter()
+              .map(|events_per_tx| {
+                events_per_tx
                   .into_iter()
                   .map(|event| {
                     Event::deserialize(
@@ -217,7 +222,9 @@ impl<'a> TemporalSerai<'a> {
                     )
                     .map_err(|_| RpcError::InvalidNode("node returned invalid event".to_string()))
                   })
-              .collect::<Result<_, _>>()?,
+                  .collect::<Result<Vec<_>, _>>()
+              })
+              .collect::<Result<Vec<_>, _>>()?,
           );
         }
       }
@@ -226,6 +233,14 @@ impl<'a> TemporalSerai<'a> {
     Ok(events)
   }
 
+  /// Fetch the events for this block.
+  ///
+  /// These will be grouped by the transactions which emitted them, including the inherent
+  /// transactions at the start and end of every block.
+  pub async fn events(&self) -> Result<Vec<Vec<Event>>, RpcError> {
+    Ok(self.events_borrowed().await?.clone().expect("`TemporalSerai::events` returned None"))
+  }
+
   /// Scope to the validator sets module.
   pub fn validator_sets(&self) -> ValidatorSets<'_> {
     ValidatorSets(self)

substrate/client/serai/src/validator_sets.rs

@@ -32,11 +32,12 @@ impl<'a> ValidatorSets<'a> {
     Ok(
       self
         .0
-        .events()
+        .events_borrowed()
         .await?
         .as_ref()
         .expect("`TemporalSerai::events` returned None")
         .iter()
+        .flat_map(IntoIterator::into_iter)
         .filter_map(|event| match event {
           serai_abi::Event::ValidatorSets(event) => Some(event.clone()),
           _ => None,
@@ -127,11 +128,7 @@ impl<'a> ValidatorSets<'a> {
       .0
       .call::<Option<String>>(
         "validator-sets/keys",
-        &format!(
+        &format!(r#", "network": {}, "session": {} "#, rpc_network(set.network)?, set.session.0),
-          r#", "network": {}, "session": {} "#,
-          rpc_network(set.network)?,
-          set.session.0
-        ),
       )
       .await?
     else {

substrate/client/serai/tests/blockchain.rs

@@ -1,3 +1,14 @@
+use std::collections::HashSet;
+
+use blake2::{Digest, Blake2b256};
+
+use serai_abi::{
+  primitives::merkle::UnbalancedMerkleTree, BLOCK_HEADER_LEAF_TAG, BLOCK_HEADER_BRANCH_TAG,
+  TRANSACTION_COMMITMENT_LEAF_TAG, TRANSACTION_COMMITMENT_BRANCH_TAG,
+  TRANSACTION_EVENTS_COMMITMENT_LEAF_TAG, TRANSACTION_EVENTS_COMMITMENT_BRANCH_TAG,
+  EVENTS_COMMITMENT_LEAF_TAG, EVENTS_COMMITMENT_BRANCH_TAG,
+};
+
 use serai_client_serai::*;
 
 #[tokio::test]
@@ -91,6 +102,99 @@ async fn blockchain() {
         test_finalized_block(next_finalized).await;
       }
 
+      // Check the blocks have the expected headers
+      {
+        let mut last_block_number = serai.latest_finalized_block_number().await.unwrap();
+        let mut observed_consensus_commitments = HashSet::new();
+        let mut tagged_block_hashes = vec![];
+        for i in 0 ..= last_block_number {
+          let block = serai.block_by_number(i).await.unwrap();
+
+          assert_eq!(block.header.number(), i);
+
+          {
+            assert_eq!(
+              UnbalancedMerkleTree::new(BLOCK_HEADER_BRANCH_TAG, tagged_block_hashes.clone()).root,
+              block.header.builds_upon().root,
+            );
+            tagged_block_hashes.push({
+              let mut tagged = vec![BLOCK_HEADER_LEAF_TAG];
+              tagged.extend(&block.header.hash().0);
+              Blake2b256::digest(tagged).into()
+            });
+          }
+
+          {
+            let mut start_transaction = [0; 32];
+            start_transaction[24 ..].copy_from_slice(&i.to_be_bytes());
+            let mut end_transaction = start_transaction;
+            end_transaction[.. 16].copy_from_slice(&[0xff; 16]);
+            let transactions_iter = core::iter::once(start_transaction)
+              .chain(block.transactions.iter().map(serai_abi::Transaction::hash))
+              .chain(core::iter::once(end_transaction));
+
+            let events = serai.as_of(block.header.hash()).await.unwrap().events().await.unwrap();
+            assert_eq!(events.len(), 2 + block.transactions.len());
+
+            let mut transaction_leaves = vec![];
+            let mut events_leaves = vec![];
+            for (transaction, events) in transactions_iter.zip(events) {
+              {
+                let mut tagged = vec![TRANSACTION_COMMITMENT_LEAF_TAG];
+                tagged.extend(&transaction);
+                transaction_leaves.push(Blake2b256::digest(tagged).into());
+              }
+              {
+                let events = UnbalancedMerkleTree::new(
+                  TRANSACTION_EVENTS_COMMITMENT_BRANCH_TAG,
+                  events
+                    .into_iter()
+                    .map(|event| {
+                      let mut tagged = vec![TRANSACTION_EVENTS_COMMITMENT_LEAF_TAG];
+                      tagged.extend(&borsh::to_vec(&event).unwrap());
+                      Blake2b256::digest(tagged).into()
+                    })
+                    .collect(),
+                )
+                .root;
+
+                let mut tagged = vec![EVENTS_COMMITMENT_LEAF_TAG];
+                tagged.extend(&transaction);
+                tagged.extend(&events);
+                events_leaves.push(Blake2b256::digest(tagged).into());
+              }
+            }
+            assert_eq!(
+              UnbalancedMerkleTree::new(TRANSACTION_COMMITMENT_BRANCH_TAG, transaction_leaves).root,
+              block.header.transactions_commitment().root
+            );
+            assert_eq!(
+              UnbalancedMerkleTree::new(EVENTS_COMMITMENT_BRANCH_TAG, events_leaves).root,
+              block.header.events_commitment().root
+            );
+          }
+
+          match block.header {
+            serai_abi::Header::V1(serai_abi::HeaderV1 {
+              unix_time_in_millis,
+              consensus_commitment,
+              ..
+            }) => {
+              if i == 0 {
+                assert_eq!(unix_time_in_millis, 0);
+              } else {
+                assert!(unix_time_in_millis != 0);
+              }
+
+              // We treat the `consensus_commitment` as opaque, but we do want to make sure it's set
+              // This check practically ensures it's being properly defined for each block
+              assert!(!observed_consensus_commitments.contains(&consensus_commitment));
+              observed_consensus_commitments.insert(consensus_commitment);
+            }
+          }
+        }
+      }
+
       println!("Finished `serai-client/blockchain` test");
     })
     .await;

substrate/coins/src/tests.rs

@@ -29,6 +29,7 @@ fn mint() {
     // test events
     let mint_events = Core::events()
       .iter()
+      .flat_map(IntoIterator::into_iter)
       .map(|event| borsh::from_slice::<serai_abi::Event>(event.as_slice()).unwrap())
       .filter_map(|event| {
         if let serai_abi::Event::Coins(e) = &event {
@@ -83,6 +84,7 @@ fn burn_with_instruction() {
 
     let burn_events = Core::events()
       .iter()
+      .flat_map(IntoIterator::into_iter)
       .map(|event| borsh::from_slice::<serai_abi::Event>(event.as_slice()).unwrap())
       .filter_map(|event| {
         if let serai_abi::Event::Coins(e) = &event {

substrate/core/src/lib.rs

@@ -14,7 +14,7 @@ pub use iumt::*;
 #[expect(clippy::cast_possible_truncation)]
 #[frame_support::pallet]
 pub mod pallet {
-  use alloc::vec::Vec;
+  use alloc::{vec::Vec, vec};
 
   use frame_support::{
     sp_runtime::traits::{Header, Block},
@@ -75,6 +75,8 @@ pub mod pallet {
   #[pallet::event]
   #[pallet::generate_deposit(pub(super) fn deposit_event)]
   pub enum Event<T: Config> {
+    /// A transaction begun.
+    BeginTransaction,
     /// An event from Serai.
     Event(Vec<u8>),
   }
@@ -130,6 +132,7 @@ pub mod pallet {
     /// The caller MUST ensure two transactions aren't simultaneously started.
     pub fn start_transaction() {
       TransactionEventsMerkle::<T>::new_expecting_none();
+      Self::deposit_event(Event::BeginTransaction);
     }
 
     /// Emit an event.
@@ -150,19 +153,24 @@ pub mod pallet {
       BlockEventsCommitmentMerkle::<T>::append(&(&transaction_hash, &transaction_events_root));
     }
 
-    /// Fetch all of Serai's events.
+    /// Fetch all of Serai's events for each transaction.
     ///
     /// This MUST NOT be called during a transaction/block's execution.
-    pub fn events() -> Vec<Vec<u8>>
+    pub fn events() -> Vec<Vec<Vec<u8>>>
     where
       T::RuntimeEvent: TryInto<Event<T>>,
     {
-      frame_system::Pallet::<T>::read_events_no_consensus()
+      let mut result = vec![];
-        .filter_map(|e| match e.event.try_into() {
+      for event in frame_system::Pallet::<T>::read_events_no_consensus() {
-          Ok(Event::Event(bytes)) => Some(bytes),
+        match event.event.try_into() {
-          _ => None,
+          Ok(Event::BeginTransaction) => result.push(vec![]),
-        })
+          Ok(Event::Event(bytes)) => {
-        .collect()
+            result.last_mut().expect("Serai event outside of a transaction").push(bytes)
+          }
+          Err(_) => {}
+        }
+      }
+      result
     }
   }
 }

substrate/node/src/rpc/blockchain.rs

@@ -66,7 +66,12 @@ pub(crate) fn module<
     let Ok(events) = client.runtime_api().events(block_hash) else {
       Err(Error::Missing("couldn't fetch the events for the requested block"))?
     };
-    Ok(events.into_iter().map(hex::encode).collect::<Vec<String>>())
+    Ok(
+      events
+        .into_iter()
+        .map(|events_per_tx| events_per_tx.into_iter().map(hex::encode).collect::<Vec<_>>())
+        .collect::<Vec<_>>(),
+    )
   })?;
 
   Ok(module)

substrate/runtime/src/lib.rs

@@ -34,7 +34,7 @@ sp_api::decl_runtime_apis! {
     fn build(genesis: GenesisConfig);
   }
   pub trait SeraiApi {
-    fn events() -> Vec<Vec<u8>>;
+    fn events() -> Vec<Vec<Vec<u8>>>;
     fn validators(network: NetworkId) -> Vec<Public>;
     fn current_session(network: NetworkId) -> Option<Session>;
     fn current_stake(network: NetworkId) -> Option<Amount>;
@@ -179,7 +179,7 @@ mod apis {
     }
 
     impl crate::SeraiApi<Block> for Runtime {
-      fn events() -> Vec<Vec<u8>> {
+      fn events() -> Vec<Vec<Vec<u8>>> {
         unimplemented!("runtime is only implemented when WASM")
       }
       fn validators(

substrate/runtime/src/wasm/mod.rs

@@ -550,7 +550,7 @@ sp_api::impl_runtime_apis! {
   }
 
   impl crate::SeraiApi<Block> for Runtime {
-    fn events() -> Vec<Vec<u8>> {
+    fn events() -> Vec<Vec<Vec<u8>>> {
       Core::events()
     }
     fn validators(network: NetworkId) -> Vec<serai_abi::primitives::crypto::Public> {