Add a crate to test the runtime can be reproducibly built

tests/reproducible-runtime/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "serai-reproducible-runtime"
+version = "0.1.0"
+description = "Tests the Serai runtimee can be reproducibly built"
+license = "AGPL-3.0-only"
+repository = "https://github.com/serai-dex/serai/tree/develop/tests/reproducible-runtime"
+authors = ["Luke Parker <lukeparker5132@gmail.com>"]
+keywords = []
+edition = "2021"
+publish = false
+
+[package.metadata.docs.rs]
+all-features = true
+rustdoc-args = ["--cfg", "docsrs"]
+
+[dependencies]
+rand_core = "0.6"
+hex = "0.4"
+
+dockertest = "0.3"
+serai-docker-tests = { path = "../docker" }
+
+tokio = { version = "1", features = ["time"] }

tests/reproducible-runtime/LICENSE

@@ -0,0 +1,15 @@
+AGPL-3.0-only license
+
+Copyright (c) 2023 Luke Parker
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License Version 3 as
+published by the Free Software Foundation.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.

tests/reproducible-runtime/src/lib.rs

@@ -0,0 +1,100 @@
+#[test]
+pub fn reproducibly_builds() {
+  use std::{collections::HashSet, process::Command};
+
+  use rand_core::{RngCore, OsRng};
+
+  use dockertest::{PullPolicy, Image, Composition, DockerTest};
+
+  const RUNS: usize = 3;
+  const TIMEOUT: u16 = 60 * 60; // 60 minutes
+
+  serai_docker_tests::build("runtime".to_string());
+
+  let mut ids = vec![[0; 8]; RUNS];
+  for id in &mut ids {
+    OsRng.fill_bytes(id);
+  }
+
+  let mut test = DockerTest::new();
+  for id in &ids {
+    test.add_composition(
+      Composition::with_image(
+        Image::with_repository("serai-dev-runtime").pull_policy(PullPolicy::Never),
+      )
+      .with_container_name(format!("runtime-build-{}", hex::encode(id)))
+      .with_cmd(vec![
+        "sh".to_string(),
+        "-c".to_string(),
+        // Sleep for a minute after building to prevent the container from closing before we
+        // retrieve the hash
+        "cd /serai/substrate/runtime && cargo clean && cargo build --release &&
+           printf \"Runtime hash: \" > hash &&
+           sha256sum /serai/target/release/wbuild/serai-runtime/serai_runtime.wasm >> hash &&
+           cat hash &&
+           sleep 60"
+          .to_string(),
+      ]),
+    );
+  }
+
+  test.run(|_| async {
+    let ids = ids;
+    let mut containers = vec![];
+    for container in String::from_utf8(
+      Command::new("docker").arg("ps").arg("--format").arg("{{.Names}}").output().unwrap().stdout,
+    )
+    .expect("output wasn't utf-8")
+    .lines()
+    {
+      for id in &ids {
+        if container.contains(&hex::encode(id)) {
+          containers.push(container.trim().to_string());
+        }
+      }
+    }
+    assert_eq!(containers.len(), RUNS, "couldn't find all containers");
+
+    let mut res = vec![None; RUNS];
+    'attempt: for _ in 0 .. (TIMEOUT / 10) {
+      tokio::time::sleep(core::time::Duration::from_secs(10)).await;
+
+      'runner: for (i, container) in containers.iter().enumerate() {
+        if res[i].is_some() {
+          continue;
+        }
+
+        let logs = Command::new("docker").arg("logs").arg(container).output().unwrap();
+        let Some(last_log) =
+          std::str::from_utf8(&logs.stdout).expect("output wasn't utf-8").lines().last() else {
+            continue 'runner;
+          };
+
+        let split = last_log.split("Runtime hash: ").collect::<Vec<_>>();
+        if split.len() == 2 {
+          res[i] = Some(split[1].to_string());
+          continue 'runner;
+        }
+      }
+
+      for item in &res {
+        if item.is_none() {
+          continue 'attempt;
+        }
+      }
+      break;
+    }
+
+    // If we didn't get resuts from all runners, panic
+    for item in &res {
+      if item.is_none() {
+        panic!("couldn't get runtime hashes within allowed time");
+      }
+    }
+    let mut identical = HashSet::new();
+    for res in res.clone() {
+      identical.insert(res.unwrap());
+    }
+    assert_eq!(identical.len(), 1, "got different runtime hashes {:?}", res);
+  });
+}