dockertest 0.4 (#406)

* Updates to modern dockertest

* More updates to latest dockertest

* Update Cargo.lock to dockertest with handle restored

* clippy coordinator tests

* clippy full-stack tests

* Remove kayabaNerve branch for official repo's latest commit hash

* Update serai-client, remove reliance on the existence of a handle fn

* Don't use the hex encoding of unique_id in dockertests

Gets our hostnames just below 64 bytes, resolving test failures on at least
Debian-based systems.

* Use Network::Isolated for all dockertest instances

* Correct error from prior commit's edits

tests/coordinator/Cargo.toml

@@ -36,6 +36,6 @@ serde_json = { version = "1", default-features = false }
 
 tokio = { version = "1", features = ["time"] }
 
-dockertest = "0.3"
+dockertest = "0.4"
 serai-docker-tests = { path = "../docker" }
 serai-message-queue-tests = { path = "../message-queue" }

tests/coordinator/src/lib.rs

@@ -18,8 +18,8 @@ use serai_message_queue::{Service, Metadata, client::MessageQueue};
 use serai_client::Serai;
 
 use dockertest::{
-  PullPolicy, Image, LogAction, LogPolicy, LogSource, LogOptions, StartPolicy, Composition,
-  DockerOperations,
+  PullPolicy, Image, LogAction, LogPolicy, LogSource, LogOptions, StartPolicy,
+  TestBodySpecification, DockerOperations,
 };
 
 #[cfg(test)]
@@ -30,13 +30,13 @@ static UNIQUE_ID: OnceLock<Mutex<u16>> = OnceLock::new();
 pub fn coordinator_instance(
   name: &str,
   message_queue_key: <Ristretto as Ciphersuite>::F,
-) -> Composition {
+) -> TestBodySpecification {
   serai_docker_tests::build("coordinator".to_string());
 
-  Composition::with_image(
+  TestBodySpecification::with_image(
     Image::with_repository("serai-dev-coordinator").pull_policy(PullPolicy::Never),
   )
-  .with_env(
+  .replace_env(
     [
       ("MESSAGE_QUEUE_KEY".to_string(), hex::encode(message_queue_key.to_repr())),
       ("DB_PATH".to_string(), "./coordinator-db".to_string()),
@@ -53,13 +53,13 @@ pub fn coordinator_instance(
   )
 }
 
-pub fn serai_composition(name: &str) -> Composition {
+pub fn serai_composition(name: &str) -> TestBodySpecification {
   serai_docker_tests::build("serai".to_string());
 
-  let mut composition = Composition::with_image(
+  TestBodySpecification::with_image(
     Image::with_repository("serai-dev-serai").pull_policy(PullPolicy::Never),
   )
-  .with_cmd(vec![
+  .replace_cmd(vec![
     "serai-node".to_string(),
     "--unsafe-rpc-external".to_string(),
     "--rpc-cors".to_string(),
@@ -67,13 +67,14 @@ pub fn serai_composition(name: &str) -> Composition {
     "--chain".to_string(),
     "local".to_string(),
     format!("--{}", name.to_lowercase()),
-  ]);
-  composition.publish_all_ports();
-  composition
+  ])
+  .set_publish_all_ports(true)
 }
 
 pub type Handles = (String, String, String);
-pub fn coordinator_stack(name: &str) -> (Handles, <Ristretto as Ciphersuite>::F, Vec<Composition>) {
+pub fn coordinator_stack(
+  name: &str,
+) -> (Handles, <Ristretto as Ciphersuite>::F, Vec<TestBodySpecification>) {
   let serai_composition = serai_composition(name);
 
   let (coord_key, message_queue_keys, message_queue_composition) =
@@ -87,7 +88,7 @@ pub fn coordinator_stack(name: &str) -> (Handles, <Ristretto as Ciphersuite>::F,
     let unique_id_mutex = UNIQUE_ID.get_or_init(|| Mutex::new(0));
     let mut unique_id_lock = unique_id_mutex.lock().unwrap();
     let first = *unique_id_lock == 0;
-    let unique_id = hex::encode(unique_id_lock.to_be_bytes());
+    let unique_id = *unique_id_lock;
     *unique_id_lock += 1;
     (first, unique_id)
   };
@@ -107,14 +108,16 @@ pub fn coordinator_stack(name: &str) -> (Handles, <Ristretto as Ciphersuite>::F,
 
   let mut compositions = vec![];
   let mut handles = vec![];
-  for composition in [serai_composition, message_queue_composition, coordinator_composition] {
-    let name = format!("{}-{}", composition.handle(), &unique_id);
+  for (name, composition) in [
+    ("serai_node", serai_composition),
+    ("message_queue", message_queue_composition),
+    ("coordinator", coordinator_composition),
+  ] {
+    let handle = format!("coordinator-{name}-{unique_id}");
 
     compositions.push(
-      composition
-        .with_start_policy(StartPolicy::Strict)
-        .with_container_name(name.clone())
-        .with_log_options(Some(LogOptions {
+      composition.set_start_policy(StartPolicy::Strict).set_handle(handle.clone()).set_log_options(
+        Some(LogOptions {
           action: if std::env::var("GITHUB_CI") == Ok("true".to_string()) {
             LogAction::Forward
           } else {
@@ -122,18 +125,19 @@ pub fn coordinator_stack(name: &str) -> (Handles, <Ristretto as Ciphersuite>::F,
           },
           policy: LogPolicy::Always,
           source: LogSource::Both,
-        })),
+        }),
+      ),
     );
 
-    handles.push(compositions.last().unwrap().handle());
+    handles.push(handle);
   }
 
   let coordinator_composition = compositions.last_mut().unwrap();
-  coordinator_composition.inject_container_name(handles.remove(0), "SERAI_HOSTNAME");
-  coordinator_composition.inject_container_name(handles.remove(0), "MESSAGE_QUEUE_RPC");
+  coordinator_composition.inject_container_name(handles[0].clone(), "SERAI_HOSTNAME");
+  coordinator_composition.inject_container_name(handles[1].clone(), "MESSAGE_QUEUE_RPC");
 
   (
-    (compositions[0].handle(), compositions[1].handle(), compositions[2].handle()),
+    (handles[0].clone(), handles[1].clone(), handles[2].clone()),
     message_queue_keys[&NetworkId::Bitcoin],
     compositions,
   )

tests/coordinator/src/tests/mod.rs

@@ -22,7 +22,7 @@ pub(crate) static ONE_AT_A_TIME: OnceLock<Mutex<()>> = OnceLock::new();
 
 pub(crate) fn new_test() -> (Vec<(Handles, <Ristretto as Ciphersuite>::F)>, DockerTest) {
   let mut coordinators = vec![];
-  let mut test = DockerTest::new();
+  let mut test = DockerTest::new().with_network(dockertest::Network::Isolated);
   for i in 0 .. COORDINATORS {
     let (handles, coord_key, compositions) = coordinator_stack(match i {
       0 => "Alice",
@@ -35,7 +35,7 @@ pub(crate) fn new_test() -> (Vec<(Handles, <Ristretto as Ciphersuite>::F)>, Dock
     });
     coordinators.push((handles, coord_key));
     for composition in compositions {
-      test.add_composition(composition);
+      test.provide_container(composition);
     }
   }
   (coordinators, test)

tests/full-stack/Cargo.toml

@@ -34,7 +34,7 @@ serai-client = { path = "../../substrate/client", features = ["serai"] }
 
 tokio = { version = "1", features = ["time"] }
 
-dockertest = "0.3"
+dockertest = "0.4"
 serai-docker-tests = { path = "../docker" }
 serai-message-queue-tests = { path = "../message-queue" }
 serai-processor-tests = { path = "../processor" }

tests/full-stack/src/lib.rs

@@ -7,7 +7,7 @@ use std::{
 use serai_client::{primitives::NetworkId, Serai};
 
 use dockertest::{
-  LogAction, LogPolicy, LogSource, LogOptions, StartPolicy, Composition, DockerOperations,
+  LogAction, LogPolicy, LogSource, LogOptions, StartPolicy, TestBodySpecification, DockerOperations,
 };
 
 #[cfg(test)]
@@ -31,7 +31,7 @@ pub struct Handles {
   serai: String,
 }
 
-pub fn full_stack(name: &str) -> (Handles, Vec<Composition>) {
+pub fn full_stack(name: &str) -> (Handles, Vec<TestBodySpecification>) {
   let (coord_key, message_queue_keys, message_queue_composition) = message_queue_instance();
 
   let (bitcoin_composition, bitcoin_port) = network_instance(NetworkId::Bitcoin);
@@ -51,7 +51,7 @@ pub fn full_stack(name: &str) -> (Handles, Vec<Composition>) {
     let unique_id_mutex = UNIQUE_ID.get_or_init(|| Mutex::new(0));
     let mut unique_id_lock = unique_id_mutex.lock().unwrap();
     let first = *unique_id_lock == 0;
-    let unique_id = hex::encode(unique_id_lock.to_be_bytes());
+    let unique_id = *unique_id_lock;
     *unique_id_lock += 1;
     (first, unique_id)
   };
@@ -71,22 +71,19 @@ pub fn full_stack(name: &str) -> (Handles, Vec<Composition>) {
 
   let mut compositions = vec![];
   let mut handles = vec![];
-  for composition in [
-    message_queue_composition,
-    bitcoin_composition,
-    bitcoin_processor_composition,
-    monero_composition,
-    monero_processor_composition,
-    coordinator_composition,
-    serai_composition,
+  for (name, composition) in [
+    ("message_queue", message_queue_composition),
+    ("bitcoin", bitcoin_composition),
+    ("bitcoin_processor", bitcoin_processor_composition),
+    ("monero", monero_composition),
+    ("monero_processor", monero_processor_composition),
+    ("coordinator", coordinator_composition),
+    ("serai", serai_composition),
   ] {
-    let name = format!("{}-{}", composition.handle(), &unique_id);
-
+    let handle = format!("full_stack-{name}-{unique_id}");
     compositions.push(
-      composition
-        .with_start_policy(StartPolicy::Strict)
-        .with_container_name(name.clone())
-        .with_log_options(Some(LogOptions {
+      composition.set_start_policy(StartPolicy::Strict).set_handle(handle.clone()).set_log_options(
+        Some(LogOptions {
           action: if std::env::var("GITHUB_CI") == Ok("true".to_string()) {
             LogAction::Forward
           } else {
@@ -94,19 +91,19 @@ pub fn full_stack(name: &str) -> (Handles, Vec<Composition>) {
           },
           policy: LogPolicy::Always,
           source: LogSource::Both,
-        })),
+        }),
+      ),
     );
-
-    handles.push(compositions.last().unwrap().handle());
+    handles.push(handle);
   }
   let handles = Handles {
-    message_queue: handles.remove(0),
-    bitcoin: (handles.remove(0), bitcoin_port),
-    bitcoin_processor: handles.remove(0),
-    monero: (handles.remove(0), monero_port),
-    monero_processor: handles.remove(0),
-    coordinator: handles.remove(0),
-    serai: handles.remove(0),
+    message_queue: handles[0].clone(),
+    bitcoin: (handles[1].clone(), bitcoin_port),
+    bitcoin_processor: handles[2].clone(),
+    monero: (handles[3].clone(), monero_port),
+    monero_processor: handles[4].clone(),
+    coordinator: handles[5].clone(),
+    serai: handles[6].clone(),
   };
 
   {

tests/full-stack/src/tests/mod.rs

@@ -13,7 +13,7 @@ pub(crate) static ONE_AT_A_TIME: OnceLock<Mutex<()>> = OnceLock::new();
 
 pub(crate) fn new_test() -> (Vec<Handles>, DockerTest) {
   let mut validators = vec![];
-  let mut test = DockerTest::new();
+  let mut test = DockerTest::new().with_network(dockertest::Network::Isolated);
   for i in 0 .. VALIDATORS {
     let (handles, compositions) = full_stack(match i {
       0 => "Alice",
@@ -26,7 +26,7 @@ pub(crate) fn new_test() -> (Vec<Handles>, DockerTest) {
     });
     validators.push(handles);
     for composition in compositions {
-      test.add_composition(composition);
+      test.provide_container(composition);
     }
   }
   (validators, test)

tests/message-queue/Cargo.toml

@@ -25,5 +25,5 @@ serai-primitives = { path = "../../substrate/primitives" }
 serai-message-queue = { path = "../../message-queue" }
 
 tokio = { version = "1", features = ["time"] }
-dockertest = "0.3"
+dockertest = "0.4"
 serai-docker-tests = { path = "../docker" }

tests/message-queue/src/lib.rs

@@ -9,11 +9,13 @@ use ciphersuite::{
 
 use serai_primitives::NetworkId;
 
-use dockertest::{PullPolicy, Image, LogAction, LogPolicy, LogSource, LogOptions, Composition};
+use dockertest::{
+  PullPolicy, Image, LogAction, LogPolicy, LogSource, LogOptions, TestBodySpecification,
+};
 
 pub type MessageQueuePrivateKey = <Ristretto as Ciphersuite>::F;
 pub fn instance(
-) -> (MessageQueuePrivateKey, HashMap<NetworkId, MessageQueuePrivateKey>, Composition) {
+) -> (MessageQueuePrivateKey, HashMap<NetworkId, MessageQueuePrivateKey>, TestBodySpecification) {
   serai_docker_tests::build("message-queue".to_string());
 
   let coord_key = <Ristretto as Ciphersuite>::F::random(&mut OsRng);
@@ -23,15 +25,15 @@ pub fn instance(
     (NetworkId::Monero, <Ristretto as Ciphersuite>::F::random(&mut OsRng)),
   ]);
 
-  let mut composition = Composition::with_image(
+  let composition = TestBodySpecification::with_image(
     Image::with_repository("serai-dev-message-queue").pull_policy(PullPolicy::Never),
   )
-  .with_log_options(Some(LogOptions {
+  .set_log_options(Some(LogOptions {
     action: LogAction::Forward,
     policy: LogPolicy::Always,
     source: LogSource::Both,
   }))
-  .with_env(
+  .replace_env(
     [
       ("COORDINATOR_KEY".to_string(), hex::encode((Ristretto::generator() * coord_key).to_bytes())),
       (
@@ -50,8 +52,8 @@ pub fn instance(
       ("RUST_LOG".to_string(), "serai_message_queue=trace,".to_string()),
     ]
     .into(),
-  );
-  composition.publish_all_ports();
+  )
+  .set_publish_all_ports(true);
 
   (coord_key, priv_keys, composition)
 }
@@ -64,9 +66,9 @@ fn basic_functionality() {
 
   use serai_message_queue::{Service, Metadata, client::MessageQueue};
 
-  let mut test = DockerTest::new();
+  let mut test = DockerTest::new().with_network(dockertest::Network::Isolated);
   let (coord_key, priv_keys, composition) = instance();
-  test.add_composition(composition);
+  test.provide_container(composition);
   test.run(|ops| async move {
     // Sleep for a second for the message-queue to boot
     // It isn't an error to start immediately, it just silences an error

tests/processor/Cargo.toml

@@ -39,6 +39,6 @@ tokio = { version = "1", features = ["time"] }
 
 processor = { package = "serai-processor", path = "../../processor", features = ["bitcoin", "monero"] }
 
-dockertest = "0.3"
+dockertest = "0.4"
 serai-docker-tests = { path = "../docker" }
 serai-message-queue-tests = { path = "../message-queue" }

tests/processor/src/lib.rs

@@ -12,8 +12,8 @@ use messages::{ProcessorMessage, CoordinatorMessage};
 use serai_message_queue::{Service, Metadata, client::MessageQueue};
 
 use dockertest::{
-  PullPolicy, Image, LogAction, LogPolicy, LogSource, LogOptions, StartPolicy, Composition,
-  DockerOperations,
+  PullPolicy, Image, LogAction, LogPolicy, LogSource, LogOptions, StartPolicy,
+  TestBodySpecification, DockerOperations,
 };
 
 mod networks;
@@ -28,16 +28,16 @@ pub fn processor_instance(
   network: NetworkId,
   port: u32,
   message_queue_key: <Ristretto as Ciphersuite>::F,
-) -> Composition {
+) -> TestBodySpecification {
   serai_docker_tests::build("processor".to_string());
 
   let mut entropy = [0; 32];
   OsRng.fill_bytes(&mut entropy);
 
-  Composition::with_image(
+  TestBodySpecification::with_image(
     Image::with_repository("serai-dev-processor").pull_policy(PullPolicy::Never),
   )
-  .with_env(
+  .replace_env(
     [
       ("MESSAGE_QUEUE_KEY".to_string(), hex::encode(message_queue_key.to_repr())),
       ("ENTROPY".to_string(), hex::encode(entropy)),
@@ -63,7 +63,7 @@ pub fn processor_instance(
 pub type Handles = (String, String, String);
 pub fn processor_stack(
   network: NetworkId,
-) -> (Handles, <Ristretto as Ciphersuite>::F, Vec<Composition>) {
+) -> (Handles, <Ristretto as Ciphersuite>::F, Vec<TestBodySpecification>) {
   let (network_composition, network_rpc_port) = network_instance(network);
 
   let (coord_key, message_queue_keys, message_queue_composition) =
@@ -77,37 +77,48 @@ pub fn processor_stack(
   let unique_id = {
     let unique_id_mutex = UNIQUE_ID.get_or_init(|| Mutex::new(0));
     let mut unique_id_lock = unique_id_mutex.lock().unwrap();
-    let unique_id = hex::encode(unique_id_lock.to_be_bytes());
+    let unique_id = *unique_id_lock;
     *unique_id_lock += 1;
     unique_id
   };
 
   let mut compositions = vec![];
   let mut handles = vec![];
-  for composition in [network_composition, message_queue_composition, processor_composition] {
-    let handle = composition.handle();
+  for (name, composition) in [
+    (
+      match network {
+        NetworkId::Serai => unreachable!(),
+        NetworkId::Bitcoin => "bitcoin",
+        NetworkId::Ethereum => "ethereum",
+        NetworkId::Monero => "monero",
+      },
+      network_composition,
+    ),
+    ("message_queue", message_queue_composition),
+    ("processor", processor_composition),
+  ] {
+    let handle = format!("processor-{name}-{unique_id}");
     compositions.push(
-      composition
-        .with_start_policy(StartPolicy::Strict)
-        .with_container_name(format!("{handle}-{}", &unique_id))
-        .with_log_options(Some(LogOptions {
+      composition.set_start_policy(StartPolicy::Strict).set_handle(handle.clone()).set_log_options(
+        Some(LogOptions {
           action: LogAction::Forward,
-          policy: if handle.contains("processor") { LogPolicy::Always } else { LogPolicy::OnError },
+          policy: if handle.contains("-processor-") {
+            LogPolicy::Always
+          } else {
+            LogPolicy::OnError
+          },
           source: LogSource::Both,
-        })),
+        }),
+      ),
     );
-    handles.push(compositions.last().unwrap().handle());
+    handles.push(handle);
   }
 
   let processor_composition = compositions.last_mut().unwrap();
-  processor_composition.inject_container_name(handles.remove(0), "NETWORK_RPC_HOSTNAME");
-  processor_composition.inject_container_name(handles.remove(0), "MESSAGE_QUEUE_RPC");
+  processor_composition.inject_container_name(handles[0].clone(), "NETWORK_RPC_HOSTNAME");
+  processor_composition.inject_container_name(handles[1].clone(), "MESSAGE_QUEUE_RPC");
 
-  (
-    (compositions[0].handle(), compositions[1].handle(), compositions[2].handle()),
-    coord_key,
-    compositions,
-  )
+  ((handles[0].clone(), handles[1].clone(), handles[2].clone()), coord_key, compositions)
 }
 
 #[derive(serde::Deserialize, Debug)]

tests/processor/src/networks.rs

@@ -11,7 +11,7 @@ use serai_client::{
   in_instructions::primitives::{InInstruction, RefundableInInstruction, Shorthand},
 };
 
-use dockertest::{PullPolicy, Image, StartPolicy, Composition, DockerOperations};
+use dockertest::{PullPolicy, Image, StartPolicy, TestBodySpecification, DockerOperations};
 
 use crate::*;
 
@@ -21,13 +21,13 @@ pub const RPC_PASS: &str = "seraidex";
 pub const BTC_PORT: u32 = 8332;
 pub const XMR_PORT: u32 = 18081;
 
-pub fn bitcoin_instance() -> (Composition, u32) {
+pub fn bitcoin_instance() -> (TestBodySpecification, u32) {
   serai_docker_tests::build("bitcoin".to_string());
 
-  let mut composition = Composition::with_image(
+  let composition = TestBodySpecification::with_image(
     Image::with_repository("serai-dev-bitcoin").pull_policy(PullPolicy::Never),
   )
-  .with_cmd(vec![
+  .replace_cmd(vec![
     "bitcoind".to_string(),
     "-txindex".to_string(),
     "-regtest".to_string(),
@@ -36,18 +36,18 @@ pub fn bitcoin_instance() -> (Composition, u32) {
     "-rpcbind=0.0.0.0".to_string(),
     "-rpcallowip=0.0.0.0/0".to_string(),
     "-rpcport=8332".to_string(),
-  ]);
-  composition.publish_all_ports();
+  ])
+  .set_publish_all_ports(true);
   (composition, BTC_PORT)
 }
 
-pub fn monero_instance() -> (Composition, u32) {
+pub fn monero_instance() -> (TestBodySpecification, u32) {
   serai_docker_tests::build("monero".to_string());
 
-  let mut composition = Composition::with_image(
+  let composition = TestBodySpecification::with_image(
     Image::with_repository("serai-dev-monero").pull_policy(PullPolicy::Never),
   )
-  .with_cmd(vec![
+  .replace_cmd(vec![
     "monerod".to_string(),
     "--regtest".to_string(),
     "--offline".to_string(),
@@ -58,12 +58,12 @@ pub fn monero_instance() -> (Composition, u32) {
     "--confirm-external-bind".to_string(),
     "--non-interactive".to_string(),
   ])
-  .with_start_policy(StartPolicy::Strict);
-  composition.publish_all_ports();
+  .set_start_policy(StartPolicy::Strict)
+  .set_publish_all_ports(true);
   (composition, XMR_PORT)
 }
 
-pub fn network_instance(network: NetworkId) -> (Composition, u32) {
+pub fn network_instance(network: NetworkId) -> (TestBodySpecification, u32) {
   match network {
     NetworkId::Bitcoin => bitcoin_instance(),
     NetworkId::Ethereum => todo!(),

tests/processor/src/tests/mod.rs

@@ -19,12 +19,12 @@ pub(crate) const THRESHOLD: usize = ((COORDINATORS * 2) / 3) + 1;
 
 fn new_test(network: NetworkId) -> (Vec<(Handles, <Ristretto as Ciphersuite>::F)>, DockerTest) {
   let mut coordinators = vec![];
-  let mut test = DockerTest::new();
+  let mut test = DockerTest::new().with_network(dockertest::Network::Isolated);
   for _ in 0 .. COORDINATORS {
     let (handles, coord_key, compositions) = processor_stack(network);
     coordinators.push((handles, coord_key));
     for composition in compositions {
-      test.add_composition(composition);
+      test.provide_container(composition);
     }
   }
   (coordinators, test)

tests/reproducible-runtime/Cargo.toml

@@ -17,7 +17,7 @@ rustdoc-args = ["--cfg", "docsrs"]
 rand_core = "0.6"
 hex = "0.4"
 
-dockertest = "0.3"
+dockertest = "0.4"
 serai-docker-tests = { path = "../docker" }
 
 tokio = { version = "1", features = ["time"] }

tests/reproducible-runtime/src/lib.rs

@@ -4,7 +4,7 @@ pub fn reproducibly_builds() {
 
   use rand_core::{RngCore, OsRng};
 
-  use dockertest::{PullPolicy, Image, Composition, DockerTest};
+  use dockertest::{PullPolicy, Image, TestBodySpecification, DockerTest};
 
   const RUNS: usize = 3;
   const TIMEOUT: u16 = 180 * 60; // 3 hours
@@ -16,14 +16,14 @@ pub fn reproducibly_builds() {
     OsRng.fill_bytes(id);
   }
 
-  let mut test = DockerTest::new();
+  let mut test = DockerTest::new().with_network(dockertest::Network::Isolated);
   for id in &ids {
-    test.add_composition(
-      Composition::with_image(
+    test.provide_container(
+      TestBodySpecification::with_image(
         Image::with_repository("serai-dev-runtime").pull_policy(PullPolicy::Never),
       )
-      .with_container_name(format!("runtime-build-{}", hex::encode(id)))
-      .with_cmd(vec![
+      .set_handle(format!("runtime-build-{}", hex::encode(id)))
+      .replace_cmd(vec![
         "sh".to_string(),
         "-c".to_string(),
         // Sleep for a minute after building to prevent the container from closing before we