Redo Dockerfile generation (#530)

Moves from concatted Dockerfiles to pseudo-templated Dockerfiles via a dedicated Rust program.

Removes the unmaintained kubernetes, not because we shouldn't have/use it, but because it's unmaintained and needs to be reworked before it's present again.

Replaces the compose with the work in the new orchestrator binary which spawns everything as expected. While this arguably re-invents the wheel, it correctly manages secrets and handles the variadic Dockerfiles.

Also adds an unrelated patch for zstd and simplifies running services a bit by greater utilizing the existing infrastructure.

---

* Delete all Dockerfile fragments, add new orchestator to generate Dockerfiles

Enables greater templating.

Also delete the unmaintained kubernetes folder *for now*. This should be
restored in the future.

* Use Dockerfiles from the orchestator

* Ignore Dockerfiles in the git repo

* Remove CI job to check Dockerfiles are as expected now that they're no longer committed

* Remove old Dockerfiles from repo

* Use Debian for monero-wallet-rpc

* Remove replace_cmds for proper usage of entry-dev

Consolidates ports a bit.

Updates serai-docker-tests from "compose" to "build".

* Only write a new dockerfile if it's distinct

Preserves the updated time metadata.

* Update serai-docker-tests

* Correct the path Dockerfiles are built from

* Correct inclusion of orchestration folder in Docker builds

* Correct debug/release flagging in the cargo command

Apparently, --debug isn't an effective NOP yet an error.

* Correct path used to run the Serai node within a Dockerfile

* Correct path in Monero Dockerfile

* Attempt storing monerod in /usr/bin

* Use sudo to move into /usr/bin in CI

* Correct 18.3.0 to 18.3.1

* Escape * with quotes

* Update deny.toml, ADD orchestration in runtime Dockerfile

* Add --detach to the Monero GH CI

* Diversify dockerfiles by network

* Fixes to network-diversified orchestration

* Bitcoin and Monero testnet scripts

* Permissions and tweaks

* Flatten scripts folders

* Add missing folder specification to Monero Dockerfile

* Have monero-wallet-rpc specify the monerod login

* Have the Docker CMD specify env variables inserted at time of Dockerfile generation

They're overrideable with the global enviornment as for tests. This enables
variable generation in orchestrator and output to productionized Docker files
without creating a life-long file within the Docker container.

* Don't add Dockerfiles into Docker containers now that they have secrets

Solely add the source code for them as needed to satisfy the workspace bounds.

* Download arm64 Monero on arm64

* Ensure constant host architecture when reproducibly building the wasm

Host architecture, for some reason, can effect the generated code despite the
target architecture always being foreign to the host architecture.

* Randomly generate infrastructure keys

* Have orchestrator generate a key, be able to create/start containers

* Ensure bash is used over sh

* Clean dated docs

* Change how quoting occurs

* Standardize to sh

* Have Docker test build the dev Dockerfiles

* Only key_gen once

* cargo update

Adds a patch for zstd and reconciles the breaking nightly change which just
occurred.

* Use a dedicated network for Serai

Also fixes SERAI_HOSTNAME passed to coordinator.

* Support providing a key over the env for the Serai node

* Enable and document running daemons for tests via serai-orchestrator

Has running containers under the dev network port forward the RPC ports.

* Use volumes for bitcoin/monero

* Use bitcoin's run.sh in GH CI

* Only use the volume for testnet (not dev)

orchestration/src/coins/bitcoin.rs

@@ -0,0 +1,56 @@
+use std::{path::Path};
+
+use crate::{Network, Os, mimalloc, os, write_dockerfile};
+
+pub fn bitcoin(orchestration_path: &Path, network: Network) {
+  #[rustfmt::skip]
+  const DOWNLOAD_BITCOIN: &str = r#"
+FROM alpine:latest as bitcoin
+
+ENV BITCOIN_VERSION=26.0
+
+RUN apk --no-cache add git gnupg
+
+# Download Bitcoin
+RUN wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/bitcoin-${BITCOIN_VERSION}-$(uname -m)-linux-gnu.tar.gz \
+  && wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/SHA256SUMS \
+  && wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/SHA256SUMS.asc
+
+# Verify all sigs and check for a valid signature from laanwj -- 71A3
+RUN git clone https://github.com/bitcoin-core/guix.sigs && \
+  cd guix.sigs/builder-keys && \
+  find . -iname '*.gpg' -exec gpg --import {} \; && \
+  gpg --verify --status-fd 1 --verify ../../SHA256SUMS.asc ../../SHA256SUMS | grep "^\[GNUPG:\] VALIDSIG.*71A3B16735405025D447E8F274810B012346C9A6"
+
+RUN grep bitcoin-${BITCOIN_VERSION}-$(uname -m)-linux-gnu.tar.gz SHA256SUMS | sha256sum -c
+
+# Prepare Image
+RUN tar xzvf bitcoin-${BITCOIN_VERSION}-$(uname -m)-linux-gnu.tar.gz
+RUN mv bitcoin-${BITCOIN_VERSION}/bin/bitcoind .
+"#;
+
+  let setup = mimalloc(Os::Debian).to_string() + DOWNLOAD_BITCOIN;
+
+  let run_bitcoin = format!(
+    r#"
+COPY --from=bitcoin --chown=bitcoin bitcoind /bin
+
+EXPOSE 8332 8333
+
+ADD /orchestration/{}/coins/bitcoin/run.sh /
+CMD ["/run.sh"]
+"#,
+    network.label()
+  );
+
+  let run =
+    os(Os::Debian, "RUN mkdir /volume && chown bitcoin:bitcoin /volume", "bitcoin") + &run_bitcoin;
+  let res = setup + &run;
+
+  let mut bitcoin_path = orchestration_path.to_path_buf();
+  bitcoin_path.push("coins");
+  bitcoin_path.push("bitcoin");
+  bitcoin_path.push("Dockerfile");
+
+  write_dockerfile(bitcoin_path, &res);
+}

orchestration/src/coins/ethereum.rs

@@ -0,0 +1,5 @@
+use std::path::Path;
+
+pub fn ethereum(_orchestration_path: &Path) {
+  // TODO
+}

orchestration/src/coins/mod.rs

@@ -0,0 +1,8 @@
+mod bitcoin;
+pub use bitcoin::*;
+
+mod ethereum;
+pub use ethereum::*;
+
+mod monero;
+pub use monero::*;

orchestration/src/coins/monero.rs

@@ -0,0 +1,87 @@
+use std::{path::Path};
+
+use crate::{Network, Os, mimalloc, write_dockerfile};
+
+fn monero_internal(
+  network: Network,
+  os: Os,
+  orchestration_path: &Path,
+  folder: &str,
+  monero_binary: &str,
+  ports: &str,
+) {
+  const MONERO_VERSION: &str = "0.18.3.1";
+
+  let arch = match std::env::consts::ARCH {
+    // We probably would run this without issues yet it's not worth needing to provide support for
+    "x86" | "arm" => panic!("unsupported architecture, please download a 64-bit OS"),
+    "x86_64" => "x64",
+    "aarch64" => "armv8",
+    _ => panic!("unsupported architecture"),
+  };
+
+  #[rustfmt::skip]
+  let download_monero = format!(r#"
+FROM alpine:latest as monero
+
+RUN apk --no-cache add gnupg
+
+# Download Monero
+RUN wget https://downloads.getmonero.org/cli/monero-linux-{arch}-v{MONERO_VERSION}.tar.bz2
+
+# Verify Binary -- fingerprint from https://github.com/monero-project/monero-site/issues/1949
+ADD orchestration/{}/coins/monero/hashes-v{MONERO_VERSION}.txt .
+RUN gpg --keyserver hkp://keyserver.ubuntu.com:80 --keyserver-options no-self-sigs-only --receive-keys 81AC591FE9C4B65C5806AFC3F0AF4D462A0BDF92 && \
+  gpg --verify hashes-v{MONERO_VERSION}.txt && \
+  grep "$(sha256sum monero-linux-{arch}-v{MONERO_VERSION}.tar.bz2 | cut -c 1-64)" hashes-v{MONERO_VERSION}.txt
+
+# Extract it
+RUN tar -xvjf monero-linux-{arch}-v{MONERO_VERSION}.tar.bz2 --strip-components=1
+"#,
+    network.label(),
+  );
+
+  let setup = mimalloc(os).to_string() + &download_monero;
+
+  let run_monero = format!(
+    r#"
+COPY --from=monero --chown=monero {monero_binary} /bin
+
+EXPOSE {ports}
+
+ADD /orchestration/{}/coins/{folder}/run.sh /
+CMD ["/run.sh"]
+"#,
+    network.label(),
+  );
+
+  let run = crate::os(
+    os,
+    &("RUN mkdir /volume && chown monero /volume\r\n".to_string() +
+      if os == Os::Alpine { "RUN apk --no-cache add gcompat" } else { "" }),
+    "monero",
+  ) + &run_monero;
+  let res = setup + &run;
+
+  let mut monero_path = orchestration_path.to_path_buf();
+  monero_path.push("coins");
+  monero_path.push(folder);
+  monero_path.push("Dockerfile");
+
+  write_dockerfile(monero_path, &res);
+}
+
+pub fn monero(orchestration_path: &Path, network: Network) {
+  monero_internal(network, Os::Alpine, orchestration_path, "monero", "monerod", "18080 18081")
+}
+
+pub fn monero_wallet_rpc(orchestration_path: &Path) {
+  monero_internal(
+    Network::Dev,
+    Os::Debian,
+    orchestration_path,
+    "monero-wallet-rpc",
+    "monero-wallet-rpc",
+    "18082",
+  )
+}

orchestration/src/coordinator.rs

@@ -0,0 +1,62 @@
+use std::{path::Path};
+
+use zeroize::Zeroizing;
+
+use ciphersuite::{group::ff::PrimeField, Ciphersuite, Ristretto};
+
+use crate::{Network, Os, mimalloc, os, build_serai_service, write_dockerfile};
+
+#[allow(clippy::needless_pass_by_value)]
+pub fn coordinator(
+  orchestration_path: &Path,
+  network: Network,
+  coordinator_key: Zeroizing<<Ristretto as Ciphersuite>::F>,
+  serai_key: Zeroizing<<Ristretto as Ciphersuite>::F>,
+) {
+  let db = network.db();
+  let longer_reattempts = if network == Network::Dev { "longer-reattempts" } else { "" };
+  let setup = mimalloc(Os::Debian).to_string() +
+    &build_serai_service(
+      network.release(),
+      &format!("{db} {longer_reattempts}"),
+      "serai-coordinator",
+    );
+
+  const ADDITIONAL_ROOT: &str = r#"
+# Install ca-certificates
+RUN apt install -y ca-certificates
+"#;
+
+  let env_vars = [
+    ("MESSAGE_QUEUE_RPC", format!("serai-{}-message-queue", network.label())),
+    ("MESSAGE_QUEUE_KEY", hex::encode(coordinator_key.to_repr())),
+    ("DB_PATH", "./coordinator-db".to_string()),
+    ("SERAI_KEY", hex::encode(serai_key.to_repr())),
+    ("SERAI_HOSTNAME", format!("serai-{}-serai", network.label())),
+    ("RUST_LOG", "serai_coordinator=debug,tributary_chain=debug,tendermint=debug".to_string()),
+  ];
+  let mut env_vars_str = String::new();
+  for (env_var, value) in env_vars {
+    env_vars_str += &format!(r#"{env_var}=${{{env_var}:="{value}"}} "#);
+  }
+
+  let run_coordinator = format!(
+    r#"
+# Copy the Coordinator binary and relevant license
+COPY --from=builder --chown=coordinator /serai/bin/serai-coordinator /bin/
+COPY --from=builder --chown=coordinator /serai/AGPL-3.0 .
+
+# Run coordinator
+CMD {env_vars_str} serai-coordinator
+"#
+  );
+
+  let run = os(Os::Debian, ADDITIONAL_ROOT, "coordinator") + &run_coordinator;
+  let res = setup + &run;
+
+  let mut coordinator_path = orchestration_path.to_path_buf();
+  coordinator_path.push("coordinator");
+  coordinator_path.push("Dockerfile");
+
+  write_dockerfile(coordinator_path, &res);
+}

orchestration/src/docker.rs

@@ -0,0 +1,47 @@
+use std::{collections::HashSet, path::Path, env, process::Command};
+
+use crate::Network;
+
+pub fn build(orchestration_path: &Path, network: Network, name: &str) {
+  let mut repo_path = env::current_exe().unwrap();
+  repo_path.pop();
+  if repo_path.as_path().ends_with("deps") {
+    repo_path.pop();
+  }
+  assert!(repo_path.as_path().ends_with("debug") || repo_path.as_path().ends_with("release"));
+  repo_path.pop();
+  assert!(repo_path.as_path().ends_with("target"));
+  repo_path.pop();
+
+  let mut dockerfile_path = orchestration_path.to_path_buf();
+  if HashSet::from(["bitcoin", "ethereum", "monero", "monero-wallet-rpc"]).contains(name) {
+    dockerfile_path = dockerfile_path.join("coins");
+  }
+  if name.contains("-processor") {
+    dockerfile_path =
+      dockerfile_path.join("processor").join(name.split('-').next().unwrap()).join("Dockerfile");
+  } else {
+    dockerfile_path = dockerfile_path.join(name).join("Dockerfile");
+  }
+
+  println!("Building {}...", &name);
+
+  if !Command::new("docker")
+    .current_dir(&repo_path)
+    .arg("build")
+    .arg("-f")
+    .arg(dockerfile_path)
+    .arg(".")
+    .arg("-t")
+    .arg(format!("serai-{}-{name}-img", network.label()))
+    .spawn()
+    .unwrap()
+    .wait()
+    .unwrap()
+    .success()
+  {
+    panic!("failed to build {name}");
+  }
+
+  println!("Built!");
+}

orchestration/src/main.rs

@@ -0,0 +1,445 @@
+// TODO: Generate randomized RPC credentials for all services
+// TODO: Generate keys for a validator and the infra
+
+use core::ops::Deref;
+use std::{collections::HashSet, env, path::PathBuf, io::Write, fs, process::Command};
+
+use zeroize::Zeroizing;
+
+use rand_core::{RngCore, SeedableRng, OsRng};
+use rand_chacha::ChaCha20Rng;
+
+use transcript::{Transcript, RecommendedTranscript};
+
+use ciphersuite::{
+  group::{
+    ff::{Field, PrimeField},
+    GroupEncoding,
+  },
+  Ciphersuite, Ristretto,
+};
+
+mod mimalloc;
+use mimalloc::mimalloc;
+
+mod coins;
+use coins::*;
+
+mod message_queue;
+use message_queue::message_queue;
+
+mod processor;
+use processor::processor;
+
+mod coordinator;
+use coordinator::coordinator;
+
+mod serai;
+use serai::serai;
+
+mod docker;
+
+#[global_allocator]
+static ALLOCATOR: zalloc::ZeroizingAlloc<std::alloc::System> =
+  zalloc::ZeroizingAlloc(std::alloc::System);
+
+#[derive(Clone, Copy, PartialEq, Eq, Debug, PartialOrd, Ord, Hash)]
+pub enum Network {
+  Dev,
+  Testnet,
+}
+
+impl Network {
+  pub fn db(&self) -> &'static str {
+    match self {
+      Network::Dev => "parity-db",
+      Network::Testnet => "rocksdb",
+    }
+  }
+
+  pub fn release(&self) -> bool {
+    match self {
+      Network::Dev => false,
+      Network::Testnet => true,
+    }
+  }
+
+  pub fn label(&self) -> &'static str {
+    match self {
+      Network::Dev => "dev",
+      Network::Testnet => "testnet",
+    }
+  }
+}
+
+#[derive(Clone, Copy, PartialEq, Eq, Debug, PartialOrd, Ord, Hash)]
+enum Os {
+  Alpine,
+  Debian,
+}
+
+fn os(os: Os, additional_root: &str, user: &str) -> String {
+  match os {
+    Os::Alpine => format!(
+      r#"
+FROM alpine:latest as image
+
+COPY --from=mimalloc-alpine libmimalloc.so /usr/lib
+ENV LD_PRELOAD=libmimalloc.so
+
+RUN apk update && apk upgrade
+
+# System user (not a human), shell of nologin, no password assigned
+RUN adduser -S -s /sbin/nologin -D {user}
+
+{additional_root}
+
+# Switch to a non-root user
+USER {user}
+
+WORKDIR /home/{user}
+"#
+    ),
+
+    Os::Debian => format!(
+      r#"
+FROM debian:bookworm-slim as image
+
+COPY --from=mimalloc-debian libmimalloc.so /usr/lib
+RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload
+
+RUN apt update && apt upgrade -y && apt autoremove -y && apt clean
+
+RUN useradd --system --create-home --shell /sbin/nologin {user}
+
+{additional_root}
+
+# Switch to a non-root user
+USER {user}
+
+WORKDIR /home/{user}
+"#
+    ),
+  }
+}
+
+fn build_serai_service(release: bool, features: &str, package: &str) -> String {
+  let profile = if release { "release" } else { "debug" };
+  let profile_flag = if release { "--release" } else { "" };
+
+  format!(
+    r#"
+FROM rust:1.75-slim-bookworm as builder
+
+COPY --from=mimalloc-debian libmimalloc.so /usr/lib
+RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload
+
+RUN apt update && apt upgrade -y && apt autoremove -y && apt clean
+
+# Add dev dependencies
+RUN apt install -y pkg-config clang
+
+# Dependencies for the Serai node
+RUN apt install -y make protobuf-compiler
+
+# Add the wasm toolchain
+RUN rustup target add wasm32-unknown-unknown
+
+# Add files for build
+ADD patches /serai/patches
+ADD common /serai/common
+ADD crypto /serai/crypto
+ADD coins /serai/coins
+ADD message-queue /serai/message-queue
+ADD processor /serai/processor
+ADD coordinator /serai/coordinator
+ADD substrate /serai/substrate
+ADD orchestration/Cargo.toml /serai/orchestration/Cargo.toml
+ADD orchestration/src /serai/orchestration/src
+ADD mini /serai/mini
+ADD tests /serai/tests
+ADD Cargo.toml /serai
+ADD Cargo.lock /serai
+ADD AGPL-3.0 /serai
+
+WORKDIR /serai
+
+# Mount the caches and build
+RUN --mount=type=cache,target=/root/.cargo \
+  --mount=type=cache,target=/usr/local/cargo/registry \
+  --mount=type=cache,target=/usr/local/cargo/git \
+  --mount=type=cache,target=/serai/target \
+  mkdir /serai/bin && \
+  cargo build {profile_flag} --features "{features}" -p {package} && \
+  mv /serai/target/{profile}/{package} /serai/bin
+"#
+  )
+}
+
+pub fn write_dockerfile(path: PathBuf, dockerfile: &str) {
+  if let Ok(existing) = fs::read_to_string(&path).as_ref() {
+    if existing == dockerfile {
+      return;
+    }
+  }
+  fs::File::create(path).unwrap().write_all(dockerfile.as_bytes()).unwrap();
+}
+
+fn orchestration_path(network: Network) -> PathBuf {
+  let mut repo_path = env::current_exe().unwrap();
+  repo_path.pop();
+  assert!(repo_path.as_path().ends_with("debug"));
+  repo_path.pop();
+  assert!(repo_path.as_path().ends_with("target"));
+  repo_path.pop();
+
+  let mut orchestration_path = repo_path.clone();
+  orchestration_path.push("orchestration");
+  orchestration_path.push(network.label());
+  orchestration_path
+}
+
+fn dockerfiles(network: Network) {
+  let orchestration_path = orchestration_path(network);
+
+  bitcoin(&orchestration_path, network);
+  ethereum(&orchestration_path);
+  monero(&orchestration_path, network);
+  if network == Network::Dev {
+    monero_wallet_rpc(&orchestration_path);
+  }
+
+  // TODO: Generate infra keys in key_gen, yet service entropy here?
+
+  // Generate entropy for the infrastructure keys
+  let mut entropy = Zeroizing::new([0; 32]);
+  // Only use actual entropy if this isn't a development environment
+  if network != Network::Dev {
+    OsRng.fill_bytes(entropy.as_mut());
+  }
+  let mut transcript = RecommendedTranscript::new(b"Serai Orchestrator Transcript");
+  transcript.append_message(b"entropy", entropy);
+  let mut new_rng = |label| ChaCha20Rng::from_seed(transcript.rng_seed(label));
+
+  let mut message_queue_keys_rng = new_rng(b"message_queue_keys");
+  let mut key_pair = || {
+    let key = Zeroizing::new(<Ristretto as Ciphersuite>::F::random(&mut message_queue_keys_rng));
+    let public = Ristretto::generator() * key.deref();
+    (key, public)
+  };
+  let coordinator_key = key_pair();
+  let bitcoin_key = key_pair();
+  let ethereum_key = key_pair();
+  let monero_key = key_pair();
+
+  message_queue(
+    &orchestration_path,
+    network,
+    coordinator_key.1,
+    bitcoin_key.1,
+    ethereum_key.1,
+    monero_key.1,
+  );
+
+  let mut processor_entropy_rng = new_rng(b"processor_entropy");
+  let mut new_entropy = || {
+    let mut res = Zeroizing::new([0; 32]);
+    processor_entropy_rng.fill_bytes(res.as_mut());
+    res
+  };
+  processor(
+    &orchestration_path,
+    network,
+    "bitcoin",
+    coordinator_key.1,
+    bitcoin_key.0,
+    new_entropy(),
+  );
+  processor(
+    &orchestration_path,
+    network,
+    "ethereum",
+    coordinator_key.1,
+    ethereum_key.0,
+    new_entropy(),
+  );
+  processor(&orchestration_path, network, "monero", coordinator_key.1, monero_key.0, new_entropy());
+
+  let serai_key = {
+    let serai_key = Zeroizing::new(
+      fs::read(home::home_dir().unwrap().join(".serai").join(network.label()).join("key"))
+        .expect("couldn't read key for this network"),
+    );
+    let mut serai_key_repr =
+      Zeroizing::new(<<Ristretto as Ciphersuite>::F as PrimeField>::Repr::default());
+    serai_key_repr.as_mut().copy_from_slice(serai_key.as_ref());
+    Zeroizing::new(<Ristretto as Ciphersuite>::F::from_repr(*serai_key_repr).unwrap())
+  };
+
+  coordinator(&orchestration_path, network, coordinator_key.0, serai_key);
+
+  serai(&orchestration_path, network);
+}
+
+fn key_gen(network: Network) {
+  let serai_dir = home::home_dir().unwrap().join(".serai").join(network.label());
+  let key_file = serai_dir.join("key");
+  if fs::File::open(&key_file).is_ok() {
+    println!("already created key");
+    return;
+  }
+
+  let key = <Ristretto as Ciphersuite>::F::random(&mut OsRng);
+
+  let _ = fs::create_dir_all(&serai_dir);
+  fs::write(key_file, key.to_repr()).expect("couldn't write key");
+
+  println!(
+    "Public Key: {}",
+    hex::encode((<Ristretto as Ciphersuite>::generator() * key).to_bytes())
+  );
+}
+
+fn start(network: Network, services: HashSet<String>) {
+  // Create the serai network
+  Command::new("docker")
+    .arg("network")
+    .arg("create")
+    .arg("--driver")
+    .arg("bridge")
+    .arg("serai")
+    .output()
+    .unwrap();
+
+  for service in services {
+    println!("Starting {service}");
+    let name = match service.as_ref() {
+      "serai" => "serai",
+      "coordinator" => "coordinator",
+      "message-queue" => "message-queue",
+      "bitcoin-daemon" => "bitcoin",
+      "bitcoin-processor" => "bitcoin-processor",
+      "monero-daemon" => "monero",
+      "monero-processor" => "monero-processor",
+      "monero-wallet-rpc" => "monero-wallet-rpc",
+      _ => panic!("starting unrecognized service"),
+    };
+
+    // Build it
+    println!("Building {service}");
+    docker::build(&orchestration_path(network), network, name);
+
+    let docker_name = format!("serai-{}-{name}", network.label());
+    let docker_image = format!("{docker_name}-img");
+    if !Command::new("docker")
+      .arg("container")
+      .arg("inspect")
+      .arg(&docker_name)
+      .status()
+      .unwrap()
+      .success()
+    {
+      // Create the docker container
+      println!("Creating new container for {service}");
+      let volume = format!("serai-{}-{name}-volume:/volume", network.label());
+      let mut command = Command::new("docker");
+      let command = command.arg("create").arg("--name").arg(&docker_name);
+      let command = command.arg("--network").arg("serai");
+      let command = match name {
+        "bitcoin" => {
+          if network == Network::Dev {
+            command.arg("-p").arg("8332:8332")
+          } else {
+            command.arg("--volume").arg(volume)
+          }
+        }
+        "monero" => {
+          if network == Network::Dev {
+            command.arg("-p").arg("18081:18081")
+          } else {
+            command.arg("--volume").arg(volume)
+          }
+        }
+        "monero-wallet-rpc" => {
+          assert_eq!(network, Network::Dev, "monero-wallet-rpc is only for dev");
+          command.arg("-p").arg("18082:18082")
+        }
+        _ => command,
+      };
+      assert!(
+        command.arg(docker_image).status().unwrap().success(),
+        "couldn't create the container"
+      );
+    }
+
+    // Start it
+    // TODO: Check it successfully started
+    println!("Starting existing container for {service}");
+    let _ = Command::new("docker").arg("start").arg(docker_name).output();
+  }
+}
+
+fn main() {
+  let help = || -> ! {
+    println!(
+      r#"
+Serai Orchestrator v0.0.1
+
+Commands:
+  key_gen *network*
+    Generates a key for the validator.
+
+  setup *network*
+    Generate infrastructure keys and the Dockerfiles for every Serai service.
+
+  start *network* [service1, service2...]
+    Start the specified services for the specified network ("dev" or "testnet").
+
+    - `serai`
+    - `coordinator`
+    - `message-queue`
+    - `bitcoin-daemon`
+    - `bitcoin-processor`
+    - `monero-daemon`
+    - `monero-processor`
+    - `monero-wallet-rpc` (if "dev")
+
+    are valid services.
+
+    `*network*-processor` will automatically start `*network*-daemon`.
+"#
+    );
+    std::process::exit(1);
+  };
+
+  let mut args = env::args();
+  args.next();
+  let command = args.next();
+  let network = match args.next().as_ref().map(AsRef::as_ref) {
+    Some("dev") => Network::Dev,
+    Some("testnet") => Network::Testnet,
+    Some(_) => panic!(r#"unrecognized network. only "dev" and "testnet" are recognized"#),
+    None => help(),
+  };
+
+  match command.as_ref().map(AsRef::as_ref) {
+    Some("key_gen") => {
+      key_gen(network);
+    }
+    Some("setup") => {
+      dockerfiles(network);
+    }
+    Some("start") => {
+      let mut services = HashSet::new();
+      for arg in args {
+        if let Some(ext_network) = arg.strip_suffix("-processor") {
+          services.insert(ext_network.to_string() + "-daemon");
+        }
+        services.insert(arg);
+      }
+
+      start(network, services);
+    }
+    _ => help(),
+  }
+}

orchestration/src/message_queue.rs

@@ -0,0 +1,51 @@
+use std::{path::Path};
+
+use ciphersuite::{group::GroupEncoding, Ciphersuite, Ristretto};
+
+use crate::{Network, Os, mimalloc, os, build_serai_service, write_dockerfile};
+
+pub fn message_queue(
+  orchestration_path: &Path,
+  network: Network,
+  coordinator_key: <Ristretto as Ciphersuite>::G,
+  bitcoin_key: <Ristretto as Ciphersuite>::G,
+  ethereum_key: <Ristretto as Ciphersuite>::G,
+  monero_key: <Ristretto as Ciphersuite>::G,
+) {
+  let setup = mimalloc(Os::Debian).to_string() +
+    &build_serai_service(network.release(), network.db(), "serai-message-queue");
+
+  let env_vars = [
+    ("COORDINATOR_KEY", hex::encode(coordinator_key.to_bytes())),
+    ("BITCOIN_KEY", hex::encode(bitcoin_key.to_bytes())),
+    ("ETHEREUM_KEY", hex::encode(ethereum_key.to_bytes())),
+    ("MONERO_KEY", hex::encode(monero_key.to_bytes())),
+    ("DB_PATH", "./message-queue-db".to_string()),
+    ("RUST_LOG", "serai_message_queue=trace".to_string()),
+  ];
+  let mut env_vars_str = String::new();
+  for (env_var, value) in env_vars {
+    env_vars_str += &format!(r#"{env_var}=${{{env_var}:="{value}"}} "#);
+  }
+
+  let run_message_queue = format!(
+    r#"
+# Copy the Message Queue binary and relevant license
+COPY --from=builder --chown=messagequeue /serai/bin/serai-message-queue /bin
+COPY --from=builder --chown=messagequeue /serai/AGPL-3.0 .
+
+# Run message-queue
+EXPOSE 2287
+CMD {env_vars_str} serai-message-queue
+"#
+  );
+
+  let run = os(Os::Debian, "", "messagequeue") + &run_message_queue;
+  let res = setup + &run;
+
+  let mut message_queue_path = orchestration_path.to_path_buf();
+  message_queue_path.push("message-queue");
+  message_queue_path.push("Dockerfile");
+
+  write_dockerfile(message_queue_path, &res);
+}

orchestration/src/mimalloc.rs

@@ -0,0 +1,36 @@
+use crate::Os;
+
+pub fn mimalloc(os: Os) -> &'static str {
+  const ALPINE_MIMALLOC: &str = r#"
+FROM alpine:latest as mimalloc-alpine
+
+RUN apk update && apk upgrade && apk --no-cache add gcc g++ libc-dev make cmake git
+RUN git clone https://github.com/microsoft/mimalloc && \
+  cd mimalloc && \
+  git checkout 43ce4bd7fd34bcc730c1c7471c99995597415488 && \
+  mkdir -p out/secure && \
+  cd out/secure && \
+  cmake -DMI_SECURE=ON ../.. && \
+  make && \
+  cp ./libmimalloc-secure.so ../../../libmimalloc.so
+"#;
+
+  const DEBIAN_MIMALLOC: &str = r#"
+FROM debian:bookworm-slim as mimalloc-debian
+
+RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git
+RUN git clone https://github.com/microsoft/mimalloc && \
+  cd mimalloc && \
+  git checkout 43ce4bd7fd34bcc730c1c7471c99995597415488 && \
+  mkdir -p out/secure && \
+  cd out/secure && \
+  cmake -DMI_SECURE=ON ../.. && \
+  make && \
+  cp ./libmimalloc-secure.so ../../../libmimalloc.so
+"#;
+
+  match os {
+    Os::Alpine => ALPINE_MIMALLOC,
+    Os::Debian => DEBIAN_MIMALLOC,
+  }
+}

orchestration/src/processor.rs

@@ -0,0 +1,78 @@
+use std::{path::Path};
+
+use zeroize::Zeroizing;
+
+use ciphersuite::{group::ff::PrimeField, Ciphersuite, Ristretto};
+
+use crate::{Network, Os, mimalloc, os, build_serai_service, write_dockerfile};
+
+#[allow(clippy::needless_pass_by_value)]
+pub fn processor(
+  orchestration_path: &Path,
+  network: Network,
+  coin: &'static str,
+  _coordinator_key: <Ristretto as Ciphersuite>::G,
+  coin_key: Zeroizing<<Ristretto as Ciphersuite>::F>,
+  entropy: Zeroizing<[u8; 32]>,
+) {
+  let setup = mimalloc(Os::Debian).to_string() +
+    &build_serai_service(
+      network.release(),
+      &format!("binaries {} {coin}", network.db()),
+      "serai-processor",
+    );
+
+  const ADDITIONAL_ROOT: &str = r#"
+# Install ca-certificates
+RUN apt install -y ca-certificates
+"#;
+
+  // TODO: Randomly generate these
+  const RPC_USER: &str = "serai";
+  const RPC_PASS: &str = "seraidex";
+  // TODO: Isolate networks
+  let hostname = format!("serai-{}-{coin}", network.label());
+  let port = match coin {
+    "bitcoin" => 8332,
+    "ethereum" => return, // TODO
+    "monero" => 18081,
+    _ => panic!("unrecognized external network"),
+  };
+
+  let env_vars = [
+    ("MESSAGE_QUEUE_RPC", format!("serai-{}-message_queue", network.label())),
+    ("MESSAGE_QUEUE_KEY", hex::encode(coin_key.to_repr())),
+    ("ENTROPY", hex::encode(entropy.as_ref())),
+    ("NETWORK", coin.to_string()),
+    ("NETWORK_RPC_LOGIN", format!("{RPC_USER}:{RPC_PASS}")),
+    ("NETWORK_RPC_HOSTNAME", hostname),
+    ("NETWORK_RPC_PORT", format!("{port}")),
+    ("DB_PATH", "./processor-db".to_string()),
+    ("RUST_LOG", "serai_processor=debug".to_string()),
+  ];
+  let mut env_vars_str = String::new();
+  for (env_var, value) in env_vars {
+    env_vars_str += &format!(r#"{env_var}=${{{env_var}:="{value}"}} "#);
+  }
+
+  let run_processor = format!(
+    r#"
+# Copy the Processor binary and relevant license
+COPY --from=builder --chown=processor /serai/bin/serai-processor /bin/
+COPY --from=builder --chown=processor /serai/AGPL-3.0 .
+
+# Run processor
+CMD {env_vars_str} serai-processor
+"#
+  );
+
+  let run = os(Os::Debian, ADDITIONAL_ROOT, "processor") + &run_processor;
+  let res = setup + &run;
+
+  let mut processor_path = orchestration_path.to_path_buf();
+  processor_path.push("processor");
+  processor_path.push(coin);
+  processor_path.push("Dockerfile");
+
+  write_dockerfile(processor_path, &res);
+}

orchestration/src/serai.rs

@@ -0,0 +1,33 @@
+use std::{path::Path};
+
+use crate::{Network, Os, mimalloc, os, build_serai_service, write_dockerfile};
+
+pub fn serai(orchestration_path: &Path, network: Network) {
+  // Always builds in release for performance reasons
+  let setup = mimalloc(Os::Debian).to_string() + &build_serai_service(true, "", "serai-node");
+
+  // TODO: Review the ports exposed here
+  let run_serai = format!(
+    r#"
+# Copy the Serai binary and relevant license
+COPY --from=builder --chown=serai /serai/bin/serai-node /bin/
+COPY --from=builder --chown=serai /serai/AGPL-3.0 .
+
+# Run the Serai node
+EXPOSE 30333 9615 9933 9944
+
+ADD /orchestration/{}/serai/run.sh /
+CMD ["/run.sh"]
+"#,
+    network.label()
+  );
+
+  let run = os(Os::Debian, "", "serai") + &run_serai;
+  let res = setup + &run;
+
+  let mut serai_path = orchestration_path.to_path_buf();
+  serai_path.push("serai");
+  serai_path.push("Dockerfile");
+
+  write_dockerfile(serai_path, &res);
+}