From b9983bf133b8d66e0f64885cd216b7e5138b9b50 Mon Sep 17 00:00:00 2001 From: Luke Parker Date: Mon, 6 Nov 2023 09:14:46 -0500 Subject: [PATCH] Replace reqwest with simple-request reqwest was replaced with hyper and hyper-rustls within monero-serai due to reqwest *solely* offering a connection pool API. In the process, it was demonstrated how quickly we can achieve equivalent functionality to reqwest for our use cases with a fraction of the code. This adds our own reqwest alternative to the tree, applying it to both bitcoin-serai and message-queue. By doing so, bitcoin-serai decreases its tree by 21 packages and the processor by 18. Cargo.lock decreases by 8 dependencies, solely adding simple-request. Notably removed is openssl-sys and openssl. One noted decrease functionality is the requirement on the system having installed CA certificates. While we could fallback to the rustls certificates if the system doesn't have any, that's blocked by https://github.com/rustls/hyper-rustls/pulls/228. --- Cargo.lock | 112 +++++------------------------------- Cargo.toml | 1 + coins/bitcoin/Cargo.toml | 4 +- coins/bitcoin/src/rpc.rs | 23 +++++--- coins/monero/Cargo.toml | 1 + common/request/Cargo.toml | 25 ++++++++ common/request/LICENSE | 21 +++++++ common/request/README.md | 7 +++ common/request/src/lib.rs | 97 +++++++++++++++++++++++++++++++ message-queue/Cargo.toml | 2 +- message-queue/src/client.rs | 30 +++++++--- 11 files changed, 204 insertions(+), 119 deletions(-) create mode 100644 common/request/Cargo.toml create mode 100644 common/request/LICENSE create mode 100644 common/request/README.md create mode 100644 common/request/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 0d1ef417..ed8dd39e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -535,10 +535,10 @@ dependencies = [ "k256", "modular-frost", "rand_core", - "reqwest", "secp256k1", "serde", "serde_json", + "simple-request", "std-shims", "thiserror", "tokio", @@ -2573,21 +2573,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "fork-tree" version = "3.0.0" @@ -3429,19 +3414,6 @@ dependencies = [ "webpki-roots", ] -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes", - "hyper", - "native-tls", - "tokio", - "tokio-native-tls", -] - [[package]] name = "hyperlocal" version = "0.8.0" @@ -5110,24 +5082,6 @@ dependencies = [ "rand", ] -[[package]] -name = "native-tls" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" -dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "netlink-packet-core" version = "0.4.2" @@ -5403,50 +5357,12 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "openssl" -version = "0.10.59" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a257ad03cd8fb16ad4172fedf8094451e1af1c4b70097636ef2eac9a5f0cc33" -dependencies = [ - "bitflags 2.4.1", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.39", -] - [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" -[[package]] -name = "openssl-sys" -version = "0.9.95" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40a4130519a360279579c2053038317e40eff64d13fd3f004f9e1b72b8a6aaf9" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "option-ext" version = "0.2.0" @@ -6617,12 +6533,10 @@ dependencies = [ "http", "http-body", "hyper", - "hyper-tls", "ipnet", "js-sys", "log", "mime", - "native-tls", "once_cell", "percent-encoding", "pin-project-lite 0.2.13", @@ -6631,7 +6545,6 @@ dependencies = [ "serde_urlencoded", "system-configuration", "tokio", - "tokio-native-tls", "tower-service", "url", "wasm-bindgen", @@ -8403,13 +8316,13 @@ dependencies = [ "log", "once_cell", "rand_core", - "reqwest", "schnorr-signatures", "serai-db", "serai-env", "serai-primitives", "serde", "serde_json", + "simple-request", "tokio", "zeroize", ] @@ -8858,6 +8771,17 @@ dependencies = [ "wide", ] +[[package]] +name = "simple-request" +version = "0.1.0" +dependencies = [ + "base64ct", + "hyper", + "hyper-rustls", + "tokio", + "zeroize", +] + [[package]] name = "simple_asn1" version = "0.6.2" @@ -10162,16 +10086,6 @@ dependencies = [ "syn 2.0.39", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.24.1" diff --git a/Cargo.toml b/Cargo.toml index ac7882b1..14add4b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "common/zalloc", "common/db", "common/env", + "common/request", "crypto/transcript", diff --git a/coins/bitcoin/Cargo.toml b/coins/bitcoin/Cargo.toml index d8ac1296..1f54c9e6 100644 --- a/coins/bitcoin/Cargo.toml +++ b/coins/bitcoin/Cargo.toml @@ -25,7 +25,7 @@ frost = { package = "modular-frost", path = "../../crypto/frost", version = "0.8 hex = { version = "0.4", default-features = false, optional = true } serde = { version = "1", default-features = false, features = ["derive"], optional = true } serde_json = { version = "1", default-features = false, optional = true } -reqwest = { version = "0.11", default-features = false, features = ["default-tls", "json"], optional = true } +simple-request = { path = "../../common/request", version = "0.1", default-features = false, features = ["basic-auth"], optional = true } [dev-dependencies] secp256k1 = { version = "0.28", default-features = false, features = ["std"] } @@ -54,7 +54,7 @@ std = [ "hex/std", "serde/std", "serde_json/std", - "reqwest", + "simple-request", ] hazmat = [] default = ["std"] diff --git a/coins/bitcoin/src/rpc.rs b/coins/bitcoin/src/rpc.rs index 7d415f2a..fc25da94 100644 --- a/coins/bitcoin/src/rpc.rs +++ b/coins/bitcoin/src/rpc.rs @@ -6,7 +6,7 @@ use thiserror::Error; use serde::{Deserialize, de::DeserializeOwned}; use serde_json::json; -use reqwest::Client; +use simple_request::{Request, Client}; use bitcoin::{ hashes::{Hash, hex::FromHex}, @@ -62,7 +62,7 @@ impl Rpc { /// provided to this library, if the RPC has an incompatible argument layout. That is not checked /// at time of RPC creation. pub async fn new(url: String) -> Result { - let rpc = Rpc { client: Client::new(), url }; + let rpc = Rpc { client: Client::with_connection_pool(), url }; // Make an RPC request to verify the node is reachable and sane let res: String = rpc.rpc_call("help", json!([])).await?; @@ -107,19 +107,26 @@ impl Rpc { method: &str, params: serde_json::Value, ) -> Result { - let res = self + let mut res = self .client - .post(&self.url) - .json(&json!({ "jsonrpc": "2.0", "method": method, "params": params })) - .send() + .request( + Request::post(&self.url) + .header("Content-Type", "application/json") + .body( + serde_json::to_vec(&json!({ "jsonrpc": "2.0", "method": method, "params": params })) + .unwrap() + .into(), + ) + .unwrap(), + ) .await .map_err(|_| RpcError::ConnectionError)? - .text() + .body() .await .map_err(|_| RpcError::ConnectionError)?; let res: RpcResponse = - serde_json::from_str(&res).map_err(|e| RpcError::InvalidJson(e.classify()))?; + serde_json::from_reader(&mut res).map_err(|e| RpcError::InvalidJson(e.classify()))?; match res { RpcResponse::Ok { result } => Ok(result), RpcResponse::Err { error } => Err(RpcError::RequestError(error)), diff --git a/coins/monero/Cargo.toml b/coins/monero/Cargo.toml index 8725f62f..5c317ed0 100644 --- a/coins/monero/Cargo.toml +++ b/coins/monero/Cargo.toml @@ -55,6 +55,7 @@ base58-monero = { version = "2", default-features = false, features = ["check"] # Used for the provided HTTP RPC digest_auth = { version = "0.3", default-features = false, optional = true } +# Deprecated here means to enable deprecated warnings, not to restore deprecated APIs hyper = { version = "0.14", default-features = false, features = ["http1", "tcp", "client", "backports", "deprecated"], optional = true } hyper-rustls = { version = "0.24", default-features = false, features = ["http1", "native-tokio"], optional = true } tokio = { version = "1", default-features = false, optional = true } diff --git a/common/request/Cargo.toml b/common/request/Cargo.toml new file mode 100644 index 00000000..0ae3789e --- /dev/null +++ b/common/request/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "simple-request" +version = "0.1.0" +description = "A simple HTTP(S) request library" +license = "MIT" +repository = "https://github.com/serai-dex/serai/tree/develop/common/simple-request" +authors = ["Luke Parker "] +keywords = ["http", "https", "async", "request", "ssl"] +edition = "2021" + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] + +[dependencies] +# Deprecated here means to enable deprecated warnings, not to restore deprecated APIs +hyper = { version = "0.14", default-features = false, features = ["http1", "tcp", "client", "backports", "deprecated"] } +hyper-rustls = { version = "0.24", default-features = false, features = ["http1", "native-tokio"] } +tokio = { version = "1", default-features = false } + +zeroize = { version = "1", optional = true } +base64ct = { version = "1", features = ["alloc"], optional = true } + +[features] +basic-auth = ["zeroize", "base64ct"] diff --git a/common/request/LICENSE b/common/request/LICENSE new file mode 100644 index 00000000..e6bff13c --- /dev/null +++ b/common/request/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Luke Parker + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/common/request/README.md b/common/request/README.md new file mode 100644 index 00000000..41eebbfd --- /dev/null +++ b/common/request/README.md @@ -0,0 +1,7 @@ +# Simple Request + +A simple alternative to reqwest, supporting HTTPS, intended to support a +majority of use cases with a fraction of the dependency tree. + +This library is built directly around `hyper`, `hyper-rustls`, and does require +`tokio`. Support for `async-std` would be welcome. diff --git a/common/request/src/lib.rs b/common/request/src/lib.rs new file mode 100644 index 00000000..edb879fb --- /dev/null +++ b/common/request/src/lib.rs @@ -0,0 +1,97 @@ +#![cfg_attr(docsrs, feature(doc_auto_cfg))] +#![doc = include_str!("../README.md")] + +use hyper_rustls::{HttpsConnectorBuilder, HttpsConnector}; +use hyper::{ + StatusCode, + header::{HeaderValue, HeaderMap}, + body::{Buf, Body}, + Response as HyperResponse, + client::HttpConnector, +}; +pub use hyper::{self, Request}; + +#[derive(Debug)] +pub struct Response(HyperResponse); +impl Response { + pub fn status(&self) -> StatusCode { + self.0.status() + } + pub fn headers(&self) -> &HeaderMap { + self.0.headers() + } + pub async fn body(self) -> Result { + Ok(hyper::body::aggregate(self.0.into_body()).await?.reader()) + } +} + +#[derive(Clone, Debug)] +enum Connection { + ConnectionPool(hyper::Client>), +} + +#[derive(Clone, Debug)] +pub struct Client { + connection: Connection, +} + +#[derive(Debug)] +pub enum Error { + InvalidHost, + Hyper(hyper::Error), +} + +impl Client { + fn https_builder() -> HttpsConnector { + HttpsConnectorBuilder::new().with_native_roots().https_or_http().enable_http1().build() + } + + pub fn with_connection_pool() -> Client { + Client { + connection: Connection::ConnectionPool(hyper::Client::builder().build(Self::https_builder())), + } + } + + /* + fn without_connection_pool() -> Client {} + */ + + pub async fn request(&self, mut request: Request) -> Result { + if request.headers().get(hyper::header::HOST).is_none() { + let host = request.uri().host().ok_or(Error::InvalidHost)?.to_string(); + request + .headers_mut() + .insert(hyper::header::HOST, HeaderValue::from_str(&host).map_err(|_| Error::InvalidHost)?); + } + + #[cfg(feature = "basic-auth")] + if request.headers().get(hyper::header::AUTHORIZATION).is_none() { + if let Some(authority) = request.uri().authority() { + let authority = authority.as_str(); + if authority.contains('@') { + // Decode the username and password from the URI + let mut userpass = authority.split('@').next().unwrap().to_string(); + // If the password is "", the URI may omit :, yet the authentication will still expect it + if !userpass.contains(':') { + userpass.push(':'); + } + + use zeroize::Zeroize; + use base64ct::{Encoding, Base64}; + + let mut encoded = Base64::encode_string(userpass.as_bytes()); + userpass.zeroize(); + request.headers_mut().insert( + hyper::header::AUTHORIZATION, + HeaderValue::from_str(&format!("Basic {encoded}")).unwrap(), + ); + encoded.zeroize(); + } + } + } + + Ok(Response(match &self.connection { + Connection::ConnectionPool(client) => client.request(request).await.map_err(Error::Hyper)?, + })) + } +} diff --git a/message-queue/Cargo.toml b/message-queue/Cargo.toml index 0185aa8f..4b6b8bd1 100644 --- a/message-queue/Cargo.toml +++ b/message-queue/Cargo.toml @@ -46,7 +46,7 @@ serai-env = { path = "../common/env" } serai-primitives = { path = "../substrate/primitives" } jsonrpsee = { version = "0.16", default-features = false, features = ["server"], optional = true } -reqwest = { version = "0.11", default-features = false, features = ["json"] } +simple-request = { path = "../common/request", default-features = false } [features] binaries = ["serai-db", "serai-db/rocksdb", "jsonrpsee"] diff --git a/message-queue/src/client.rs b/message-queue/src/client.rs index f1bf29d0..9adcc2cd 100644 --- a/message-queue/src/client.rs +++ b/message-queue/src/client.rs @@ -11,7 +11,7 @@ use schnorr_signatures::SchnorrSignature; use serde::{Serialize, Deserialize}; -use reqwest::Client; +use simple_request::{Request, Client}; use serai_env as env; @@ -45,7 +45,7 @@ impl MessageQueue { service, pub_key: Ristretto::generator() * priv_key.deref(), priv_key, - client: Client::new(), + client: Client::with_connection_pool(), url, } } @@ -81,18 +81,30 @@ impl MessageQueue { id: u64, } - let res = loop { + let mut res = loop { // Make the request match self .client - .post(&self.url) - .json(&JsonRpcRequest { jsonrpc: "2.0", method, params: params.clone(), id: 0 }) - .send() + .request( + Request::post(&self.url) + .header("Content-Type", "application/json") + .body( + serde_json::to_vec(&JsonRpcRequest { + jsonrpc: "2.0", + method, + params: params.clone(), + id: 0, + }) + .unwrap() + .into(), + ) + .unwrap(), + ) .await { Ok(req) => { // Get the response - match req.text().await { + match req.body().await { Ok(res) => break res, Err(e) => { dbg!(e); @@ -108,8 +120,8 @@ impl MessageQueue { tokio::time::sleep(core::time::Duration::from_secs(1)).await; }; - let json = - serde_json::from_str::(&res).expect("message-queue returned invalid JSON"); + let json: serde_json::Value = + serde_json::from_reader(&mut res).expect("message-queue returned invalid JSON"); if json.get("result").is_none() { panic!("call failed: {json}"); }