Switch from reqwest to bitreq HTTP client

.github/workflows/build.yml

@@ -6,11 +6,11 @@ jobs:
   build:
     strategy:
       matrix:
-        toolchain: [ stable, beta, 1.75.0 ] # 1.75.0 is current MSRV for vss-client-ng
+        toolchain: [ stable, beta, 1.85.0 ] # 1.85.0 is current MSRV for vss-client-ng
         include:
           - toolchain: stable
             check-fmt: true
-          - toolchain: 1.75.0
+          - toolchain: 1.85.0
             msrv: true
     runs-on: ubuntu-latest
     steps:
@@ -22,11 +22,6 @@ jobs:
         run: |
           curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --default-toolchain ${{ matrix.toolchain }}
           rustup override set ${{ matrix.toolchain }}
-      - name: Pin packages to allow for MSRV
-        if: matrix.msrv
-        run: |
-          cargo update -p idna_adapter --precise 1.1.0 # This has us use `unicode-normalization` which has a more conservative MSRV
-          cargo update -p proptest --precise "1.8.0" --verbose # proptest 1.9.0 requires rustc 1.82.0
       - name: Build on Rust ${{ matrix.toolchain }}
         run: cargo build --verbose --color always
       - name: Check formatting

Cargo.toml

@@ -2,7 +2,7 @@
 name = "vss-client-ng"
 version = "0.4.1"
 authors = ["Leo Nash <hello@leonash.net>", "Elias Rohrer <dev@tnull.de>"]
-rust-version = "1.75.0"
+rust-version = "1.85.0"
 license = "MIT OR Apache-2.0"
 edition = "2021"
 homepage = "https://lightningdevkit.org/"
@@ -15,28 +15,28 @@ build = "build.rs"
 
 [features]
 default = ["lnurl-auth", "sigs-auth"]
-lnurl-auth = ["dep:bitcoin", "dep:url", "dep:serde", "dep:serde_json", "reqwest/json"]
+lnurl-auth = ["dep:bitcoin", "dep:url", "dep:serde", "dep:serde_json"]
 sigs-auth = ["dep:bitcoin"]
 
 [dependencies]
 prost = "0.11.6"
-reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
+bitreq = { version = "0.3", default-features = false, features = ["async-https"] }
 tokio = { version = "1", default-features = false, features = ["time"] }
 rand = "0.8.5"
 async-trait = "0.1.77"
 bitcoin = { version = "0.32.2", default-features = false, features = ["std", "rand-std"], optional = true }
 url = { version = "2.5.0", default-features = false, optional = true }
-base64 = { version = "0.22", default-features = false}
+base64 = { version = "0.22", default-features = false, features = ["std"]}
 serde = { version = "1.0.196", default-features = false, features = ["serde_derive"], optional = true }
-serde_json = { version = "1.0.113", default-features = false, optional = true }
+serde_json = { version = "1.0.113", default-features = false, features = ["std"], optional = true }
 
 bitcoin_hashes = "0.14.0"
 chacha20-poly1305 = "0.1.2"
 log = { version = "0.4.29", default-features = false, features = ["std"]}
 
 [target.'cfg(genproto)'.build-dependencies]
 prost-build = { version = "0.11.3" }
-reqwest =  { version = "0.12", default-features = false, features = ["rustls-tls", "blocking"] }
+bitreq = { version = "0.3", default-features = false, features = ["std", "https"] }
 
 [dev-dependencies]
 mockito = "0.28.0"

README.md

@@ -9,4 +9,4 @@ and manage the essential state required for Lightning Network (LN) operations.
 Learn more [here](https://github.com/lightningdevkit/vss-server/blob/main/README.md).
 
 ## MSRV
-The Minimum Supported Rust Version (MSRV) is currently 1.75.0.
+The Minimum Supported Rust Version (MSRV) is currently 1.85.0.

build.rs

@@ -1,6 +1,8 @@
 #[cfg(genproto)]
 extern crate prost_build;
 #[cfg(genproto)]
+use std::io::Write;
+#[cfg(genproto)]
 use std::{env, fs, fs::File, path::Path};
 
 /// To generate updated proto objects:
@@ -14,7 +16,7 @@ fn main() {
 #[cfg(genproto)]
 fn generate_protos() {
 	download_file(
-				"https://raw.githubusercontent.com/lightningdevkit/vss-server/7f492fcac0c561b212f49ca40f7d16075822440f/app/src/main/proto/vss.proto",
+				"https://raw.githubusercontent.com/lightningdevkit/vss-server/022ee5e92debb60516438af0a369966495bfe595/proto/vss.proto",
 				"src/proto/vss.proto",
 		).unwrap();
 
@@ -25,9 +27,9 @@ fn generate_protos() {
 
 #[cfg(genproto)]
 fn download_file(url: &str, save_to: &str) -> Result<(), Box<dyn std::error::Error>> {
-	let mut response = reqwest::blocking::get(url)?;
+	let response = bitreq::get(url).send()?;
 	fs::create_dir_all(Path::new(save_to).parent().unwrap())?;
 	let mut out_file = File::create(save_to)?;
-	response.copy_to(&mut out_file)?;
+	out_file.write_all(&response.into_bytes())?;
 	Ok(())
 }

src/client.rs

@@ -1,14 +1,13 @@
+use bitreq::Client;
 use prost::Message;
-use reqwest::header::CONTENT_TYPE;
-use reqwest::Client;
 use std::collections::HashMap;
 use std::default::Default;
 use std::sync::Arc;
 
 use log::trace;
 
 use crate::error::VssError;
-use crate::headers::{get_headermap, FixedHeaders, VssHeaderProvider};
+use crate::headers::{FixedHeaders, VssHeaderProvider};
 use crate::types::{
 	DeleteObjectRequest, DeleteObjectResponse, GetObjectRequest, GetObjectResponse,
 	ListKeyVersionsRequest, ListKeyVersionsResponse, PutObjectRequest, PutObjectResponse,
@@ -17,7 +16,10 @@ use crate::util::retry::{retry, RetryPolicy};
 use crate::util::KeyValueVecKeyPrinter;
 
 const APPLICATION_OCTET_STREAM: &str = "application/octet-stream";
-const DEFAULT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
+const CONTENT_TYPE: &str = "content-type";
+const DEFAULT_TIMEOUT_SECS: u64 = 10;
+const MAX_RESPONSE_BODY_SIZE: usize = 1024 * 1024 * 1024; // 1GB
+const DEFAULT_CLIENT_CAPACITY: usize = 10;
 
 /// Thin-client to access a hosted instance of Versioned Storage Service (VSS).
 /// The provided [`VssClient`] API is minimalistic and is congruent to the VSS server-side API.
@@ -35,11 +37,11 @@ where
 impl<R: RetryPolicy<E = VssError>> VssClient<R> {
 	/// Constructs a [`VssClient`] using `base_url` as the VSS server endpoint.
 	pub fn new(base_url: String, retry_policy: R) -> Self {
-		let client = build_client();
+		let client = Client::new(DEFAULT_CLIENT_CAPACITY);
 		Self::from_client(base_url, client, retry_policy)
 	}
 
-	/// Constructs a [`VssClient`] from a given [`reqwest::Client`], using `base_url` as the VSS server endpoint.
+	/// Constructs a [`VssClient`] from a given [`bitreq::Client`], using `base_url` as the VSS server endpoint.
 	pub fn from_client(base_url: String, client: Client, retry_policy: R) -> Self {
 		Self {
 			base_url,
@@ -49,7 +51,7 @@ impl<R: RetryPolicy<E = VssError>> VssClient<R> {
 		}
 	}
 
-	/// Constructs a [`VssClient`] from a given [`reqwest::Client`], using `base_url` as the VSS server endpoint.
+	/// Constructs a [`VssClient`] from a given [`bitreq::Client`], using `base_url` as the VSS server endpoint.
 	///
 	/// HTTP headers will be provided by the given `header_provider`.
 	pub fn from_client_and_headers(
@@ -65,7 +67,7 @@ impl<R: RetryPolicy<E = VssError>> VssClient<R> {
 	pub fn new_with_headers(
 		base_url: String, retry_policy: R, header_provider: Arc<dyn VssHeaderProvider>,
 	) -> Self {
-		let client = build_client();
+		let client = Client::new(DEFAULT_CLIENT_CAPACITY);
 		Self { base_url, client, retry_policy, header_provider }
 	}
 
@@ -85,15 +87,17 @@ impl<R: RetryPolicy<E = VssError>> VssClient<R> {
 		let res = retry(
 			|| async {
 				let url = format!("{}/getObject", self.base_url);
-				self.post_request(request, &url).await.and_then(|response: GetObjectResponse| {
-					if response.value.is_none() {
-						Err(VssError::InternalServerError(
+				self.post_request(request, &url, true).await.and_then(
+					|response: GetObjectResponse| {
+						if response.value.is_none() {
+							Err(VssError::InternalServerError(
 							"VSS Server API Violation, expected value in GetObjectResponse but found none".to_string(),
 						))
-					} else {
-						Ok(response)
-					}
-				})
+						} else {
+							Ok(response)
+						}
+					},
+				)
 			},
 			&self.retry_policy,
 		)
@@ -121,7 +125,7 @@ impl<R: RetryPolicy<E = VssError>> VssClient<R> {
 		let res = retry(
 			|| async {
 				let url = format!("{}/putObjects", self.base_url);
-				self.post_request(request, &url).await
+				self.post_request(request, &url, false).await
 			},
 			&self.retry_policy,
 		)
@@ -147,7 +151,7 @@ impl<R: RetryPolicy<E = VssError>> VssClient<R> {
 		let res = retry(
 			|| async {
 				let url = format!("{}/deleteObject", self.base_url);
-				self.post_request(request, &url).await
+				self.post_request(request, &url, false).await
 			},
 			&self.retry_policy,
 		)
@@ -175,7 +179,7 @@ impl<R: RetryPolicy<E = VssError>> VssClient<R> {
 		let res = retry(
 			|| async {
 				let url = format!("{}/listKeyVersions", self.base_url);
-				self.post_request(request, &url).await
+				self.post_request(request, &url, true).await
 			},
 			&self.retry_policy,
 		)
@@ -187,40 +191,36 @@ impl<R: RetryPolicy<E = VssError>> VssClient<R> {
 	}
 
 	async fn post_request<Rq: Message, Rs: Message + Default>(
-		&self, request: &Rq, url: &str,
+		&self, request: &Rq, url: &str, enable_pipelining: bool,
 	) -> Result<Rs, VssError> {
 		let request_body = request.encode_to_vec();
-		let headermap = self
+		let headers = self
 			.header_provider
 			.get_headers(&request_body)
 			.await
-			.and_then(|h| get_headermap(&h))
 			.map_err(|e| VssError::AuthError(e.to_string()))?;
-		let response_raw = self
-			.client
-			.post(url)
-			.header(CONTENT_TYPE, APPLICATION_OCTET_STREAM)
-			.headers(headermap)
-			.body(request_body)
-			.send()
-			.await?;
-		let status = response_raw.status();
-		let payload = response_raw.bytes().await?;
-
-		if status.is_success() {
+
+		let mut http_request = bitreq::post(url)
+			.with_header(CONTENT_TYPE, APPLICATION_OCTET_STREAM)
+			.with_headers(headers)
+			.with_body(request_body)
+			.with_timeout(DEFAULT_TIMEOUT_SECS)
+			.with_max_body_size(Some(MAX_RESPONSE_BODY_SIZE));
+
+		if enable_pipelining {
+			http_request = http_request.with_pipelining();
+		}
+
+		let response = self.client.send_async(http_request).await?;
+
+		let status_code = response.status_code;
+		let payload = response.into_bytes();
+
+		if (200..300).contains(&status_code) {
 			let response = Rs::decode(&payload[..])?;
 			Ok(response)
 		} else {
-			Err(VssError::new(status, payload))
+			Err(VssError::new(status_code, payload))
 		}
 	}
 }
-
-fn build_client() -> Client {
-	Client::builder()
-		.timeout(DEFAULT_TIMEOUT)
-		.connect_timeout(DEFAULT_TIMEOUT)
-		.read_timeout(DEFAULT_TIMEOUT)
-		.build()
-		.unwrap()
-}

src/error.rs

@@ -1,7 +1,5 @@
 use crate::types::{ErrorCode, ErrorResponse};
-use prost::bytes::Bytes;
 use prost::{DecodeError, Message};
-use reqwest::StatusCode;
 use std::error::Error;
 use std::fmt::{Display, Formatter};
 
@@ -32,13 +30,13 @@ pub enum VssError {
 
 impl VssError {
 	/// Create new instance of `VssError`
-	pub fn new(status: StatusCode, payload: Bytes) -> VssError {
+	pub fn new(status_code: i32, payload: Vec<u8>) -> VssError {
 		match ErrorResponse::decode(&payload[..]) {
 			Ok(error_response) => VssError::from(error_response),
 			Err(e) => {
 				let message = format!(
 					"Unable to decode ErrorResponse from server, HttpStatusCode: {}, DecodeErr: {}",
-					status, e
+					status_code, e
 				);
 				VssError::InternalError(message)
 			},
@@ -99,8 +97,8 @@ impl From<DecodeError> for VssError {
 	}
 }
 
-impl From<reqwest::Error> for VssError {
-	fn from(err: reqwest::Error) -> Self {
+impl From<bitreq::Error> for VssError {
+	fn from(err: bitreq::Error) -> Self {
 		VssError::InternalError(err.to_string())
 	}
 }