From db5a3099235f83eb12eb3c9042799e31b57ef07f Mon Sep 17 00:00:00 2001 From: lcian <17258265+lcian@users.noreply.github.com> Date: Wed, 4 Mar 2026 15:04:14 +0100 Subject: [PATCH 1/4] build(deps): Bump objectstore-client to 0.1.2 --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 72afa7f2a2..ec1ba5eb31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2322,9 +2322,9 @@ dependencies = [ [[package]] name = "objectstore-client" -version = "0.0.19" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bcaf8bc0ea7c50905631df108126c6a075ce5a9c16713ec4b68cc872a408e08" +checksum = "033eedf125e31b30962c0172842e964fc9983bbccd99d9ff033e7e413946861c" dependencies = [ "async-compression", "async-stream", @@ -2346,9 +2346,9 @@ dependencies = [ [[package]] name = "objectstore-types" -version = "0.0.19" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f190038e8988112a4e593f1796612941117d88753574ef6476f99324d4605aae" +checksum = "956cbdef3971ea108a15e5248625d6229870da3a3c637b6e7aada213526f8014" dependencies = [ "http", "humantime", diff --git a/Cargo.toml b/Cargo.toml index 728aea4c73..a54e6d5f3f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,7 +44,7 @@ java-properties = "2.0.0" lazy_static = "1.4.0" libc = "0.2.139" log = { version = "0.4.17", features = ["std"] } -objectstore-client = { version = "0.0.19" , default-features = false, features = ["native-tls"] } +objectstore-client = { version = "0.1.2" , default-features = false, features = ["native-tls"] } open = "3.2.0" parking_lot = "0.12.1" percent-encoding = "2.2.0" From 8555c09a1a627f6eb643141a8e13cda6a8c4498c Mon Sep 17 00:00:00 2001 From: lcian <17258265+lcian@users.noreply.github.com> Date: Wed, 4 Mar 2026 17:54:21 +0100 Subject: [PATCH 2/4] ref(snapshots): Use file-based upload for objectstore puts Use `put_file` instead of `put` when uploading images to objectstore, streaming file contents from disk rather than loading entire files into memory. Hash computation also now uses buffered reads. Co-Authored-By: Claude Opus 4.6 --- src/commands/build/snapshots.rs | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/src/commands/build/snapshots.rs b/src/commands/build/snapshots.rs index 89f2d387c4..a6e4176bae 100644 --- a/src/commands/build/snapshots.rs +++ b/src/commands/build/snapshots.rs @@ -1,5 +1,4 @@ use std::collections::HashMap; -use std::fs; use std::path::{Path, PathBuf}; use std::str::FromStr as _; @@ -10,6 +9,7 @@ use log::{debug, info, warn}; use objectstore_client::{ClientBuilder, ExpirationPolicy, Usecase}; use secrecy::ExposeSecret as _; use sha2::{Digest as _, Sha256}; +use tokio::fs::File; use walkdir::WalkDir; use crate::api::{Api, CreateSnapshotResponse, ImageMetadata, SnapshotsManifest}; @@ -174,11 +174,24 @@ fn collect_image_info(dir: &Path, path: &Path) -> Option { }) } -fn compute_sha256_hash(data: &[u8]) -> String { +fn compute_sha256_hash(path: &Path) -> Result { + use std::io::Read as _; + + let mut file = std::fs::File::open(path) + .with_context(|| format!("Failed to open image for hashing: {}", path.display()))?; let mut hasher = Sha256::new(); - hasher.update(data); + let mut buffer = [0u8; 8192]; + loop { + let bytes_read = file + .read(&mut buffer) + .with_context(|| format!("Failed to read image for hashing: {}", path.display()))?; + if bytes_read == 0 { + break; + } + hasher.update(&buffer[..bytes_read]); + } let result = hasher.finalize(); - format!("{result:x}") + Ok(format!("{result:x}")) } fn is_hidden(root: &Path, path: &Path) -> bool { @@ -236,15 +249,16 @@ fn upload_images( for image in images { debug!("Processing image: {}", image.path.display()); - let contents = fs::read(&image.path) - .with_context(|| format!("Failed to read image: {}", image.path.display()))?; - let hash = compute_sha256_hash(&contents); + let hash = compute_sha256_hash(&image.path)?; + let file = runtime.block_on(File::open(&image.path)).with_context(|| { + format!("Failed to open image for upload: {}", image.path.display()) + })?; info!("Queueing {} as {hash}", image.relative_path.display()); many_builder = many_builder.push( session - .put(contents) + .put_file(file) .key(&hash) .expiration_policy(expiration), ); From 805e22908984083d9dc20f2ff042ab856800b35b Mon Sep 17 00:00:00 2001 From: lcian <17258265+lcian@users.noreply.github.com> Date: Wed, 4 Mar 2026 18:46:36 +0100 Subject: [PATCH 3/4] fix(snapshots): Use org/project-scoped object keys for snapshot uploads Co-Authored-By: Claude Opus 4.6 --- src/commands/build/snapshots.rs | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/commands/build/snapshots.rs b/src/commands/build/snapshots.rs index 9e7b1f9ae2..f32b5a3ea2 100644 --- a/src/commands/build/snapshots.rs +++ b/src/commands/build/snapshots.rs @@ -267,9 +267,22 @@ fn upload_images( .build()?; let mut scope = Usecase::new("preprod").scope(); - for (key, value) in &options.objectstore.scopes { - scope = scope.push(key, value); + let (mut org_id, mut project_id): (Option, Option) = (None, None); + for (key, value) in options.objectstore.scopes.into_iter() { + scope = scope.push(&key, value.clone()); + if key == "org" { + org_id = Some(value); + } else if key == "project" { + project_id = Some(value); + } } + let Some(org_id) = org_id else { + anyhow::bail!("Missing org in UploadOptions scope"); + }; + let Some(project_id) = project_id else { + anyhow::bail!("Missing project in UploadOptions scope"); + }; + let session = scope.session(&client)?; let runtime = tokio::runtime::Builder::new_current_thread() @@ -289,12 +302,13 @@ fn upload_images( format!("Failed to open image for upload: {}", image.path.display()) })?; - info!("Queueing {} as {hash}", image.relative_path.display()); + let key = format!("{org_id}/{project_id}/{hash}"); + info!("Queueing {} as {key}", image.relative_path.display()); many_builder = many_builder.push( session .put_file(file) - .key(&hash) + .key(&key) .expiration_policy(expiration), ); From e94eefaeb6575d75739a15a78592c2e766e50dcf Mon Sep 17 00:00:00 2001 From: lcian <17258265+lcian@users.noreply.github.com> Date: Wed, 4 Mar 2026 19:05:48 +0100 Subject: [PATCH 4/4] ref(snapshots): Use async I/O for snapshot image processing Enter the tokio runtime early with a single `block_on` call instead of calling it multiple times in the upload loop. Convert `compute_sha256_hash` to async using `tokio::fs::File` and `AsyncReadExt`, and use async file opens for uploads. Wrap `manifest_entries` in `Arc` for safe mutation inside the async block. Co-Authored-By: Claude Opus 4.6 --- Cargo.toml | 2 +- src/commands/build/snapshots.rs | 116 +++++++++++++++++--------------- 2 files changed, 63 insertions(+), 55 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f154c06100..47f60be6ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,7 +69,7 @@ sha2 = "0.10.9" sourcemap = { version = "9.3.0", features = ["ram_bundle"] } symbolic = { version = "12.13.3", features = ["debuginfo-serde", "il2cpp"] } thiserror = "1.0.38" -tokio = { version = "1.47", features = ["rt"] } +tokio = { version = "1.47", features = ["rt", "fs", "io-util"] } url = "2.3.1" uuid = { version = "1.3.0", features = ["v4", "serde"] } walkdir = "2.3.2" diff --git a/src/commands/build/snapshots.rs b/src/commands/build/snapshots.rs index f32b5a3ea2..3249ddd0a1 100644 --- a/src/commands/build/snapshots.rs +++ b/src/commands/build/snapshots.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::str::FromStr as _; +use std::sync::Arc; use anyhow::{Context as _, Result}; use clap::{Arg, ArgMatches, Command}; @@ -10,7 +11,8 @@ use log::{debug, info, warn}; use objectstore_client::{ClientBuilder, ExpirationPolicy, Usecase}; use secrecy::ExposeSecret as _; use sha2::{Digest as _, Sha256}; -use tokio::fs::File; +use tokio::io::AsyncReadExt as _; +use tokio::sync::Mutex; use walkdir::WalkDir; use crate::api::{Api, CreateSnapshotResponse, ImageMetadata, SnapshotsManifest}; @@ -209,16 +211,16 @@ fn validate_image_sizes(images: &[ImageInfo]) -> Result<()> { Ok(()) } -fn compute_sha256_hash(path: &Path) -> Result { - use std::io::Read as _; - - let mut file = std::fs::File::open(path) +async fn compute_sha256_hash(path: &Path) -> Result { + let mut file = tokio::fs::File::open(path) + .await .with_context(|| format!("Failed to open image for hashing: {}", path.display()))?; let mut hasher = Sha256::new(); let mut buffer = [0u8; 8192]; loop { let bytes_read = file .read(&mut buffer) + .await .with_context(|| format!("Failed to read image for hashing: {}", path.display()))?; if bytes_read == 0 { break; @@ -290,66 +292,72 @@ fn upload_images( .build() .context("Failed to create tokio runtime")?; - let mut many_builder = session.many(); - let mut manifest_entries = HashMap::new(); let image_count = images.len(); + let manifest_entries = Arc::new(Mutex::new(HashMap::new())); - for image in images { - debug!("Processing image: {}", image.path.display()); - - let hash = compute_sha256_hash(&image.path)?; - let file = runtime.block_on(File::open(&image.path)).with_context(|| { - format!("Failed to open image for upload: {}", image.path.display()) - })?; + runtime.block_on(async { + let mut many_builder = session.many(); - let key = format!("{org_id}/{project_id}/{hash}"); - info!("Queueing {} as {key}", image.relative_path.display()); + for image in images { + debug!("Processing image: {}", image.path.display()); - many_builder = many_builder.push( - session - .put_file(file) - .key(&key) - .expiration_policy(expiration), - ); + let hash = compute_sha256_hash(&image.path).await?; + let file = tokio::fs::File::open(&image.path).await.with_context(|| { + format!("Failed to open image for upload: {}", image.path.display()) + })?; - let image_file_name = image - .relative_path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(); - manifest_entries.insert( - hash, - ImageMetadata { - image_file_name, - width: image.width, - height: image.height, - }, - ); - } + let key = format!("{org_id}/{project_id}/{hash}"); + info!("Queueing {} as {key}", image.relative_path.display()); - let result = runtime.block_on(async { many_builder.send().error_for_failures().await }); + many_builder = many_builder.push( + session + .put_file(file) + .key(&key) + .expiration_policy(expiration), + ); - match result { - Ok(()) => { - println!( - "{} Uploaded {} image {}", - style(">").dim(), - style(image_count).yellow(), - if image_count == 1 { "file" } else { "files" } + let image_file_name = image + .relative_path + .file_name() + .unwrap_or_default() + .to_string_lossy() + .into_owned(); + manifest_entries.lock().await.insert( + hash, + ImageMetadata { + image_file_name, + width: image.width, + height: image.height, + }, ); - Ok(manifest_entries) } - Err(errors) => { - eprintln!("There were errors uploading images:"); - let mut error_count = 0; - for error in errors { - eprintln!(" {}", style(error).red()); - error_count += 1; + + let result = many_builder.send().error_for_failures().await; + match result { + Ok(()) => { + println!( + "{} Uploaded {} image {}", + style(">").dim(), + style(image_count).yellow(), + if image_count == 1 { "file" } else { "files" } + ); + Ok(()) + } + Err(errors) => { + eprintln!("There were errors uploading images:"); + let mut error_count = 0; + for error in errors { + eprintln!(" {}", style(error).red()); + error_count += 1; + } + anyhow::bail!("Failed to upload {error_count} out of {image_count} images") } - anyhow::bail!("Failed to upload {error_count} out of {image_count} images") } - } + })?; + + Ok(Arc::try_unwrap(manifest_entries) + .expect("all references should be dropped after runtime completes") + .into_inner()) } #[cfg(test)]