From fdb34e2b4260ad38cfc5b9ca3a86493564685a2e Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Wed, 3 Dec 2025 10:45:24 +0000 Subject: [PATCH 1/9] wip- resumable upload finalize --- .../lib/google/apis/core/storage_upload.rb | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index da31878d3a3..cfc6d8562e9 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -176,13 +176,19 @@ def send_upload_command(client) request_header = header.dup request_header[CONTENT_RANGE_HEADER] = get_content_range_header current_chunk_size request_header[CONTENT_LENGTH_HEADER] = current_chunk_size.to_s + last_chunk= remaining_content_size <= current_chunk_size + hash_data= JSON.parse(body) + target_keys = ["crc32c", "md5Hash"] + formatted_string = hash_data.slice(*target_keys).map { |key, value| "#{key}=#{value}" }.join(',') + request_header['X-Goog-Hash'] = formatted_string if last_chunk + chunk_body = if @upload_chunk_size == 0 upload_io else StringIO.new(upload_io.read(current_chunk_size)) end - + binding.pry if last_chunk response = client.put(@upload_url, chunk_body, request_header) result = process_response(response.status.to_i, response.headers, response.body) @@ -191,7 +197,7 @@ def send_upload_command(client) success(result) rescue => e logger.warn { - "error occured please use uploadId-#{response.headers['X-GUploader-UploadID']} to resume your upload" + "error occurred please use uploadId-#{response.headers['X-GUploader-UploadID']} to resume your upload , error==> #{e}" } unless response.nil? upload_io.pos = @offset error(e, rethrow: true) @@ -246,7 +252,7 @@ def cancel_resumable_upload(client) def handle_resumable_upload_http_response_codes(response) code = response.status.to_i - + binding.pry case code when 308 if response.headers['Range'] @@ -262,6 +268,7 @@ def handle_resumable_upload_http_response_codes(response) @upload_incomplete = false when 200, 201 # Upload is complete. + binding.pry @upload_incomplete = false else logger.debug { sprintf("Unexpected response: #{response.status.to_i} - #{response.body}") } From bbcbceccddd37350440c3ff24db3410444a04edd Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Thu, 4 Dec 2025 09:22:21 +0000 Subject: [PATCH 2/9] wip- adding integrity check at final upload --- .../lib/google/apis/core/storage_upload.rb | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index cfc6d8562e9..19dcb85a0b0 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -32,6 +32,7 @@ class StorageUploadCommand < ApiCommand CONTENT_RANGE_HEADER = "Content-Range" RESUMABLE = "resumable" OK_STATUS = 200 + UPLOAD_MISMATCH = "The uploaded data did not match the data from the server." # File name or IO containing the content to upload # @return [String, File, #read] @@ -178,8 +179,13 @@ def send_upload_command(client) request_header[CONTENT_LENGTH_HEADER] = current_chunk_size.to_s last_chunk= remaining_content_size <= current_chunk_size hash_data= JSON.parse(body) - target_keys = ["crc32c", "md5Hash"] - formatted_string = hash_data.slice(*target_keys).map { |key, value| "#{key}=#{value}" }.join(',') + + target_keys = ["crc32c", "md5Hash", "md5"] + selected_keys = hash_data.slice(*target_keys) + formatted_string = selected_keys.map do |key, value| + output_key = (key == "md5Hash") ? "md5" : key + "#{output_key}=#{value}" + end.join(',') request_header['X-Goog-Hash'] = formatted_string if last_chunk chunk_body = @@ -188,13 +194,24 @@ def send_upload_command(client) else StringIO.new(upload_io.read(current_chunk_size)) end - binding.pry if last_chunk response = client.put(@upload_url, chunk_body, request_header) - + # binding.pry result = process_response(response.status.to_i, response.headers, response.body) + @upload_incomplete = false if response.status.to_i.eql? OK_STATUS @offset += current_chunk_size if @upload_incomplete - success(result) + if last_chunk + response_data = JSON.parse(response.body) + target_keys= selected_keys.keys + selected_keys_res= response_data.slice(*target_keys) + + if selected_keys_res == selected_keys + success(result) + else + raise error UPLOAD_MISMATCH + end + end + rescue => e logger.warn { "error occurred please use uploadId-#{response.headers['X-GUploader-UploadID']} to resume your upload , error==> #{e}" @@ -252,7 +269,6 @@ def cancel_resumable_upload(client) def handle_resumable_upload_http_response_codes(response) code = response.status.to_i - binding.pry case code when 308 if response.headers['Range'] @@ -268,7 +284,6 @@ def handle_resumable_upload_http_response_codes(response) @upload_incomplete = false when 200, 201 # Upload is complete. - binding.pry @upload_incomplete = false else logger.debug { sprintf("Unexpected response: #{response.status.to_i} - #{response.body}") } From 5475c48a6fdbd71c7c416318ecf83c730d8405b9 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Mon, 8 Dec 2025 08:21:36 +0000 Subject: [PATCH 3/9] undo integrity check --- .../lib/google/apis/core/storage_upload.rb | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index 19dcb85a0b0..0dde366ba4a 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -195,21 +195,10 @@ def send_upload_command(client) StringIO.new(upload_io.read(current_chunk_size)) end response = client.put(@upload_url, chunk_body, request_header) - # binding.pry result = process_response(response.status.to_i, response.headers, response.body) - @upload_incomplete = false if response.status.to_i.eql? OK_STATUS @offset += current_chunk_size if @upload_incomplete - if last_chunk - response_data = JSON.parse(response.body) - target_keys= selected_keys.keys - selected_keys_res= response_data.slice(*target_keys) - - if selected_keys_res == selected_keys - success(result) - else - raise error UPLOAD_MISMATCH - end + success(result) end rescue => e From a88e676bfb82c89836f514cca1b0bdbf7d089782 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Mon, 8 Dec 2025 08:29:29 +0000 Subject: [PATCH 4/9] cleaning --- google-apis-core/lib/google/apis/core/storage_upload.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index 0dde366ba4a..97668db2e12 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -32,7 +32,6 @@ class StorageUploadCommand < ApiCommand CONTENT_RANGE_HEADER = "Content-Range" RESUMABLE = "resumable" OK_STATUS = 200 - UPLOAD_MISMATCH = "The uploaded data did not match the data from the server." # File name or IO containing the content to upload # @return [String, File, #read] @@ -199,7 +198,6 @@ def send_upload_command(client) @upload_incomplete = false if response.status.to_i.eql? OK_STATUS @offset += current_chunk_size if @upload_incomplete success(result) - end rescue => e logger.warn { From 9ca62b82e092435af731b2ee76f53a1903d41886 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Mon, 8 Dec 2025 11:18:34 +0000 Subject: [PATCH 5/9] refactor --- google-apis-core/lib/google/apis/core/storage_upload.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index 97668db2e12..4a791737a6d 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -177,7 +177,7 @@ def send_upload_command(client) request_header[CONTENT_RANGE_HEADER] = get_content_range_header current_chunk_size request_header[CONTENT_LENGTH_HEADER] = current_chunk_size.to_s last_chunk= remaining_content_size <= current_chunk_size - hash_data= JSON.parse(body) + hash_data = body.to_s.empty? ? {} : JSON.parse(body) target_keys = ["crc32c", "md5Hash", "md5"] selected_keys = hash_data.slice(*target_keys) From 9b3796b99c749b8335a9b0bd8644459615dafb3a Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Thu, 11 Dec 2025 11:51:47 +0000 Subject: [PATCH 6/9] adding tests --- .../lib/google/apis/core/storage_upload.rb | 37 ++- .../google/apis/core/storage_upload_spec.rb | 236 +++++++++++++++++- 2 files changed, 261 insertions(+), 12 deletions(-) diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index 4a791737a6d..aeb12e8a8ee 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -136,6 +136,8 @@ def initiate_resumable_upload(client) request_header[CONTENT_LENGTH_HEADER] = upload_io.size.to_s request_header[CONTENT_TYPE_HEADER] = JSON_CONTENT_TYPE request_header[UPLOAD_CONTENT_TYPE_HEADER] = upload_content_type unless upload_content_type.nil? + formatted_string = formatted_checksum_header + request_header['X-Goog-Hash'] = formatted_string unless formatted_string.empty? response = client.post(url.to_s, body, request_header) do |request| request.params.replace(request_query) @@ -177,15 +179,8 @@ def send_upload_command(client) request_header[CONTENT_RANGE_HEADER] = get_content_range_header current_chunk_size request_header[CONTENT_LENGTH_HEADER] = current_chunk_size.to_s last_chunk= remaining_content_size <= current_chunk_size - hash_data = body.to_s.empty? ? {} : JSON.parse(body) - - target_keys = ["crc32c", "md5Hash", "md5"] - selected_keys = hash_data.slice(*target_keys) - formatted_string = selected_keys.map do |key, value| - output_key = (key == "md5Hash") ? "md5" : key - "#{output_key}=#{value}" - end.join(',') - request_header['X-Goog-Hash'] = formatted_string if last_chunk + formatted_string = formatted_checksum_header + request_header['X-Goog-Hash'] = formatted_string if (last_chunk && !formatted_string.empty?) chunk_body = if @upload_chunk_size == 0 @@ -299,6 +294,30 @@ def get_content_range_header current_chunk_size end sprintf('bytes %s/%d', numerator, upload_io.size) end + + # Generates a formatted checksum header string from the request body. + # + # Parses the body as JSON and extracts checksum values for the keys "crc32c", "md5Hash", and "md5". + # The "md5Hash" key is renamed to "md5" in the output. + # Returns a comma-separated string in the format "key=value" for each present checksum. + # + # @example + # If the body contains: + # { "md5Hash": "1B2M2Y8AsgTpgAmY7PhCfg==", + # "crc32c": "AAAAAA==" } + # The method returns: + # "crc32c=AAAAAA==,md5=1B2M2Y8AsgTpgAmY7PhCfg==" + # @return [String] the formatted checksum header, or an empty string if no relevant keys are present + def formatted_checksum_header + hash_data = body.to_s.empty? ? {} : JSON.parse(body) + target_keys = ["crc32c", "md5Hash", "md5"] + selected_keys = hash_data.slice(*target_keys) + formatted_string = selected_keys.map do |key, value| + output_key = (key == "md5Hash") ? "md5" : key + "#{output_key}=#{value}" + end.join(',') + formatted_string + end end end end diff --git a/google-apis-core/spec/google/apis/core/storage_upload_spec.rb b/google-apis-core/spec/google/apis/core/storage_upload_spec.rb index 7435f9cafe8..26541aba25b 100644 --- a/google-apis-core/spec/google/apis/core/storage_upload_spec.rb +++ b/google-apis-core/spec/google/apis/core/storage_upload_spec.rb @@ -12,9 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -require 'spec_helper' +require_relative '../../../spec_helper' require 'google/apis/core/storage_upload' require 'google/apis/core/json_representation' +require 'pry' +require 'digest/crc32c' RSpec.describe Google::Apis::Core::StorageUploadCommand do include TestHelpers @@ -117,7 +119,9 @@ before(:example) do stub_request(:put, 'https://www.googleapis.com/zoo/animals') - .with(headers: { 'Content-Range' => 'bytes 11-21/22' }) + .with(headers: { + 'Content-Range' => 'bytes 11-21/22' + }) .to_return(body: %(OK)) end @@ -307,4 +311,230 @@ expect { command.execute(client) }.to raise_error Google::Apis::ServerError end end -end + context 'when uploading with md5 checksum' do + + let(:file) { StringIO.new(file_content) } + let(:md5_checksum) {"md5_checksum" } + let(:body_with_md5) { { "md5Hash" => md5_checksum }.to_json } + + context 'with single shot upload' do + let(:file_content) { "Hello world" } + + before(:example) do + command.body = body_with_md5 + allow(command).to receive(:formatted_checksum_header).and_call_original + + stub_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') + .with { |req| req.headers['X-Goog-Hash'] == "md5=#{md5_checksum}" } + .to_return(headers: { 'Location' => 'https://www.googleapis.com/zoo/animals' }, body: %(OK)) + stub_request(:put, 'https://www.googleapis.com/zoo/animals') + .with { |req| req.headers['X-Goog-Hash'] == "md5=#{md5_checksum}" } + .to_return(body: %(OK)) + end + + it 'includes md5 checksum in X-Goog-Hash header during initiation' do + command.execute(client) + expect(a_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') + .with { |req| req.headers['X-Goog-Hash'] == "md5=#{md5_checksum}" }).to have_been_made + end + + it 'calls formatted_checksum_header and returns correct value' do + expect(command.formatted_checksum_header).to eq("md5=#{md5_checksum}") + end + end + + context 'with chunked upload' do + let(:file_content) { "Hello world" * 2 } + + before(:example) do + command.body = body_with_md5 + allow(command).to receive(:formatted_checksum_header).and_call_original + + stub_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') + .with { |req| req.headers['X-Goog-Hash'] == "md5=#{md5_checksum}" } + .to_return(headers: { 'Location' => 'https://www.googleapis.com/zoo/animals' }, body: %(OK)) + stub_request(:put, 'https://www.googleapis.com/zoo/animals') + .with(headers: { 'Content-Range' => 'bytes 0-10/22' }) + .to_return(status: [308, 'Resume Incomplete']) + stub_request(:put, 'https://www.googleapis.com/zoo/animals') + .with(headers: { + 'Content-Range' => 'bytes 11-21/22', + 'X-Goog-Hash' => "md5=#{md5_checksum}" + }) + .to_return(body: %(OK)) + end + + it 'includes md5 checksum in X-Goog-Hash header during initiation' do + command.options.upload_chunk_size = 11 + command.execute(client) + expect(a_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') + .with { |req| req.headers['X-Goog-Hash'] == "md5=#{md5_checksum}" }).to have_been_made + end + + it 'includes md5 checksum in X-Goog-Hash header only in the last chunk' do + command.options.upload_chunk_size = 11 + command.execute(client) + + # First chunk should NOT have the X-Goog-Hash header + expect(a_request(:put, 'https://www.googleapis.com/zoo/animals') + .with { |req| req.headers['Content-Range'] == 'bytes 0-10/22' && !req.headers.key?('X-Goog-Hash') }).to have_been_made + + # Last chunk should have the X-Goog-Hash header + expect(a_request(:put, 'https://www.googleapis.com/zoo/animals') + .with { |req| req.headers['Content-Range'] == 'bytes 11-21/22' && req.headers['X-Goog-Hash'] == "md5=#{md5_checksum}" }).to have_been_made + end + end + end + + context 'when uploading with crc32c checksum' do + + let(:file) { StringIO.new(file_content) } + let(:crc32c_checksum) { "abc_checksum" } + let(:body_with_crc32c) { { "crc32c" => crc32c_checksum }.to_json } + + context 'with single shot upload' do + let(:file_content) { "Hello world" } + + before(:example) do + command.body = body_with_crc32c + allow(command).to receive(:formatted_checksum_header).and_call_original + + stub_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') + .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum}" } + .to_return(headers: { 'Location' => 'https://www.googleapis.com/zoo/animals' }, body: %(OK)) + stub_request(:put, 'https://www.googleapis.com/zoo/animals') + .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum}" } + .to_return(body: %(OK)) + end + + it 'includes crc32c checksum in X-Goog-Hash header during initiation' do + command.execute(client) + expect(a_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') + .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum}" }).to have_been_made + end + + it 'calls formatted_checksum_header and returns correct value' do + expect(command.formatted_checksum_header).to eq("crc32c=#{crc32c_checksum}") + end + end + + context 'with chunked upload' do + let(:file_content) { "Hello world" * 2 } + + before(:example) do + command.body = body_with_crc32c + allow(command).to receive(:formatted_checksum_header).and_call_original + + stub_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') + .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum}" } + .to_return(headers: { 'Location' => 'https://www.googleapis.com/zoo/animals' }, body: %(OK)) + stub_request(:put, 'https://www.googleapis.com/zoo/animals') + .with(headers: { 'Content-Range' => 'bytes 0-10/22' }) + .to_return(status: [308, 'Resume Incomplete']) + stub_request(:put, 'https://www.googleapis.com/zoo/animals') + .with(headers: { + 'Content-Range' => 'bytes 11-21/22', + 'X-Goog-Hash' => "crc32c=#{crc32c_checksum}" + }) + .to_return(body: %(OK)) + end + + it 'includes crc32c checksum in X-Goog-Hash header during initiation' do + command.options.upload_chunk_size = 11 + command.execute(client) + expect(a_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') + .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum}" }).to have_been_made + end + + it 'includes md5 checksum in X-Goog-Hash header only in the last chunk' do + command.options.upload_chunk_size = 11 + command.execute(client) + + # First chunk should NOT have the X-Goog-Hash header + expect(a_request(:put, 'https://www.googleapis.com/zoo/animals') + .with { |req| req.headers['Content-Range'] == 'bytes 0-10/22' && !req.headers.key?('X-Goog-Hash') }).to have_been_made + + # Last chunk should have the X-Goog-Hash header + expect(a_request(:put, 'https://www.googleapis.com/zoo/animals') + .with { |req| req.headers['Content-Range'] == 'bytes 11-21/22' && req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum}" }).to have_been_made + end + + end + end + + context 'when uploading with md5 and crc32c checksum' do + let(:file) { StringIO.new(file_content) } + let(:md5_checksum) { "md5_checksum"} + let(:crc32c_checksum) { "crc32c_checksum" } + let(:body_with_md5_crc32c) { { "md5Hash" => md5_checksum, "crc32c" => crc32c_checksum }.to_json } + + context 'with single shot upload' do + let(:file_content) { "Hello world" } + + before(:example) do + command.body = body_with_md5_crc32c + allow(command).to receive(:formatted_checksum_header).and_call_original + stub_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') + .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum},md5=#{md5_checksum}" } + .to_return(headers: { 'Location' => 'https://www.googleapis.com/zoo/animals' }, body: %(OK)) + stub_request(:put, 'https://www.googleapis.com/zoo/animals') + .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum},md5=#{md5_checksum}" } + .to_return(body: %(OK)) + end + + it 'includes md5 checksum in X-Goog-Hash header during initiation' do + command.execute(client) + expect(a_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') + .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum},md5=#{md5_checksum}" }).to have_been_made + end + + it 'calls formatted_checksum_header and returns correct value' do + expect(command.formatted_checksum_header).to eq("crc32c=#{crc32c_checksum},md5=#{md5_checksum}") + end + end + + context 'with chunked upload' do + let(:file_content) { "Hello world" * 2 } + + before(:example) do + command.body = body_with_md5_crc32c + allow(command).to receive(:formatted_checksum_header).and_call_original + + stub_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') + .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum},md5=#{md5_checksum}" } + .to_return(headers: { 'Location' => 'https://www.googleapis.com/zoo/animals' }, body: %(OK)) + stub_request(:put, 'https://www.googleapis.com/zoo/animals') + .with(headers: { 'Content-Range' => 'bytes 0-10/22' }) + .to_return(status: [308, 'Resume Incomplete']) + stub_request(:put, 'https://www.googleapis.com/zoo/animals') + .with(headers: { + 'Content-Range' => 'bytes 11-21/22', + 'X-Goog-Hash' => "crc32c=#{crc32c_checksum},md5=#{md5_checksum}" + }) + .to_return(body: %(OK)) + end + + it 'includes md5 and crc32c checksum in X-Goog-Hash header during initiation' do + command.options.upload_chunk_size = 11 + command.execute(client) + expect(a_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') + .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum},md5=#{md5_checksum}" }).to have_been_made + end + + it 'includes md5 and crc32c checksum in X-Goog-Hash header only in the last chunk' do + command.options.upload_chunk_size = 11 + command.execute(client) + + # First chunk should NOT have the X-Goog-Hash header + expect(a_request(:put, 'https://www.googleapis.com/zoo/animals') + .with { |req| req.headers['Content-Range'] == 'bytes 0-10/22' && !req.headers.key?('X-Goog-Hash') }).to have_been_made + + # Last chunk should have the X-Goog-Hash header + expect(a_request(:put, 'https://www.googleapis.com/zoo/animals') + .with { |req| req.headers['Content-Range'] == 'bytes 11-21/22' && req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum},md5=#{md5_checksum}" }).to have_been_made + end + + end + end + +end \ No newline at end of file From 7e0935798586c2403f2eb604d24363ec5a59070d Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Fri, 12 Dec 2025 10:04:41 +0000 Subject: [PATCH 7/9] undo -adding header while initiating --- .../lib/google/apis/core/storage_upload.rb | 3 -- .../google/apis/core/storage_upload_spec.rb | 34 +++++++------------ 2 files changed, 13 insertions(+), 24 deletions(-) diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index aeb12e8a8ee..36260bdce5d 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -136,9 +136,6 @@ def initiate_resumable_upload(client) request_header[CONTENT_LENGTH_HEADER] = upload_io.size.to_s request_header[CONTENT_TYPE_HEADER] = JSON_CONTENT_TYPE request_header[UPLOAD_CONTENT_TYPE_HEADER] = upload_content_type unless upload_content_type.nil? - formatted_string = formatted_checksum_header - request_header['X-Goog-Hash'] = formatted_string unless formatted_string.empty? - response = client.post(url.to_s, body, request_header) do |request| request.params.replace(request_query) end diff --git a/google-apis-core/spec/google/apis/core/storage_upload_spec.rb b/google-apis-core/spec/google/apis/core/storage_upload_spec.rb index 26541aba25b..e8a0f6c4d6f 100644 --- a/google-apis-core/spec/google/apis/core/storage_upload_spec.rb +++ b/google-apis-core/spec/google/apis/core/storage_upload_spec.rb @@ -12,11 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -require_relative '../../../spec_helper' +require 'spec_helper' require 'google/apis/core/storage_upload' require 'google/apis/core/json_representation' -require 'pry' -require 'digest/crc32c' RSpec.describe Google::Apis::Core::StorageUploadCommand do include TestHelpers @@ -325,17 +323,16 @@ allow(command).to receive(:formatted_checksum_header).and_call_original stub_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') - .with { |req| req.headers['X-Goog-Hash'] == "md5=#{md5_checksum}" } .to_return(headers: { 'Location' => 'https://www.googleapis.com/zoo/animals' }, body: %(OK)) stub_request(:put, 'https://www.googleapis.com/zoo/animals') .with { |req| req.headers['X-Goog-Hash'] == "md5=#{md5_checksum}" } .to_return(body: %(OK)) end - it 'includes md5 checksum in X-Goog-Hash header during initiation' do + it 'should not include X-Goog-Hash header during initiation' do command.execute(client) expect(a_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') - .with { |req| req.headers['X-Goog-Hash'] == "md5=#{md5_checksum}" }).to have_been_made + .with { |req| req.headers['X-Goog-Hash'] == "md5=#{md5_checksum}" }).to_not have_been_made end it 'calls formatted_checksum_header and returns correct value' do @@ -351,7 +348,6 @@ allow(command).to receive(:formatted_checksum_header).and_call_original stub_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') - .with { |req| req.headers['X-Goog-Hash'] == "md5=#{md5_checksum}" } .to_return(headers: { 'Location' => 'https://www.googleapis.com/zoo/animals' }, body: %(OK)) stub_request(:put, 'https://www.googleapis.com/zoo/animals') .with(headers: { 'Content-Range' => 'bytes 0-10/22' }) @@ -364,11 +360,11 @@ .to_return(body: %(OK)) end - it 'includes md5 checksum in X-Goog-Hash header during initiation' do + it 'should not include X-Goog-Hash header during initiation' do command.options.upload_chunk_size = 11 command.execute(client) expect(a_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') - .with { |req| req.headers['X-Goog-Hash'] == "md5=#{md5_checksum}" }).to have_been_made + .with { |req| req.headers['X-Goog-Hash'] == "md5=#{md5_checksum}" }).to_not have_been_made end it 'includes md5 checksum in X-Goog-Hash header only in the last chunk' do @@ -400,17 +396,16 @@ allow(command).to receive(:formatted_checksum_header).and_call_original stub_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') - .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum}" } .to_return(headers: { 'Location' => 'https://www.googleapis.com/zoo/animals' }, body: %(OK)) stub_request(:put, 'https://www.googleapis.com/zoo/animals') .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum}" } .to_return(body: %(OK)) end - it 'includes crc32c checksum in X-Goog-Hash header during initiation' do + it 'should not include X-Goog-Hash header during initiation' do command.execute(client) expect(a_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') - .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum}" }).to have_been_made + .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum}" }).to_not have_been_made end it 'calls formatted_checksum_header and returns correct value' do @@ -426,7 +421,6 @@ allow(command).to receive(:formatted_checksum_header).and_call_original stub_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') - .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum}" } .to_return(headers: { 'Location' => 'https://www.googleapis.com/zoo/animals' }, body: %(OK)) stub_request(:put, 'https://www.googleapis.com/zoo/animals') .with(headers: { 'Content-Range' => 'bytes 0-10/22' }) @@ -439,11 +433,11 @@ .to_return(body: %(OK)) end - it 'includes crc32c checksum in X-Goog-Hash header during initiation' do + it 'should not include X-Goog-Hash header during initiation' do command.options.upload_chunk_size = 11 command.execute(client) expect(a_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') - .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum}" }).to have_been_made + .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum}" }).to_not have_been_made end it 'includes md5 checksum in X-Goog-Hash header only in the last chunk' do @@ -475,17 +469,16 @@ command.body = body_with_md5_crc32c allow(command).to receive(:formatted_checksum_header).and_call_original stub_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') - .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum},md5=#{md5_checksum}" } .to_return(headers: { 'Location' => 'https://www.googleapis.com/zoo/animals' }, body: %(OK)) stub_request(:put, 'https://www.googleapis.com/zoo/animals') .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum},md5=#{md5_checksum}" } .to_return(body: %(OK)) end - it 'includes md5 checksum in X-Goog-Hash header during initiation' do + it 'should not include X-Goog-Hash header during initiation' do command.execute(client) expect(a_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') - .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum},md5=#{md5_checksum}" }).to have_been_made + .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum},md5=#{md5_checksum}" }).to_not have_been_made end it 'calls formatted_checksum_header and returns correct value' do @@ -501,7 +494,6 @@ allow(command).to receive(:formatted_checksum_header).and_call_original stub_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') - .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum},md5=#{md5_checksum}" } .to_return(headers: { 'Location' => 'https://www.googleapis.com/zoo/animals' }, body: %(OK)) stub_request(:put, 'https://www.googleapis.com/zoo/animals') .with(headers: { 'Content-Range' => 'bytes 0-10/22' }) @@ -514,11 +506,11 @@ .to_return(body: %(OK)) end - it 'includes md5 and crc32c checksum in X-Goog-Hash header during initiation' do + it 'should not includeX-Goog-Hash header during initiation' do command.options.upload_chunk_size = 11 command.execute(client) expect(a_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') - .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum},md5=#{md5_checksum}" }).to have_been_made + .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum},md5=#{md5_checksum}" }).to_not have_been_made end it 'includes md5 and crc32c checksum in X-Goog-Hash header only in the last chunk' do From 23e4ff1a298287e56e661cc4e1ab78dea1995055 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Fri, 12 Dec 2025 10:34:18 +0000 Subject: [PATCH 8/9] cleanup --- google-apis-core/lib/google/apis/core/storage_upload.rb | 6 +++++- .../spec/google/apis/core/storage_upload_spec.rb | 4 +--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index 36260bdce5d..b8cd0372fc2 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -136,6 +136,7 @@ def initiate_resumable_upload(client) request_header[CONTENT_LENGTH_HEADER] = upload_io.size.to_s request_header[CONTENT_TYPE_HEADER] = JSON_CONTENT_TYPE request_header[UPLOAD_CONTENT_TYPE_HEADER] = upload_content_type unless upload_content_type.nil? + response = client.post(url.to_s, body, request_header) do |request| request.params.replace(request_query) end @@ -175,7 +176,7 @@ def send_upload_command(client) request_header = header.dup request_header[CONTENT_RANGE_HEADER] = get_content_range_header current_chunk_size request_header[CONTENT_LENGTH_HEADER] = current_chunk_size.to_s - last_chunk= remaining_content_size <= current_chunk_size + last_chunk = remaining_content_size <= current_chunk_size formatted_string = formatted_checksum_header request_header['X-Goog-Hash'] = formatted_string if (last_chunk && !formatted_string.empty?) @@ -185,7 +186,9 @@ def send_upload_command(client) else StringIO.new(upload_io.read(current_chunk_size)) end + response = client.put(@upload_url, chunk_body, request_header) + result = process_response(response.status.to_i, response.headers, response.body) @upload_incomplete = false if response.status.to_i.eql? OK_STATUS @offset += current_chunk_size if @upload_incomplete @@ -248,6 +251,7 @@ def cancel_resumable_upload(client) def handle_resumable_upload_http_response_codes(response) code = response.status.to_i + case code when 308 if response.headers['Range'] diff --git a/google-apis-core/spec/google/apis/core/storage_upload_spec.rb b/google-apis-core/spec/google/apis/core/storage_upload_spec.rb index e8a0f6c4d6f..c0d6c311464 100644 --- a/google-apis-core/spec/google/apis/core/storage_upload_spec.rb +++ b/google-apis-core/spec/google/apis/core/storage_upload_spec.rb @@ -117,9 +117,7 @@ before(:example) do stub_request(:put, 'https://www.googleapis.com/zoo/animals') - .with(headers: { - 'Content-Range' => 'bytes 11-21/22' - }) + .with(headers: { 'Content-Range' => 'bytes 11-21/22'}) .to_return(body: %(OK)) end From 3d35d10ea2243690914ce8522cdba82c401e2171 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Fri, 12 Dec 2025 12:15:31 +0000 Subject: [PATCH 9/9] clean --- google-apis-core/lib/google/apis/core/storage_upload.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index b8cd0372fc2..1517709d104 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -136,7 +136,7 @@ def initiate_resumable_upload(client) request_header[CONTENT_LENGTH_HEADER] = upload_io.size.to_s request_header[CONTENT_TYPE_HEADER] = JSON_CONTENT_TYPE request_header[UPLOAD_CONTENT_TYPE_HEADER] = upload_content_type unless upload_content_type.nil? - + response = client.post(url.to_s, body, request_header) do |request| request.params.replace(request_query) end @@ -193,7 +193,6 @@ def send_upload_command(client) @upload_incomplete = false if response.status.to_i.eql? OK_STATUS @offset += current_chunk_size if @upload_incomplete success(result) - rescue => e logger.warn { "error occurred please use uploadId-#{response.headers['X-GUploader-UploadID']} to resume your upload , error==> #{e}" @@ -251,7 +250,7 @@ def cancel_resumable_upload(client) def handle_resumable_upload_http_response_codes(response) code = response.status.to_i - + case code when 308 if response.headers['Range']