From 09435e2e42c4eaa63f25f107cec5d9bbcc2ca17c Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Mon, 16 Mar 2026 15:59:11 +0100 Subject: [PATCH 01/15] CLDSRV-863: emit trailer in TrailingChecksumTransform --- .../streamingV4/trailingChecksumTransform.js | 76 +++++- .../raw-node/test/trailingChecksums.js | 3 +- tests/unit/auth/TrailingChecksumTransform.js | 232 +++++++++++++++++- 3 files changed, 293 insertions(+), 18 deletions(-) diff --git a/lib/auth/streamingV4/trailingChecksumTransform.js b/lib/auth/streamingV4/trailingChecksumTransform.js index 48870c80e7..687e2befac 100644 --- a/lib/auth/streamingV4/trailingChecksumTransform.js +++ b/lib/auth/streamingV4/trailingChecksumTransform.js @@ -3,10 +3,11 @@ const { errors } = require('arsenal'); const { maximumAllowedPartSize } = require('../../../constants'); /** - * This class is designed to handle the chunks sent in a streaming - * unsigned playload trailer request. In this iteration, we are not checking - * the checksums, but we are removing them from the stream. - * S3C-9732 will deal with checksum verification. + * This class handles the chunked-upload body format used by + * STREAMING-UNSIGNED-PAYLOAD-TRAILER requests. It strips the chunk-size + * headers and trailing checksum trailer from the stream, forwarding only + * the raw object data. The trailer name and value are emitted via a + * 'trailer' event so that ChecksumTransform can validate the checksum. */ class TrailingChecksumTransform extends Transform { /** @@ -20,6 +21,10 @@ class TrailingChecksumTransform extends Transform { this.bytesToDiscard = 0; // when trailing \r\n are present, we discard them but they can be in different chunks this.bytesToRead = 0; // when a chunk is advertised, the size is put here and we forward all bytes this.streamClosed = false; + this.readingTrailer = false; + this.trailerBuffer = Buffer.alloc(0); + this.trailerName = null; + this.trailerValue = null; } /** @@ -30,9 +35,18 @@ class TrailingChecksumTransform extends Transform { * @return {function} executes callback with err if applicable */ _flush(callback) { - if (!this.streamClosed) { + if (!this.streamClosed && this.readingTrailer && this.trailerBuffer.length === 0) { + // Nothing came after "0\r\n", don't fail. + // If the x-amz-trailer header was present then the trailer is required and ChecksumTransform will fail. + return callback(); + } else if (!this.streamClosed && this.readingTrailer && this.trailerBuffer.length !== 0) { + this.log.error('stream ended without trailer "\r\n"'); + return callback(errors.IncompleteBody.customizeDescription( + 'The request body terminated unexpectedly')); + } else if (!this.streamClosed && !this.readingTrailer) { this.log.error('stream ended without closing chunked encoding'); - return callback(errors.InvalidArgument); + return callback(errors.IncompleteBody.customizeDescription( + 'The request body terminated unexpectedly')); } return callback(); } @@ -66,6 +80,50 @@ class TrailingChecksumTransform extends Transform { continue; } + // after the 0-size chunk, read the trailer line (e.g. "x-amz-checksum-crc32:YABb/g==") + if (this.readingTrailer) { + const combined = Buffer.concat([this.trailerBuffer, chunk]); + const lineBreakIndex = combined.indexOf('\r\n'); + if (lineBreakIndex === -1) { + if (combined.byteLength > 1024) { + this.log.error('trailer line too long'); + return callback(errors.MalformedTrailerError); + } + // The trailer is not complete yet, continue. + this.trailerBuffer = combined; + return callback(); + } + this.trailerBuffer = Buffer.alloc(0); + const fullTrailer = combined.subarray(0, lineBreakIndex); + if (fullTrailer.length === 0) { + // The trailer is empty, stop reading. + this.readingTrailer = false; + this.streamClosed = true; + return callback(); + } + let trailerLine = fullTrailer.toString(); + // Some clients terminate the trailer with \n\r\n instead of + // just \r\n, producing a trailing \n in the parsed line. + if (trailerLine.endsWith('\n')) { + trailerLine = trailerLine.slice(0, -1); + } + const colonIndex = trailerLine.indexOf(':'); + if (colonIndex > 0) { + this.trailerName = trailerLine.slice(0, colonIndex).trim(); + this.trailerValue = trailerLine.slice(colonIndex + 1).trim(); + this.emit('trailer', this.trailerName, this.trailerValue); + } else { + this.log.error('incomplete trailer missing ":"', { trailerLine }); + return callback(errors.IncompleteBody.customizeDescription( + 'The request body terminated unexpectedly')); + } + this.readingTrailer = false; + this.streamClosed = true; + // The trailer \r\n is the last bytes of the stream per the AWS + // chunked upload format, so any remaining bytes are discarded. + return callback(); + } + // we are now looking for the chunk size field // no need to look further than 10 bytes since the field cannot be bigger: the max // chunk size is 5GB (see constants.maximumAllowedPartSize) @@ -100,9 +158,9 @@ class TrailingChecksumTransform extends Transform { } this.chunkSizeBuffer = Buffer.alloc(0); if (dataSize === 0) { - // TODO: check if the checksum is correct (S3C-9732) - // last chunk, no more data to read, the stream is closed - this.streamClosed = true; + // last chunk, no more data to read; enter trailer-reading mode + // bytesToDiscard = 2 below will consume the \r\n after "0" + this.readingTrailer = true; } if (dataSize > maximumAllowedPartSize) { this.log.error('chunk size too big', { dataSize }); diff --git a/tests/functional/raw-node/test/trailingChecksums.js b/tests/functional/raw-node/test/trailingChecksums.js index bad429c3c8..925d959a7f 100644 --- a/tests/functional/raw-node/test/trailingChecksums.js +++ b/tests/functional/raw-node/test/trailingChecksums.js @@ -6,10 +6,9 @@ const HttpRequestAuthV4 = require('../utils/HttpRequestAuthV4'); const bucket = 'testunsupportedchecksumsbucket'; const objectKey = 'key'; const objData = Buffer.alloc(1024, 'a'); -// note this is not the correct checksum in objDataWithTrailingChecksum const objDataWithTrailingChecksum = '10\r\n0123456789abcdef\r\n' + '10\r\n0123456789abcdef\r\n' + - '0\r\nx-amz-checksum-crc64nvme:YeIDuLa7tU0=\r\n'; + '0\r\nx-amz-checksum-crc64nvme:skQv82y5rgE=\r\n'; const objDataWithoutTrailingChecksum = '0123456789abcdef0123456789abcdef'; const config = require('../../config.json'); diff --git a/tests/unit/auth/TrailingChecksumTransform.js b/tests/unit/auth/TrailingChecksumTransform.js index dd88b3719b..cbb39a0309 100644 --- a/tests/unit/auth/TrailingChecksumTransform.js +++ b/tests/unit/auth/TrailingChecksumTransform.js @@ -4,9 +4,39 @@ const async = require('async'); const { Readable } = require('stream'); const TrailingChecksumTransform = require('../../../lib/auth/streamingV4/trailingChecksumTransform'); -const { stripTrailingChecksumStream } = require('../../../lib/api/apiUtils/object/prepareStream'); const { DummyRequestLogger } = require('../helpers'); +// Helper: pipe input chunks through TrailingChecksumTransform, collect output and trailer events +function runTransform(inputChunks) { + const stream = new TrailingChecksumTransform(new DummyRequestLogger()); + return new Promise((resolve, reject) => { + const output = []; + const trailerEvents = []; + stream.on('data', chunk => output.push(Buffer.from(chunk))); + stream.on('trailer', (name, value) => trailerEvents.push({ name, value })); + stream.on('finish', () => resolve({ data: Buffer.concat(output), trailers: trailerEvents, stream })); + stream.on('error', reject); + for (const chunk of inputChunks) { + stream.write(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + stream.end(); + }); +} + +// Helper: expect a stream error from the given input chunks +function expectError(inputChunks) { + const stream = new TrailingChecksumTransform(new DummyRequestLogger()); + return new Promise((resolve, reject) => { + stream.on('error', resolve); + stream.on('finish', () => reject(new Error('expected error but stream finished cleanly'))); + for (const chunk of inputChunks) { + stream.write(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + stream.end(); + stream.resume(); + }); +} + const log = new DummyRequestLogger(); // note this is not the correct checksum in objDataWithTrailingChecksum @@ -174,30 +204,33 @@ describe('TrailingChecksumTransform class', () => { it('should propagate _flush error via errCb when stream closes without chunked encoding', done => { const incompleteData = '10\r\n01234\r6789abcd\r\n\r\n'; const source = new ChunkedReader([Buffer.from(incompleteData)]); - source.headers = { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER' }; - const stream = stripTrailingChecksumStream(source, log, err => { - assert.deepStrictEqual(err, errors.InvalidArgument); + const stream = new TrailingChecksumTransform(log); + stream.on('error', err => { + assert.deepStrictEqual(err, errors.IncompleteBody); done(); }); + source.pipe(stream); stream.resume(); }); it('should propagate _transform error via errCb for invalid chunk size', done => { const badData = '500000000000\r\n'; const source = new ChunkedReader([Buffer.from(badData)]); - source.headers = { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER' }; - const stream = stripTrailingChecksumStream(source, log, err => { + const stream = new TrailingChecksumTransform(log); + stream.on('error', err => { assert.deepStrictEqual(err, errors.InvalidArgument); done(); }); + source.pipe(stream); stream.resume(); }); - it('should return early if supplied with an out of specification chunk size', done => { + it('should return early if supplied with an out-of-specification chunk size', done => { const trailingChecksumTransform = new TrailingChecksumTransform(log); const chunks = [ Buffer.from('500000'), Buffer.from('000000\r\n'), + Buffer.alloc(1000000), Buffer.alloc(1000000), Buffer.alloc(1000000), @@ -227,3 +260,188 @@ describe('TrailingChecksumTransform class', () => { chunkedReader.pipe(trailingChecksumTransform); }); }); + +describe('TrailingChecksumTransform trailer parsing and emitting', () => { + describe('happy path', () => { + it('single chunk with data and trailer: forwards data, emits trailer name and value', async () => { + const input = '5\r\nhello\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; + const { data, trailers } = await runTransform([input]); + assert.strictEqual(data.toString(), 'hello'); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); + assert.strictEqual(trailers[0].value, 'AAAAAA=='); + }); + + it('multiple data chunks followed by trailer: forwards all data, emits trailer once', async () => { + const input = '5\r\nhello\r\n5\r\nworld\r\n0\r\nx-amz-checksum-sha256:AAAAAA==\r\n'; + const { data, trailers } = await runTransform([input]); + assert.strictEqual(data.toString(), 'helloworld'); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-sha256'); + }); + + it('data chunk containing \\r\\n in payload is forwarded correctly', async () => { + // 7 bytes: h e l \r \n l o + const input = '7\r\nhel\r\nlo\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; + const { data } = await runTransform([input]); + assert.strictEqual(data.toString(), 'hel\r\nlo'); + }); + + it('trailer with whitespace around name and value: name and value are trimmed', async () => { + const input = '0\r\n x-amz-checksum-crc32 : AAAAAA== \r\n'; + const { trailers } = await runTransform([input]); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); + assert.strictEqual(trailers[0].value, 'AAAAAA=='); + }); + + it('trailer terminated with \\n\\r\\n: trailing \\n stripped from parsed line', async () => { + const input = '0\r\nx-amz-checksum-crc32:AAAAAA==\n\r\n'; + const { trailers } = await runTransform([input]); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); + assert.strictEqual(trailers[0].value, 'AAAAAA=='); + }); + + it('trailer value containing a colon: only first colon used as separator', async () => { + const input = '0\r\nx-amz-checksum-crc32:AA:BB==\r\n'; + const { trailers } = await runTransform([input]); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); + assert.strictEqual(trailers[0].value, 'AA:BB=='); + }); + + it('bytes after trailer \\r\\n are silently discarded', async () => { + const input = '5\r\nhello\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\nextra bytes ignored'; + const { data, trailers } = await runTransform([input]); + assert.strictEqual(data.toString(), 'hello'); + assert.strictEqual(trailers.length, 1); + }); + }); + + describe('chunk boundary edge cases', () => { + it('chunk size field split across two input chunks: parsed correctly', async () => { + // size 'a' (hex) = 10 bytes; split the size field across two chunks + const c1 = 'a'; + const c2 = '\r\nAAAAAAAAAA\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; + const { data, trailers } = await runTransform([c1, c2]); + assert.strictEqual(data.toString(), 'AAAAAAAAAA'); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); + }); + + it('data bytes split across two input chunks: all forwarded', async () => { + // 5 bytes 'hello', split after 'hel' + const c1 = '5\r\nhel'; + const c2 = 'lo\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; + const { data, trailers } = await runTransform([c1, c2]); + assert.strictEqual(data.toString(), 'hello'); + assert.strictEqual(trailers.length, 1); + }); + + it('\\r\\n delimiter after chunk size split across two input chunks: parsed correctly', async () => { + // '5\r' in chunk1, '\nhello\r\n0\r\n...' in chunk2 + const c1 = '5\r'; + const c2 = '\nhello\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; + const { data, trailers } = await runTransform([c1, c2]); + assert.strictEqual(data.toString(), 'hello'); + assert.strictEqual(trailers.length, 1); + }); + + it('trailer line split across two input chunks: emits trailer correctly', async () => { + const c1 = '5\r\nhello\r\n0\r\nx-amz-checksum-'; + const c2 = 'crc32:AAAAAA==\r\n'; + const { data, trailers } = await runTransform([c1, c2]); + assert.strictEqual(data.toString(), 'hello'); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); + assert.strictEqual(trailers[0].value, 'AAAAAA=='); + }); + + it('trailer \\r\\n split across two input chunks: emits trailer correctly', async () => { + const c1 = '5\r\nhello\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r'; + const c2 = '\n'; + const { data, trailers } = await runTransform([c1, c2]); + assert.strictEqual(data.toString(), 'hello'); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); + assert.strictEqual(trailers[0].value, 'AAAAAA=='); + }); + }); + + describe('zero-size terminator and trailer', () => { + it('empty trailer line (0\\r\\n\\r\\n): no trailer event emitted, stream closes cleanly', async () => { + const input = '5\r\nhello\r\n0\r\n\r\n'; + const { data, trailers } = await runTransform([input]); + assert.strictEqual(data.toString(), 'hello'); + assert.strictEqual(trailers.length, 0); + }); + + it('zero data chunks (only terminator + trailer): no data forwarded, trailer emitted', async () => { + const input = '0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; + const { data, trailers } = await runTransform([input]); + assert.strictEqual(data.length, 0); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); + assert.strictEqual(trailers[0].value, 'AAAAAA=='); + }); + }); + + describe('_flush error cases', () => { + it('stream ends mid-data (no zero-chunk): IncompleteBody error', async () => { + // 5 bytes declared but stream ends after only 3 + const err = await expectError(['5\r\nhel']); + assert.deepStrictEqual(err, errors.IncompleteBody); + }); + + it('stream ends after zero-chunk with partial trailer content: IncompleteBody error', async () => { + // zero-chunk received, trailer starts but no \r\n terminator + const err = await expectError(['0\r\nx-amz-checksum-crc32:AAAAAA==']); + assert.deepStrictEqual(err, errors.IncompleteBody); + }); + + it('stream ends after zero-chunk with no trailer content: no error', async () => { + // only '0\r\n' — readingTrailer=true, trailerBuffer empty → no error + const { data, trailers } = await runTransform(['0\r\n']); + assert.strictEqual(data.length, 0); + assert.strictEqual(trailers.length, 0); + }); + }); + + describe('_transform error cases', () => { + it('chunk size field larger than 10 bytes: InvalidArgument error', async () => { + // 11 hex digits — exceeds the 10-byte field size limit + const err = await expectError(['12345678901\r\n']); + assert.deepStrictEqual(err, errors.InvalidArgument); + }); + + it('chunk size is not valid hex: InvalidArgument error', async () => { + // 2 chars, short enough to pass size check, but not valid hex + const err = await expectError(['zz\r\n']); + assert.deepStrictEqual(err, errors.InvalidArgument); + }); + + it('chunk size exceeds maximumAllowedPartSize: EntityTooLarge error', async () => { + // 0x200000000 = 8589934592 > maximumAllowedPartSize (5GB = 0x140000000) + const err = await expectError(['200000000\r\n']); + assert.deepStrictEqual(err, errors.EntityTooLarge); + }); + + it('trailer line longer than 1024 bytes: MalformedTrailerError', async () => { + // send zero-chunk then a trailer line > 1024 bytes with no \r\n + const longTrailer = 'x'.repeat(1025); + const err = await expectError([`0\r\n${longTrailer}`]); + assert.deepStrictEqual(err, errors.MalformedTrailerError); + }); + + it('trailer line missing colon: IncompleteBody error', async () => { + const err = await expectError(['0\r\nnocolon\r\n']); + assert.deepStrictEqual(err, errors.IncompleteBody); + }); + + it('trailer line with colon at position 0 (empty name): IncompleteBody error', async () => { + const err = await expectError(['0\r\n:value\r\n']); + assert.deepStrictEqual(err, errors.IncompleteBody); + }); + }); +}); From 7c7367dbac5f7ae381a17c311586ee758be9658a Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Mon, 16 Mar 2026 16:00:30 +0100 Subject: [PATCH 02/15] CLDSRV-863: handle trailer headers and add new trailer errors --- .../apiUtils/integrity/validateChecksums.js | 155 +++++++++- .../apiUtils/integrity/validateChecksums.js | 281 +++++++++++++++++- 2 files changed, 423 insertions(+), 13 deletions(-) diff --git a/lib/api/apiUtils/integrity/validateChecksums.js b/lib/api/apiUtils/integrity/validateChecksums.js index 9643a71258..e440389ab2 100644 --- a/lib/api/apiUtils/integrity/validateChecksums.js +++ b/lib/api/apiUtils/integrity/validateChecksums.js @@ -37,6 +37,12 @@ const ChecksumError = Object.freeze({ MultipleChecksumTypes: 'MultipleChecksumTypes', MissingCorresponding: 'MissingCorresponding', MalformedChecksum: 'MalformedChecksum', + TrailerAlgoMismatch: 'TrailerAlgoMismatch', + TrailerChecksumMalformed: 'TrailerChecksumMalformed', + TrailerMissing: 'TrailerMissing', + TrailerUnexpected: 'TrailerUnexpected', + TrailerAndChecksum: 'TrailerAndChecksum', + TrailerNotSupported: 'TrailerNotSupported', }); const base64Regex = /^[A-Za-z0-9+/]*={0,2}$/; @@ -56,35 +62,51 @@ const algorithms = Object.freeze({ const result = await crc.digest(); return Buffer.from(result).toString('base64'); }, + digestFromHash: async hash => { + const result = await hash.digest(); + return Buffer.from(result).toString('base64'); + }, isValidDigest: expected => typeof expected === 'string' && expected.length === 12 && base64Regex.test(expected), + createHash: () => new CrtCrc64Nvme() }, crc32: { digest: data => { const input = Buffer.isBuffer(data) ? data : Buffer.from(data); return uint32ToBase64(new Crc32().update(input).digest() >>> 0); // >>> 0 coerce number to uint32 }, + digestFromHash: hash => { + const result = hash.digest(); + return uint32ToBase64(result >>> 0); + }, isValidDigest: expected => typeof expected === 'string' && expected.length === 8 && base64Regex.test(expected), + createHash: () => new Crc32() }, crc32c: { digest: data => { const input = Buffer.isBuffer(data) ? data : Buffer.from(data); return uint32ToBase64(new Crc32c().update(input).digest() >>> 0); // >>> 0 coerce number to uint32 }, + digestFromHash: hash => uint32ToBase64(hash.digest() >>> 0), isValidDigest: expected => typeof expected === 'string' && expected.length === 8 && base64Regex.test(expected), + createHash: () => new Crc32c() }, sha1: { digest: data => { const input = Buffer.isBuffer(data) ? data : Buffer.from(data); return crypto.createHash('sha1').update(input).digest('base64'); }, + digestFromHash: hash => hash.digest('base64'), isValidDigest: expected => typeof expected === 'string' && expected.length === 28 && base64Regex.test(expected), + createHash: () => crypto.createHash('sha1') }, sha256: { digest: data => { const input = Buffer.isBuffer(data) ? data : Buffer.from(data); return crypto.createHash('sha256').update(input).digest('base64'); }, + digestFromHash: hash => hash.digest('base64'), isValidDigest: expected => typeof expected === 'string' && expected.length === 44 && base64Regex.test(expected), + createHash: () => crypto.createHash('sha256') } }); @@ -132,7 +154,7 @@ async function validateXAmzChecksums(headers, body) { return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } }; } - // If AWS there is a mismatch, AWS returns the same error as if the algo was invalid. + // If there is a mismatch, AWS returns the same error as if the algo was invalid. if (sdkLowerAlgo !== algo) { return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } }; } @@ -141,6 +163,94 @@ async function validateXAmzChecksums(headers, body) { return null; } +function getChecksumDataFromHeaders(headers) { + const checkSdk = algo => { + if (!('x-amz-sdk-checksum-algorithm' in headers)) { + return null; + } + + const sdkAlgo = headers['x-amz-sdk-checksum-algorithm']; + if (typeof sdkAlgo !== 'string') { + return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } }; + } + + const sdkLowerAlgo = sdkAlgo.toLowerCase(); + if (!(sdkLowerAlgo in algorithms)) { + return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } }; + } + + // If there is a mismatch, AWS returns the same error as if the algo was invalid. + if (sdkLowerAlgo !== algo) { + return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } }; + } + + return null; + }; + + const checksumHeaders = Object.keys(headers).filter(header => header.startsWith('x-amz-checksum-')); + const xAmzChecksumCnt = checksumHeaders.length; + if (xAmzChecksumCnt > 1) { + return { error: ChecksumError.MultipleChecksumTypes, details: { algorithms: checksumHeaders } }; + } + + if (xAmzChecksumCnt === 0 && !('x-amz-trailer' in headers) && 'x-amz-sdk-checksum-algorithm' in headers) { + return { + error: ChecksumError.MissingCorresponding, + details: { expected: headers['x-amz-sdk-checksum-algorithm'] } + }; + } + + if ('x-amz-trailer' in headers) { + if (xAmzChecksumCnt !== 0) { + return { + error: ChecksumError.TrailerAndChecksum, + details: { trailer: headers['x-amz-trailer'], checksum: checksumHeaders }, + }; + } + + const trailer = headers['x-amz-trailer']; + if (!trailer.startsWith('x-amz-checksum-')) { + return { error: ChecksumError.TrailerNotSupported, details: { value: trailer } }; + } + + const trailerAlgo = trailer.slice('x-amz-checksum-'.length); + if (!(trailerAlgo in algorithms)) { + return { error: ChecksumError.TrailerNotSupported, details: { value: trailer } }; + } + + const err = checkSdk(trailerAlgo); + if (err) { + return err; + } + + return { algorithm: trailerAlgo, isTrailer: true, expected: undefined }; + } + + if (xAmzChecksumCnt === 0) { + // There was no x-amz-checksum- or x-amz-trailer return crc64nvme. + // The calculated crc64nvme will be stored in the object metadata. + return { algorithm: 'crc64nvme', isTrailer: false, expected: undefined }; + } + + // No x-amz-sdk-checksum-algorithm we expect one x-amz-checksum-[crc64nvme, crc32, crc32C, sha1, sha256]. + const algo = checksumHeaders[0].slice('x-amz-checksum-'.length); + if (!(algo in algorithms)) { + return { error: ChecksumError.AlgoNotSupported, details: { algorithm: algo } }; + } + + const expected = headers[`x-amz-checksum-${algo}`]; + if (!algorithms[algo].isValidDigest(expected)) { + return { error: ChecksumError.MalformedChecksum, details: { algorithm: algo, expected } }; + } + + const err = checkSdk(algo); + if (err) { + return err; + } + + return { algorithm: algo, isTrailer: false, expected }; +} + /** * validateChecksumsNoChunking - Validate the checksums of a request. * @param {object} headers - http headers @@ -183,16 +293,7 @@ async function validateChecksumsNoChunking(headers, body) { return err; } -async function defaultValidationFunc(request, body, log) { - const err = await validateChecksumsNoChunking(request.headers, body); - if (!err) { - return null; - } - - if (err.error !== ChecksumError.MissingChecksum) { - log.debug('failed checksum validation', { method: request.apiMethod }, err); - } - +function arsenalErrorFromChecksumError(err) { switch (err.error) { case ChecksumError.MissingChecksum: return null; @@ -225,11 +326,40 @@ async function defaultValidationFunc(request, body, log) { ); case ChecksumError.MD5Invalid: return ArsenalErrors.InvalidDigest; + case ChecksumError.TrailerAlgoMismatch: + return ArsenalErrors.MalformedTrailerError; + case ChecksumError.TrailerMissing: + return ArsenalErrors.MalformedTrailerError; + case ChecksumError.TrailerUnexpected: + return ArsenalErrors.MalformedTrailerError; + case ChecksumError.TrailerChecksumMalformed: + return ArsenalErrors.InvalidRequest.customizeDescription( + `Value for x-amz-checksum-${err.details.algorithm} trailing header is invalid.` + ); + case ChecksumError.TrailerAndChecksum: + return ArsenalErrors.InvalidRequest.customizeDescription('Expecting a single x-amz-checksum- header'); + case ChecksumError.TrailerNotSupported: + return ArsenalErrors.InvalidRequest.customizeDescription( + 'The value specified in the x-amz-trailer header is not supported' + ); default: return ArsenalErrors.BadDigest; } } +async function defaultValidationFunc(request, body, log) { + const err = await validateChecksumsNoChunking(request.headers, body); + if (!err) { + return null; + } + + if (err.error !== ChecksumError.MissingChecksum) { + log.debug('failed checksum validation', { method: request.apiMethod }, err); + } + + return arsenalErrorFromChecksumError(err); +} + /** * validateMethodChecksumsNoChunking - Validate the checksums of a request. * @param {object} request - http request @@ -253,5 +383,8 @@ module.exports = { ChecksumError, validateChecksumsNoChunking, validateMethodChecksumNoChunking, + getChecksumDataFromHeaders, + arsenalErrorFromChecksumError, + algorithms, checksumedMethods, }; diff --git a/tests/unit/api/apiUtils/integrity/validateChecksums.js b/tests/unit/api/apiUtils/integrity/validateChecksums.js index a234dcfd60..925d79a713 100644 --- a/tests/unit/api/apiUtils/integrity/validateChecksums.js +++ b/tests/unit/api/apiUtils/integrity/validateChecksums.js @@ -2,8 +2,14 @@ const assert = require('assert'); const crypto = require('crypto'); const sinon = require('sinon'); -const { validateChecksumsNoChunking, ChecksumError, validateMethodChecksumNoChunking, checksumedMethods } = - require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); +const { + validateChecksumsNoChunking, + ChecksumError, + validateMethodChecksumNoChunking, + checksumedMethods, + getChecksumDataFromHeaders, + arsenalErrorFromChecksumError, +} = require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); const { errors: ArsenalErrors } = require('arsenal'); const { config } = require('../../../../../lib/Config'); @@ -453,3 +459,274 @@ describe('validateMethodChecksumNoChunking', () => { }); }); }); + +describe('getChecksumDataFromHeaders', () => { + // Valid-format digests (correct length and base64, content not verified by getChecksumDataFromHeaders) + const validDigests = { + crc32: 'AAAAAA==', // 8 chars + crc32c: 'AAAAAA==', // 8 chars + sha1: 'AAAAAAAAAAAAAAAAAAAAAAAAAAA=', // 28 chars + sha256: 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', // 44 chars + crc64nvme: 'AAAAAAAAAAA=', // 12 chars + }; + + it('no headers: returns crc64nvme with isTrailer=false and expected=undefined', () => { + const result = getChecksumDataFromHeaders({}); + assert.deepStrictEqual(result, { algorithm: 'crc64nvme', isTrailer: false, expected: undefined }); + }); + + it('no checksum headers, no trailer, no sdk algo: returns crc64nvme default', () => { + const result = getChecksumDataFromHeaders({ 'content-type': 'application/octet-stream' }); + assert.deepStrictEqual(result, { algorithm: 'crc64nvme', isTrailer: false, expected: undefined }); + }); + + for (const [algo, digest] of Object.entries(validDigests)) { + it(`x-amz-checksum-${algo} with valid digest: returns algorithm, isTrailer=false, expected`, () => { + const result = getChecksumDataFromHeaders({ [`x-amz-checksum-${algo}`]: digest }); + assert.deepStrictEqual(result, { algorithm: algo, isTrailer: false, expected: digest }); + }); + } + + it('x-amz-checksum-unknown-algo: returns AlgoNotSupported error', () => { + const result = getChecksumDataFromHeaders({ 'x-amz-checksum-md4': 'AAAAAA==' }); + assert.strictEqual(result.error, ChecksumError.AlgoNotSupported); + assert.strictEqual(result.details.algorithm, 'md4'); + }); + + it('x-amz-checksum-crc32 with malformed digest (wrong length): returns MalformedChecksum error', () => { + const result = getChecksumDataFromHeaders({ 'x-amz-checksum-crc32': 'AAAAA==' }); // 7 chars, crc32 needs 8 + assert.strictEqual(result.error, ChecksumError.MalformedChecksum); + assert.strictEqual(result.details.algorithm, 'crc32'); + }); + + it('x-amz-checksum-crc32 with malformed digest (invalid base64): returns MalformedChecksum error', () => { + const result = getChecksumDataFromHeaders({ 'x-amz-checksum-crc32': '!!!!!!!!' }); + assert.strictEqual(result.error, ChecksumError.MalformedChecksum); + assert.strictEqual(result.details.algorithm, 'crc32'); + }); + + it('two x-amz-checksum- headers: returns MultipleChecksumTypes error', () => { + const result = getChecksumDataFromHeaders({ + 'x-amz-checksum-crc32': validDigests.crc32, + 'x-amz-checksum-sha256': validDigests.sha256, + }); + assert.strictEqual(result.error, ChecksumError.MultipleChecksumTypes); + }); + + it('x-amz-sdk-checksum-algorithm with no x-amz-checksum- and no x-amz-trailer: returns MissingCorresponding', + () => { + const result = getChecksumDataFromHeaders({ 'x-amz-sdk-checksum-algorithm': 'crc32' }); + assert.strictEqual(result.error, ChecksumError.MissingCorresponding); + assert.strictEqual(result.details.expected, 'crc32'); + }); + + it('x-amz-checksum-crc32 with matching x-amz-sdk-checksum-algorithm CRC32: returns success', () => { + const result = getChecksumDataFromHeaders({ + 'x-amz-checksum-crc32': validDigests.crc32, + 'x-amz-sdk-checksum-algorithm': 'crc32', + }); + assert.deepStrictEqual(result, { algorithm: 'crc32', isTrailer: false, expected: validDigests.crc32 }); + }); + + it('x-amz-checksum-crc32 with mismatched x-amz-sdk-checksum-algorithm SHA256: returns AlgoNotSupportedSDK', () => { + const result = getChecksumDataFromHeaders({ + 'x-amz-checksum-crc32': validDigests.crc32, + 'x-amz-sdk-checksum-algorithm': 'sha256', + }); + assert.strictEqual(result.error, ChecksumError.AlgoNotSupportedSDK); + assert.strictEqual(result.details.algorithm, 'sha256'); + }); + + it('x-amz-checksum-crc32 with non-string x-amz-sdk-checksum-algorithm: returns AlgoNotSupportedSDK', () => { + const result = getChecksumDataFromHeaders({ + 'x-amz-checksum-crc32': validDigests.crc32, + 'x-amz-sdk-checksum-algorithm': 1234, + }); + assert.strictEqual(result.error, ChecksumError.AlgoNotSupportedSDK); + assert.strictEqual(result.details.algorithm, 1234); + }); + + it('x-amz-checksum-crc32 with unknown x-amz-sdk-checksum-algorithm: returns AlgoNotSupportedSDK', () => { + const result = getChecksumDataFromHeaders({ + 'x-amz-checksum-crc32': validDigests.crc32, + 'x-amz-sdk-checksum-algorithm': 'md4', + }); + assert.strictEqual(result.error, ChecksumError.AlgoNotSupportedSDK); + assert.strictEqual(result.details.algorithm, 'md4'); + }); + + it('x-amz-trailer: x-amz-checksum-crc32: returns isTrailer=true', () => { + const result = getChecksumDataFromHeaders({ 'x-amz-trailer': 'x-amz-checksum-crc32' }); + assert.deepStrictEqual(result, { algorithm: 'crc32', isTrailer: true, expected: undefined }); + }); + + it('x-amz-trailer: x-amz-checksum-crc64nvme: returns isTrailer=true', () => { + const result = getChecksumDataFromHeaders({ 'x-amz-trailer': 'x-amz-checksum-crc64nvme' }); + assert.deepStrictEqual(result, { algorithm: 'crc64nvme', isTrailer: true, expected: undefined }); + }); + + it('x-amz-trailer with unsupported value (not x-amz-checksum- prefix): returns TrailerNotSupported', () => { + const result = getChecksumDataFromHeaders({ 'x-amz-trailer': 'x-custom-header' }); + assert.strictEqual(result.error, ChecksumError.TrailerNotSupported); + assert.strictEqual(result.details.value, 'x-custom-header'); + }); + + it('x-amz-trailer: x-amz-checksum-unknown-algo: returns TrailerNotSupported', () => { + const result = getChecksumDataFromHeaders({ 'x-amz-trailer': 'x-amz-checksum-md4' }); + assert.strictEqual(result.error, ChecksumError.TrailerNotSupported); + assert.strictEqual(result.details.value, 'x-amz-checksum-md4'); + }); + + it('x-amz-trailer with also an x-amz-checksum- header: returns TrailerAndChecksum', () => { + const result = getChecksumDataFromHeaders({ + 'x-amz-trailer': 'x-amz-checksum-crc32', + 'x-amz-checksum-crc32': validDigests.crc32, + }); + assert.strictEqual(result.error, ChecksumError.TrailerAndChecksum); + }); + + it('x-amz-trailer with matching x-amz-sdk-checksum-algorithm: returns success', () => { + const result = getChecksumDataFromHeaders({ + 'x-amz-trailer': 'x-amz-checksum-crc32', + 'x-amz-sdk-checksum-algorithm': 'crc32', + }); + assert.deepStrictEqual(result, { algorithm: 'crc32', isTrailer: true, expected: undefined }); + }); + + it('x-amz-trailer with mismatched x-amz-sdk-checksum-algorithm: returns AlgoNotSupportedSDK', () => { + const result = getChecksumDataFromHeaders({ + 'x-amz-trailer': 'x-amz-checksum-crc32', + 'x-amz-sdk-checksum-algorithm': 'sha256', + }); + assert.strictEqual(result.error, ChecksumError.AlgoNotSupportedSDK); + assert.strictEqual(result.details.algorithm, 'sha256'); + }); +}); + +describe('arsenalErrorFromChecksumError', () => { + it('MissingChecksum: returns null', () => { + const result = arsenalErrorFromChecksumError({ error: ChecksumError.MissingChecksum, details: null }); + assert.strictEqual(result, null); + }); + + it('XAmzMismatch (crc32): returns BadDigest mentioning CRC32', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.XAmzMismatch, + details: { algorithm: 'crc32', calculated: 'a', expected: 'b' }, + }); + assert(result.is.BadDigest); + assert.strictEqual(result.description, 'The CRC32 you specified did not match the calculated checksum.'); + }); + + it('XAmzMismatch (sha256): returns BadDigest mentioning SHA256', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.XAmzMismatch, + details: { algorithm: 'sha256', calculated: 'a', expected: 'b' }, + }); + assert(result.is.BadDigest); + assert.strictEqual(result.description, 'The SHA256 you specified did not match the calculated checksum.'); + }); + + it('AlgoNotSupported: returns InvalidRequest', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.AlgoNotSupported, + details: { algorithm: 'md4' }, + }); + assert(result.is.InvalidRequest); + }); + + it('AlgoNotSupportedSDK: returns InvalidRequest', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.AlgoNotSupportedSDK, + details: { algorithm: 'md4' }, + }); + assert(result.is.InvalidRequest); + }); + + it('MissingCorresponding: returns InvalidRequest', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.MissingCorresponding, + details: { expected: 'crc32' }, + }); + assert(result.is.InvalidRequest); + }); + + it('MultipleChecksumTypes: returns InvalidRequest', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.MultipleChecksumTypes, + details: { algorithms: ['x-amz-checksum-crc32', 'x-amz-checksum-sha256'] }, + }); + assert(result.is.InvalidRequest); + }); + + it('MalformedChecksum (crc32): returns InvalidRequest mentioning crc32', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.MalformedChecksum, + details: { algorithm: 'crc32', expected: 'bad' }, + }); + assert(result.is.InvalidRequest); + assert.strictEqual(result.description, 'Value for x-amz-checksum-crc32 header is invalid.'); + }); + + it('MD5Invalid: returns InvalidDigest', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.MD5Invalid, + details: { expected: 'bad' }, + }); + assert.deepStrictEqual(result, ArsenalErrors.InvalidDigest); + }); + + it('TrailerAlgoMismatch: returns MalformedTrailerError', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.TrailerAlgoMismatch, + details: { algorithm: 'crc32' }, + }); + assert.deepStrictEqual(result, ArsenalErrors.MalformedTrailerError); + }); + + it('TrailerMissing: returns MalformedTrailerError', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.TrailerMissing, + details: { expectedTrailer: 'x-amz-checksum-crc32' }, + }); + assert.deepStrictEqual(result, ArsenalErrors.MalformedTrailerError); + }); + + it('TrailerUnexpected: returns MalformedTrailerError', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.TrailerUnexpected, + details: { name: 'x-amz-checksum-crc32', val: 'AAAAAA==' }, + }); + assert.deepStrictEqual(result, ArsenalErrors.MalformedTrailerError); + }); + + it('TrailerChecksumMalformed: returns InvalidRequest mentioning the algorithm', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.TrailerChecksumMalformed, + details: { algorithm: 'sha256', expected: 'bad' }, + }); + assert(result.is.InvalidRequest); + assert.strictEqual(result.description, 'Value for x-amz-checksum-sha256 trailing header is invalid.'); + }); + + it('TrailerAndChecksum: returns InvalidRequest', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.TrailerAndChecksum, + details: { trailer: 'x-amz-checksum-crc32', checksum: ['x-amz-checksum-crc32'] }, + }); + assert(result.is.InvalidRequest); + }); + + it('TrailerNotSupported: returns InvalidRequest', () => { + const result = arsenalErrorFromChecksumError({ + error: ChecksumError.TrailerNotSupported, + details: { value: 'x-custom-header' }, + }); + assert(result.is.InvalidRequest); + }); + + it('unknown error type (default): returns BadDigest', () => { + const result = arsenalErrorFromChecksumError({ error: 'SomeUnknownError', details: null }); + assert.deepStrictEqual(result, ArsenalErrors.BadDigest); + }); +}); From 588f37e424166dd43dbf5cab9734be4966417d11 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Mon, 16 Mar 2026 16:00:58 +0100 Subject: [PATCH 03/15] CLDSRV-863: add ChecksumTransform to calculate and verify stream checksums --- lib/auth/streamingV4/ChecksumTransform.js | 92 +++++++++++ tests/unit/auth/ChecksumTransform.js | 182 ++++++++++++++++++++++ 2 files changed, 274 insertions(+) create mode 100644 lib/auth/streamingV4/ChecksumTransform.js create mode 100644 tests/unit/auth/ChecksumTransform.js diff --git a/lib/auth/streamingV4/ChecksumTransform.js b/lib/auth/streamingV4/ChecksumTransform.js new file mode 100644 index 0000000000..38ad6f1d8b --- /dev/null +++ b/lib/auth/streamingV4/ChecksumTransform.js @@ -0,0 +1,92 @@ +const { errors } = require('arsenal'); +const { algorithms, ChecksumError } = require('../../api/apiUtils/integrity/validateChecksums'); +const { Transform } = require('stream'); + +class ChecksumTransform extends Transform { + constructor(algoName, expectedDigest, isTrailer, log) { + super({}); + this.log = log; + this.algoName = algoName; + this.algo = algorithms[algoName]; + this.hash = this.algo.createHash(); + this.digest = undefined; + this.expectedDigest = expectedDigest; + this.isTrailer = isTrailer; + this.trailerChecksumName = undefined; + this.trailerChecksumValue = undefined; + } + + setExpectedChecksum(name, value) { + this.trailerChecksumName = name; + this.trailerChecksumValue = value; + } + + validateChecksum() { + if (this.isTrailer) { + // x-amz-trailer in headers but no trailer in body. + if (this.trailerChecksumValue === undefined) { + return { + error: ChecksumError.TrailerMissing, + details: { expectedTrailer: `x-amz-checksum-${this.algoName}` }, + }; + } + + if (this.trailerChecksumName !== `x-amz-checksum-${this.algoName}`) { + return { error: ChecksumError.TrailerAlgoMismatch, details: { algorithm: this.algoName } }; + } + + const expected = this.trailerChecksumValue; + if (!this.algo.isValidDigest(expected)) { + return { + error: ChecksumError.TrailerChecksumMalformed, + details: { algorithm: this.algoName, expected }, + }; + } + + if (this.digest !== this.trailerChecksumValue) { + return { + error: ChecksumError.XAmzMismatch, + details: { algorithm: this.algoName, calculated: this.digest, expected }, + }; + } + + return null; + } + + if (this.trailerChecksumValue) { + // Trailer found in the body but no x-amz-trailer in the headers. + return { + error: ChecksumError.TrailerUnexpected, + details: { name: this.trailerChecksumName, val: this.trailerChecksumValue }, + }; + } + + if (this.expectedDigest) { + if (this.digest !== this.expectedDigest) { + return { + error: ChecksumError.XAmzMismatch, + details: { algorithm: this.algoName, calculated: this.digest, expected: this.expectedDigest }, + }; + } + } + + return null; + } + + _flush(callback) { + Promise.resolve(this.algo.digestFromHash(this.hash)) + .then(digest => { this.digest = digest; }) + .then(() => callback(), err => { + this.log.error('failed to compute checksum digest', { error: err, algorithm: this.algoName }); + callback(errors.InternalError); + }); + } + + _transform(chunk, encoding, callback) { + const input = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + this.hash.update(input, encoding); + callback(null, input); + } +} + +module.exports = ChecksumTransform; diff --git a/tests/unit/auth/ChecksumTransform.js b/tests/unit/auth/ChecksumTransform.js new file mode 100644 index 0000000000..732d7f8cd4 --- /dev/null +++ b/tests/unit/auth/ChecksumTransform.js @@ -0,0 +1,182 @@ +const assert = require('assert'); +const { errors } = require('arsenal'); + +const { algorithms, ChecksumError } = require('../../../lib/api/apiUtils/integrity/validateChecksums'); +const ChecksumTransform = require('../../../lib/auth/streamingV4/ChecksumTransform'); +const { DummyRequestLogger } = require('../helpers'); + +const log = new DummyRequestLogger(); +const testData = Buffer.from('hello world'); +const algos = ['crc32', 'crc32c', 'sha1', 'sha256', 'crc64nvme']; + +// Helper: pipe chunks into a ChecksumTransform, collect output, resolve on finish +function runTransform(stream, chunks) { + return new Promise((resolve, reject) => { + const output = []; + stream.on('data', chunk => output.push(chunk)); + stream.on('end', () => resolve(Buffer.concat(output))); + stream.on('error', reject); + for (const chunk of chunks) { + stream.write(chunk); + } + stream.end(); + }); +} + +// Helper: pipe data through and wait for finish without collecting output +function drainTransform(stream, chunks) { + return new Promise((resolve, reject) => { + stream.resume(); + stream.on('finish', resolve); + stream.on('error', reject); + for (const chunk of chunks) { + stream.write(chunk); + } + stream.end(); + }); +} + +describe('ChecksumTransform basic behaviour', () => { + let expectedDigests; + + before(async () => { + expectedDigests = {}; + for (const algo of algos) { + expectedDigests[algo] = await Promise.resolve(algorithms[algo].digest(testData)); + } + }); + + for (const algo of algos) { + it(`[${algo}] passes data through unchanged`, async () => { + const stream = new ChecksumTransform(algo, undefined, false, log); + const output = await runTransform(stream, [testData]); + assert.deepStrictEqual(output, testData); + }); + + it(`[${algo}] computes digest correctly after stream ends`, async () => { + const stream = new ChecksumTransform(algo, undefined, false, log); + await drainTransform(stream, [testData]); + assert.strictEqual(stream.digest, expectedDigests[algo]); + }); + + it(`[${algo}] handles multi-chunk input: digest matches single-chunk equivalent`, async () => { + const half = Math.floor(testData.length / 2); + const stream = new ChecksumTransform(algo, undefined, false, log); + await drainTransform(stream, [testData.subarray(0, half), testData.subarray(half)]); + assert.strictEqual(stream.digest, expectedDigests[algo]); + }); + + it(`[${algo}] handles Buffer and string chunks equally`, async () => { + const streamBuf = new ChecksumTransform(algo, undefined, false, log); + const streamStr = new ChecksumTransform(algo, undefined, false, log); + await drainTransform(streamBuf, [testData]); + await drainTransform(streamStr, [testData.toString()]); + assert.strictEqual(streamBuf.digest, streamStr.digest); + }); + } + + it('emits error via stream error event if digestFromHash fails', done => { + const stream = new ChecksumTransform('crc32', undefined, false, log); + // Replace digestFromHash to return a rejected Promise + stream.algo = Object.assign({}, stream.algo, { + digestFromHash: () => Promise.reject(new Error('simulated digest failure')), + }); + stream.on('error', err => { + assert.deepStrictEqual(err, errors.InternalError); + done(); + }); + stream.write(testData); + stream.end(); + stream.resume(); + }); +}); + +describe('ChecksumTransform validateChecksum — non-trailer mode (isTrailer=false)', () => { + let crc32Digest; + + before(async () => { + crc32Digest = await Promise.resolve(algorithms.crc32.digest(testData)); + }); + + it('returns null when no expectedDigest and no trailer received', async () => { + const stream = new ChecksumTransform('crc32', undefined, false, log); + await drainTransform(stream, [testData]); + assert.strictEqual(stream.validateChecksum(), null); + }); + + it('returns null when expectedDigest matches computed digest', async () => { + const stream = new ChecksumTransform('crc32', crc32Digest, false, log); + await drainTransform(stream, [testData]); + assert.strictEqual(stream.validateChecksum(), null); + }); + + it('returns XAmzMismatch when expectedDigest does not match computed digest', async () => { + const stream = new ChecksumTransform('crc32', 'AAAAAA==', false, log); + await drainTransform(stream, [testData]); + const result = stream.validateChecksum(); + assert.strictEqual(result.error, ChecksumError.XAmzMismatch); + assert.strictEqual(result.details.algorithm, 'crc32'); + assert.strictEqual(result.details.calculated, crc32Digest); + assert.strictEqual(result.details.expected, 'AAAAAA=='); + }); + + it('returns TrailerUnexpected when setExpectedChecksum was called but isTrailer=false', async () => { + const stream = new ChecksumTransform('crc32', undefined, false, log); + stream.setExpectedChecksum('x-amz-checksum-crc32', crc32Digest); + await drainTransform(stream, [testData]); + const result = stream.validateChecksum(); + assert.strictEqual(result.error, ChecksumError.TrailerUnexpected); + }); +}); + +describe('ChecksumTransform validateChecksum — trailer mode (isTrailer=true)', () => { + let crc32Digest; + + before(async () => { + crc32Digest = await Promise.resolve(algorithms.crc32.digest(testData)); + }); + + it('returns TrailerMissing when setExpectedChecksum was never called', async () => { + const stream = new ChecksumTransform('crc32', undefined, true, log); + await drainTransform(stream, [testData]); + const result = stream.validateChecksum(); + assert.strictEqual(result.error, ChecksumError.TrailerMissing); + assert.strictEqual(result.details.expectedTrailer, 'x-amz-checksum-crc32'); + }); + + it('returns TrailerAlgoMismatch when trailer name does not match algo', async () => { + const stream = new ChecksumTransform('crc32', undefined, true, log); + stream.setExpectedChecksum('x-amz-checksum-sha256', 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='); + await drainTransform(stream, [testData]); + const result = stream.validateChecksum(); + assert.strictEqual(result.error, ChecksumError.TrailerAlgoMismatch); + assert.strictEqual(result.details.algorithm, 'crc32'); + }); + + it('returns TrailerChecksumMalformed when trailer value is not a valid digest for the algo', async () => { + const stream = new ChecksumTransform('crc32', undefined, true, log); + stream.setExpectedChecksum('x-amz-checksum-crc32', 'not-valid!'); + await drainTransform(stream, [testData]); + const result = stream.validateChecksum(); + assert.strictEqual(result.error, ChecksumError.TrailerChecksumMalformed); + assert.strictEqual(result.details.algorithm, 'crc32'); + }); + + it('returns XAmzMismatch when trailer value is valid but does not match computed digest', async () => { + const stream = new ChecksumTransform('crc32', undefined, true, log); + stream.setExpectedChecksum('x-amz-checksum-crc32', 'AAAAAA=='); + await drainTransform(stream, [testData]); + const result = stream.validateChecksum(); + assert.strictEqual(result.error, ChecksumError.XAmzMismatch); + assert.strictEqual(result.details.algorithm, 'crc32'); + assert.strictEqual(result.details.calculated, crc32Digest); + assert.strictEqual(result.details.expected, 'AAAAAA=='); + }); + + it('returns null when trailer name and value match computed digest', async () => { + const stream = new ChecksumTransform('crc32', undefined, true, log); + stream.setExpectedChecksum('x-amz-checksum-crc32', crc32Digest); + await drainTransform(stream, [testData]); + assert.strictEqual(stream.validateChecksum(), null); + }); +}); From 176ee41f7be34c8f7fbf2ffa21d5adfbe689a8b4 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Mon, 16 Mar 2026 16:01:49 +0100 Subject: [PATCH 04/15] CLDSRV-863: pipe requests to ChecksumTransform and validate checksum after upload --- lib/api/apiUtils/object/prepareStream.js | 144 ++++++--- lib/api/apiUtils/object/storeObject.js | 71 +++- .../unit/api/apiUtils/object/prepareStream.js | 200 ++++++++++++ tests/unit/api/apiUtils/object/storeObject.js | 303 ++++++++++++++++++ 4 files changed, 661 insertions(+), 57 deletions(-) create mode 100644 tests/unit/api/apiUtils/object/prepareStream.js create mode 100644 tests/unit/api/apiUtils/object/storeObject.js diff --git a/lib/api/apiUtils/object/prepareStream.js b/lib/api/apiUtils/object/prepareStream.js index be3f6c69ca..043a296049 100644 --- a/lib/api/apiUtils/object/prepareStream.js +++ b/lib/api/apiUtils/object/prepareStream.js @@ -1,50 +1,114 @@ const V4Transform = require('../../../auth/streamingV4/V4Transform'); const TrailingChecksumTransform = require('../../../auth/streamingV4/trailingChecksumTransform'); +const ChecksumTransform = require('../../../auth/streamingV4/ChecksumTransform'); +const { + getChecksumDataFromHeaders, + arsenalErrorFromChecksumError, +} = require('../../apiUtils/integrity/validateChecksums'); +const { errors, jsutil } = require('arsenal'); +const { unsupportedSignatureChecksums } = require('../../../../constants'); /** - * Prepares the stream if the chunks are sent in a v4 Auth request - * @param {object} stream - stream containing the data - * @param {object | null } streamingV4Params - if v4 auth, object containing - * accessKey, signatureFromRequest, region, scopeDate, timestamp, and - * credentialScope (to be used for streaming v4 auth if applicable) - * @param {RequestLogger} log - the current request logger - * @param {function} errCb - callback called if an error occurs - * @return {object|null} - V4Transform object if v4 Auth request, or - * the original stream, or null if the request has no V4 params but - * the type of request requires them + * Prepares the request stream for data storage by wrapping it in the + * appropriate transform pipeline based on the x-amz-content-sha256 header. + * Always returns a ChecksumTransform as the final stream. + * If no checksum was sent by the client a CRC64NVME ChecksumTransform is returned. + * + * @param {object} request - incoming HTTP request with headers and body stream + * @param {object|null} streamingV4Params - v4 streaming auth params (accessKey, + * signatureFromRequest, region, scopeDate, timestamp, credentialScope), or + * null/undefined for non-v4 requests + * @param {RequestLogger} log - request logger + * @param {function} errCb - error callback invoked if a stream error occurs + * @return {{ error: Arsenal.Error|null, stream: ChecksumTransform|null }} + * error is set and stream is null if the request headers are invalid; + * otherwise error is null and stream is the ChecksumTransform to read from */ -function prepareStream(stream, streamingV4Params, log, errCb) { - if (stream.headers['x-amz-content-sha256'] === - 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD') { - if (typeof streamingV4Params !== 'object') { - // this might happen if the user provided a valid V2 - // Authentication header, while the chunked upload method - // requires V4: in such case we don't get any V4 params - // and we should return an error to the client. - return null; - } - const v4Transform = new V4Transform(streamingV4Params, log, errCb); - stream.pipe(v4Transform); - v4Transform.headers = stream.headers; - return v4Transform; - } - return stream; -} +function prepareStream(request, streamingV4Params, log, errCb) { + const xAmzContentSHA256 = request.headers['x-amz-content-sha256']; -function stripTrailingChecksumStream(stream, log, errCb) { - // don't do anything if we are not in the correct integrity check mode - if (stream.headers['x-amz-content-sha256'] !== 'STREAMING-UNSIGNED-PAYLOAD-TRAILER') { - return stream; + const checksumAlgo = getChecksumDataFromHeaders(request.headers); + if (checksumAlgo.error) { + log.debug('prepareStream invalid checksum headers', checksumAlgo); + return { error: arsenalErrorFromChecksumError(checksumAlgo), stream: null }; } - const trailingChecksumTransform = new TrailingChecksumTransform(log); - trailingChecksumTransform.on('error', errCb); - stream.pipe(trailingChecksumTransform); - trailingChecksumTransform.headers = stream.headers; - return trailingChecksumTransform; + let stream = request; + switch (xAmzContentSHA256) { + case 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD': { + if (streamingV4Params === null || typeof streamingV4Params !== 'object') { + // this might happen if the user provided a valid V2 + // Authentication header, while the chunked upload method + // requires V4: in such case we don't get any V4 params + // and we should return an error to the client. + log.error('missing v4 streaming params for chunked upload', { + method: 'prepareStream', + streamingV4ParamsType: typeof streamingV4Params, + streamingV4Params, + }); + return { error: errors.InvalidArgument, stream: null }; + } + // Use a once-guard so that if both V4Transform and ChecksumTransform + // independently error, errCb is only called once. + const onStreamError = jsutil.once(errCb); + const v4Transform = new V4Transform(streamingV4Params, log, onStreamError); + request.pipe(v4Transform); + v4Transform.headers = request.headers; + stream = v4Transform; + + const checksumedStream = new ChecksumTransform( + checksumAlgo.algorithm, + checksumAlgo.expected, + checksumAlgo.isTrailer, + log, + ); + checksumedStream.on('error', onStreamError); + stream.pipe(checksumedStream); + return { error: null, stream: checksumedStream }; + } + case 'STREAMING-UNSIGNED-PAYLOAD-TRAILER': { + // Use a once-guard so that auto-destroying both piped streams + // when one errors does not result in errCb being called twice. + const onStreamError = jsutil.once(errCb); + const trailingChecksumTransform = new TrailingChecksumTransform(log); + trailingChecksumTransform.on('error', onStreamError); + request.pipe(trailingChecksumTransform); + trailingChecksumTransform.headers = request.headers; + stream = trailingChecksumTransform; + + const checksumedStream = new ChecksumTransform( + checksumAlgo.algorithm, + checksumAlgo.expected, + checksumAlgo.isTrailer, + log, + ); + checksumedStream.on('error', onStreamError); + trailingChecksumTransform.on('trailer', (name, value) => { + checksumedStream.setExpectedChecksum(name, value); + }); + stream.pipe(checksumedStream); + return { error: null, stream: checksumedStream }; + } + case 'UNSIGNED-PAYLOAD': // Fallthrough + default: { + if (unsupportedSignatureChecksums.has(xAmzContentSHA256)) { + return { + error: errors.BadRequest.customizeDescription(`${xAmzContentSHA256} is not supported`), + stream: null, + }; + } + + const checksumedStream = new ChecksumTransform( + checksumAlgo.algorithm, + checksumAlgo.expected, + checksumAlgo.isTrailer, + log, + ); + checksumedStream.on('error', errCb); + stream.pipe(checksumedStream); + return { error: null, stream: checksumedStream }; + } + } } -module.exports = { - prepareStream, - stripTrailingChecksumStream, -}; +module.exports = { prepareStream }; diff --git a/lib/api/apiUtils/object/storeObject.js b/lib/api/apiUtils/object/storeObject.js index 8beea03ecb..68a5312c4f 100644 --- a/lib/api/apiUtils/object/storeObject.js +++ b/lib/api/apiUtils/object/storeObject.js @@ -1,7 +1,8 @@ const { errors, jsutil } = require('arsenal'); const { data } = require('../../../data/wrapper'); -const { prepareStream, stripTrailingChecksumStream } = require('./prepareStream'); +const { prepareStream } = require('./prepareStream'); +const { arsenalErrorFromChecksumError } = require('../../apiUtils/integrity/validateChecksums'); /** * Check that `hashedStream.completedHash` matches header `stream.contentMD5` @@ -58,31 +59,67 @@ function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cb) { function dataStore(objectContext, cipherBundle, stream, size, streamingV4Params, backendInfo, log, cb) { const cbOnce = jsutil.once(cb); - const dataStreamTmp = prepareStream(stream, streamingV4Params, log, cbOnce); - if (!dataStreamTmp) { - return process.nextTick(() => cb(errors.InvalidArgument)); + + // errCb is delegated through a mutable reference so it can be upgraded to + // include batchDelete once data.put has actually stored data. + let onStreamError = cbOnce; + const errCb = err => onStreamError(err); + + const checksumedStream = prepareStream(stream, streamingV4Params, log, errCb); + if (checksumedStream.error) { + log.debug('dataStore failed to prepare stream', checksumedStream); + return process.nextTick(() => cbOnce(checksumedStream.error)); } - const dataStream = stripTrailingChecksumStream(dataStreamTmp, log, cbOnce); return data.put( - cipherBundle, dataStream, size, objectContext, backendInfo, log, + cipherBundle, checksumedStream.stream, size, objectContext, backendInfo, log, (err, dataRetrievalInfo, hashedStream) => { if (err) { - log.error('error in datastore', { - error: err, - }); + log.error('error in datastore', { error: err }); return cbOnce(err); } if (!dataRetrievalInfo) { - log.fatal('data put returned neither an error nor a key', { - method: 'storeObject::dataStore', - }); + log.fatal('data put returned neither an error nor a key', { method: 'storeObject::dataStore' }); return cbOnce(errors.InternalError); } - log.trace('dataStore: backend stored key', { - dataRetrievalInfo, - }); - return checkHashMatchMD5(stream, hashedStream, - dataRetrievalInfo, log, cbOnce); + log.trace('dataStore: backend stored key', { dataRetrievalInfo }); + + // Data is now stored. Upgrade the error handler so that any stream + // error from this point on cleans up the stored data first. + onStreamError = streamErr => { + log.error('checksum stream error after data.put', { error: streamErr }); + return data.batchDelete([dataRetrievalInfo], null, null, log, deleteErr => { + if (deleteErr) { + log.error('dataStore failed to delete old data', { error: deleteErr }); + } + cbOnce(streamErr); + }); + }; + + const doValidate = () => { + const checksumErr = checksumedStream.stream.validateChecksum(); + if (checksumErr) { + log.debug('failed checksum validation stream', checksumErr); + return data.batchDelete([dataRetrievalInfo], null, null, log, deleteErr => { + if (deleteErr) { + // Failure of batch delete is only logged. + log.error('dataStore failed to delete old data', { error: deleteErr }); + } + return cbOnce(arsenalErrorFromChecksumError(checksumErr)); + }); + } + return checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cbOnce); + }; + + // ChecksumTransform._flush computes the digest asynchronously for + // some algorithms (e.g. crc64nvme). writableFinished is true once + // _flush has called its callback, guaranteeing this.digest is set. + // Stream piping ordering means this is virtually always true here, + // but we wait for 'finish' explicitly to be safe. + if (checksumedStream.stream.writableFinished) { + return doValidate(); + } + checksumedStream.stream.once('finish', doValidate); + return undefined; }); } diff --git a/tests/unit/api/apiUtils/object/prepareStream.js b/tests/unit/api/apiUtils/object/prepareStream.js new file mode 100644 index 0000000000..3ae5768f02 --- /dev/null +++ b/tests/unit/api/apiUtils/object/prepareStream.js @@ -0,0 +1,200 @@ +const assert = require('assert'); +const { errors } = require('arsenal'); + +const { prepareStream } = require('../../../../../lib/api/apiUtils/object/prepareStream'); +const ChecksumTransform = require('../../../../../lib/auth/streamingV4/ChecksumTransform'); +const { DummyRequestLogger } = require('../../../helpers'); +const DummyRequest = require('../../../DummyRequest'); + +const log = new DummyRequestLogger(); + +function makeRequest(headers, body) { + return new DummyRequest({ headers }, body != null ? Buffer.from(body) : undefined); +} + +const mockV4Params = { + accessKey: 'AKIAIOSFODNN7EXAMPLE', + signatureFromRequest: 'abc123', + region: 'us-east-1', + scopeDate: '20210101', + timestamp: '20210101T000000Z', + credentialScope: '20210101/us-east-1/s3/aws4_request', +}; + +describe('prepareStream return value shape', () => { + it('returns { error: null, stream: ChecksumTransform } for UNSIGNED-PAYLOAD', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + const result = prepareStream(request, null, log, () => {}); + assert.strictEqual(result.error, null); + assert(result.stream instanceof ChecksumTransform); + }); + + it('returns { error: InvalidRequest, stream: null } on invalid checksum headers', () => { + const request = makeRequest({ + 'x-amz-checksum-crc32': 'AAAAAA==', + 'x-amz-checksum-sha256': 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', + }); + const result = prepareStream(request, null, log, () => {}); + assert(result.error.is.InvalidRequest); + assert.strictEqual(result.stream, null); + }); + + it('returns { error: BadRequest, stream: null } for unsupported x-amz-content-sha256', () => { + const request = makeRequest({ + 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER', + }); + const result = prepareStream(request, null, log, () => {}); + assert(result.error.is.BadRequest); + assert.strictEqual(result.stream, null); + }); +}); + +describe('prepareStream STREAMING-AWS4-HMAC-SHA256-PAYLOAD', () => { + it('with valid streamingV4Params: returns ChecksumTransform as final stream', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' }); + const result = prepareStream(request, mockV4Params, log, () => {}); + assert.strictEqual(result.error, null); + assert(result.stream instanceof ChecksumTransform); + }); + + it('with valid streamingV4Params: returned stream uses crc64nvme by default', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' }); + const result = prepareStream(request, mockV4Params, log, () => {}); + assert.strictEqual(result.stream.algoName, 'crc64nvme'); + }); + + it('with x-amz-checksum-crc32c header: ChecksumTransform uses crc32c algorithm', () => { + const request = makeRequest({ + 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD', + 'x-amz-checksum-crc32c': 'AAAAAA==', + }); + const result = prepareStream(request, mockV4Params, log, () => {}); + assert.strictEqual(result.error, null); + assert.strictEqual(result.stream.algoName, 'crc32c'); + assert.strictEqual(result.stream.expectedDigest, 'AAAAAA=='); + }); + + it('with null streamingV4Params: returns InvalidArgument error', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' }); + const result = prepareStream(request, null, log, () => {}); + assert.deepStrictEqual(result.error, errors.InvalidArgument); + assert.strictEqual(result.stream, null); + }); + + it('with non-object streamingV4Params (string): returns InvalidArgument error', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' }); + const result = prepareStream(request, 'not-an-object', log, () => {}); + assert.deepStrictEqual(result.error, errors.InvalidArgument); + assert.strictEqual(result.stream, null); + }); +}); + +describe('prepareStream STREAMING-UNSIGNED-PAYLOAD-TRAILER', () => { + it('returns ChecksumTransform with isTrailer=true', () => { + const request = makeRequest({ + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-crc32', + }); + const result = prepareStream(request, null, log, () => {}); + assert.strictEqual(result.error, null); + assert(result.stream instanceof ChecksumTransform); + assert.strictEqual(result.stream.isTrailer, true); + assert.strictEqual(result.stream.algoName, 'crc32'); + }); + + it('trailer event from TrailingChecksumTransform calls setExpectedChecksum on ChecksumTransform', done => { + const body = '0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; + const request = makeRequest({ + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-crc32', + }, body); + const result = prepareStream(request, null, log, done); + result.stream.resume(); + result.stream.on('finish', () => { + assert.strictEqual(result.stream.trailerChecksumName, 'x-amz-checksum-crc32'); + assert.strictEqual(result.stream.trailerChecksumValue, 'AAAAAA=='); + done(); + }); + result.stream.on('error', done); + }); + + it('errCb is called when TrailingChecksumTransform emits an error', done => { + // malformed chunked data triggers an error in TrailingChecksumTransform + const request = makeRequest({ + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-crc32', + }, 'zz\r\n'); // invalid hex chunk size + prepareStream(request, null, log, err => { + assert(err.is.InvalidArgument); + done(); + }); + }); + + it('errCb is called when ChecksumTransform emits an error', done => { + const request = makeRequest({ + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-crc32', + }); + const result = prepareStream(request, null, log, err => { + assert.deepStrictEqual(err, errors.InternalError); + done(); + }); + result.stream.emit('error', errors.InternalError); + }); +}); + +describe('prepareStream UNSIGNED-PAYLOAD', () => { + it('returns ChecksumTransform as final stream', () => { + const request = makeRequest({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + const result = prepareStream(request, null, log, () => {}); + assert.strictEqual(result.error, null); + assert(result.stream instanceof ChecksumTransform); + }); + + it('ChecksumTransform receives the algorithm and expected digest from headers', () => { + const request = makeRequest({ + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + 'x-amz-checksum-crc32': 'AAAAAA==', + }); + const result = prepareStream(request, null, log, () => {}); + assert.strictEqual(result.stream.algoName, 'crc32'); + assert.strictEqual(result.stream.expectedDigest, 'AAAAAA=='); + }); + + it('errCb is called when ChecksumTransform emits an error', done => { + const request = makeRequest({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + const result = prepareStream(request, null, log, err => { + assert.deepStrictEqual(err, errors.InternalError); + done(); + }); + result.stream.emit('error', errors.InternalError); + }); +}); + +describe('prepareStream default (no x-amz-content-sha256)', () => { + it('no x-amz-content-sha256 header: returns ChecksumTransform with crc64nvme algorithm', () => { + const request = makeRequest({}); + const result = prepareStream(request, null, log, () => {}); + assert.strictEqual(result.error, null); + assert(result.stream instanceof ChecksumTransform); + assert.strictEqual(result.stream.algoName, 'crc64nvme'); + }); + + it('unsupported x-amz-content-sha256 (STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER): returns BadRequest', () => { + const request = makeRequest({ + 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER', + }); + const result = prepareStream(request, null, log, () => {}); + assert(result.error.is.BadRequest); + assert.strictEqual(result.stream, null); + }); + + it('errCb is called when ChecksumTransform emits an error', done => { + const request = makeRequest({}); + const result = prepareStream(request, null, log, err => { + assert.deepStrictEqual(err, errors.InternalError); + done(); + }); + result.stream.emit('error', errors.InternalError); + }); +}); diff --git a/tests/unit/api/apiUtils/object/storeObject.js b/tests/unit/api/apiUtils/object/storeObject.js new file mode 100644 index 0000000000..31fd8c6847 --- /dev/null +++ b/tests/unit/api/apiUtils/object/storeObject.js @@ -0,0 +1,303 @@ +const assert = require('assert'); +const sinon = require('sinon'); +const { errors } = require('arsenal'); + +const { dataStore } = require('../../../../../lib/api/apiUtils/object/storeObject'); +const dataWrapper = require('../../../../../lib/data/wrapper'); +const { DummyRequestLogger } = require('../../../helpers'); +const DummyRequest = require('../../../DummyRequest'); + +const log = new DummyRequestLogger(); + +const fakeDataRetrievalInfo = { key: 'test-key', dataStoreName: 'mem' }; + +function makeStream(headers = {}, body = '') { + return new DummyRequest({ headers }, body ? Buffer.from(body) : undefined); +} + +describe('dataStore', () => { + let putStub, batchDeleteStub; + + beforeEach(() => { + putStub = sinon.stub(dataWrapper.data, 'put'); + batchDeleteStub = sinon.stub(dataWrapper.data, 'batchDelete'); + }); + + afterEach(() => { + sinon.restore(); + }); + + // Stub data.put to drain the readable side and succeed once the stream finishes. + function putSucceeds(completedHash = null) { + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + stream.resume(); + stream.once('finish', () => cb(null, fakeDataRetrievalInfo, { completedHash })); + }); + } + + // Stub data.batchDelete to succeed. + function batchDeleteSucceeds() { + batchDeleteStub.callsFake((keys, a, b, log2, cb) => cb(null)); + } + + describe('normal behaviour', () => { + it('calls data.put with the stream returned by prepareStream', done => { + putSucceeds(); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, err => { + assert.strictEqual(err, null); + assert(putStub.calledOnce); + done(); + }); + }); + + it('calls cb with (null, dataRetrievalInfo, completedHash) on success', done => { + putSucceeds('abc123'); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, (err, dataInfo, completedHash) => { + assert.strictEqual(err, null); + assert.strictEqual(dataInfo, fakeDataRetrievalInfo); + assert.strictEqual(completedHash, 'abc123'); + done(); + }); + }); + + it('calls cb with the error from data.put when data.put fails', done => { + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + stream.resume(); + cb(errors.InternalError); + }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, err => { + assert.deepStrictEqual(err, errors.InternalError); + done(); + }); + }); + + it('does not delete stored data when data.put fails', done => { + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + stream.resume(); + cb(errors.InternalError); + }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, () => { + assert(batchDeleteStub.notCalled); + done(); + }); + }); + + it('calls cb with InternalError when data.put returns neither error nor dataRetrievalInfo', done => { + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + stream.resume(); + cb(null, null, null); + }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, err => { + assert.deepStrictEqual(err, errors.InternalError); + done(); + }); + }); + + it('calls cb with BadDigest and deletes stored data when content-md5 does not match', done => { + batchDeleteSucceeds(); + putSucceeds('correct-md5'); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + request.contentMD5 = 'wrong-md5'; + dataStore({}, null, request, 0, null, {}, log, err => { + assert.deepStrictEqual(err, errors.BadDigest); + assert(batchDeleteStub.calledOnce); + done(); + }); + }); + + it('calls cb with (null, ...) and does not delete stored data when content-md5 matches', done => { + putSucceeds('abc123'); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + request.contentMD5 = 'abc123'; + dataStore({}, null, request, 0, null, {}, log, err => { + assert.strictEqual(err, null); + assert(batchDeleteStub.notCalled); + done(); + }); + }); + + it('cb is called exactly once on success', done => { + putSucceeds(); + let cbCount = 0; + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, () => { + cbCount++; + setImmediate(() => { + assert.strictEqual(cbCount, 1); + done(); + }); + }); + }); + }); + + describe('checksum behaviour', () => { + it('calls cb with error from prepareStream when stream headers are invalid', done => { + const request = makeStream({ + 'x-amz-checksum-crc32': 'AAAAAA==', + 'x-amz-checksum-sha256': 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', + }); + dataStore({}, null, request, 0, null, {}, log, err => { + assert(err.is.InvalidRequest); + done(); + }); + }); + + it('does not call data.put when prepareStream returns an error', done => { + const request = makeStream({ + 'x-amz-checksum-crc32': 'AAAAAA==', + 'x-amz-checksum-sha256': 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', + }); + dataStore({}, null, request, 0, null, {}, log, () => { + assert(putStub.notCalled); + done(); + }); + }); + + it('calls cb with BadDigest and deletes stored data when checksum validation fails', done => { + batchDeleteSucceeds(); + putSucceeds(); + // CRC32 of 'hello world' is not 0x00000000 (AAAAAA==) + const request = makeStream({ + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + 'x-amz-checksum-crc32': 'AAAAAA==', + }, 'hello world'); + dataStore({}, null, request, 0, null, {}, log, err => { + assert(err.is.BadDigest); + assert(batchDeleteStub.calledOnce); + done(); + }); + }); + + it('does not delete stored data when checksum validation passes', done => { + putSucceeds(); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, err => { + assert.strictEqual(err, null); + assert(batchDeleteStub.notCalled); + done(); + }); + }); + + it('when checksumedStream is not yet writableFinished after data.put, waits for finish before validating', + done => { + let capturedStream; + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + capturedStream = stream; + stream.resume(); + // Call cb synchronously — _flush uses Promise.resolve().then() so + // writableFinished is false here, exercising the finish-wait path. + cb(null, fakeDataRetrievalInfo, { completedHash: null }); + }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, err => { + assert.strictEqual(err, null); + assert(capturedStream.writableFinished); + done(); + }); + }); + + it('when checksumedStream emits error after data.put, deletes stored data and calls cb with the error', + done => { + batchDeleteSucceeds(); + let capturedStream; + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + capturedStream = stream; + // Do not resume — keeps writableFinished false, so onError listener is registered. + cb(null, fakeDataRetrievalInfo, { completedHash: null }); + }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, err => { + assert.deepStrictEqual(err, errors.InternalError); + assert(batchDeleteStub.calledOnce); + done(); + }); + // process.nextTick fires before Promise microtasks, so the error arrives + // before _flush resolves, ensuring onError fires rather than onFinish. + process.nextTick(() => capturedStream.emit('error', errors.InternalError)); + }); + + it('cb is called exactly once when finish fires (no double callback)', done => { + let cbCount = 0; + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + stream.resume(); + // Synchronous cb → writableFinished is false → finish-wait path. + cb(null, fakeDataRetrievalInfo, { completedHash: null }); + }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, () => { + cbCount++; + setImmediate(() => { + assert.strictEqual(cbCount, 1); + done(); + }); + }); + }); + + it('cb is called exactly once when stream errors after data.put (no double callback)', done => { + batchDeleteSucceeds(); + let cbCount = 0; + let capturedStream; + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + capturedStream = stream; + cb(null, fakeDataRetrievalInfo, { completedHash: null }); + }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, () => { + cbCount++; + setImmediate(() => { + assert.strictEqual(cbCount, 1); + done(); + }); + }); + process.nextTick(() => capturedStream.emit('error', errors.InternalError)); + }); + }); + + describe('batchDelete failure paths', () => { + it('validateChecksum fails and batchDelete fails: cb called with checksum error, not delete error', done => { + batchDeleteStub.callsFake((keys, a, b, log2, cb) => cb(errors.InternalError)); + putSucceeds(); + const request = makeStream({ + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + 'x-amz-checksum-crc32': 'AAAAAA==', + }, 'hello world'); + dataStore({}, null, request, 0, null, {}, log, err => { + assert(err.is.BadDigest); + done(); + }); + }); + + it('content-md5 mismatch and batchDelete fails: cb called with BadDigest, not delete error', done => { + batchDeleteStub.callsFake((keys, a, b, log2, cb) => cb(errors.InternalError)); + putSucceeds('correct-md5'); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + request.contentMD5 = 'wrong-md5'; + dataStore({}, null, request, 0, null, {}, log, err => { + assert.deepStrictEqual(err, errors.BadDigest); + done(); + }); + }); + + it('checksumedStream emits error after data.put and batchDelete fails: ' + + 'cb called with stream error, not delete error', + done => { + batchDeleteStub.callsFake((keys, a, b, log2, cb) => cb(errors.BadRequest)); + let capturedStream; + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + capturedStream = stream; + cb(null, fakeDataRetrievalInfo, { completedHash: null }); + }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, log, err => { + assert.deepStrictEqual(err, errors.InternalError); + done(); + }); + process.nextTick(() => capturedStream.emit('error', errors.InternalError)); + }); + }); +}); From 888ea4333d5b1ed6af940cdc5ab6bf945eebb595 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Mon, 16 Mar 2026 20:49:27 +0100 Subject: [PATCH 05/15] CLDSRV-863: PutObject/UploadPart checksum service tests --- .../test/checksumPutObjectUploadPart.js | 610 ++++++++++++++++++ 1 file changed, 610 insertions(+) create mode 100644 tests/functional/raw-node/test/checksumPutObjectUploadPart.js diff --git a/tests/functional/raw-node/test/checksumPutObjectUploadPart.js b/tests/functional/raw-node/test/checksumPutObjectUploadPart.js new file mode 100644 index 0000000000..132d8b1a60 --- /dev/null +++ b/tests/functional/raw-node/test/checksumPutObjectUploadPart.js @@ -0,0 +1,610 @@ +const assert = require('assert'); +const crypto = require('crypto'); +const async = require('async'); + +const { makeS3Request } = require('../utils/makeRequest'); +const HttpRequestAuthV4 = require('../utils/HttpRequestAuthV4'); + +const bucket = 'checksumrejectionbucket'; +const objectKey = 'key'; +const objData = Buffer.alloc(1, 'a'); +const objDataSha256Hex = crypto.createHash('sha256').update(objData).digest('hex'); + +const authCredentials = { + accessKey: 'accessKey1', + secretKey: 'verySecretKey1', +}; + +const itSkipIfAWS = process.env.AWS_ON_AIR ? it.skip : it; + +const algos = [ + { name: 'crc32', wrongDigest: 'AAAAAA==' }, + { name: 'crc32c', wrongDigest: 'AAAAAA==' }, + { name: 'crc64nvme', wrongDigest: 'AAAAAAAAAAA=' }, + { name: 'sha1', wrongDigest: 'AAAAAAAAAAAAAAAAAAAAAAAAAAA=' }, + { name: 'sha256', wrongDigest: 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=' }, +]; + +// Build the raw chunked body for STREAMING-UNSIGNED-PAYLOAD-TRAILER +function buildTrailerBody(algoName, wrongDigest) { + const chunkSize = objData.length.toString(16); + return `${chunkSize}\r\n${objData.toString()}\r\n0\r\nx-amz-checksum-${algoName}:${wrongDigest}\r\n`; +} + +function doPutRequest(url, headers, body, callback) { + const req = new HttpRequestAuthV4( + url, + Object.assign({ method: 'PUT', headers }, authCredentials), + res => { + let data = ''; + res.on('data', chunk => { data += chunk; }); + res.on('end', () => callback(null, { statusCode: res.statusCode, body: data })); + } + ); + req.on('error', callback); + req.write(body); + req.end(); +} + +const protocols = [ + { + name: 'UNSIGNED-PAYLOAD', + buildHeaders: algo => ({ + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + 'content-length': objData.length, + [`x-amz-checksum-${algo.name}`]: algo.wrongDigest, + }), + buildBody: () => objData, + }, + { + name: 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD', + buildHeaders: algo => ({ + // No x-amz-content-sha256: HttpRequestAuthV4 defaults to + // STREAMING-AWS4-HMAC-SHA256-PAYLOAD and handles chunk signing. + 'content-length': objData.length, + [`x-amz-checksum-${algo.name}`]: algo.wrongDigest, + }), + buildBody: () => objData, + }, + { + name: 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + buildHeaders: algo => { + const body = buildTrailerBody(algo.name, algo.wrongDigest); + return { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': `x-amz-checksum-${algo.name}`, + 'x-amz-decoded-content-length': objData.length, + 'content-length': body.length, + }; + }, + buildBody: algo => buildTrailerBody(algo.name, algo.wrongDigest), + }, + { + name: 'valid x-amz-content-sha256', + buildHeaders: algo => ({ + 'x-amz-content-sha256': objDataSha256Hex, + 'content-length': objData.length, + [`x-amz-checksum-${algo.name}`]: algo.wrongDigest, + }), + buildBody: () => objData, + }, +]; + +function assertBadDigest(err, res, done) { + assert.ifError(err); + assert.strictEqual(res.statusCode, 400, `expected 400, got ${res.statusCode}: ${res.body}`); + assert(res.body.includes('BadDigest'), `missing BadDigest in "${res.body}"`); + done(); +} + +// Constants for protocol scenario tests + +const trailerContent = Buffer.from('trailer content'); // 15 bytes, hex 'f' +const trailerContentSha256 = '4x74k2oA6j6knXzpDwogNkS6E3MM49tPpJMjfD+ES68='; +// md5("trailer content") in base64 — used in testS3PutTrailerWithContentMD5 +const trailerContentMd5B64 = crypto.createHash('md5').update(trailerContent).digest('base64'); + +const testContent2 = Buffer.from('test content'); +const testContent2Sha256Hex = crypto.createHash('sha256').update(testContent2).digest('hex'); +const testContent2Sha256B64 = crypto.createHash('sha256').update(testContent2).digest('base64'); + +// Build a STREAMING-UNSIGNED-PAYLOAD-TRAILER body with "trailer content" as the +// data and a sha256 trailer. Uses the '\n\r\n\r\n\r\n' ending that AWS SDK sends +// (TrailingChecksumTransform strips the trailing \n from the parsed trailer line). +function buildOkTrailerBody() { + const hexLen = trailerContent.length.toString(16); // 'f' + return `${hexLen}\r\ntrailer content\r\n0\r\nx-amz-checksum-sha256:${trailerContentSha256}\n\r\n\r\n\r\n`; +} + +// Assert that the response has the given HTTP status code and (optionally) +// that the body contains the expected error code string. +// Returns a (err, res, done) callback suitable for use with doPutRequest. +function assertStatus(expectedStatus, expectedCode, expectedMessage) { + return (err, res, done) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, expectedStatus, + `expected ${expectedStatus}, got ${res.statusCode}: ${res.body}`); + if (expectedCode) { + assert(res.body.includes(expectedCode), + `expected "${expectedCode}" in body: "${res.body}"`); + } + if (expectedMessage) { + assert(res.body.includes(expectedMessage), + `expected "${expectedMessage}" in body: "${res.body}"`); + } + done(); + }; +} + +const msgMalformedTrailer = 'The request contained trailing data that was not well-formed' + + ' or did not conform to our published schema.'; +const msgSdkMissingTrailer = 'x-amz-sdk-checksum-algorithm specified, but no corresponding' + + ' x-amz-checksum-* or x-amz-trailer headers were found.'; + +// Create the 24 common protocol-scenario tests for a given URL factory. +// urlFn() is called lazily at test runtime so that uploadId is available. +function makeScenarioTests(urlFn) { + itSkipIfAWS( + 'testS3PutNoChecksum: signed sha256 in x-amz-content-sha256, no x-amz-checksum header -> 200 OK', + done => { + doPutRequest(urlFn(), { + 'x-amz-content-sha256': testContent2Sha256Hex, + 'content-length': testContent2.length, + }, testContent2, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutChecksum: correct sha256 checksum with x-amz-sdk-checksum-algorithm -> 200 OK', + done => { + doPutRequest(urlFn(), { + 'x-amz-content-sha256': testContent2Sha256Hex, + 'x-amz-sdk-checksum-algorithm': 'SHA256', + 'x-amz-checksum-sha256': testContent2Sha256B64, + 'content-length': testContent2.length, + }, testContent2, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutChecksumKo: wrong sha256 checksum with x-amz-sdk-checksum-algorithm -> 400 BadDigest', + done => { + doPutRequest(urlFn(), { + 'x-amz-content-sha256': testContent2Sha256Hex, + 'x-amz-sdk-checksum-algorithm': 'SHA256', + 'x-amz-checksum-sha256': 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', + 'content-length': testContent2.length, + }, testContent2, (err, res) => assertStatus(400, 'BadDigest', + 'The SHA256 you specified did not match the calculated checksum.')(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutChecksumUnsignedPayload: UNSIGNED-PAYLOAD with correct sha256 checksum -> 200 OK', + done => { + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + 'x-amz-sdk-checksum-algorithm': 'SHA256', + 'x-amz-checksum-sha256': testContent2Sha256B64, + 'content-length': testContent2.length, + }, testContent2, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerNoBody: TRAILER with empty body -> 400 IncompleteBody', + done => { + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': 0, + }, Buffer.alloc(0), (err, res) => assertStatus(400, 'IncompleteBody', + 'The request body terminated unexpectedly')(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerOk: TRAILER with correct sha256 checksum -> 200 OK', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerBadDecodedLen: wrong x-amz-decoded-content-length (7 but actual is 32) -> 500 InternalError', + done => { + // Two chunks of 16 bytes each with a valid crc64nvme trailer. + const body = + '10\r\n0123456789abcdef\r\n' + + '10\r\n0123456789abcdef\r\n' + + '0\r\nx-amz-checksum-crc64nvme:skQv82y5rgE=\r\n\r\n\r\n'; + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-crc64nvme', + 'x-amz-decoded-content-length': 7, // wrong: actual content is 32 bytes + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(500, 'InternalError', + 'We encountered an internal error. Please try again.')(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerAlgoMismatch: x-amz-trailer says sha1 but body trailer has sha256 ' + + ' -> 400 MalformedTrailerError', + done => { + // Header announces sha1 but the actual trailer line carries sha256. + const body = + `f\r\ntrailer content\r\n0\r\nx-amz-checksum-sha256:${trailerContentSha256}\n\r\n\r\n\r\n`; + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha1', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'MalformedTrailerError', + msgMalformedTrailer)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerBadDigest: TRAILER with wrong sha256 checksum -> 400 BadDigest', + done => { + const wrongSha256 = 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='; + const body = + `f\r\ntrailer content\r\n0\r\nx-amz-checksum-sha256:${wrongSha256}\n\r\n\r\n\r\n`; + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'BadDigest', + 'The SHA256 you specified did not match the calculated checksum.')(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerAndChecksum: x-amz-trailer + x-amz-checksum-crc32 header -> 400 InvalidRequest', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-checksum-crc32': 'H+Yzmw==', // crc32("trailer content") + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'InvalidRequest', + 'Expecting a single x-amz-checksum- header')(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerNoTrailerInHeader: no x-amz-trailer header but body has trailer -> 400 MalformedTrailerError', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + // no x-amz-trailer header + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'MalformedTrailerError', + msgMalformedTrailer)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerContentLength: TRAILER with explicit Content-Length -> 200 OK', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerSDK: TRAILER with matching x-amz-sdk-checksum-algorithm:SHA256 -> 200 OK', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-sdk-checksum-algorithm': 'SHA256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerSDKMismatch: x-amz-sdk-checksum-algorithm:SHA1 but x-amz-trailer is sha256 ' + + '-> 400 InvalidRequest', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-sdk-checksum-algorithm': 'SHA1', // mismatch + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'InvalidRequest', + 'Value for x-amz-sdk-checksum-algorithm header is invalid.')(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerUnknownTrailerAlgo: x-amz-trailer:x-amz-checksum-sha3 (unknown) -> 400 InvalidRequest', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha3', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'InvalidRequest', + 'The value specified in the x-amz-trailer header is not supported')(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerInvalidTrailerHeader: x-amz-trailer with non-checksum value -> 400 InvalidRequest', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'AAAAAAAAAAAAAAAAAAAAA', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'InvalidRequest', + 'The value specified in the x-amz-trailer header is not supported')(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerInvalidTrailerBody: trailer body has invalid base64 checksum value -> 400 InvalidRequest', + done => { + const body = 'f\r\ntrailer content\r\n0\r\nx-amz-checksum-sha256:BAD\n\r\n\r\n\r\n'; + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'InvalidRequest', + 'Value for x-amz-checksum-sha256 trailing header is invalid.')(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerSDKMissingTrailer: x-amz-sdk-checksum-algorithm without x-amz-trailer -> 400 InvalidRequest', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + // no x-amz-trailer + 'x-amz-sdk-checksum-algorithm': 'SHA256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'InvalidRequest', + msgSdkMissingTrailer)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerNoTrailerInBody: x-amz-trailer header but body has no trailer -> 400 MalformedTrailerError', + done => { + // Body ends with "0\r\n\r\n" — empty trailer section, no checksum line. + const body = 'f\r\ntrailer content\r\n0\r\n\r\n'; + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(400, 'MalformedTrailerError', + msgMalformedTrailer)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerNoTrailerInBodyAndHeader: no x-amz-trailer, no body trailer -> 200 OK', + done => { + // No x-amz-trailer header; body just has chunked data with no trailer. + const body = 'f\r\ntrailer content\r\n0\r\n\r\n'; + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + // no x-amz-trailer + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerDataAfterCRLF: data after final CRLF is ignored -> 200 OK', + done => { + // No x-amz-trailer; after the terminating CRLF there is extra data. + // TrailingChecksumTransform discards everything after streamClosed=true. + const body = 'f\r\ntrailer content\r\n0\r\n\r\nRANDOM DATA IGNORED'; + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + // no x-amz-trailer + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerWithContentMD5: TRAILER + correct Content-MD5 header -> 200 OK', + done => { + const body = buildOkTrailerBody(); + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-sdk-checksum-algorithm': 'SHA256', + 'content-md5': trailerContentMd5B64, + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(200)(err, res, done)); + }); + + itSkipIfAWS( + 'testS3PutTrailerWithWhitespace: trailer line with whitespace around name and value -> 200 OK', + done => { + // TrailingChecksumTransform trims both name and value, so whitespace is accepted. + const body = + `f\r\ntrailer content\r\n0\r\n x-amz-checksum-sha256 : ${trailerContentSha256} \n\r\n\r\n\r\n`; + doPutRequest(urlFn(), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, body, (err, res) => assertStatus(200)(err, res, done)); + }); +} + +describe('PutObject: bad checksum is rejected', () => { + before(done => { + makeS3Request({ method: 'PUT', authCredentials, bucket }, err => { + assert.ifError(err); + done(); + }); + }); + + after(done => { + // Delete the object key first (defensive: clears any state left by a previous run). + makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => { + makeS3Request({ method: 'DELETE', authCredentials, bucket }, err => { + assert.ifError(err); + done(); + }); + }); + }); + + for (const protocol of protocols) { + for (const algo of algos) { + itSkipIfAWS( + `${protocol.name} with wrong x-amz-checksum-${algo.name}: returns 400 BadDigest`, + done => { + const url = `http://localhost:8000/${bucket}/${objectKey}`; + doPutRequest(url, protocol.buildHeaders(algo), protocol.buildBody(algo), + (err, res) => assertBadDigest(err, res, done)); + } + ); + } + } +}); + +describe('UploadPart: bad checksum is rejected', () => { + let uploadId; + + before(done => { + async.series([ + next => makeS3Request({ method: 'PUT', authCredentials, bucket }, next), + next => makeS3Request({ + method: 'POST', + authCredentials, + bucket, + objectKey, + queryObj: { uploads: '' }, + }, (err, res) => { + if (err) { return next(err); } + const match = res.body.match(/([^<]+)<\/UploadId>/); + assert(match, `missing UploadId in response: ${res.body}`); + uploadId = match[1]; + return next(); + }), + ], err => { + assert.ifError(err); + done(); + }); + }); + + after(done => { + async.series([ + next => makeS3Request({ + method: 'DELETE', + authCredentials, + bucket, + objectKey, + queryObj: { uploadId }, + }, next), + // Delete the object key first (defensive: clears any state left by a previous run). + next => makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => next()), + next => makeS3Request({ method: 'DELETE', authCredentials, bucket }, next), + ], err => { + assert.ifError(err); + done(); + }); + }); + + for (const protocol of protocols) { + for (const algo of algos) { + itSkipIfAWS( + `${protocol.name} with wrong x-amz-checksum-${algo.name}: returns 400 BadDigest`, + done => { + const url = `http://localhost:8000/${bucket}/${objectKey}` + + `?partNumber=1&uploadId=${uploadId}`; + doPutRequest(url, protocol.buildHeaders(algo), protocol.buildBody(algo), + (err, res) => assertBadDigest(err, res, done)); + } + ); + } + } +}); + +describe('PutObject: trailer and checksum protocol scenarios', () => { + before(done => { + makeS3Request({ method: 'PUT', authCredentials, bucket }, err => { + assert.ifError(err); + done(); + }); + }); + + after(done => { + // Some scenario tests store objects; delete the object key before removing the bucket. + makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => { + makeS3Request({ method: 'DELETE', authCredentials, bucket }, err => { + assert.ifError(err); + done(); + }); + }); + }); + + makeScenarioTests(() => `http://localhost:8000/${bucket}/${objectKey}`); + +}); + +describe('UploadPart: trailer and checksum protocol scenarios', () => { + let uploadId2; + + before(done => { + async.series([ + next => makeS3Request({ method: 'PUT', authCredentials, bucket }, next), + next => makeS3Request({ + method: 'POST', + authCredentials, + bucket, + objectKey, + queryObj: { uploads: '' }, + }, (err, res) => { + if (err) { return next(err); } + const match = res.body.match(/([^<]+)<\/UploadId>/); + assert(match, `missing UploadId in response: ${res.body}`); + uploadId2 = match[1]; + return next(); + }), + ], err => { + assert.ifError(err); + done(); + }); + }); + + after(done => { + async.series([ + next => makeS3Request({ + method: 'DELETE', + authCredentials, + bucket, + objectKey, + queryObj: { uploadId: uploadId2 }, + }, next), + // Delete the object key first (defensive: clears any state left by a previous run). + next => makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => next()), + next => makeS3Request({ method: 'DELETE', authCredentials, bucket }, next), + ], err => { + assert.ifError(err); + done(); + }); + }); + + makeScenarioTests( + () => `http://localhost:8000/${bucket}/${objectKey}?partNumber=1&uploadId=${uploadId2}` + ); +}); From 661e8d9d7b5249a9389eb20932549c30cb66f925 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Tue, 17 Mar 2026 15:59:44 +0100 Subject: [PATCH 06/15] CLDSRV-872: store checksum in metadata for objects uploaded with PutObject --- .../apiUtils/object/createAndStoreObject.js | 7 +- lib/api/apiUtils/object/storeObject.js | 10 ++- lib/services.js | 6 +- .../aws-node-sdk/test/object/putVersion.js | 20 ++--- .../raw-node/test/routes/routeMetadata.js | 81 ++++++++++++++++++- tests/unit/api/objectPut.js | 47 +++++++++++ 6 files changed, 152 insertions(+), 19 deletions(-) diff --git a/lib/api/apiUtils/object/createAndStoreObject.js b/lib/api/apiUtils/object/createAndStoreObject.js index 542b9f5296..d1b17b0e44 100644 --- a/lib/api/apiUtils/object/createAndStoreObject.js +++ b/lib/api/apiUtils/object/createAndStoreObject.js @@ -217,7 +217,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, if (size === 0) { if (!dontSkipBackend[locationType]) { metadataStoreParams.contentMD5 = constants.emptyFileMd5; - return next(null, null, null); + return next(null, null, null, null); } // Handle mdOnlyHeader as a metadata only operation. If @@ -243,14 +243,14 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, dataStoreVersionId: versionId, dataStoreMD5: _md5, }; - return next(null, dataGetInfo, _md5); + return next(null, dataGetInfo, _md5, null); } } return dataStore(objectKeyContext, cipherBundle, request, size, streamingV4Params, backendInfo, log, next); }, - function processDataResult(dataGetInfo, calculatedHash, next) { + function processDataResult(dataGetInfo, calculatedHash, checksum, next) { if (dataGetInfo === null || dataGetInfo === undefined) { return next(null, null); } @@ -275,6 +275,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, dataGetInfoArr[0].size = mdOnlySize; } metadataStoreParams.contentMD5 = calculatedHash; + metadataStoreParams.checksum = checksum; return next(null, dataGetInfoArr); }, function getVersioningInfo(infoArr, next) { diff --git a/lib/api/apiUtils/object/storeObject.js b/lib/api/apiUtils/object/storeObject.js index 68a5312c4f..7963e97aa5 100644 --- a/lib/api/apiUtils/object/storeObject.js +++ b/lib/api/apiUtils/object/storeObject.js @@ -17,7 +17,7 @@ const { arsenalErrorFromChecksumError } = require('../../apiUtils/integrity/vali * @return {function} - calls callback with arguments: * error, dataRetrievalInfo, and completedHash (if any) */ -function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cb) { +function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, checksumStream, log, cb) { const contentMD5 = stream.contentMD5; const completedHash = hashedStream.completedHash; if (contentMD5 && completedHash && contentMD5 !== completedHash) { @@ -37,7 +37,10 @@ function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cb) { return cb(errors.BadDigest); }); } - return cb(null, dataRetrievalInfo, completedHash); + const checksum = checksumStream.digest + ? { algorithm: checksumStream.algoName, value: checksumStream.digest, type: 'FULL_OBJECT' } + : null; + return cb(null, dataRetrievalInfo, completedHash, checksum); } /** @@ -107,7 +110,8 @@ function dataStore(objectContext, cipherBundle, stream, size, return cbOnce(arsenalErrorFromChecksumError(checksumErr)); }); } - return checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cbOnce); + return checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, + checksumedStream.stream, log, cbOnce); }; // ChecksumTransform._flush computes the digest asynchronously for diff --git a/lib/services.js b/lib/services.js index f2980f1fc5..43b41c9548 100644 --- a/lib/services.js +++ b/lib/services.js @@ -6,6 +6,7 @@ const { errors, s3middleware } = require('arsenal'); const ObjectMD = require('arsenal').models.ObjectMD; const BucketInfo = require('arsenal').models.BucketInfo; const ObjectMDArchive = require('arsenal').models.ObjectMDArchive; +const ObjectMDChecksum = require('arsenal').models.ObjectMDChecksum; const { versioning } = require('arsenal'); const acl = require('./metadata/acl'); const constants = require('../constants'); @@ -102,7 +103,7 @@ const services = { * @return {function} executes callback with err or ETag as arguments */ metadataStoreObject(bucketName, dataGetInfo, cipherBundle, params, cb) { - const { objectKey, authInfo, size, contentMD5, metaHeaders, + const { objectKey, authInfo, size, contentMD5, checksum, metaHeaders, contentType, cacheControl, contentDisposition, contentEncoding, expires, multipart, headers, overrideMetadata, log, lastModifiedDate, versioning, versionId, uploadId, @@ -138,6 +139,9 @@ const services = { // CreationTime needs to be carried over so that it remains static .setCreationTime(creationTime) .setOriginOp(originOp); + if (checksum) { + md.setChecksum(new ObjectMDChecksum(checksum.algorithm, checksum.value, checksum.type)); + } // Sending in last modified date in object put copy since need // to return the exact date in the response if (lastModifiedDate) { diff --git a/tests/functional/aws-node-sdk/test/object/putVersion.js b/tests/functional/aws-node-sdk/test/object/putVersion.js index efa68708be..cbea941a8c 100644 --- a/tests/functional/aws-node-sdk/test/object/putVersion.js +++ b/tests/functional/aws-node-sdk/test/object/putVersion.js @@ -258,7 +258,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { assert.deepStrictEqual(versionsAfter, versionsBefore); checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'originOp']); + 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'originOp', 'checksum']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -309,7 +309,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { assert.deepStrictEqual(versionsAfter, versionsBefore); checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); + 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -360,7 +360,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { assert.deepStrictEqual(versionsAfter, versionsBefore); checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); + 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -408,7 +408,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { assert.deepStrictEqual(versionsAfter, versionsBefore); checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); + 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -460,7 +460,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { assert.deepStrictEqual(versionsAfter, versionsBefore); checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); + 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -515,7 +515,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { assert.deepStrictEqual(versionsAfter, versionsBefore); checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); + 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -568,7 +568,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { assert.deepStrictEqual(versionsAfter, versionsBefore); checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); + 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -620,7 +620,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { assert.deepStrictEqual(versionsAfter, versionsBefore); checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); + 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -679,7 +679,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { assert.deepStrictEqual(versionsAfter, versionsBefore); checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); + 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -726,7 +726,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { assert.deepStrictEqual(versionsAfter, versionsBefore); checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName']); + 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); diff --git a/tests/functional/raw-node/test/routes/routeMetadata.js b/tests/functional/raw-node/test/routes/routeMetadata.js index 81b24a2bfa..84f698a7cc 100644 --- a/tests/functional/raw-node/test/routes/routeMetadata.js +++ b/tests/functional/raw-node/test/routes/routeMetadata.js @@ -1,7 +1,9 @@ const assert = require('assert'); +const crypto = require('crypto'); const http = require('http'); -const { CreateBucketCommand, - PutObjectCommand, +const { CreateBucketCommand, + PutObjectCommand, + DeleteObjectCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const { makeRequest } = require('../../utils/makeRequest'); @@ -144,3 +146,78 @@ describe('metadata routes with metadata', () => { }); }); }); + +describe('checksum stored in object metadata after PutObject', () => { + const bucketUtil = new BucketUtility('default', { signatureVersion: 'v4' }); + const s3 = bucketUtil.s3; + + const bucket = 'bucket-checksum-test'; + const objectBody = 'hello checksum'; + const sha256Key = 'object-with-sha256-checksum'; + const defaultKey = 'object-with-default-checksum'; + + // CRC32 of 'hello checksum' in base64 — the AWS SDK v3 injects x-amz-checksum-crc32 by default + const expectedCrc32 = 'EyV5Tg=='; + + before(async function () { + if (!process.env.S3_END_TO_END) { + this.skip(); + } + await s3.send(new CreateBucketCommand({ Bucket: bucket })); + + const sha256Value = crypto.createHash('sha256').update(objectBody).digest('base64'); + await s3.send(new PutObjectCommand({ + Bucket: bucket, + Key: sha256Key, + Body: objectBody, + ChecksumSHA256: sha256Value, + })); + await s3.send(new PutObjectCommand({ + Bucket: bucket, + Key: defaultKey, + Body: objectBody, + })); + }); + + after(async () => { + if (!process.env.S3_END_TO_END) { + return; + } + await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: sha256Key })); + await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: defaultKey })); + await s3.send(new DeleteBucketCommand({ Bucket: bucket })); + }); + + it('stores sha256 checksum in metadata when x-amz-checksum-sha256 is provided', done => { + const expectedValue = crypto.createHash('sha256').update(objectBody).digest('base64'); + makeMetadataRequest({ + method: 'GET', + authCredentials: metadataAuthCredentials, + path: `/_/metadata/default/bucket/${bucket}/${sha256Key}`, + }, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200); + const md = JSON.parse(res.body); + assert.strictEqual(md.checksum.checksumAlgorithm, 'sha256'); + assert.strictEqual(md.checksum.checksumValue, expectedValue); + assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT'); + return done(); + }); + }); + + it('stores crc32 checksum in metadata when no explicit checksum header is provided (AWS SDK default)', done => { + makeMetadataRequest({ + method: 'GET', + authCredentials: metadataAuthCredentials, + path: `/_/metadata/default/bucket/${bucket}/${defaultKey}`, + }, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200); + const md = JSON.parse(res.body); + assert.strictEqual(md.checksum.checksumAlgorithm, 'crc32'); + assert.strictEqual(md.checksum.checksumValue, expectedCrc32); + assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT'); + return done(); + }); + }); +}); diff --git a/tests/unit/api/objectPut.js b/tests/unit/api/objectPut.js index 3f4c458d60..4e219c53e7 100644 --- a/tests/unit/api/objectPut.js +++ b/tests/unit/api/objectPut.js @@ -1,5 +1,6 @@ const assert = require('assert'); const async = require('async'); +const crypto = require('crypto'); const moment = require('moment'); const { s3middleware, storage, versioning } = require('arsenal'); const sinon = require('sinon'); @@ -865,6 +866,52 @@ describe('objectPut API', () => { }); }); }); + + it('should store sha256 checksum in metadata when x-amz-checksum-sha256 header is provided', done => { + const sha256Value = crypto.createHash('sha256').update(postBody).digest('base64'); + const request = new DummyRequest({ + bucketName, + namespace, + objectKey: objectName, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-checksum-sha256': sha256Value, + }, + url: '/', + }, postBody); + + bucketPut(authInfo, testPutBucketRequest, log, err => { + assert.ifError(err); + objectPut(authInfo, request, undefined, log, err => { + assert.ifError(err); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.ifError(err); + assert(md.checksum, 'checksum should be set in metadata'); + assert.strictEqual(md.checksum.checksumAlgorithm, 'sha256'); + assert.strictEqual(md.checksum.checksumValue, sha256Value); + assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT'); + done(); + }); + }); + }); + }); + + it('should store crc64nvme checksum in metadata when no checksum header is provided', done => { + bucketPut(authInfo, testPutBucketRequest, log, err => { + assert.ifError(err); + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.ifError(err); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.ifError(err); + assert(md.checksum, 'checksum should be set in metadata'); + assert.strictEqual(md.checksum.checksumAlgorithm, 'crc64nvme'); + assert(md.checksum.checksumValue, 'checksumValue should be set'); + assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT'); + done(); + }); + }); + }); + }); }); describe('objectPut API with versioning', () => { From e0566e0c3e30ed33cb14648cf795a975e32265f9 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Tue, 17 Mar 2026 16:31:35 +0100 Subject: [PATCH 07/15] tmp bump arsenal --- package.json | 2 +- yarn.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index eaee9d3d60..c8786e5d68 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "@azure/storage-blob": "^12.28.0", "@hapi/joi": "^17.1.1", "@smithy/node-http-handler": "^3.0.0", - "arsenal": "git+https://github.com/scality/Arsenal#8.3.6", + "arsenal": "git+https://github.com/scality/Arsenal#improvement/ARSN-557-add-checksum-to-object-metadata", "async": "2.6.4", "bucketclient": "scality/bucketclient#8.2.7", "bufferutil": "^4.0.8", diff --git a/yarn.lock b/yarn.lock index 82bf9887f0..85e2947183 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5766,9 +5766,9 @@ arraybuffer.prototype.slice@^1.0.4: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/Arsenal#8.3.6": +"arsenal@git+https://github.com/scality/Arsenal#improvement/ARSN-557-add-checksum-to-object-metadata": version "8.3.6" - resolved "git+https://github.com/scality/Arsenal#21f6a54ac70f2f6c94b592760780b3ca3bab7a95" + resolved "git+https://github.com/scality/Arsenal#af4442f234f0c460ea7607f8f7f9eaac22fe8fa4" dependencies: "@aws-sdk/client-kms" "^3.975.0" "@aws-sdk/client-s3" "^3.975.0" From b80b2ade71f30b8819efc33cd3edda44260656a7 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Tue, 17 Mar 2026 18:15:51 +0100 Subject: [PATCH 08/15] CLDSRV-872: return stored checksum in PutObject response --- lib/api/objectPut.js | 4 + lib/services.js | 1 + .../test/checksumPutObjectUploadPart.js | 196 ++++++++++++++++-- tests/unit/api/objectPut.js | 8 +- 4 files changed, 193 insertions(+), 16 deletions(-) diff --git a/lib/api/objectPut.js b/lib/api/objectPut.js index 8b92e0e9a6..3b1641ca88 100644 --- a/lib/api/objectPut.js +++ b/lib/api/objectPut.js @@ -225,6 +225,10 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) { if (storingResult) { // ETag's hex should always be enclosed in quotes responseHeaders.ETag = `"${storingResult.contentMD5}"`; + if (storingResult.checksum) { + const { checksumAlgorithm, checksumValue } = storingResult.checksum; + responseHeaders[`x-amz-checksum-${checksumAlgorithm}`] = checksumValue; + } } const vcfg = bucket.getVersioningConfiguration(); const isVersionedObj = vcfg && vcfg.Status === 'Enabled'; diff --git a/lib/services.js b/lib/services.js index 43b41c9548..966b8d4665 100644 --- a/lib/services.js +++ b/lib/services.js @@ -333,6 +333,7 @@ const services = { tags: md.getTags(), contentMD5, versionId, + checksum: md.getChecksum(), }); }); }, diff --git a/tests/functional/raw-node/test/checksumPutObjectUploadPart.js b/tests/functional/raw-node/test/checksumPutObjectUploadPart.js index 132d8b1a60..fd8db840df 100644 --- a/tests/functional/raw-node/test/checksumPutObjectUploadPart.js +++ b/tests/functional/raw-node/test/checksumPutObjectUploadPart.js @@ -1,6 +1,7 @@ const assert = require('assert'); const crypto = require('crypto'); const async = require('async'); +const { algorithms } = require('../../../../lib/api/apiUtils/integrity/validateChecksums'); const { makeS3Request } = require('../utils/makeRequest'); const HttpRequestAuthV4 = require('../utils/HttpRequestAuthV4'); @@ -38,7 +39,11 @@ function doPutRequest(url, headers, body, callback) { res => { let data = ''; res.on('data', chunk => { data += chunk; }); - res.on('end', () => callback(null, { statusCode: res.statusCode, body: data })); + res.on('end', () => callback(null, { + statusCode: res.statusCode, + body: data, + headers: res.headers, + })); } ); req.on('error', callback); @@ -141,16 +146,38 @@ const msgMalformedTrailer = 'The request contained trailing data that was not we const msgSdkMissingTrailer = 'x-amz-sdk-checksum-algorithm specified, but no corresponding' + ' x-amz-checksum-* or x-amz-trailer headers were found.'; +// Module-level variables for computed crc64nvme checksums (filled in before hook) +let crc64nvmeOfTestContent2; +let crc64nvmeOfTrailerContent; + // Create the 24 common protocol-scenario tests for a given URL factory. // urlFn() is called lazily at test runtime so that uploadId is available. -function makeScenarioTests(urlFn) { +// checkChecksumResponse: if true, assert x-amz-checksum-* response headers on 200 OK tests. +function makeScenarioTests(urlFn, checkChecksumResponse = false) { + before(async () => { + if (!crc64nvmeOfTestContent2) { + crc64nvmeOfTestContent2 = await algorithms.crc64nvme.digest(testContent2); + } + if (!crc64nvmeOfTrailerContent) { + crc64nvmeOfTrailerContent = await algorithms.crc64nvme.digest(trailerContent); + } + }); + itSkipIfAWS( 'testS3PutNoChecksum: signed sha256 in x-amz-content-sha256, no x-amz-checksum header -> 200 OK', done => { doPutRequest(urlFn(), { 'x-amz-content-sha256': testContent2Sha256Hex, 'content-length': testContent2.length, - }, testContent2, (err, res) => assertStatus(200)(err, res, done)); + }, testContent2, (err, res) => { + assertStatus(200)(err, res, () => { + if (checkChecksumResponse) { + assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], crc64nvmeOfTestContent2, + `expected x-amz-checksum-crc64nvme: ${crc64nvmeOfTestContent2}`); + } + done(); + }); + }); }); itSkipIfAWS( @@ -161,7 +188,15 @@ function makeScenarioTests(urlFn) { 'x-amz-sdk-checksum-algorithm': 'SHA256', 'x-amz-checksum-sha256': testContent2Sha256B64, 'content-length': testContent2.length, - }, testContent2, (err, res) => assertStatus(200)(err, res, done)); + }, testContent2, (err, res) => { + assertStatus(200)(err, res, () => { + if (checkChecksumResponse) { + assert.strictEqual(res.headers['x-amz-checksum-sha256'], testContent2Sha256B64, + `expected x-amz-checksum-sha256: ${testContent2Sha256B64}`); + } + done(); + }); + }); }); itSkipIfAWS( @@ -184,7 +219,15 @@ function makeScenarioTests(urlFn) { 'x-amz-sdk-checksum-algorithm': 'SHA256', 'x-amz-checksum-sha256': testContent2Sha256B64, 'content-length': testContent2.length, - }, testContent2, (err, res) => assertStatus(200)(err, res, done)); + }, testContent2, (err, res) => { + assertStatus(200)(err, res, () => { + if (checkChecksumResponse) { + assert.strictEqual(res.headers['x-amz-checksum-sha256'], testContent2Sha256B64, + `expected x-amz-checksum-sha256: ${testContent2Sha256B64}`); + } + done(); + }); + }); }); itSkipIfAWS( @@ -208,7 +251,15 @@ function makeScenarioTests(urlFn) { 'x-amz-trailer': 'x-amz-checksum-sha256', 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(200)(err, res, done)); + }, body, (err, res) => { + assertStatus(200)(err, res, () => { + if (checkChecksumResponse) { + assert.strictEqual(res.headers['x-amz-checksum-sha256'], trailerContentSha256, + `expected x-amz-checksum-sha256: ${trailerContentSha256}`); + } + done(); + }); + }); }); itSkipIfAWS( @@ -295,7 +346,15 @@ function makeScenarioTests(urlFn) { 'x-amz-trailer': 'x-amz-checksum-sha256', 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(200)(err, res, done)); + }, body, (err, res) => { + assertStatus(200)(err, res, () => { + if (checkChecksumResponse) { + assert.strictEqual(res.headers['x-amz-checksum-sha256'], trailerContentSha256, + `expected x-amz-checksum-sha256: ${trailerContentSha256}`); + } + done(); + }); + }); }); itSkipIfAWS( @@ -308,7 +367,15 @@ function makeScenarioTests(urlFn) { 'x-amz-sdk-checksum-algorithm': 'SHA256', 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(200)(err, res, done)); + }, body, (err, res) => { + assertStatus(200)(err, res, () => { + if (checkChecksumResponse) { + assert.strictEqual(res.headers['x-amz-checksum-sha256'], trailerContentSha256, + `expected x-amz-checksum-sha256: ${trailerContentSha256}`); + } + done(); + }); + }); }); itSkipIfAWS( @@ -403,7 +470,15 @@ function makeScenarioTests(urlFn) { // no x-amz-trailer 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(200)(err, res, done)); + }, body, (err, res) => { + assertStatus(200)(err, res, () => { + if (checkChecksumResponse) { + assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], crc64nvmeOfTrailerContent, + `expected x-amz-checksum-crc64nvme: ${crc64nvmeOfTrailerContent}`); + } + done(); + }); + }); }); itSkipIfAWS( @@ -417,7 +492,15 @@ function makeScenarioTests(urlFn) { // no x-amz-trailer 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(200)(err, res, done)); + }, body, (err, res) => { + assertStatus(200)(err, res, () => { + if (checkChecksumResponse) { + assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], crc64nvmeOfTrailerContent, + `expected x-amz-checksum-crc64nvme: ${crc64nvmeOfTrailerContent}`); + } + done(); + }); + }); }); itSkipIfAWS( @@ -431,7 +514,15 @@ function makeScenarioTests(urlFn) { 'content-md5': trailerContentMd5B64, 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(200)(err, res, done)); + }, body, (err, res) => { + assertStatus(200)(err, res, () => { + if (checkChecksumResponse) { + assert.strictEqual(res.headers['x-amz-checksum-sha256'], trailerContentSha256, + `expected x-amz-checksum-sha256: ${trailerContentSha256}`); + } + done(); + }); + }); }); itSkipIfAWS( @@ -445,7 +536,15 @@ function makeScenarioTests(urlFn) { 'x-amz-trailer': 'x-amz-checksum-sha256', 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(200)(err, res, done)); + }, body, (err, res) => { + assertStatus(200)(err, res, () => { + if (checkChecksumResponse) { + assert.strictEqual(res.headers['x-amz-checksum-sha256'], trailerContentSha256, + `expected x-amz-checksum-sha256: ${trailerContentSha256}`); + } + done(); + }); + }); }); } @@ -557,7 +656,7 @@ describe('PutObject: trailer and checksum protocol scenarios', () => { }); }); - makeScenarioTests(() => `http://localhost:8000/${bucket}/${objectKey}`); + makeScenarioTests(() => `http://localhost:8000/${bucket}/${objectKey}`, true); }); @@ -608,3 +707,74 @@ describe('UploadPart: trailer and checksum protocol scenarios', () => { () => `http://localhost:8000/${bucket}/${objectKey}?partNumber=1&uploadId=${uploadId2}` ); }); + +describe('PutObject: checksum response header per algorithm', () => { + const url = `http://localhost:8000/${bucket}/${objectKey}`; + const body = testContent2; + const sha256Hex = testContent2Sha256Hex; + + let expectedCrc64nvme; + + before(async () => { + await new Promise((resolve, reject) => + makeS3Request({ method: 'PUT', authCredentials, bucket }, + err => (err ? reject(err) : resolve()))); + expectedCrc64nvme = await algorithms.crc64nvme.digest(body); + }); + + after(done => { + makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => { + makeS3Request({ method: 'DELETE', authCredentials, bucket }, err => { + assert.ifError(err); + done(); + }); + }); + }); + + const checksumAlgos = [ + { name: 'crc32', computeExpected: () => algorithms.crc32.digest(body) }, + { name: 'crc32c', computeExpected: () => algorithms.crc32c.digest(body) }, + { name: 'crc64nvme', computeExpected: () => expectedCrc64nvme }, + { name: 'sha1', computeExpected: () => algorithms.sha1.digest(body) }, + { name: 'sha256', computeExpected: () => algorithms.sha256.digest(body) }, + ]; + + for (const algo of checksumAlgos) { + itSkipIfAWS( + `returns x-amz-checksum-${algo.name} response header with correct value`, + done => { + const expectedValue = algo.computeExpected(); + const headerName = `x-amz-checksum-${algo.name}`; + doPutRequest(url, { + 'x-amz-content-sha256': sha256Hex, + [headerName]: expectedValue, + 'content-length': body.length, + }, body, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200, + `expected 200, got ${res.statusCode}: ${res.body}`); + assert.strictEqual(res.headers[headerName], expectedValue, + `expected ${headerName}: ${expectedValue}`); + done(); + }); + } + ); + } + + itSkipIfAWS( + 'returns x-amz-checksum-crc64nvme response header when no checksum header is sent', + done => { + doPutRequest(url, { + 'x-amz-content-sha256': sha256Hex, + 'content-length': body.length, + }, body, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200, + `expected 200, got ${res.statusCode}: ${res.body}`); + assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], expectedCrc64nvme, + `expected x-amz-checksum-crc64nvme: ${expectedCrc64nvme}`); + done(); + }); + } + ); +}); diff --git a/tests/unit/api/objectPut.js b/tests/unit/api/objectPut.js index 4e219c53e7..4b3e3ec8bc 100644 --- a/tests/unit/api/objectPut.js +++ b/tests/unit/api/objectPut.js @@ -882,8 +882,9 @@ describe('objectPut API', () => { bucketPut(authInfo, testPutBucketRequest, log, err => { assert.ifError(err); - objectPut(authInfo, request, undefined, log, err => { + objectPut(authInfo, request, undefined, log, (err, resHeaders) => { assert.ifError(err); + assert.strictEqual(resHeaders['x-amz-checksum-sha256'], sha256Value); metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { assert.ifError(err); assert(md.checksum, 'checksum should be set in metadata'); @@ -899,13 +900,14 @@ describe('objectPut API', () => { it('should store crc64nvme checksum in metadata when no checksum header is provided', done => { bucketPut(authInfo, testPutBucketRequest, log, err => { assert.ifError(err); - objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { assert.ifError(err); + assert(resHeaders['x-amz-checksum-crc64nvme'], 'crc64nvme response header should be set'); metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { assert.ifError(err); assert(md.checksum, 'checksum should be set in metadata'); assert.strictEqual(md.checksum.checksumAlgorithm, 'crc64nvme'); - assert(md.checksum.checksumValue, 'checksumValue should be set'); + assert.strictEqual(md.checksum.checksumValue, resHeaders['x-amz-checksum-crc64nvme']); assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT'); done(); }); From 20da7020e11e4d53e953b234653bda87f4e425f4 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Tue, 17 Mar 2026 19:52:40 +0100 Subject: [PATCH 09/15] CLDSRV-872: handle empty object checksum --- .../apiUtils/object/createAndStoreObject.js | 52 ++++++- .../test/checksumPutObjectUploadPart.js | 131 +++++++++++++++++- 2 files changed, 176 insertions(+), 7 deletions(-) diff --git a/lib/api/apiUtils/object/createAndStoreObject.js b/lib/api/apiUtils/object/createAndStoreObject.js index d1b17b0e44..274c78ae23 100644 --- a/lib/api/apiUtils/object/createAndStoreObject.js +++ b/lib/api/apiUtils/object/createAndStoreObject.js @@ -14,11 +14,50 @@ const { config } = require('../../../Config'); const validateWebsiteHeader = require('./websiteServing') .validateWebsiteHeader; const applyZenkoUserMD = require('./applyZenkoUserMD'); +const { + algorithms, + getChecksumDataFromHeaders, + arsenalErrorFromChecksumError, +} = require('../integrity/validateChecksums'); const { externalBackends, versioningNotImplBackends } = constants; const externalVersioningErrorMessage = 'We do not currently support putting ' + -'a versioned object to a location-constraint of type Azure or GCP.'; + 'a versioned object to a location-constraint of type Azure or GCP.'; + +/** + * Validate and compute the checksum for a zero-size object body. + * Parses the checksum headers, validates the client-supplied digest against + * the empty-body hash, sets metadataStoreParams.checksum on success, and + * calls back with an error on mismatch or invalid headers. + * + * @param {object} headers - request headers + * @param {object} metadataStoreParams - metadata params (checksum field set in-place) + * @param {function} callback - (err) callback + * @return {undefined} + */ +function zeroSizeBodyChecksumCheck(headers, metadataStoreParams, callback) { + const checksumData = getChecksumDataFromHeaders(headers); + if (checksumData.error) { + return callback(arsenalErrorFromChecksumError(checksumData)); + } + // For trailer format with zero decoded bytes, the trailer in the body is + // never read (stream bypassed), so expected is always undefined here. + // We still compute and store the empty-body hash for the announced algorithm. + const { algorithm, expected } = checksumData; + return Promise.resolve(algorithms[algorithm].digest(Buffer.alloc(0))) + .then(value => { + if (expected !== undefined && expected !== value) { + return callback(errors.BadDigest.customizeDescription( + `The ${algorithm.toUpperCase()} you specified did not match the calculated checksum.` + )); + } + // eslint-disable-next-line no-param-reassign + metadataStoreParams.checksum = { algorithm, value, type: 'FULL_OBJECT' }; + return callback(null); + }) + .catch(err => callback(err)); +} function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle, metadataStoreParams, dataToDelete, log, requestMethod, callback) { @@ -217,7 +256,13 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, if (size === 0) { if (!dontSkipBackend[locationType]) { metadataStoreParams.contentMD5 = constants.emptyFileMd5; - return next(null, null, null, null); + // Delete markers are zero-byte versioned tombstones with + // no body, ETag, or checksum — skip checksum handling. + if (isDeleteMarker) { + return next(null, null, null, null); + } + return zeroSizeBodyChecksumCheck(request.headers, metadataStoreParams, + err => next(err, null, null, null)); } // Handle mdOnlyHeader as a metadata only operation. If @@ -264,7 +309,8 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, : `1:${calculatedHash}`; const dataGetInfoArr = [{ key, size, start: 0, dataStoreName, dataStoreType, dataStoreETag: prefixedDataStoreETag, - dataStoreVersionId }]; + dataStoreVersionId + }]; if (cipherBundle) { dataGetInfoArr[0].cryptoScheme = cipherBundle.cryptoScheme; dataGetInfoArr[0].cipheredDataKey = diff --git a/tests/functional/raw-node/test/checksumPutObjectUploadPart.js b/tests/functional/raw-node/test/checksumPutObjectUploadPart.js index fd8db840df..0a4fc09ce3 100644 --- a/tests/functional/raw-node/test/checksumPutObjectUploadPart.js +++ b/tests/functional/raw-node/test/checksumPutObjectUploadPart.js @@ -732,11 +732,11 @@ describe('PutObject: checksum response header per algorithm', () => { }); const checksumAlgos = [ - { name: 'crc32', computeExpected: () => algorithms.crc32.digest(body) }, - { name: 'crc32c', computeExpected: () => algorithms.crc32c.digest(body) }, + { name: 'crc32', computeExpected: () => algorithms.crc32.digest(body) }, + { name: 'crc32c', computeExpected: () => algorithms.crc32c.digest(body) }, { name: 'crc64nvme', computeExpected: () => expectedCrc64nvme }, - { name: 'sha1', computeExpected: () => algorithms.sha1.digest(body) }, - { name: 'sha256', computeExpected: () => algorithms.sha256.digest(body) }, + { name: 'sha1', computeExpected: () => algorithms.sha1.digest(body) }, + { name: 'sha256', computeExpected: () => algorithms.sha256.digest(body) }, ]; for (const algo of checksumAlgos) { @@ -778,3 +778,126 @@ describe('PutObject: checksum response header per algorithm', () => { } ); }); + +describe('PutObject: zero-byte object checksum handling', () => { + const zeroBucket = 'checksumzerobytebucket'; + const zeroUrl = `http://localhost:8000/${zeroBucket}/${objectKey}`; + const emptyBody = Buffer.alloc(0); + + // Known empty-body checksums (well-known constants except crc64nvme which is async) + // crc32("") = crc32c("") = 0x00000000 => AAAAAA== + // sha1("") = 2jmj7l5rSw0yVb/vlWAYkK/YBwk= + // sha256("") = 47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU= + const expectedEmptyChecksums = { + crc32: 'AAAAAA==', + crc32c: 'AAAAAA==', + sha1: '2jmj7l5rSw0yVb/vlWAYkK/YBwk=', + sha256: '47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=', + crc64nvme: null, // filled in before hook (async) + }; + + // Wrong digests for empty body: valid format but not the correct empty-body hash. + // crc32/crc32c of "" = AAAAAA==, so we use a different value for those. + const wrongEmptyDigests = { + crc32: 'EyV5Tg==', + crc32c: 'EyV5Tg==', + crc64nvme: 'skQv82y5rgE=', + sha1: 'AAAAAAAAAAAAAAAAAAAAAAAAAAA=', + sha256: 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', + }; + + before(async () => { + await new Promise((resolve, reject) => + makeS3Request({ method: 'PUT', authCredentials, bucket: zeroBucket }, + err => (err ? reject(err) : resolve()))); + expectedEmptyChecksums.crc64nvme = await algorithms.crc64nvme.digest(emptyBody); + }); + + after(done => { + makeS3Request({ method: 'DELETE', authCredentials, bucket: zeroBucket, objectKey }, () => { + makeS3Request({ method: 'DELETE', authCredentials, bucket: zeroBucket }, err => { + assert.ifError(err); + done(); + }); + }); + }); + + itSkipIfAWS( + 'no checksum header: returns 200 with x-amz-checksum-crc64nvme of empty body', + done => { + doPutRequest(zeroUrl, { + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + 'content-length': 0, + }, emptyBody, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200, + `expected 200, got ${res.statusCode}: ${res.body}`); + assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], + expectedEmptyChecksums.crc64nvme, + `expected x-amz-checksum-crc64nvme: ${expectedEmptyChecksums.crc64nvme}`); + done(); + }); + }); + + for (const algoName of ['crc32', 'crc32c', 'crc64nvme', 'sha1', 'sha256']) { + itSkipIfAWS( + `correct empty-body ${algoName} checksum: returns 200 with echoed response header`, + done => { + const expected = expectedEmptyChecksums[algoName]; + const headerName = `x-amz-checksum-${algoName}`; + doPutRequest(zeroUrl, { + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + [headerName]: expected, + 'content-length': 0, + }, emptyBody, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200, + `expected 200, got ${res.statusCode}: ${res.body}`); + assert.strictEqual(res.headers[headerName], expected, + `expected ${headerName}: ${expected}`); + done(); + }); + }); + + itSkipIfAWS( + `wrong empty-body ${algoName} checksum: returns 400 BadDigest`, + done => { + const wrong = wrongEmptyDigests[algoName]; + const headerName = `x-amz-checksum-${algoName}`; + doPutRequest(zeroUrl, { + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + [headerName]: wrong, + 'content-length': 0, + }, emptyBody, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 400, + `expected 400, got ${res.statusCode}: ${res.body}`); + assert(res.body.includes('BadDigest'), + `expected BadDigest in: ${res.body}`); + done(); + }); + }); + } + + itSkipIfAWS( + 'TRAILER with zero decoded content length: returns 200 with checksum for the trailer algorithm', + done => { + // x-amz-decoded-content-length: 0 → parsedContentLength === 0, hits the zero-byte path. + // The trailer body is never consumed; server computes and stores the empty-body hash itself. + const emptySha256 = expectedEmptyChecksums.sha256; + const trailerBody = `0\r\nx-amz-checksum-sha256:${emptySha256}\n\r\n\r\n\r\n`; + doPutRequest(zeroUrl, { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': 0, + 'content-length': Buffer.byteLength(trailerBody), + }, trailerBody, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200, + `expected 200, got ${res.statusCode}: ${res.body}`); + assert.strictEqual(res.headers['x-amz-checksum-sha256'], emptySha256, + `expected x-amz-checksum-sha256: ${emptySha256}`); + done(); + }); + }); +}); From 3ad3dafd191217a81e2ceb2132ea833f3f5ca018 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Tue, 17 Mar 2026 20:37:52 +0100 Subject: [PATCH 10/15] CLDSRV-872: fix failing test, checksum lost when restoring object --- tests/functional/aws-node-sdk/test/object/mpuVersion.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/functional/aws-node-sdk/test/object/mpuVersion.js b/tests/functional/aws-node-sdk/test/object/mpuVersion.js index 98c3155e95..0cdeee830f 100644 --- a/tests/functional/aws-node-sdk/test/object/mpuVersion.js +++ b/tests/functional/aws-node-sdk/test/object/mpuVersion.js @@ -141,6 +141,14 @@ function checkObjMdAndUpdate(objMDBefore, objMDAfter, props) { // eslint-disable-next-line no-param-reassign delete objMDBefore['content-type']; } + if (objMDBefore.checksum && !objMDAfter.checksum) { + // The initial PutObject stores a checksum, but the MPU restore path does not + // (CompleteMultipartUpload checksum storage is not yet implemented). + // Once it is, the restored object should carry a checksum and this workaround + // should be removed. + // eslint-disable-next-line no-param-reassign + delete objMDBefore.checksum; + } } function clearUploadIdAndRestoreStatusFromVersions(versions) { From a3d952d3d71d39e71e192b9d67c2e16135a2c9d3 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Wed, 18 Mar 2026 12:52:13 +0100 Subject: [PATCH 11/15] CLDSRV-872: test checksums are respected in backbeat routes --- lib/routes/routeBackbeat.js | 10 ++ tests/multipleBackend/routes/routeBackbeat.js | 103 ++++++++++++++++++ 2 files changed, 113 insertions(+) diff --git a/lib/routes/routeBackbeat.js b/lib/routes/routeBackbeat.js index ba834e5e3e..12605d947f 100644 --- a/lib/routes/routeBackbeat.js +++ b/lib/routes/routeBackbeat.js @@ -447,6 +447,11 @@ function putData(request, response, bucketInfo, objMd, log, callback) { } return dataStore( context, cipherBundle, request, payloadLen, {}, + // checksum (4th arg) intentionally ignored: any x-amz-checksum-* + // header sent by Backbeat is already validated inside dataStore by + // ChecksumTransform. The computed value is not stored here because + // this is a data-only write — metadata is written separately by + // Backbeat, which should propagate the source object's checksum. backendInfo, log, (err, retrievalInfo, md5) => { if (err) { log.error('error putting data', { @@ -852,6 +857,11 @@ function putObject(request, response, log, callback) { } const payloadLen = parseInt(request.headers['content-length'], 10); const backendInfo = new BackendInfo(config, storageLocation); + // checksum (4th arg) intentionally ignored: any x-amz-checksum-* header + // sent by Backbeat is already validated inside dataStore by ChecksumTransform. + // The computed value is not stored here because this is a data-only write to + // an external backend — metadata is managed separately by Backbeat, which + // should propagate the source object's checksum. return dataStore(context, CIPHER, request, payloadLen, {}, backendInfo, log, (err, retrievalInfo, md5) => { if (err) { diff --git a/tests/multipleBackend/routes/routeBackbeat.js b/tests/multipleBackend/routes/routeBackbeat.js index d8a43b7b75..2cec218254 100644 --- a/tests/multipleBackend/routes/routeBackbeat.js +++ b/tests/multipleBackend/routes/routeBackbeat.js @@ -3642,4 +3642,107 @@ describe('backbeat routes', () => { ], done); }); }); + + describe('checksum validation', () => { + const testDataSha256B64 = crypto.createHash('sha256') + .update(testData, 'utf-8').digest('base64'); + // A valid-length but wrong sha256 digest (44 base64 chars). + const wrongSha256B64 = 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='; + + describe('putData', () => { + it('should return 400 BadDigest when x-amz-checksum-sha256 does not match body', done => { + makeBackbeatRequest({ + method: 'PUT', + resourceType: 'data', + bucket: TEST_BUCKET, + objectKey: TEST_KEY, + headers: { + 'x-scal-canonical-id': testMd['owner-id'], + 'content-md5': testDataMd5, + 'content-length': testData.length, + 'x-amz-checksum-sha256': wrongSha256B64, + }, + requestBody: testData, + authCredentials: backbeatAuthCredentials, + }, err => { + assert(err, 'expected an error response'); + assert.strictEqual(err.statusCode, 400); + assert.strictEqual(err.code, 'BadDigest'); + done(); + }); + }); + + it('should return 200 when x-amz-checksum-sha256 matches body', done => { + makeBackbeatRequest({ + method: 'PUT', + resourceType: 'data', + bucket: TEST_BUCKET, + objectKey: TEST_KEY, + headers: { + 'x-scal-canonical-id': testMd['owner-id'], + 'content-md5': testDataMd5, + 'content-length': testData.length, + 'x-amz-checksum-sha256': testDataSha256B64, + }, + requestBody: testData, + authCredentials: backbeatAuthCredentials, + }, (err, data) => { + assert.ifError(err); + assert.strictEqual(data.statusCode, 200); + done(); + }); + }); + }); + + describe('putObject (multiplebackenddata)', () => { + itIfLocationAws('should return 400 BadDigest when x-amz-checksum-sha256 does not match body', done => { + makeBackbeatRequest({ + method: 'PUT', + resourceType: 'multiplebackenddata', + bucket: TEST_BUCKET, + objectKey: TEST_KEY, + queryObj: { operation: 'putobject' }, + headers: { + 'x-scal-canonical-id': testMd['owner-id'], + 'x-scal-storage-type': 'aws_s3', + 'x-scal-storage-class': awsLocation, + 'content-md5': testDataMd5, + 'content-length': testData.length, + 'x-amz-checksum-sha256': wrongSha256B64, + }, + requestBody: testData, + authCredentials: backbeatAuthCredentials, + }, err => { + assert(err, 'expected an error response'); + assert.strictEqual(err.statusCode, 400); + assert.strictEqual(err.code, 'BadDigest'); + done(); + }); + }); + + itIfLocationAws('should return 200 when x-amz-checksum-sha256 matches body', done => { + makeBackbeatRequest({ + method: 'PUT', + resourceType: 'multiplebackenddata', + bucket: TEST_BUCKET, + objectKey: TEST_KEY, + queryObj: { operation: 'putobject' }, + headers: { + 'x-scal-canonical-id': testMd['owner-id'], + 'x-scal-storage-type': 'aws_s3', + 'x-scal-storage-class': awsLocation, + 'content-md5': testDataMd5, + 'content-length': testData.length, + 'x-amz-checksum-sha256': testDataSha256B64, + }, + requestBody: testData, + authCredentials: backbeatAuthCredentials, + }, (err, data) => { + assert.ifError(err); + assert.strictEqual(data.statusCode, 200); + done(); + }); + }); + }); + }); }); From e2419b7f10cf2a0d2c11be82efef75c0f7011d5d Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Wed, 18 Mar 2026 12:58:05 +0100 Subject: [PATCH 12/15] CLDSRV-872: add missing jsdoc --- lib/api/apiUtils/object/storeObject.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/api/apiUtils/object/storeObject.js b/lib/api/apiUtils/object/storeObject.js index 7963e97aa5..f4a90e8066 100644 --- a/lib/api/apiUtils/object/storeObject.js +++ b/lib/api/apiUtils/object/storeObject.js @@ -12,6 +12,7 @@ const { arsenalErrorFromChecksumError } = require('../../apiUtils/integrity/vali * @param {object} dataRetrievalInfo - object containing the keys of stored data * @param {number} dataRetrievalInfo.key - key of the stored data * @param {string} dataRetrievalInfo.dataStoreName - the implName of the data + * @param {object} checksumStream - checksum transform stream with digest/algoName properties * @param {object} log - request logger instance * @param {function} cb - callback to send error or move to next task * @return {function} - calls callback with arguments: From 3d60da36909c66e706186efdcc9400244ce71443 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Wed, 18 Mar 2026 13:09:18 +0100 Subject: [PATCH 13/15] fixup! CLDSRV-872: test checksums are respected in backbeat routes --- lib/routes/routeBackbeat.js | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/lib/routes/routeBackbeat.js b/lib/routes/routeBackbeat.js index 12605d947f..718d19d9eb 100644 --- a/lib/routes/routeBackbeat.js +++ b/lib/routes/routeBackbeat.js @@ -447,12 +447,14 @@ function putData(request, response, bucketInfo, objMd, log, callback) { } return dataStore( context, cipherBundle, request, payloadLen, {}, - // checksum (4th arg) intentionally ignored: any x-amz-checksum-* - // header sent by Backbeat is already validated inside dataStore by - // ChecksumTransform. The computed value is not stored here because - // this is a data-only write — metadata is written separately by - // Backbeat, which should propagate the source object's checksum. - backendInfo, log, (err, retrievalInfo, md5) => { + backendInfo, log, + // The callback's 4th arg (checksum) is intentionally ignored: any + // x-amz-checksum-* header sent by Backbeat is already validated + // inside dataStore by ChecksumTransform. The computed value is not + // stored here because this is a data-only write — metadata is + // written separately by Backbeat, which should propagate the source + // object's checksum. + (err, retrievalInfo, md5) => { if (err) { log.error('error putting data', { error: err, @@ -857,12 +859,13 @@ function putObject(request, response, log, callback) { } const payloadLen = parseInt(request.headers['content-length'], 10); const backendInfo = new BackendInfo(config, storageLocation); - // checksum (4th arg) intentionally ignored: any x-amz-checksum-* header - // sent by Backbeat is already validated inside dataStore by ChecksumTransform. - // The computed value is not stored here because this is a data-only write to - // an external backend — metadata is managed separately by Backbeat, which - // should propagate the source object's checksum. return dataStore(context, CIPHER, request, payloadLen, {}, backendInfo, log, + // The callback's 4th arg (checksum) is intentionally ignored: any + // x-amz-checksum-* header sent by Backbeat is already validated inside + // dataStore by ChecksumTransform. The computed value is not stored here + // because this is a data-only write to an external backend — metadata + // is managed separately by Backbeat, which should propagate the source + // object's checksum. (err, retrievalInfo, md5) => { if (err) { log.error('error putting data', { From 180d9402b8b5975bc3702af7a2e4d3ba38025162 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Wed, 18 Mar 2026 13:18:43 +0100 Subject: [PATCH 14/15] fixup! CLDSRV-872: handle empty object checksum --- lib/api/apiUtils/object/createAndStoreObject.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/api/apiUtils/object/createAndStoreObject.js b/lib/api/apiUtils/object/createAndStoreObject.js index 274c78ae23..f306863a8e 100644 --- a/lib/api/apiUtils/object/createAndStoreObject.js +++ b/lib/api/apiUtils/object/createAndStoreObject.js @@ -55,8 +55,7 @@ function zeroSizeBodyChecksumCheck(headers, metadataStoreParams, callback) { // eslint-disable-next-line no-param-reassign metadataStoreParams.checksum = { algorithm, value, type: 'FULL_OBJECT' }; return callback(null); - }) - .catch(err => callback(err)); + }, err => callback(err)); } function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle, From 86267367ad0000ac1c76e3b06f1975fbd58a358d Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Wed, 18 Mar 2026 15:18:06 +0100 Subject: [PATCH 15/15] CLDSRV-873: return checksum in HeadObject --- lib/api/objectHead.js | 15 +++ .../aws-node-sdk/test/object/objectHead.js | 87 ++++++++++++++- tests/unit/api/objectHead.js | 101 ++++++++++++++++++ .../unit/api/utils/metadataMockColdStorage.js | 1 + 4 files changed, 203 insertions(+), 1 deletion(-) diff --git a/lib/api/objectHead.js b/lib/api/objectHead.js index 2d4c524f21..80b7613371 100644 --- a/lib/api/objectHead.js +++ b/lib/api/objectHead.js @@ -32,6 +32,12 @@ function objectHead(authInfo, request, log, callback) { const bucketName = request.bucketName; const objectKey = request.objectKey; + const checksumMode = request.headers['x-amz-checksum-mode']; + if (checksumMode !== undefined && checksumMode !== 'ENABLED') { + log.debug('invalid x-amz-checksum-mode', { checksumMode }); + return callback(errors.InvalidArgument); + } + const decodedVidResult = decodeVersionId(request.query); if (decodedVidResult instanceof Error) { log.trace('invalid versionId query', { @@ -97,6 +103,15 @@ function objectHead(authInfo, request, log, callback) { const responseHeaders = collectResponseHeaders(objMD, corsHeaders, verCfg); + if (checksumMode === 'ENABLED') { + const checksum = objMD.checksum; + if (checksum) { + responseHeaders[`x-amz-checksum-${checksum.checksumAlgorithm}`] + = checksum.checksumValue; + responseHeaders['x-amz-checksum-type'] = checksum.checksumType; + } + } + setExpirationHeaders(responseHeaders, { lifecycleConfig: bucket.getLifecycleConfiguration(), objectParams: { diff --git a/tests/functional/aws-node-sdk/test/object/objectHead.js b/tests/functional/aws-node-sdk/test/object/objectHead.js index 92e2b64799..b59f54400d 100644 --- a/tests/functional/aws-node-sdk/test/object/objectHead.js +++ b/tests/functional/aws-node-sdk/test/object/objectHead.js @@ -14,7 +14,14 @@ const { UploadPartCommand, CompleteMultipartUploadCommand, } = require('@aws-sdk/client-s3'); - +// Register the CRT-based CRC64NVME implementation with the SDK middleware. +// The crc64-nvme-crt package sets its own container but the flexible-checksums +// middleware has a separate container that must be patched explicitly. +const { CrtCrc64Nvme } = require('@aws-sdk/crc64-nvme-crt'); +const { crc64NvmeCrtContainer } = require('@aws-sdk/middleware-flexible-checksums'); +crc64NvmeCrtContainer.CrtCrc64Nvme = CrtCrc64Nvme; + +const { algorithms } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); const changeObjectLock = require('../../../../utilities/objectLock-util'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); @@ -538,6 +545,84 @@ describe('HEAD object, conditions', () => { }); }); +describe('HEAD object checksum mode', () => { + withV4(sigCfg => { + let bucketUtil; + let s3; + const checksumBucket = 'checksum-headobject-test'; + const checksumKey = 'checksum-test-object'; + const body = Buffer.from('checksum test body'); + + // Expected base64-encoded digests of `body` for each algorithm, + // computed once in the before hook (crc64nvme digest is async). + const expectedDigests = {}; + + before(async () => { + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; + await s3.send(new CreateBucketCommand({ Bucket: checksumBucket })); + + for (const { internalName } of checksumAlgorithms) { + // algorithms[internalName].digest() returns a base64 string + expectedDigests[internalName] = + await algorithms[internalName].digest(body); + } + }); + + after(async () => { + await bucketUtil.empty(checksumBucket); + await s3.send(new DeleteBucketCommand({ Bucket: checksumBucket })); + }); + + const checksumAlgorithms = [ + { algorithm: 'SHA256', responseField: 'ChecksumSHA256', internalName: 'sha256' }, + { algorithm: 'SHA1', responseField: 'ChecksumSHA1', internalName: 'sha1' }, + { algorithm: 'CRC32', responseField: 'ChecksumCRC32', internalName: 'crc32' }, + { algorithm: 'CRC32C', responseField: 'ChecksumCRC32C', internalName: 'crc32c' }, + { algorithm: 'CRC64NVME', responseField: 'ChecksumCRC64NVME', internalName: 'crc64nvme' }, + ]; + + checksumAlgorithms.forEach(({ algorithm, responseField, internalName }) => { + it(`should return ${responseField} and ChecksumType when ChecksumMode is ENABLED`, async () => { + const putRes = await s3.send(new PutObjectCommand({ + Bucket: checksumBucket, + Key: checksumKey, + Body: body, + ChecksumAlgorithm: algorithm, + })); + const storedChecksum = putRes[responseField]; + assert(storedChecksum, `Expected ${responseField} in PutObject response`); + + const headRes = await s3.send(new HeadObjectCommand({ + Bucket: checksumBucket, + Key: checksumKey, + ChecksumMode: 'ENABLED', + })); + assert.strictEqual(headRes[responseField], expectedDigests[internalName], + `${responseField} value mismatch`); + assert.strictEqual(headRes[responseField], storedChecksum); + assert.strictEqual(headRes.ChecksumType, 'FULL_OBJECT'); + }); + }); + + it('should not return checksum headers when ChecksumMode is not set', async () => { + await s3.send(new PutObjectCommand({ + Bucket: checksumBucket, + Key: checksumKey, + Body: body, + ChecksumAlgorithm: 'SHA256', + })); + + const headRes = await s3.send(new HeadObjectCommand({ + Bucket: checksumBucket, + Key: checksumKey, + })); + assert.strictEqual(headRes.ChecksumSHA256, undefined); + assert.strictEqual(headRes.ChecksumType, undefined); + }); + }); +}); + const isCEPH = process.env.CI_CEPH !== undefined; const describeSkipIfCeph = isCEPH ? describe.skip : describe; diff --git a/tests/unit/api/objectHead.js b/tests/unit/api/objectHead.js index 511a39037c..e5dc669cbb 100644 --- a/tests/unit/api/objectHead.js +++ b/tests/unit/api/objectHead.js @@ -1,4 +1,7 @@ const assert = require('assert'); +const { models } = require('arsenal'); +const { ObjectMD, ObjectMDChecksum } = models; +const { algorithms } = require('../../../lib/api/apiUtils/integrity/validateChecksums'); const { bucketPut } = require('../../../lib/api/bucketPut'); const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); @@ -565,6 +568,104 @@ describe('objectHead API', () => { }); }); + describe('x-amz-checksum-mode', () => { + const checksumAlgorithms = [ + { name: 'sha256', header: 'x-amz-checksum-sha256' }, + { name: 'sha1', header: 'x-amz-checksum-sha1' }, + { name: 'crc32', header: 'x-amz-checksum-crc32' }, + { name: 'crc32c', header: 'x-amz-checksum-crc32c' }, + { name: 'crc64nvme', header: 'x-amz-checksum-crc64nvme' }, + ]; + + // Digests of postBody ("I am a body") for each algorithm, computed once. + const expectedDigests = {}; + + before(done => { + Promise.all(checksumAlgorithms.map(async ({ name }) => { + expectedDigests[name] = await algorithms[name].digest(postBody); + })).then(() => done(), done); + }); + + checksumAlgorithms.forEach(({ name, header }) => { + it(`should return ${header} and x-amz-checksum-type when mode is ENABLED`, done => { + const md = new ObjectMD(mdColdHelper.baseMd) + .setChecksum(new ObjectMDChecksum(name, expectedDigests[name], 'FULL_OBJECT')); + mdColdHelper.putBucketMock(bucketName, null, () => + mdColdHelper.putObjectMock(bucketName, objectName, md, () => { + const req = { + bucketName, + namespace, + objectKey: objectName, + headers: { 'x-amz-checksum-mode': 'ENABLED' }, + url: `/${bucketName}/${objectName}`, + }; + objectHead(authInfo, req, log, (err, res) => { + assert.ifError(err); + assert.strictEqual(res[header], expectedDigests[name]); + assert.strictEqual(res['x-amz-checksum-type'], 'FULL_OBJECT'); + done(); + }); + })); + }); + }); + + it('should not return checksum headers when mode is ENABLED but object has no checksum', done => { + mdColdHelper.putBucketMock(bucketName, null, () => + mdColdHelper.putObjectMock(bucketName, objectName, undefined, () => { + const req = { + bucketName, + namespace, + objectKey: objectName, + headers: { 'x-amz-checksum-mode': 'ENABLED' }, + url: `/${bucketName}/${objectName}`, + }; + objectHead(authInfo, req, log, (err, res) => { + assert.ifError(err); + checksumAlgorithms.forEach(({ header }) => + assert.strictEqual(res[header], undefined)); + assert.strictEqual(res['x-amz-checksum-type'], undefined); + done(); + }); + })); + }); + + it('should not return checksum headers when x-amz-checksum-mode is not set', done => { + const md = new ObjectMD(mdColdHelper.baseMd) + .setChecksum(new ObjectMDChecksum('sha256', expectedDigests.sha256, 'FULL_OBJECT')); + mdColdHelper.putBucketMock(bucketName, null, () => + mdColdHelper.putObjectMock(bucketName, objectName, md, () => { + const req = { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + }; + objectHead(authInfo, req, log, (err, res) => { + assert.ifError(err); + checksumAlgorithms.forEach(({ header }) => + assert.strictEqual(res[header], undefined)); + assert.strictEqual(res['x-amz-checksum-type'], undefined); + done(); + }); + })); + }); + + it('should return InvalidArgument when x-amz-checksum-mode is not ENABLED', done => { + const req = { + bucketName, + namespace, + objectKey: objectName, + headers: { 'x-amz-checksum-mode': 'DISABLED' }, + url: `/${bucketName}/${objectName}`, + }; + objectHead(authInfo, req, log, err => { + assert.strictEqual(err.is.InvalidArgument, true); + done(); + }); + }); + }); + [ { name: 'should return content-length of 0 when requesting part 1 of empty object', diff --git a/tests/unit/api/utils/metadataMockColdStorage.js b/tests/unit/api/utils/metadataMockColdStorage.js index b1f6496418..e2052e5277 100644 --- a/tests/unit/api/utils/metadataMockColdStorage.js +++ b/tests/unit/api/utils/metadataMockColdStorage.js @@ -202,6 +202,7 @@ function getDeleteMarkerObjectMD(versionId) { } module.exports = { + baseMd, putObjectMock, getArchivedObjectMD, getRestoringObjectMD,